20260408_2 增加CIR sequence,未验证drop analysis,遇到问题是static pool保存太慢,拖慢整体录制,待解决

This commit is contained in:
2026-04-09 16:08:11 +08:00
parent c9ef5aaf4c
commit e083fe4daa
26 changed files with 3233 additions and 15 deletions

View File

@@ -2,6 +2,7 @@ use std::collections::BTreeMap;
use std::collections::BTreeSet;
use std::path::Path;
use crate::audit::{AuditObjectResult, PublicationPointAudit};
use crate::cir::encode::{CirEncodeError, encode_cir};
use crate::cir::model::{
CIR_VERSION_V1, CanonicalInputRepresentation, CirHashAlgorithm, CirObject, CirTal,
@@ -13,6 +14,14 @@ use crate::cir::static_pool::{
use crate::data_model::ta::TrustAnchor;
use crate::storage::{RepositoryViewState, RocksStore};
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct CirExportTiming {
pub build_cir_ms: u64,
pub static_pool_ms: u64,
pub write_cir_ms: u64,
pub total_ms: u64,
}
#[derive(Debug, thiserror::Error)]
pub enum CirExportError {
#[error("list repository_view entries failed: {0}")]
@@ -42,6 +51,7 @@ pub struct CirExportSummary {
pub object_count: usize,
pub tal_count: usize,
pub static_pool: CirStaticPoolExportSummary,
pub timing: CirExportTiming,
}
pub fn build_cir_from_run(
@@ -49,6 +59,7 @@ pub fn build_cir_from_run(
trust_anchor: &TrustAnchor,
tal_uri: &str,
validation_time: time::OffsetDateTime,
publication_points: &[PublicationPointAudit],
) -> Result<CanonicalInputRepresentation, CirExportError> {
if !(tal_uri.starts_with("https://") || tal_uri.starts_with("http://")) {
return Err(CirExportError::InvalidTalUri(tal_uri.to_string()));
@@ -69,6 +80,26 @@ pub fn build_cir_from_run(
}
}
// CIR must describe the actual input world used by validation. When a
// publication point falls back to the latest validated current instance,
// repository_view may not contain the reused manifest/object set. Pull
// those object hashes from the audit so replay can reconstruct the same
// world state.
for pp in publication_points {
if pp.source != "vcir_current_instance" {
continue;
}
for obj in &pp.objects {
if obj.result != AuditObjectResult::Ok {
continue;
}
if !obj.rsync_uri.starts_with("rsync://") {
continue;
}
objects.insert(obj.rsync_uri.clone(), obj.sha256_hex.to_ascii_lowercase());
}
}
let ta_hash = ta_sha256_hex(&trust_anchor.ta_certificate.raw_der);
let mut saw_rsync_uri = false;
for uri in &trust_anchor.tal.ta_uris {
@@ -149,17 +180,41 @@ pub fn export_cir_from_run(
trust_anchor: &TrustAnchor,
tal_uri: &str,
validation_time: time::OffsetDateTime,
publication_points: &[PublicationPointAudit],
cir_out: &Path,
static_root: &Path,
capture_date_utc: time::Date,
) -> Result<CirExportSummary, CirExportError> {
let cir = build_cir_from_run(store, trust_anchor, tal_uri, validation_time)?;
let total_started = std::time::Instant::now();
let started = std::time::Instant::now();
let cir = build_cir_from_run(
store,
trust_anchor,
tal_uri,
validation_time,
publication_points,
)?;
let build_cir_ms = started.elapsed().as_millis() as u64;
let started = std::time::Instant::now();
let static_pool = export_cir_static_pool(store, static_root, capture_date_utc, &cir, trust_anchor)?;
let static_pool_ms = started.elapsed().as_millis() as u64;
let started = std::time::Instant::now();
write_cir_file(cir_out, &cir)?;
let write_cir_ms = started.elapsed().as_millis() as u64;
Ok(CirExportSummary {
object_count: cir.objects.len(),
tal_count: cir.tals.len(),
static_pool,
timing: CirExportTiming {
build_cir_ms,
static_pool_ms,
write_cir_ms,
total_ms: total_started.elapsed().as_millis() as u64,
},
})
}
@@ -222,7 +277,13 @@ mod tests {
.unwrap();
let ta = sample_trust_anchor();
let cir = build_cir_from_run(&store, &ta, "https://example.test/root.tal", sample_time())
let cir = build_cir_from_run(
&store,
&ta,
"https://example.test/root.tal",
sample_time(),
&[],
)
.expect("build cir");
assert_eq!(cir.version, CIR_VERSION_V1);
assert_eq!(cir.tals.len(), 1);
@@ -267,6 +328,7 @@ mod tests {
&ta,
"https://example.test/root.tal",
sample_time(),
&[],
&cir_path,
&static_root,
sample_date(),
@@ -282,4 +344,48 @@ mod tests {
let object_path = static_pool_path(&static_root, sample_date(), &hash).unwrap();
assert_eq!(std::fs::read(object_path).unwrap(), bytes);
}
#[test]
fn build_cir_from_run_includes_vcir_current_instance_objects_from_audit() {
let td = tempfile::tempdir().unwrap();
let store = RocksStore::open(td.path()).unwrap();
let ta = sample_trust_anchor();
let mut pp = PublicationPointAudit {
source: "vcir_current_instance".to_string(),
..PublicationPointAudit::default()
};
pp.objects.push(crate::audit::ObjectAuditEntry {
rsync_uri: "rsync://example.test/repo/fallback.mft".to_string(),
sha256_hex: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa".to_string(),
kind: crate::audit::AuditObjectKind::Manifest,
result: crate::audit::AuditObjectResult::Ok,
detail: None,
});
pp.objects.push(crate::audit::ObjectAuditEntry {
rsync_uri: "rsync://example.test/repo/fallback.roa".to_string(),
sha256_hex: "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string(),
kind: crate::audit::AuditObjectKind::Roa,
result: crate::audit::AuditObjectResult::Ok,
detail: None,
});
let cir = build_cir_from_run(
&store,
&ta,
"https://example.test/root.tal",
sample_time(),
&[pp],
)
.expect("build cir");
assert!(cir
.objects
.iter()
.any(|item| item.rsync_uri == "rsync://example.test/repo/fallback.mft"));
assert!(cir
.objects
.iter()
.any(|item| item.rsync_uri == "rsync://example.test/repo/fallback.roa"));
}
}

View File

@@ -2,6 +2,7 @@ pub mod decode;
pub mod encode;
pub mod materialize;
pub mod model;
pub mod sequence;
#[cfg(feature = "full")]
pub mod export;
#[cfg(feature = "full")]
@@ -16,6 +17,7 @@ pub use materialize::{
pub use model::{
CIR_VERSION_V1, CirHashAlgorithm, CirObject, CirTal, CanonicalInputRepresentation,
};
pub use sequence::{CirSequenceManifest, CirSequenceStep, CirSequenceStepKind};
#[cfg(feature = "full")]
pub use export::{CirExportError, CirExportSummary, build_cir_from_run, export_cir_from_run, write_cir_file};
#[cfg(feature = "full")]

147
src/cir/sequence.rs Normal file
View File

@@ -0,0 +1,147 @@
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum CirSequenceStepKind {
Full,
Delta,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct CirSequenceStep {
pub step_id: String,
pub kind: CirSequenceStepKind,
pub validation_time: String,
pub cir_path: String,
pub ccr_path: String,
pub report_path: String,
pub previous_step_id: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub struct CirSequenceManifest {
pub version: u32,
pub static_root: String,
pub steps: Vec<CirSequenceStep>,
}
impl CirSequenceManifest {
pub fn validate(&self) -> Result<(), String> {
if self.version == 0 {
return Err("sequence.version must be positive".to_string());
}
if self.static_root.trim().is_empty() {
return Err("sequence.static_root must not be empty".to_string());
}
if self.steps.is_empty() {
return Err("sequence.steps must not be empty".to_string());
}
let mut previous_ids = std::collections::BTreeSet::new();
for (idx, step) in self.steps.iter().enumerate() {
if step.step_id.trim().is_empty() {
return Err(format!("sequence.steps[{idx}].step_id must not be empty"));
}
if !previous_ids.insert(step.step_id.clone()) {
return Err(format!("sequence.steps[{idx}].step_id must be unique"));
}
if step.validation_time.trim().is_empty() {
return Err(format!(
"sequence.steps[{idx}].validation_time must not be empty"
));
}
if step.cir_path.trim().is_empty()
|| step.ccr_path.trim().is_empty()
|| step.report_path.trim().is_empty()
{
return Err(format!(
"sequence.steps[{idx}] output paths must not be empty"
));
}
match step.kind {
CirSequenceStepKind::Full => {
if idx != 0 {
return Err("full step must be the first step".to_string());
}
if step.previous_step_id.is_some() {
return Err("full step must not reference previous_step_id".to_string());
}
}
CirSequenceStepKind::Delta => {
if idx == 0 {
return Err("delta step cannot be the first step".to_string());
}
let previous = step.previous_step_id.as_ref().ok_or_else(|| {
format!("sequence.steps[{idx}] delta step must set previous_step_id")
})?;
if !previous_ids.contains(previous) {
return Err(format!(
"sequence.steps[{idx}] previous_step_id must reference an earlier step"
));
}
}
}
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::{CirSequenceManifest, CirSequenceStep, CirSequenceStepKind};
fn sample_manifest() -> CirSequenceManifest {
CirSequenceManifest {
version: 1,
static_root: "static".to_string(),
steps: vec![
CirSequenceStep {
step_id: "full".to_string(),
kind: CirSequenceStepKind::Full,
validation_time: "2026-04-09T00:00:00Z".to_string(),
cir_path: "full/input.cir".to_string(),
ccr_path: "full/result.ccr".to_string(),
report_path: "full/report.json".to_string(),
previous_step_id: None,
},
CirSequenceStep {
step_id: "delta-001".to_string(),
kind: CirSequenceStepKind::Delta,
validation_time: "2026-04-09T00:10:00Z".to_string(),
cir_path: "delta-001/input.cir".to_string(),
ccr_path: "delta-001/result.ccr".to_string(),
report_path: "delta-001/report.json".to_string(),
previous_step_id: Some("full".to_string()),
},
],
}
}
#[test]
fn sequence_manifest_validate_accepts_minimal_chain() {
sample_manifest().validate().expect("valid sequence");
}
#[test]
fn sequence_manifest_validate_rejects_bad_order_and_duplicates() {
let mut bad = sample_manifest();
bad.steps.swap(0, 1);
let err = bad.validate().expect_err("full must be first");
assert!(
err.contains("delta step cannot be the first step")
|| err.contains("full step must be the first step")
);
let mut dup = sample_manifest();
dup.steps[1].step_id = "full".to_string();
let err = dup.validate().expect_err("duplicate id must fail");
assert!(err.contains("must be unique"));
}
#[test]
fn sequence_manifest_validate_rejects_missing_previous_reference() {
let mut bad = sample_manifest();
bad.steps[1].previous_step_id = Some("missing".to_string());
let err = bad.validate().expect_err("missing previous must fail");
assert!(err.contains("previous_step_id"));
}
}