From 8cf3d09a232c232cd5933fdb67604c00b210c822 Mon Sep 17 00:00:00 2001 From: Sharang Parnerkar <30073382+mighty840@users.noreply.github.com> Date: Tue, 21 Jul 2026 13:12:01 +0200 Subject: [PATCH] feat(agent): cache control embedding index + auto-wire semantic pass (gated) Item 2 of the semantic-mapping follow-ups. Caching (2a): ControlIndex::load_or_build persists the embedded corpus to snapshot_dir keyed by a corpus hash (control ids + requirement text). A later scan reuses the embeddings unless the catalog changed, turning the per-scan re-embed of the ~13.6k master-control corpus into a one-time cost. Atomic write (temp + rename); self-invalidates on catalog change. ControlCheckSpec gains Serialize/Deserialize to persist. Auto-wire (2b): orchestrator Stage 5c runs semantic_stamp_findings after control triage, gated on breakpilot.semantic_mapping (env BREAKPILOT_SEMANTIC_MAPPING, default off). Stays off until verified live against a deployed master-controls catalog; safe no-op meanwhile (the endpoint 400s pre-deploy and the pass warns + returns 0). Co-Authored-By: Claude Fable 5 --- compliance-agent/src/config.rs | 3 + compliance-agent/src/controls/index.rs | 135 ++++++++++++++++++ compliance-agent/src/controls/scan_triage.rs | 12 +- compliance-agent/src/pipeline/orchestrator.rs | 22 +++ compliance-core/src/config.rs | 6 + compliance-core/src/control_check.rs | 3 +- 6 files changed, 176 insertions(+), 5 deletions(-) diff --git a/compliance-agent/src/config.rs b/compliance-agent/src/config.rs index 257f554..a18044a 100644 --- a/compliance-agent/src/config.rs +++ b/compliance-agent/src/config.rs @@ -100,5 +100,8 @@ fn load_breakpilot_config() -> BreakpilotConfig { base_url: env_var_opt("BREAKPILOT_BASE_URL"), token: env_secret_opt("BREAKPILOT_TOKEN"), snapshot_dir: env_var_opt("BREAKPILOT_SNAPSHOT_DIR").unwrap_or(d.snapshot_dir), + semantic_mapping: env_var_opt("BREAKPILOT_SEMANTIC_MAPPING") + .map(|v| v == "1" || v.eq_ignore_ascii_case("true")) + .unwrap_or(d.semantic_mapping), } } diff --git a/compliance-agent/src/controls/index.rs b/compliance-agent/src/controls/index.rs index 494e918..35c06cc 100644 --- a/compliance-agent/src/controls/index.rs +++ b/compliance-agent/src/controls/index.rs @@ -6,6 +6,11 @@ //! requirement text once, then for a code region pull the top-K nearest controls //! to hand to the grounded judge. This is the retrieval half of the semantic path. +use std::path::Path; + +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + use compliance_core::control_check::ControlCheckSpec; use compliance_core::error::CoreError; @@ -16,6 +21,34 @@ pub struct ControlIndex { entries: Vec<(ControlCheckSpec, Vec)>, } +/// On-disk form of the index: the corpus identity hash plus every spec+embedding. +/// The hash lets a later scan reuse the embeddings only if the corpus is unchanged. +#[derive(Serialize, Deserialize)] +struct PersistedIndex { + corpus_hash: String, + entries: Vec, +} + +#[derive(Serialize, Deserialize)] +struct PersistedEntry { + spec: ControlCheckSpec, + embedding: Vec, +} + +/// Stable hash of the corpus identity (each control's id + requirement text, in +/// order). Same catalog → same hash → the cached embeddings are reused instead of +/// re-embedding the whole corpus. +fn corpus_hash(specs: &[ControlCheckSpec]) -> String { + let mut hasher = Sha256::new(); + for s in specs { + hasher.update(s.control_id.as_bytes()); + hasher.update([0u8]); + hasher.update(s.requirement.as_bytes()); + hasher.update([0u8]); + } + format!("{:x}", hasher.finalize()) +} + impl ControlIndex { /// Build directly from precomputed embeddings (used by tests + callers that /// already embedded the corpus). @@ -23,6 +56,70 @@ impl ControlIndex { Self { entries } } + /// Load the index from `cache_path` if it still matches the current corpus, + /// otherwise embed the corpus and persist it there. This turns the per-scan + /// re-embed of the whole (~13.6k) master-control corpus into a one-time cost + /// that survives across scans; the cache self-invalidates when the catalog + /// changes (its [`corpus_hash`] no longer matches). + pub async fn load_or_build( + llm: &LlmClient, + specs: Vec, + cache_path: &Path, + ) -> Result { + let hash = corpus_hash(&specs); + if let Some(index) = Self::load_cache(cache_path, &hash).await { + tracing::debug!( + controls = index.len(), + "reusing cached control embedding index" + ); + return Ok(index); + } + let index = Self::build(llm, specs).await?; + if let Err(e) = index.write_cache(cache_path, &hash).await { + tracing::warn!(error = %e, "failed to persist control embedding index"); + } + Ok(index) + } + + /// Read a persisted index, returning it only if its corpus hash matches. + async fn load_cache(path: &Path, hash: &str) -> Option { + let raw = tokio::fs::read(path).await.ok()?; + let persisted: PersistedIndex = serde_json::from_slice(&raw).ok()?; + if persisted.corpus_hash != hash { + return None; + } + Some(Self { + entries: persisted + .entries + .into_iter() + .map(|e| (e.spec, e.embedding)) + .collect(), + }) + } + + /// Persist the index atomically (temp file + rename) keyed by corpus hash. + async fn write_cache(&self, path: &Path, hash: &str) -> Result<(), CoreError> { + if let Some(parent) = path.parent() { + tokio::fs::create_dir_all(parent).await?; + } + let persisted = PersistedIndex { + corpus_hash: hash.to_string(), + entries: self + .entries + .iter() + .map(|(spec, emb)| PersistedEntry { + spec: spec.clone(), + embedding: emb.clone(), + }) + .collect(), + }; + let raw = serde_json::to_vec(&persisted)?; + let tmp = path.with_extension("json.tmp"); + tokio::fs::write(&tmp, &raw).await?; + tokio::fs::rename(&tmp, path).await?; + Ok(()) + } + /// Build by embedding each control's requirement text. pub async fn build(llm: &LlmClient, specs: Vec) -> Result { if specs.is_empty() { @@ -107,4 +204,42 @@ mod tests { assert_eq!(cosine(&[0.0, 0.0], &[1.0, 1.0]), 0.0); // zero vector assert!((cosine(&[1.0, 0.0], &[1.0, 0.0]) - 1.0).abs() < 1e-9); // identical } + + #[test] + fn corpus_hash_is_stable_and_identity_sensitive() { + let a = corpus_hash(&[spec("x"), spec("y")]); + assert_eq!(a, corpus_hash(&[spec("x"), spec("y")])); // same corpus → same hash + assert_ne!(a, corpus_hash(&[spec("y"), spec("x")])); // reorder → different + assert_ne!(a, corpus_hash(&[spec("x")])); // fewer controls → different + } + + #[tokio::test] + #[allow(clippy::unwrap_used)] + async fn cache_round_trips_and_misses_on_corpus_change() { + let dir = std::env::temp_dir().join(format!("cidx-{}", uuid::Uuid::new_v4())); + let path = dir.join("control-index.json"); + let specs = [spec("a"), spec("b")]; + let hash = corpus_hash(&specs); + let index = ControlIndex::from_embeddings(vec![ + (spec("a"), vec![1.0, 0.0]), + (spec("b"), vec![0.0, 1.0]), + ]); + index.write_cache(&path, &hash).await.unwrap(); + + // matching corpus hash → hit + let loaded = ControlIndex::load_cache(&path, &hash).await.unwrap(); + assert_eq!(loaded.len(), 2); + assert_eq!(loaded.nearest(&[0.9, 0.1], 1)[0].control_id, "a"); + // corpus changed → miss (forces a rebuild) + assert!(ControlIndex::load_cache(&path, "differenthash") + .await + .is_none()); + // absent file → miss, not an error + assert!( + ControlIndex::load_cache(dir.join("nope.json").as_path(), &hash) + .await + .is_none() + ); + let _ = std::fs::remove_dir_all(&dir); + } } diff --git a/compliance-agent/src/controls/scan_triage.rs b/compliance-agent/src/controls/scan_triage.rs index 3c7e62c..83c1b82 100644 --- a/compliance-agent/src/controls/scan_triage.rs +++ b/compliance-agent/src/controls/scan_triage.rs @@ -128,9 +128,11 @@ fn fetch_region(repo_path: &Path, file: &str, line: u32) -> Option, @@ -164,7 +166,9 @@ pub async fn semantic_stamp_findings( severity: Severity::Medium, }) .collect(); - let index = match ControlIndex::build(&llm, specs).await { + let cache_path = + Path::new(&config.breakpilot.snapshot_dir).join("control-index-master-controls.json"); + let index = match ControlIndex::load_or_build(&llm, specs, &cache_path).await { Ok(i) if !i.is_empty() => i, Ok(_) => return 0, Err(e) => { diff --git a/compliance-agent/src/pipeline/orchestrator.rs b/compliance-agent/src/pipeline/orchestrator.rs index 1e7b265..3614b37 100644 --- a/compliance-agent/src/pipeline/orchestrator.rs +++ b/compliance-agent/src/pipeline/orchestrator.rs @@ -230,6 +230,28 @@ impl PipelineOrchestrator { tracing::info!("[{repo_id}] Control triage tagged {tagged} findings with control refs"); } + // Stage 5c: semantic control mapping — scale path for the master-controls + // corpus (no CWE to LUT on): embed each finding's region, retrieve the + // nearest master controls, grounded-judge, and stamp confirmed refs. Gated + // (default off) as the corpus embedding + per-finding judging is the heavy + // path; enabled once verified live against a deployed master-controls catalog. + if self.config.breakpilot.semantic_mapping { + self.update_phase(scan_run_id, "semantic_control_mapping") + .await; + let sem = crate::controls::semantic_stamp_findings( + &self.config, + self.llm.clone(), + &repo_path, + &mut all_findings, + ) + .await; + if sem > 0 { + tracing::info!( + "[{repo_id}] Semantic mapping tagged {sem} findings with master-control refs" + ); + } + } + // Dedup against existing findings and insert new ones let mut new_count = 0u32; let mut new_findings: Vec = Vec::new(); diff --git a/compliance-core/src/config.rs b/compliance-core/src/config.rs index 4eadad1..222f89e 100644 --- a/compliance-core/src/config.rs +++ b/compliance-core/src/config.rs @@ -75,6 +75,11 @@ pub struct BreakpilotConfig { pub token: Option, /// Directory for catalog snapshots. pub snapshot_dir: String, + /// Enable the master-controls **semantic** mapping pass (embed regions, + /// retrieve nearest controls, grounded-judge). Off by default: it is the + /// scale path and stays gated until verified live against a deployed + /// master-controls catalog. + pub semantic_mapping: bool, } impl Default for BreakpilotConfig { @@ -83,6 +88,7 @@ impl Default for BreakpilotConfig { base_url: None, token: None, snapshot_dir: "/data/compliance-scanner/oscal".to_string(), + semantic_mapping: false, } } } diff --git a/compliance-core/src/control_check.rs b/compliance-core/src/control_check.rs index af8ecb6..4876124 100644 --- a/compliance-core/src/control_check.rs +++ b/compliance-core/src/control_check.rs @@ -14,13 +14,14 @@ //! The LLM supplies cross-language / cross-stack pattern recognition; this module //! supplies the determinism. +use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use crate::models::finding::{Finding, Severity}; use crate::models::scan::ScanType; /// A control rendered as a check the LLM judges code against. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct ControlCheckSpec { /// Stable control id, e.g. `"cra-ai-8"`. pub control_id: String, -- 2.54.0