feat(agent): cache control embedding index + auto-wire semantic pass (gated) (#217)

compliance-agent/src/config.rs

@@ -100,5 +100,8 @@ fn load_breakpilot_config() -> BreakpilotConfig {
         base_url: env_var_opt("BREAKPILOT_BASE_URL"),
         token: env_secret_opt("BREAKPILOT_TOKEN"),
         snapshot_dir: env_var_opt("BREAKPILOT_SNAPSHOT_DIR").unwrap_or(d.snapshot_dir),
+        semantic_mapping: env_var_opt("BREAKPILOT_SEMANTIC_MAPPING")
+            .map(|v| v == "1" || v.eq_ignore_ascii_case("true"))
+            .unwrap_or(d.semantic_mapping),
     }
 }

compliance-agent/src/controls/index.rs

@@ -6,6 +6,11 @@
 //! requirement text once, then for a code region pull the top-K nearest controls
 //! to hand to the grounded judge. This is the retrieval half of the semantic path.
 
+use std::path::Path;
+
+use serde::{Deserialize, Serialize};
+use sha2::{Digest, Sha256};
+
 use compliance_core::control_check::ControlCheckSpec;
 use compliance_core::error::CoreError;
 
@@ -16,6 +21,34 @@ pub struct ControlIndex {
     entries: Vec<(ControlCheckSpec, Vec<f64>)>,
 }
 
+/// On-disk form of the index: the corpus identity hash plus every spec+embedding.
+/// The hash lets a later scan reuse the embeddings only if the corpus is unchanged.
+#[derive(Serialize, Deserialize)]
+struct PersistedIndex {
+    corpus_hash: String,
+    entries: Vec<PersistedEntry>,
+}
+
+#[derive(Serialize, Deserialize)]
+struct PersistedEntry {
+    spec: ControlCheckSpec,
+    embedding: Vec<f64>,
+}
+
+/// Stable hash of the corpus identity (each control's id + requirement text, in
+/// order). Same catalog → same hash → the cached embeddings are reused instead of
+/// re-embedding the whole corpus.
+fn corpus_hash(specs: &[ControlCheckSpec]) -> String {
+    let mut hasher = Sha256::new();
+    for s in specs {
+        hasher.update(s.control_id.as_bytes());
+        hasher.update([0u8]);
+        hasher.update(s.requirement.as_bytes());
+        hasher.update([0u8]);
+    }
+    format!("{:x}", hasher.finalize())
+}
+
 impl ControlIndex {
     /// Build directly from precomputed embeddings (used by tests + callers that
     /// already embedded the corpus).
@@ -23,6 +56,70 @@ impl ControlIndex {
         Self { entries }
     }
 
+    /// Load the index from `cache_path` if it still matches the current corpus,
+    /// otherwise embed the corpus and persist it there. This turns the per-scan
+    /// re-embed of the whole (~13.6k) master-control corpus into a one-time cost
+    /// that survives across scans; the cache self-invalidates when the catalog
+    /// changes (its [`corpus_hash`] no longer matches).
+    pub async fn load_or_build(
+        llm: &LlmClient,
+        specs: Vec<ControlCheckSpec>,
+        cache_path: &Path,
+    ) -> Result<Self, CoreError> {
+        let hash = corpus_hash(&specs);
+        if let Some(index) = Self::load_cache(cache_path, &hash).await {
+            tracing::debug!(
+                controls = index.len(),
+                "reusing cached control embedding index"
+            );
+            return Ok(index);
+        }
+        let index = Self::build(llm, specs).await?;
+        if let Err(e) = index.write_cache(cache_path, &hash).await {
+            tracing::warn!(error = %e, "failed to persist control embedding index");
+        }
+        Ok(index)
+    }
+
+    /// Read a persisted index, returning it only if its corpus hash matches.
+    async fn load_cache(path: &Path, hash: &str) -> Option<Self> {
+        let raw = tokio::fs::read(path).await.ok()?;
+        let persisted: PersistedIndex = serde_json::from_slice(&raw).ok()?;
+        if persisted.corpus_hash != hash {
+            return None;
+        }
+        Some(Self {
+            entries: persisted
+                .entries
+                .into_iter()
+                .map(|e| (e.spec, e.embedding))
+                .collect(),
+        })
+    }
+
+    /// Persist the index atomically (temp file + rename) keyed by corpus hash.
+    async fn write_cache(&self, path: &Path, hash: &str) -> Result<(), CoreError> {
+        if let Some(parent) = path.parent() {
+            tokio::fs::create_dir_all(parent).await?;
+        }
+        let persisted = PersistedIndex {
+            corpus_hash: hash.to_string(),
+            entries: self
+                .entries
+                .iter()
+                .map(|(spec, emb)| PersistedEntry {
+                    spec: spec.clone(),
+                    embedding: emb.clone(),
+                })
+                .collect(),
+        };
+        let raw = serde_json::to_vec(&persisted)?;
+        let tmp = path.with_extension("json.tmp");
+        tokio::fs::write(&tmp, &raw).await?;
+        tokio::fs::rename(&tmp, path).await?;
+        Ok(())
+    }
+
     /// Build by embedding each control's requirement text.
     pub async fn build(llm: &LlmClient, specs: Vec<ControlCheckSpec>) -> Result<Self, CoreError> {
         if specs.is_empty() {
@@ -107,4 +204,42 @@ mod tests {
         assert_eq!(cosine(&[0.0, 0.0], &[1.0, 1.0]), 0.0); // zero vector
         assert!((cosine(&[1.0, 0.0], &[1.0, 0.0]) - 1.0).abs() < 1e-9); // identical
     }
+
+    #[test]
+    fn corpus_hash_is_stable_and_identity_sensitive() {
+        let a = corpus_hash(&[spec("x"), spec("y")]);
+        assert_eq!(a, corpus_hash(&[spec("x"), spec("y")])); // same corpus → same hash
+        assert_ne!(a, corpus_hash(&[spec("y"), spec("x")])); // reorder → different
+        assert_ne!(a, corpus_hash(&[spec("x")])); // fewer controls → different
+    }
+
+    #[tokio::test]
+    #[allow(clippy::unwrap_used)]
+    async fn cache_round_trips_and_misses_on_corpus_change() {
+        let dir = std::env::temp_dir().join(format!("cidx-{}", uuid::Uuid::new_v4()));
+        let path = dir.join("control-index.json");
+        let specs = [spec("a"), spec("b")];
+        let hash = corpus_hash(&specs);
+        let index = ControlIndex::from_embeddings(vec![
+            (spec("a"), vec![1.0, 0.0]),
+            (spec("b"), vec![0.0, 1.0]),
+        ]);
+        index.write_cache(&path, &hash).await.unwrap();
+
+        // matching corpus hash → hit
+        let loaded = ControlIndex::load_cache(&path, &hash).await.unwrap();
+        assert_eq!(loaded.len(), 2);
+        assert_eq!(loaded.nearest(&[0.9, 0.1], 1)[0].control_id, "a");
+        // corpus changed → miss (forces a rebuild)
+        assert!(ControlIndex::load_cache(&path, "differenthash")
+            .await
+            .is_none());
+        // absent file → miss, not an error
+        assert!(
+            ControlIndex::load_cache(dir.join("nope.json").as_path(), &hash)
+                .await
+                .is_none()
+        );
+        let _ = std::fs::remove_dir_all(&dir);
+    }
 }

compliance-agent/src/controls/scan_triage.rs

@@ -128,9 +128,11 @@ fn fetch_region(repo_path: &Path, file: &str, line: u32) -> Option<CandidateRegi
 /// ~13.6k master-control corpus (which has no CWE to LUT on). Returns the number
 /// of findings that gained a master-control ref.
 ///
-/// Opt-in: the orchestrator does not run this yet. It builds the control embedding
-/// index per call (embeds the whole corpus) — production should cache/persist that
-/// index rather than rebuild it each scan.
+/// Gated: the orchestrator runs this only when `breakpilot.semantic_mapping` is
+/// set (default off, flipped on once the master-controls catalog is live). The
+/// control embedding index is built once and cached to `snapshot_dir` keyed by
+/// corpus hash ([`ControlIndex::load_or_build`]), so only the first scan after a
+/// catalog change pays the embedding cost.
 pub async fn semantic_stamp_findings(
     config: &AgentConfig,
     llm: Arc<LlmClient>,
@@ -164,7 +166,9 @@ pub async fn semantic_stamp_findings(
             severity: Severity::Medium,
         })
         .collect();
-    let index = match ControlIndex::build(&llm, specs).await {
+    let cache_path =
+        Path::new(&config.breakpilot.snapshot_dir).join("control-index-master-controls.json");
+    let index = match ControlIndex::load_or_build(&llm, specs, &cache_path).await {
         Ok(i) if !i.is_empty() => i,
         Ok(_) => return 0,
         Err(e) => {

compliance-agent/src/pipeline/orchestrator.rs

@@ -230,6 +230,28 @@ impl PipelineOrchestrator {
             tracing::info!("[{repo_id}] Control triage tagged {tagged} findings with control refs");
         }
 
+        // Stage 5c: semantic control mapping — scale path for the master-controls
+        // corpus (no CWE to LUT on): embed each finding's region, retrieve the
+        // nearest master controls, grounded-judge, and stamp confirmed refs. Gated
+        // (default off) as the corpus embedding + per-finding judging is the heavy
+        // path; enabled once verified live against a deployed master-controls catalog.
+        if self.config.breakpilot.semantic_mapping {
+            self.update_phase(scan_run_id, "semantic_control_mapping")
+                .await;
+            let sem = crate::controls::semantic_stamp_findings(
+                &self.config,
+                self.llm.clone(),
+                &repo_path,
+                &mut all_findings,
+            )
+            .await;
+            if sem > 0 {
+                tracing::info!(
+                    "[{repo_id}] Semantic mapping tagged {sem} findings with master-control refs"
+                );
+            }
+        }
+
         // Dedup against existing findings and insert new ones
         let mut new_count = 0u32;
         let mut new_findings: Vec<Finding> = Vec::new();

compliance-core/src/config.rs

@@ -75,6 +75,11 @@ pub struct BreakpilotConfig {
     pub token: Option<SecretString>,
     /// Directory for catalog snapshots.
     pub snapshot_dir: String,
+    /// Enable the master-controls **semantic** mapping pass (embed regions,
+    /// retrieve nearest controls, grounded-judge). Off by default: it is the
+    /// scale path and stays gated until verified live against a deployed
+    /// master-controls catalog.
+    pub semantic_mapping: bool,
 }
 
 impl Default for BreakpilotConfig {
@@ -83,6 +88,7 @@ impl Default for BreakpilotConfig {
             base_url: None,
             token: None,
             snapshot_dir: "/data/compliance-scanner/oscal".to_string(),
+            semantic_mapping: false,
         }
     }
 }

compliance-core/src/control_check.rs

@@ -14,13 +14,14 @@
 //! The LLM supplies cross-language / cross-stack pattern recognition; this module
 //! supplies the determinism.
 
+use serde::{Deserialize, Serialize};
 use sha2::{Digest, Sha256};
 
 use crate::models::finding::{Finding, Severity};
 use crate::models::scan::ScanType;
 
 /// A control rendered as a check the LLM judges code against.
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct ControlCheckSpec {
     /// Stable control id, e.g. `"cra-ai-8"`.
     pub control_id: String,