From 31f1635ee8d01df5e8ab6153dcc57d4fd8c0608c Mon Sep 17 00:00:00 2001 From: Sharang Parnerkar <30073382+mighty840@users.noreply.github.com> Date: Wed, 22 Jul 2026 09:06:18 +0200 Subject: [PATCH] feat(controls): enrich semantic retrieval query with finding intent + C5 live test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tuning from the C5 live run against the real 2,882-control master corpus. The semantic pass retrieved on the code region alone, so two findings in one file (overlapping windows, both md5/password tokens) collapsed onto the SAME controls — a brute-force finding wrongly matched password-hashing controls. Fix: build the retrieval query from the finding's title + description + region, so retrieval keys on what the finding is *about*. The raw region still goes to the judge for snippet grounding. Verified live (same fixture, cached corpus index): - 'Weak password hash (md5)' -> mc-23149 (eliminate weak unsalted hashes) now RANK 1 - 'Login without brute-force prot' -> newly surfaces mc-19984 (brute_force_protection) + mc-23186 (account_lockout) — the correct controls, absent under region-only retrieval. Also commits the gated live regression test (tests/c5_semantic_live.rs, #[ignore]d, not run by CI's --lib): ingest-only + full semantic-stamping checks against api-dev. Co-Authored-By: Claude Fable 5 --- compliance-agent/src/controls/scan_triage.rs | 15 +- compliance-agent/src/controls/semantic.rs | 12 +- compliance-agent/tests/c5_semantic_live.rs | 145 +++++++++++++++++++ 3 files changed, 164 insertions(+), 8 deletions(-) create mode 100644 compliance-agent/tests/c5_semantic_live.rs diff --git a/compliance-agent/src/controls/scan_triage.rs b/compliance-agent/src/controls/scan_triage.rs index edfbbf9..202e1a1 100644 --- a/compliance-agent/src/controls/scan_triage.rs +++ b/compliance-agent/src/controls/scan_triage.rs @@ -234,13 +234,22 @@ pub async fn semantic_stamp_findings( let Some(region) = fetch_region(repo_path, &file, line) else { continue; }; - let region_emb = match llm.embed(vec![region.content.clone()]).await { + // Retrieve on the finding's intent + the code, not the region alone: two + // findings in one file share overlapping windows and otherwise embed alike, + // collapsing onto the same controls. The finding's title/description carry + // the discriminating signal (e.g. "brute-force protection" vs "weak hash"). + // The raw `region` still goes to the judge for snippet grounding. + let query = format!( + "{}\n{}\n\n{}", + finding.title, finding.description, region.content + ); + let query_emb = match llm.embed(vec![query]).await { Ok(mut embs) => match embs.pop() { Some(v) => v, None => continue, }, Err(e) => { - tracing::warn!(error = %e, "region embed failed; skipping finding"); + tracing::warn!(error = %e, "query embed failed; skipping finding"); continue; } }; @@ -248,7 +257,7 @@ pub async fn semantic_stamp_findings( .check( &index, ®ion, - ®ion_emb, + &query_emb, SEMANTIC_TOP_K, &finding.repo_id, ) diff --git a/compliance-agent/src/controls/semantic.rs b/compliance-agent/src/controls/semantic.rs index 1584de7..f66d481 100644 --- a/compliance-agent/src/controls/semantic.rs +++ b/compliance-agent/src/controls/semantic.rs @@ -22,18 +22,20 @@ impl SemanticControlChecker { Self { judge } } - /// Map a code region to the controls it violates. `region_embedding` is the - /// region's embedding (the caller computes it via the LLM); the top-`k` - /// nearest controls in `index` are judged and grounded. + /// Map a code region to the controls it violates. `query_embedding` is the + /// caller-supplied retrieval embedding — typically the finding's intent + /// (title/description) plus the region, so retrieval keys on what the finding + /// is *about*, not just the ambient code. The top-`k` nearest controls in + /// `index` are then judged against the raw `region` and grounded. pub async fn check( &self, index: &ControlIndex, region: &CandidateRegion, - region_embedding: &[f64], + query_embedding: &[f64], k: usize, repo_id: &str, ) -> Vec { - let candidates = index.nearest(region_embedding, k); + let candidates = index.nearest(query_embedding, k); let mut findings = Vec::new(); for spec in &candidates { let verdict = self.judge.judge(spec, region).await; diff --git a/compliance-agent/tests/c5_semantic_live.rs b/compliance-agent/tests/c5_semantic_live.rs new file mode 100644 index 0000000..37a422c --- /dev/null +++ b/compliance-agent/tests/c5_semantic_live.rs @@ -0,0 +1,145 @@ +//! C5 live verification — the semantic master-controls path end to end against the +//! deployed api-dev catalog. Ignored (hits api-dev + LiteLLM). Run explicitly: +//! +//! set -a; . ./.env; set +a +//! BREAKPILOT_BASE_URL=https://api-dev.breakpilot.ai \ +//! cargo test -p compliance-agent --test c5_semantic_live -- --ignored --nocapture +//! +//! Pulls the live master-controls catalog, embeds the corpus (chunked), then for a +//! couple of real vulnerable findings retrieves the nearest master controls and +//! grounded-judges them, stamping master-control refs. + +mod common; + +use std::sync::Arc; + +use compliance_agent::llm::LlmClient; +use compliance_core::config::BreakpilotConfig; +use compliance_core::models::finding::{Finding, Severity}; +use compliance_core::models::scan::ScanType; +use secrecy::SecretString; + +fn env(k: &str) -> String { + std::env::var(k).unwrap_or_else(|_| panic!("env {k} must be set for the live C5 test")) +} + +fn mk_finding(file: &str, line: u32, title: &str) -> Finding { + let mut f = Finding::new( + "repo-c5".into(), + format!("{file}:{line}"), + "semgrep".into(), + ScanType::Sast, + title.into(), + title.into(), + Severity::High, + ); + f.file_path = Some(file.into()); + f.line_number = Some(line); + f +} + +#[tokio::test] +#[ignore = "live: requires deployed api-dev master-controls (fetch+parse only, no LLM)"] +async fn c5_ingest_master_controls_catalog() { + use compliance_agent::controls::OscalControlsProvider; + + let provider = OscalControlsProvider::new( + reqwest::Client::new(), + env("BREAKPILOT_BASE_URL"), + None, + std::env::temp_dir().join("c5-ingest-snap"), + ); + let doc = provider + .load_master_controls() + .await + .expect("pull + parse master-controls catalog"); + let controls = doc.to_controls(); + println!( + "\n=== C5 ingest: {} master controls parsed ===", + controls.len() + ); + for c in controls.iter().take(4) { + let text: String = c.text.chars().take(90).collect(); + println!(" {} | {} | {}", c.id, c.title, text); + } + assert!( + !controls.is_empty(), + "expected a non-empty master-control corpus" + ); +} + +#[tokio::test] +#[ignore = "live: requires deployed api-dev master-controls + LiteLLM"] +async fn c5_semantic_stamps_master_control_refs() { + let llm = Arc::new(LlmClient::new( + env("LITELLM_URL"), + SecretString::from(env("LITELLM_API_KEY")), + env("LITELLM_MODEL"), + env("LITELLM_EMBED_MODEL"), + )); + + let mut config = common::dev_config("mongodb://unused".into(), "c5".into()); + let snapshot = std::env::temp_dir().join("c5-oscal-snap"); + config.breakpilot = BreakpilotConfig { + base_url: Some(env("BREAKPILOT_BASE_URL")), + token: None, + snapshot_dir: snapshot.to_string_lossy().into_owned(), + semantic_mapping: true, + grounded_control_checks: false, + }; + + // Fixture repo with recognizable code-checkable surfaces. + let repo = std::env::temp_dir().join("c5-fixture-repo"); + let _ = std::fs::remove_dir_all(&repo); + std::fs::create_dir_all(repo.join("app")).expect("mkdir"); + std::fs::write( + repo.join("app/auth.py"), + concat!( + "import hashlib\n", + "\n", + "def store_password(user, password):\n", + " # weak, unsalted password hashing\n", + " digest = hashlib.md5(password.encode()).hexdigest()\n", + " db.save(user, digest)\n", + "\n", + "@app.route('/login', methods=['POST'])\n", + "def login():\n", + " u = request.form['username']\n", + " p = request.form['password']\n", + " return 'ok' if check(u, p) else ('bad', 401)\n", + ), + ) + .expect("write fixture"); + + let mut findings = vec![ + mk_finding("app/auth.py", 5, "Weak password hash (md5, unsalted)"), + mk_finding( + "app/auth.py", + 9, + "Login endpoint without brute-force protection", + ), + ]; + + let tagged = + compliance_agent::controls::semantic_stamp_findings(&config, llm, &repo, &mut findings) + .await; + + println!("\n=== C5 semantic master-controls stamping ==="); + for f in &findings { + println!( + " {:50} {}:{:?} -> {:?}", + f.title, + f.file_path.as_deref().unwrap_or(""), + f.line_number, + f.control_refs + ); + } + println!("findings that gained >=1 master-control ref: {tagged}"); + let _ = std::fs::remove_dir_all(&repo); + + // Live corpus — assert only that the path runs and stamps at least one ref. + assert!( + tagged >= 1, + "expected at least one finding to gain a master-control ref" + ); +} -- 2.54.0