Compare commits

..
Author SHA1 Message Date
Sharang ParnerkarandClaude Fable 5 0d83859bcf feat(migrate): per-tenant onboarding backfill + revert + CLI (#132 part 2)
CI / Check (pull_request) Successful in 8m15s
CI / Detect Changes (pull_request) Has been skipped
CI / Deploy Agent (pull_request) Has been skipped
CI / Deploy Dashboard (pull_request) Has been skipped
CI / Deploy Docs (pull_request) Has been skipped
CI / Deploy MCP (pull_request) Has been skipped
Wire the id-preserving mappers into a runnable, idempotent, reversible migration:
- backfill_onboarded_targets(db, dry_run): repositories -> onboarded_targets;
  dast_targets fold into the linked repo target (append LiveUrl, promote type,
  relink dast_scan_runs/pentest_sessions history) or migrate standalone. Skips
  existing (by _id), writes a schema_migrations marker; dry-run computes the
  report without writing. Legacy collections are never deleted.
- revert(db): replay the relink log, drop onboarded_targets + the log, clear the
  marker — restores the pre-migration state.
- CLI: `compliance-agent migrate onboarding [--all | --tenant <id>] [--dry-run]
  [--revert]`, with DatabasePool::list_tenant_ids for --all.
- database.rs: collection_named<T> accessor + list_tenant_ids helper.

Integration test (real Mongo, local-only — CI is --lib) covers fold + relink +
idempotency + revert end to end. 5 mapper unit tests run in CI.

Closes #132.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-10 19:49:52 +02:00
Sharang ParnerkarandClaude Fable 5 fa3a6b71fb feat(migrate): id-preserving mapping core for onboarding backfill (#132 part 1)
Pure transforms folding legacy records into the unified model, preserving _id so
every downstream collection keyed by that hex id keeps resolving:
- repo_to_target: TrackedRepository -> OnboardedTarget (GitRepo artifact with
  branch/watermark/auth; tracker config -> scan_config; webhook/schedule/counts).
- dast_to_artifact / dast_to_standalone_target: DastTarget -> LiveUrl artifact
  (crawl config + auth), or a standalone target when repo-less.
- fold_dast_into_target: append a LiveUrl to a repo-derived target (idempotent),
  promoting BackendService -> WebApp for a browser app.

5 unit tests. The per-tenant DB orchestration (idempotent backfill + revert) and
the `migrate onboarding` CLI subcommand are the next increment.

Refs #132.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-10 18:25:46 +02:00
sharang ef6ee3dcd1 feat(onboarding): artifact ingest + classifier + native tramiton + suite seams (#138)
CI / Check (push) Has been skipped
CI / Detect Changes (push) Successful in 3s
CI / Deploy Agent (push) Failing after 5s
CI / Deploy Dashboard (push) Failing after 5s
CI / Deploy Docs (push) Has been skipped
CI / Deploy MCP (push) Failing after 4s
2026-07-10 16:00:51 +00:00
sharang 675c4ef699 feat(onboarding): unified multi-target model + scan matrix foundation (#134)
CI / Check (push) Has been skipped
CI / Detect Changes (push) Successful in 5s
CI / Deploy Agent (push) Successful in 7m36s
CI / Deploy Dashboard (push) Successful in 7m9s
CI / Deploy Docs (push) Has been skipped
CI / Deploy MCP (push) Successful in 1m44s
2026-07-10 13:41:25 +00:00
12 changed files with 818 additions and 115 deletions
+16
View File
@@ -13,6 +13,9 @@ env:
# both --features server and --features web shares common crate work.
RUSTC_WRAPPER: /usr/local/bin/sccache
SCCACHE_DIR: /tmp/sccache
# compliance-agent depends on tramiton-core via git; use the system git so the
# credential rewrite below (see "Configure git auth ...") is honored on fetch.
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
# Cancel in-progress runs for the same branch/PR
concurrency:
@@ -46,6 +49,19 @@ jobs:
env:
RUSTC_WRAPPER: ""
# compliance-agent has a git dependency on tramiton-core (a private repo on
# this Gitea instance). Rewrite its SSH URL to HTTPS + a PAT so the runner
# can fetch it. Requires the repo secret TRAMITON_FETCH_TOKEN (a Gitea PAT
# with read:repository, owned by a user with access to sharang/tramiton).
# (Honored on fetch because CARGO_NET_GIT_FETCH_WITH_CLI=true uses system git.)
- name: Configure git auth for private tramiton dependency
run: |
git config --global \
url."https://sharang:${{ secrets.TRAMITON_FETCH_TOKEN }}@gitea.meghsakha.com/".insteadOf \
"ssh://git@gitea.meghsakha.com:22222/"
env:
RUSTC_WRAPPER: ""
# Format (no compilation needed)
- name: Format
run: cargo fmt --all --check
Generated
+65 -2
View File
@@ -692,6 +692,7 @@ dependencies = [
"tower-http",
"tracing",
"tracing-subscriber",
"tramiton-core",
"urlencoding",
"uuid",
"walkdir",
@@ -4197,7 +4198,7 @@ version = "3.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983"
dependencies = [
"toml_edit",
"toml_edit 0.23.10+spec-1.0.0",
]
[[package]]
@@ -4996,6 +4997,15 @@ dependencies = [
"syn",
]
[[package]]
name = "serde_spanned"
version = "0.6.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"
dependencies = [
"serde",
]
[[package]]
name = "serde_urlencoded"
version = "0.7.1"
@@ -5810,6 +5820,27 @@ dependencies = [
"tokio",
]
[[package]]
name = "toml"
version = "0.8.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
dependencies = [
"serde",
"serde_spanned",
"toml_datetime 0.6.11",
"toml_edit 0.22.27",
]
[[package]]
name = "toml_datetime"
version = "0.6.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
dependencies = [
"serde",
]
[[package]]
name = "toml_datetime"
version = "0.7.5+spec-1.1.0"
@@ -5819,6 +5850,20 @@ dependencies = [
"serde_core",
]
[[package]]
name = "toml_edit"
version = "0.22.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
dependencies = [
"indexmap 2.13.0",
"serde",
"serde_spanned",
"toml_datetime 0.6.11",
"toml_write",
"winnow",
]
[[package]]
name = "toml_edit"
version = "0.23.10+spec-1.0.0"
@@ -5826,7 +5871,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269"
dependencies = [
"indexmap 2.13.0",
"toml_datetime",
"toml_datetime 0.7.5+spec-1.1.0",
"toml_parser",
"winnow",
]
@@ -5840,6 +5885,12 @@ dependencies = [
"winnow",
]
[[package]]
name = "toml_write"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
[[package]]
name = "tonic"
version = "0.12.3"
@@ -6086,6 +6137,18 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "tramiton-core"
version = "0.4.0"
source = "git+ssh://git@gitea.meghsakha.com:22222/sharang/tramiton.git?tag=v0.4.0#e3dc1bf7027a2f6d7b1fe43043d6dfa887ce4af3"
dependencies = [
"serde",
"tempfile",
"thiserror 1.0.69",
"toml",
"walkdir",
]
[[package]]
name = "tree-sitter"
version = "0.24.7"
+5
View File
@@ -10,6 +10,11 @@ workspace = true
compliance-core = { workspace = true, features = ["mongodb", "telemetry", "axum"] }
compliance-graph = { path = "../compliance-graph" }
compliance-dast = { path = "../compliance-dast" }
# Native firmware build/target detection for bare-metal & RTOS artifacts.
# Same-company IP, used directly (not via CLI) so the whole tramiton suite is
# available to the onboarding classifier. NOTE: CI must be able to fetch this
# private repo (see the git-auth step in .gitea/workflows/ci.yml).
tramiton-core = { git = "ssh://git@gitea.meghsakha.com:22222/sharang/tramiton.git", tag = "v0.4.0" }
serde = { workspace = true }
serde_json = { workspace = true }
tokio = { workspace = true }
+75 -104
View File
@@ -1,55 +1,40 @@
//! Firmware classification via tramiton.
//!
//! tramiton is the company's firmware build/repro engine; we do not re-implement
//! its detection. This module shells out to `tramiton detect --json` behind a
//! [`FirmwareDetector`] port (so a future in-process or cloud impl can slot in)
//! and maps the resulting build plan onto a [`TargetType`]. A deterministic
//! [`MockFirmwareDetector`] backs the tests so CI never needs the binary.
//!
//! The parsed structs mirror a *subset* of tramiton's `BuildPlan` JSON — we
//! deliberately do not depend on the proprietary `tramiton-core` crate.
//! its detection. We depend on `tramiton-core` directly (same-company IP) and run
//! its provider analysis in-process behind a [`FirmwareDetector`] port, mapping
//! tramiton's `BuildPlan` onto a [`TargetType`]. A deterministic
//! [`MockFirmwareDetector`] backs the tests so CI unit tests need neither the
//! tramiton sources nor a real firmware tree.
use std::path::Path;
use serde::Deserialize;
use compliance_core::error::CoreError;
use compliance_core::models::{DetectedFact, TargetType};
use compliance_core::traits::ClassifierVerdict;
/// The top-level `tramiton detect --json` document (fields we use).
#[derive(Debug, Clone, Deserialize)]
pub struct TramitonDetect {
/// The selected build plan, if tramiton could form one.
#[serde(default)]
pub plan: Option<TramitonBuildPlan>,
}
/// The subset of tramiton's `BuildPlan` we consume for classification.
#[derive(Debug, Clone, Default, Deserialize)]
pub struct TramitonBuildPlan {
/// A minimal firmware-detection summary, mapped from tramiton's `BuildPlan`.
/// Kept small and tramiton-independent so the classifier and the test mock don't
/// need to construct a full tramiton plan.
#[derive(Debug, Clone, Default)]
pub struct FirmwareDetection {
/// The detecting provider (e.g. `zephyr`, `cmake`, `source-archaeology`).
#[serde(default)]
pub provider: String,
/// Detection confidence: `low` | `medium` | `high`.
#[serde(default)]
pub confidence: String,
/// Build system (kebab-case: `zephyr`, `esp-idf`, `cmake`, `make`, ...).
#[serde(default)]
/// Build-system label (e.g. `Zephyr`, `ESP-IDF`, `CMake`).
pub build_system: String,
/// Framework, when known (`zephyr`, `esp-idf`, `mbed`, `bare-metal`, ...).
/// Framework, when known (`zephyr`, `esp-idf`, `bare-metal`, ...).
pub framework: Option<String>,
/// Target board / MCU / arch.
#[serde(default)]
pub target: TramitonTarget,
pub target: FirmwareTarget,
/// Unresolved gaps in the plan.
#[serde(default)]
pub gaps: Vec<String>,
}
/// tramiton's target descriptor.
#[derive(Debug, Clone, Default, Deserialize)]
pub struct TramitonTarget {
/// The detected firmware target (board / MCU / arch).
#[derive(Debug, Clone, Default)]
pub struct FirmwareTarget {
/// Board name.
pub board: Option<String>,
/// MCU part.
@@ -61,57 +46,52 @@ pub struct TramitonTarget {
/// A source of tramiton firmware detection.
#[allow(async_fn_in_trait)]
pub trait FirmwareDetector: Send + Sync {
/// Run detection over a path, returning tramiton's build plan if any.
async fn detect(&self, path: &Path) -> Result<Option<TramitonBuildPlan>, CoreError>;
/// Run detection over a path, returning a firmware detection if tramiton
/// could form a build plan.
async fn detect(&self, path: &Path) -> Result<Option<FirmwareDetection>, CoreError>;
}
/// Shells out to the `tramiton` CLI. A missing binary or a non-zero exit is
/// treated as "no detection" rather than an error, so firmware classification
/// degrades gracefully when tramiton is not installed.
pub struct TramitonCli {
/// The `tramiton` binary to invoke.
pub bin: String,
}
/// Uses `tramiton-core` in-process. The analysis is blocking (filesystem walk),
/// so it runs on a blocking thread to avoid stalling the async runtime. A path
/// with no recognizable build system yields `Ok(None)`.
pub struct TramitonNative;
impl TramitonCli {
/// Construct from `TRAMITON_BIN` (default `tramiton`).
pub fn from_env() -> Self {
Self {
bin: std::env::var("TRAMITON_BIN").unwrap_or_else(|_| "tramiton".to_string()),
}
impl FirmwareDetector for TramitonNative {
async fn detect(&self, path: &Path) -> Result<Option<FirmwareDetection>, CoreError> {
let path = path.to_path_buf();
let plan = tokio::task::spawn_blocking(move || {
let repo = tramiton_core::Repo::new(&path);
tramiton_core::provider::analyze(&repo)
})
.await
.map_err(|e| CoreError::Other(format!("tramiton detect task join error: {e}")))?
.map_err(|e| CoreError::Other(format!("tramiton analyze error: {e}")))?;
Ok(plan.map(|bp| detection_from_build_plan(&bp)))
}
}
impl FirmwareDetector for TramitonCli {
async fn detect(&self, path: &Path) -> Result<Option<TramitonBuildPlan>, CoreError> {
let output = tokio::process::Command::new(&self.bin)
.arg("detect")
.arg("--json")
.arg(path)
.output()
.await;
match output {
Ok(o) if o.status.success() => {
let parsed: TramitonDetect = serde_json::from_slice(&o.stdout)?;
Ok(parsed.plan)
}
// Non-zero exit: tramiton ran but formed no plan.
Ok(_) => Ok(None),
// Binary not found / not executable: degrade gracefully.
Err(_) => Ok(None),
}
/// Map tramiton's `BuildPlan` onto our minimal detection summary.
fn detection_from_build_plan(bp: &tramiton_core::BuildPlan) -> FirmwareDetection {
FirmwareDetection {
provider: bp.provider.clone(),
confidence: bp.confidence.to_string(),
build_system: bp.build_system.label().to_string(),
framework: bp.framework.clone(),
target: FirmwareTarget {
board: bp.target.board.clone(),
mcu: bp.target.mcu.clone(),
arch: bp.target.arch.clone(),
},
gaps: bp.gaps.clone(),
}
}
/// Map a tramiton build plan to a target type. Framework/build-system signals
/// Map a firmware detection to a target type. Framework/build-system signals
/// distinguish RTOS from bare-metal from Yocto.
pub fn plan_to_target_type(plan: &TramitonBuildPlan) -> TargetType {
let framework = plan.framework.as_deref().unwrap_or("").to_lowercase();
let build_system = plan.build_system.to_lowercase();
let signal = format!(
"{framework} {build_system} {}",
plan.provider.to_lowercase()
);
pub fn detection_to_target_type(det: &FirmwareDetection) -> TargetType {
let framework = det.framework.as_deref().unwrap_or("").to_lowercase();
let build_system = det.build_system.to_lowercase();
let signal = format!("{framework} {build_system} {}", det.provider.to_lowercase());
const RTOS: [&str; 6] = ["zephyr", "esp-idf", "freertos", "nuttx", "riot", "chibios"];
if signal.contains("bitbake") || signal.contains("yocto") || signal.contains("openembedded") {
@@ -133,35 +113,35 @@ fn confidence_score(label: &str) -> f32 {
}
}
/// Turn a tramiton build plan into a classifier verdict, carrying the MCU / board
/// Turn a firmware detection into a classifier verdict, carrying the MCU / board
/// / build-system as facts.
pub fn plan_to_verdict(plan: &TramitonBuildPlan) -> ClassifierVerdict {
let target_type = plan_to_target_type(plan);
pub fn detection_to_verdict(det: &FirmwareDetection) -> ClassifierVerdict {
let target_type = detection_to_target_type(det);
let mut facts = vec![DetectedFact::new(
"build_system",
plan.build_system.clone(),
det.build_system.clone(),
"tramiton",
)];
if let Some(fw) = &plan.framework {
if let Some(fw) = &det.framework {
facts.push(DetectedFact::new("framework", fw.clone(), "tramiton"));
}
if let Some(mcu) = &plan.target.mcu {
if let Some(mcu) = &det.target.mcu {
facts.push(DetectedFact::new("mcu", mcu.clone(), "tramiton"));
}
if let Some(board) = &plan.target.board {
if let Some(board) = &det.target.board {
facts.push(DetectedFact::new("board", board.clone(), "tramiton"));
}
if let Some(arch) = &plan.target.arch {
if let Some(arch) = &det.target.arch {
facts.push(DetectedFact::new("arch", arch.clone(), "tramiton"));
}
ClassifierVerdict {
target_type,
confidence: confidence_score(&plan.confidence),
confidence: confidence_score(&det.confidence),
facts,
rationale: format!(
"tramiton detected build system '{}'{}",
plan.build_system,
plan.framework
det.build_system,
det.framework
.as_ref()
.map(|f| format!(" (framework {f})"))
.unwrap_or_default()
@@ -169,15 +149,15 @@ pub fn plan_to_verdict(plan: &TramitonBuildPlan) -> ClassifierVerdict {
}
}
/// A deterministic [`FirmwareDetector`] for tests — returns a preset plan.
/// A deterministic [`FirmwareDetector`] for tests — returns a preset detection.
pub struct MockFirmwareDetector {
/// The plan to return (or `None` for "no detection").
pub plan: Option<TramitonBuildPlan>,
/// The detection to return (or `None` for "no detection").
pub detection: Option<FirmwareDetection>,
}
impl FirmwareDetector for MockFirmwareDetector {
async fn detect(&self, _path: &Path) -> Result<Option<TramitonBuildPlan>, CoreError> {
Ok(self.plan.clone())
async fn detect(&self, _path: &Path) -> Result<Option<FirmwareDetection>, CoreError> {
Ok(self.detection.clone())
}
}
@@ -186,13 +166,13 @@ impl FirmwareDetector for MockFirmwareDetector {
mod tests {
use super::*;
fn plan(build_system: &str, framework: Option<&str>) -> TramitonBuildPlan {
TramitonBuildPlan {
fn detection(build_system: &str, framework: Option<&str>) -> FirmwareDetection {
FirmwareDetection {
provider: build_system.to_string(),
confidence: "high".to_string(),
build_system: build_system.to_string(),
framework: framework.map(|s| s.to_string()),
target: TramitonTarget {
target: FirmwareTarget {
mcu: Some("stm32f429".to_string()),
..Default::default()
},
@@ -203,7 +183,7 @@ mod tests {
#[test]
fn zephyr_maps_to_rtos() {
assert_eq!(
plan_to_target_type(&plan("zephyr", Some("zephyr"))),
detection_to_target_type(&detection("zephyr", Some("zephyr"))),
TargetType::FirmwareRtos
);
}
@@ -211,7 +191,7 @@ mod tests {
#[test]
fn bare_cmake_maps_to_bare_metal() {
assert_eq!(
plan_to_target_type(&plan("cmake", Some("bare-metal"))),
detection_to_target_type(&detection("cmake", Some("bare-metal"))),
TargetType::FirmwareBareMetal
);
}
@@ -219,14 +199,14 @@ mod tests {
#[test]
fn bitbake_maps_to_yocto() {
assert_eq!(
plan_to_target_type(&plan("bitbake", None)),
detection_to_target_type(&detection("bitbake", None)),
TargetType::EmbeddedLinuxYocto
);
}
#[test]
fn verdict_carries_mcu_fact_and_confidence() {
let v = plan_to_verdict(&plan("esp-idf", Some("esp-idf")));
let v = detection_to_verdict(&detection("esp-idf", Some("esp-idf")));
assert_eq!(v.target_type, TargetType::FirmwareRtos);
assert!((v.confidence - 0.9).abs() < f32::EPSILON);
assert!(v
@@ -234,13 +214,4 @@ mod tests {
.iter()
.any(|f| f.key == "mcu" && f.value == "stm32f429"));
}
#[test]
fn detect_json_parses() {
let json = r#"{"repo":"/x","detections":[],"plan":{"provider":"zephyr","confidence":"high","build_system":"zephyr","framework":"zephyr","target":{"mcu":"nrf52840","board":"nrf52840dk","arch":"arm"},"gaps":[]}}"#;
let parsed: TramitonDetect = serde_json::from_str(json).expect("parse");
let plan = parsed.plan.expect("plan present");
assert_eq!(plan.target.mcu.as_deref(), Some("nrf52840"));
assert_eq!(plan_to_target_type(&plan), TargetType::FirmwareRtos);
}
}
+7 -8
View File
@@ -9,8 +9,7 @@ mod firmware;
mod language;
pub use firmware::{
FirmwareDetector, MockFirmwareDetector, TramitonBuildPlan, TramitonCli, TramitonDetect,
TramitonTarget,
FirmwareDetection, FirmwareDetector, FirmwareTarget, MockFirmwareDetector, TramitonNative,
};
pub use language::HeuristicClassifier;
@@ -23,7 +22,7 @@ use compliance_core::models::{
};
use compliance_core::traits::{ClassificationInput, ClassifierVerdict, TargetClassifier};
use firmware::plan_to_verdict;
use firmware::detection_to_verdict;
/// Classify a target from its artifacts and their ingested working paths, using
/// the heuristic classifier plus the tramiton firmware detector. Verdicts are
@@ -60,8 +59,8 @@ pub async fn classify_target<D: FirmwareDetector>(
let Some(path) = working_paths.get(&artifact.id) else {
continue;
};
if let Some(plan) = firmware_detector.detect(path).await? {
verdicts.push(plan_to_verdict(&plan));
if let Some(detection) = firmware_detector.detect(path).await? {
verdicts.push(detection_to_verdict(&detection));
tramiton_used = true;
}
}
@@ -155,7 +154,7 @@ mod tests {
}
fn no_firmware() -> MockFirmwareDetector {
MockFirmwareDetector { plan: None }
MockFirmwareDetector { detection: None }
}
#[tokio::test]
@@ -190,12 +189,12 @@ mod tests {
target.artifacts.push(artifact);
let detector = MockFirmwareDetector {
plan: Some(TramitonBuildPlan {
detection: Some(FirmwareDetection {
provider: "zephyr".to_string(),
confidence: "high".to_string(),
build_system: "zephyr".to_string(),
framework: Some("zephyr".to_string()),
target: TramitonTarget {
target: FirmwareTarget {
mcu: Some("nrf52840".to_string()),
..Default::default()
},
+24
View File
@@ -179,6 +179,23 @@ impl DatabasePool {
.collect())
}
/// Tenant ids for every provisioned tenant database, derived by stripping
/// the `<prefix>_` from the database names. Skips the admin database
/// (`<prefix>__admin`). Hash-fallback names (very long tenant_ids) are lost
/// at the cluster level and cannot be recovered here — in practice tenant
/// ids are UUIDs and never hit that path. Used by the migration CLI's
/// `--all` mode.
pub async fn list_tenant_ids(&self) -> Result<Vec<String>, AgentError> {
let prefix = format!("{}_", self.db_prefix);
Ok(self
.list_tenant_db_names()
.await?
.into_iter()
.filter_map(|n| n.strip_prefix(&prefix).map(str::to_string))
.filter(|id| !id.starts_with('_'))
.collect())
}
/// Drop the database for a specific tenant. Used by GDPR delete
/// and tenant offboarding. Idempotent — dropping a non-existent
/// database is a no-op at the driver level.
@@ -521,6 +538,13 @@ impl Database {
self.inner.collection("onboarded_targets")
}
/// A typed handle to an arbitrary collection by name. For bookkeeping
/// collections without a dedicated model (e.g. `schema_migrations`,
/// `onboarding_migration_log`).
pub fn collection_named<T: Send + Sync>(&self, name: &str) -> Collection<T> {
self.inner.collection(name)
}
pub fn dast_scan_runs(&self) -> Collection<DastScanRun> {
self.inner.collection("dast_scan_runs")
}
+1
View File
@@ -8,6 +8,7 @@ pub mod database;
pub mod error;
pub mod ingest;
pub mod llm;
pub mod migrate;
pub mod pentest;
pub mod pipeline;
pub mod rag;
+54 -1
View File
@@ -1,4 +1,50 @@
use compliance_agent::{agent, api, config, database, scheduler, ssh, webhooks};
use compliance_agent::{agent, api, config, database, migrate, scheduler, ssh, webhooks};
/// Run the `migrate onboarding` subcommand and exit. Backfills (or reverts) the
/// unified `onboarded_targets` collection per tenant.
///
/// Usage: `compliance-agent migrate onboarding [--all | --tenant <id>] [--dry-run] [--revert]`
async fn run_migration(
args: &[String],
pool: &database::DatabasePool,
) -> Result<(), compliance_agent::error::AgentError> {
if args.get(2).map(String::as_str) != Some("onboarding") {
eprintln!(
"usage: compliance-agent migrate onboarding [--all | --tenant <id>] [--dry-run] [--revert]"
);
std::process::exit(2);
}
let has = |flag: &str| args.iter().any(|a| a == flag);
let dry_run = has("--dry-run");
let revert = has("--revert");
let tenant = args
.iter()
.position(|a| a == "--tenant")
.and_then(|i| args.get(i + 1))
.cloned();
let tenants: Vec<String> = if has("--all") {
pool.list_tenant_ids().await?
} else if let Some(t) = tenant {
vec![t]
} else {
eprintln!("specify --all or --tenant <id>");
std::process::exit(2);
};
for tenant_id in tenants {
let db = pool.for_tenant_id(&tenant_id).await?;
if revert {
migrate::onboarding::revert(&db).await?;
println!("[{tenant_id}] reverted onboarding backfill");
} else {
let report = migrate::onboarding::backfill_onboarded_targets(&db, dry_run).await?;
let prefix = if dry_run { "(dry-run) " } else { "" };
println!("[{tenant_id}] {prefix}{report:?}");
}
}
Ok(())
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
@@ -31,6 +77,13 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
let db_pool =
database::DatabasePool::connect(&config.mongodb_uri, &config.mongodb_database).await?;
// One-shot subcommands run and exit without starting the servers.
let args: Vec<String> = std::env::args().collect();
if args.get(1).map(String::as_str) == Some("migrate") {
run_migration(&args, &db_pool).await?;
return Ok(());
}
let agent = agent::ComplianceAgent::new(config.clone(), db_pool);
tracing::info!("Starting scheduler...");
+8
View File
@@ -0,0 +1,8 @@
//! One-time data migrations.
//!
//! Currently just the onboarding backfill ([`onboarding`]), which folds the
//! legacy `repositories` and `dast_targets` collections into the unified
//! `onboarded_targets` collection, preserving `_id` so every downstream record
//! keyed by `repo_id` / `target_id` keeps resolving.
pub mod onboarding;
+406
View File
@@ -0,0 +1,406 @@
//! Backfill: legacy `repositories` + `dast_targets` → `onboarded_targets`.
//!
//! The transforms here are **id-preserving**: an [`OnboardedTarget`] keeps the
//! same `_id` as the `TrackedRepository` / `DastTarget` it came from, so every
//! downstream collection keyed by that hex id (findings, sbom, scan_runs,
//! graph, dast_*, pentest_*) keeps resolving with zero row rewrites, and
//! existing webhook URLs keep working. The mapping functions are pure and unit
//! tested; the DB orchestration (idempotent per-tenant backfill + revert) is a
//! thin driver over them.
use compliance_core::models::{
Artifact, ArtifactKind, DastTarget, DastTargetType, GitArtifactConfig, IssueTrackerConfig,
OnboardedTarget, TargetType, TrackedRepository, WebArtifactConfig,
};
use futures_util::TryStreamExt;
use mongodb::bson::{doc, Document};
use crate::database::Database;
use crate::error::AgentError;
/// Marker id in `schema_migrations` recording that the backfill has run.
const MIGRATION_MARKER: &str = "onboarding_backfill_v1";
/// Summary of a backfill run.
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct MigrationReport {
/// Repositories turned into onboarded targets.
pub repos_migrated: u64,
/// DAST targets folded into an existing (repo-linked) target as a LiveUrl.
pub dast_targets_folded: u64,
/// DAST targets with no repo link, migrated as standalone targets.
pub dast_targets_standalone: u64,
/// Records skipped because a target with that `_id` already existed.
pub skipped_existing: u64,
}
/// Map a legacy `DastTargetType` to a unified [`TargetType`]. REST/GraphQL APIs
/// are backend services; a browser app is a web app.
fn target_type_for_dast(kind: &DastTargetType) -> TargetType {
match kind {
DastTargetType::WebApp => TargetType::WebApp,
DastTargetType::RestApi | DastTargetType::GraphQl => TargetType::BackendService,
}
}
/// Build the LiveUrl artifact for a DAST target (its base URL + crawl config +
/// auth). Shared by fold-in and standalone migration.
pub fn dast_to_artifact(dast: &DastTarget) -> Artifact {
let mut artifact = Artifact::live_url(dast.base_url.clone());
artifact.web = Some(WebArtifactConfig {
target_kind: dast.target_type.clone(),
excluded_paths: dast.excluded_paths.clone(),
max_crawl_depth: dast.max_crawl_depth,
rate_limit: dast.rate_limit,
allow_destructive: dast.allow_destructive,
});
artifact.auth = dast.auth_config.clone().map(Into::into);
artifact
}
/// Map a `TrackedRepository` to an onboarded target, preserving `_id`. The git
/// remote becomes a `GitRepo` artifact carrying the repo's branch, watermark,
/// and auth; tracker config folds into `scan_config`.
///
/// `target_type` is a safe default (`BackendService`) — the classifier can
/// refine it later; `classification` is left `None` (unconfirmed).
pub fn repo_to_target(repo: &TrackedRepository) -> OnboardedTarget {
let mut target = OnboardedTarget::new(repo.name.clone(), TargetType::BackendService);
target.id = repo.id;
let mut artifact = Artifact::git_repo(repo.git_url.clone(), repo.default_branch.clone());
artifact.git = Some(GitArtifactConfig {
default_branch: repo.default_branch.clone(),
last_scanned_commit: repo.last_scanned_commit.clone(),
local_path: repo.local_path.clone(),
});
if repo.auth_token.is_some() || repo.auth_username.is_some() {
artifact.auth = Some(compliance_core::models::ArtifactAuth {
method: "token".to_string(),
username: repo.auth_username.clone(),
secret: repo.auth_token.clone(),
..Default::default()
});
}
target.artifacts.push(artifact);
if repo.tracker_type.is_some() {
target.scan_config.issue_tracker = Some(IssueTrackerConfig {
tracker_type: repo.tracker_type.clone(),
owner: repo.tracker_owner.clone(),
repo: repo.tracker_repo.clone(),
token: repo.tracker_token.clone(),
});
}
target.scan_schedule = repo.scan_schedule.clone();
target.webhook_enabled = repo.webhook_enabled;
target.webhook_secret = repo.webhook_secret.clone();
target.findings_count = repo.findings_count;
target.created_at = repo.created_at;
target.updated_at = repo.updated_at;
target
}
/// Append a DAST target's LiveUrl artifact onto an existing (repo-derived)
/// target. If the repo default was `BackendService` but the DAST target is a
/// browser web app, promote the type to `WebApp`.
pub fn fold_dast_into_target(target: &mut OnboardedTarget, dast: &DastTarget) {
if matches!(dast.target_type, DastTargetType::WebApp)
&& target.target_type == TargetType::BackendService
{
target.target_type = TargetType::WebApp;
}
if !target.has(ArtifactKind::LiveUrl) {
target.artifacts.push(dast_to_artifact(dast));
}
}
/// Map a repo-less DAST target to a standalone onboarded target, preserving `_id`.
pub fn dast_to_standalone_target(dast: &DastTarget) -> OnboardedTarget {
let mut target =
OnboardedTarget::new(dast.name.clone(), target_type_for_dast(&dast.target_type));
target.id = dast.id;
target.artifacts.push(dast_to_artifact(dast));
target.created_at = dast.created_at;
target.updated_at = dast.updated_at;
target
}
/// Whether the onboarding backfill has already been applied to this database.
pub async fn already_applied(db: &Database) -> Result<bool, AgentError> {
let found = db
.collection_named::<Document>("schema_migrations")
.find_one(doc! { "_id": MIGRATION_MARKER })
.await?;
Ok(found.is_some())
}
/// Backfill `onboarded_targets` from `repositories` + `dast_targets` for one
/// tenant database.
///
/// Id-preserving and **idempotent**: targets that already exist (by `_id`) are
/// skipped, so re-running is safe. With `dry_run`, computes the report without
/// writing. The legacy collections are never deleted; the only mutation outside
/// `onboarded_targets` is the history relink of folded DAST targets, which is
/// logged so [`revert`] can undo it.
pub async fn backfill_onboarded_targets(
db: &Database,
dry_run: bool,
) -> Result<MigrationReport, AgentError> {
let mut report = MigrationReport::default();
// 1. repositories -> onboarded_targets (preserve _id, skip existing).
let mut repos = db.repositories().find(doc! {}).await?;
while let Some(repo) = repos.try_next().await? {
let Some(id) = repo.id else { continue };
if db
.onboarded_targets()
.find_one(doc! { "_id": id })
.await?
.is_some()
{
report.skipped_existing += 1;
continue;
}
if !dry_run {
db.onboarded_targets()
.insert_one(repo_to_target(&repo))
.await?;
}
report.repos_migrated += 1;
}
// 2. dast_targets -> fold into the linked repo target, or migrate standalone.
let mut dasts = db.dast_targets().find(doc! {}).await?;
while let Some(dast) = dasts.try_next().await? {
let Some(dast_id) = dast.id else { continue };
let repo_oid = dast
.repo_id
.as_deref()
.and_then(|r| mongodb::bson::oid::ObjectId::parse_str(r).ok());
let linked = match repo_oid {
Some(oid) => db.onboarded_targets().find_one(doc! { "_id": oid }).await?,
None => None,
};
match (linked, repo_oid) {
// Fold into an existing repo-derived target.
(Some(mut target), Some(oid)) => {
if target.has(ArtifactKind::LiveUrl) {
report.skipped_existing += 1; // already folded on a prior run
continue;
}
fold_dast_into_target(&mut target, &dast);
if !dry_run {
db.onboarded_targets()
.replace_one(doc! { "_id": oid }, &target)
.await?;
relink_history(db, &dast_id.to_hex(), &oid.to_hex()).await?;
}
report.dast_targets_folded += 1;
}
// No linked repo target: migrate as a standalone target (keeps _id).
_ => {
if db
.onboarded_targets()
.find_one(doc! { "_id": dast_id })
.await?
.is_some()
{
report.skipped_existing += 1;
continue;
}
if !dry_run {
db.onboarded_targets()
.insert_one(dast_to_standalone_target(&dast))
.await?;
}
report.dast_targets_standalone += 1;
}
}
}
if !dry_run {
db.collection_named::<Document>("schema_migrations")
.update_one(
doc! { "_id": MIGRATION_MARKER },
doc! { "$set": { "applied_at": mongodb::bson::DateTime::now() } },
)
.upsert(true)
.await?;
}
Ok(report)
}
/// Relink DAST scan runs and pentest sessions from the old DAST target id to the
/// unified target id, logging each move so [`revert`] can undo it.
///
/// Note: if multiple DAST targets fold into the same repo target, revert
/// restores only the last-logged mapping — a rare edge. The source collections
/// (`repositories`, `dast_targets`) are never deleted, so no data is lost.
async fn relink_history(db: &Database, old_id: &str, new_id: &str) -> Result<(), AgentError> {
db.dast_scan_runs()
.update_many(
doc! { "target_id": old_id },
doc! { "$set": { "target_id": new_id } },
)
.await?;
db.pentest_sessions()
.update_many(
doc! { "target_id": old_id },
doc! { "$set": { "target_id": new_id } },
)
.await?;
db.collection_named::<Document>("onboarding_migration_log")
.insert_one(doc! { "old_target_id": old_id, "new_target_id": new_id })
.await?;
Ok(())
}
/// Undo the backfill: replay the relink log in reverse, drop `onboarded_targets`
/// and the log, and clear the marker. The legacy collections are untouched, so
/// this restores the pre-migration state.
pub async fn revert(db: &Database) -> Result<(), AgentError> {
let log = db.collection_named::<Document>("onboarding_migration_log");
let mut cursor = log.find(doc! {}).await?;
while let Some(entry) = cursor.try_next().await? {
if let (Ok(old), Ok(new)) = (
entry.get_str("old_target_id"),
entry.get_str("new_target_id"),
) {
db.dast_scan_runs()
.update_many(
doc! { "target_id": new },
doc! { "$set": { "target_id": old } },
)
.await?;
db.pentest_sessions()
.update_many(
doc! { "target_id": new },
doc! { "$set": { "target_id": old } },
)
.await?;
}
}
db.onboarded_targets().drop().await?;
log.drop().await?;
db.collection_named::<Document>("schema_migrations")
.delete_one(doc! { "_id": MIGRATION_MARKER })
.await?;
Ok(())
}
#[cfg(test)]
#[allow(clippy::expect_used, clippy::unwrap_used)]
mod tests {
use super::*;
use compliance_core::models::{DastAuthConfig, TrackerType};
fn repo() -> TrackedRepository {
let mut r = TrackedRepository::new("acme".to_string(), "https://git/acme.git".to_string());
r.id = Some(mongodb::bson::oid::ObjectId::new());
r.default_branch = "develop".to_string();
r.last_scanned_commit = Some("abc123".to_string());
r.auth_token = Some("pat".to_string());
r.auth_username = Some("bob".to_string());
r.tracker_type = Some(TrackerType::Gitea);
r.tracker_owner = Some("acme".to_string());
r.findings_count = 7;
r
}
fn dast(repo_id: Option<String>, kind: DastTargetType) -> DastTarget {
let mut d = DastTarget::new(
"acme-web".to_string(),
"https://acme.example.com".to_string(),
kind,
);
d.id = Some(mongodb::bson::oid::ObjectId::new());
d.repo_id = repo_id;
d.max_crawl_depth = 5;
d.auth_config = Some(DastAuthConfig {
method: "bearer".to_string(),
login_url: None,
username: None,
password: None,
token: Some("tok".to_string()),
headers: None,
});
d
}
#[test]
fn repo_maps_preserving_id_and_git_artifact() {
let r = repo();
let t = repo_to_target(&r);
assert_eq!(t.id, r.id); // id preserved
assert_eq!(t.findings_count, 7);
assert_eq!(t.scan_schedule, r.scan_schedule);
let git = t.code_artifact().expect("git artifact");
assert_eq!(git.kind, ArtifactKind::GitRepo);
assert_eq!(git.source_ref, "https://git/acme.git");
let gc = git.git.as_ref().expect("git config");
assert_eq!(gc.default_branch, "develop");
assert_eq!(gc.last_scanned_commit.as_deref(), Some("abc123"));
let auth = git.auth.as_ref().expect("auth");
assert_eq!(auth.secret.as_deref(), Some("pat"));
assert_eq!(auth.username.as_deref(), Some("bob"));
assert_eq!(
t.scan_config
.issue_tracker
.as_ref()
.and_then(|it| it.tracker_type.clone()),
Some(TrackerType::Gitea)
);
}
#[test]
fn standalone_dast_maps_preserving_id_and_live_url() {
let d = dast(None, DastTargetType::WebApp);
let t = dast_to_standalone_target(&d);
assert_eq!(t.id, d.id);
assert_eq!(t.target_type, TargetType::WebApp);
let url = t.live_url().expect("live url");
assert_eq!(url.source_ref, "https://acme.example.com");
let web = url.web.as_ref().expect("web config");
assert_eq!(web.max_crawl_depth, 5);
assert_eq!(
url.auth.as_ref().and_then(|a| a.secret.clone()),
Some("tok".to_string())
);
}
#[test]
fn rest_api_dast_maps_to_backend_service() {
let d = dast(None, DastTargetType::RestApi);
assert_eq!(
dast_to_standalone_target(&d).target_type,
TargetType::BackendService
);
}
#[test]
fn fold_adds_live_url_and_promotes_webapp() {
let mut t = repo_to_target(&repo());
assert_eq!(t.target_type, TargetType::BackendService);
fold_dast_into_target(&mut t, &dast(Some("x".to_string()), DastTargetType::WebApp));
assert_eq!(t.target_type, TargetType::WebApp); // promoted
assert!(t.has(ArtifactKind::LiveUrl));
assert!(t.has(ArtifactKind::GitRepo));
}
#[test]
fn fold_is_idempotent_on_live_url() {
let mut t = repo_to_target(&repo());
let d = dast(Some("x".to_string()), DastTargetType::WebApp);
fold_dast_into_target(&mut t, &d);
fold_dast_into_target(&mut t, &d);
let live_urls = t
.artifacts
.iter()
.filter(|a| a.kind == ArtifactKind::LiveUrl)
.count();
assert_eq!(live_urls, 1);
}
}
@@ -0,0 +1,156 @@
// Integration tests for the onboarding backfill migration.
//
// Requires MongoDB (set TEST_MONGODB_URI if not at the default).
// Not run in CI (which is `--lib` only) — run locally:
// cargo test -p compliance-agent --test e2e migration
use compliance_agent::database::{Database, DatabasePool};
use compliance_agent::migrate::onboarding;
use compliance_core::models::{
ArtifactKind, DastTarget, DastTargetType, TargetType, TrackedRepository,
};
use mongodb::bson::{doc, Document};
async fn fresh_db() -> (DatabasePool, String, Database) {
let uri = std::env::var("TEST_MONGODB_URI")
.unwrap_or_else(|_| "mongodb://root:example@localhost:27017/?authSource=admin".into());
// Prefix must fit the pool's 30-char cap (`<prefix>_<32 hex>` <= 63).
let prefix = format!("t_{}", &uuid::Uuid::new_v4().simple().to_string()[..16]);
let pool = DatabasePool::connect(&uri, &prefix)
.await
.expect("connect mongo");
let db = pool.for_tenant_id("t1").await.expect("tenant db");
(pool, prefix, db)
}
async fn cleanup(pool: &DatabasePool, prefix: &str) {
if let Ok(names) = pool.client().list_database_names().await {
for n in names {
if n.starts_with(prefix) {
pool.client().database(&n).drop().await.ok();
}
}
}
}
#[tokio::test]
async fn backfill_folds_relinks_is_idempotent_and_reversible() {
let (pool, prefix, db) = fresh_db().await;
// Seed a repo.
let repo = TrackedRepository::new("acme".into(), "https://git/acme.git".into());
let repo_id = db
.repositories()
.insert_one(repo)
.await
.expect("insert repo")
.inserted_id
.as_object_id()
.expect("repo oid");
// A DAST target linked to the repo (folds + promotes to WebApp + relinks).
let mut linked = DastTarget::new(
"acme-web".into(),
"https://acme.example.com".into(),
DastTargetType::WebApp,
);
linked.repo_id = Some(repo_id.to_hex());
let linked_id = db
.dast_targets()
.insert_one(linked)
.await
.expect("insert linked dast")
.inserted_id
.as_object_id()
.expect("linked oid");
// A repo-less DAST target (standalone).
let standalone = DastTarget::new(
"acme-api".into(),
"https://api.acme.com".into(),
DastTargetType::RestApi,
);
let standalone_id = db
.dast_targets()
.insert_one(standalone)
.await
.expect("insert standalone dast")
.inserted_id
.as_object_id()
.expect("standalone oid");
// A DAST scan run pointing at the linked target — should be relinked to the repo.
db.collection_named::<Document>("dast_scan_runs")
.insert_one(doc! { "target_id": linked_id.to_hex(), "status": "completed" })
.await
.expect("insert dast run");
// --- Backfill ---
assert!(!onboarding::already_applied(&db).await.unwrap());
let report = onboarding::backfill_onboarded_targets(&db, false)
.await
.expect("backfill");
assert_eq!(report.repos_migrated, 1);
assert_eq!(report.dast_targets_folded, 1);
assert_eq!(report.dast_targets_standalone, 1);
assert!(onboarding::already_applied(&db).await.unwrap());
// Repo target: preserved _id, has git + folded live-url, promoted to WebApp.
let repo_target = db
.onboarded_targets()
.find_one(doc! { "_id": repo_id })
.await
.unwrap()
.expect("repo target");
assert!(repo_target.has(ArtifactKind::GitRepo));
assert!(repo_target.has(ArtifactKind::LiveUrl));
assert_eq!(repo_target.target_type, TargetType::WebApp);
// Standalone target: preserved _id, live-url, backend service.
let standalone_target = db
.onboarded_targets()
.find_one(doc! { "_id": standalone_id })
.await
.unwrap()
.expect("standalone target");
assert!(standalone_target.has(ArtifactKind::LiveUrl));
assert_eq!(standalone_target.target_type, TargetType::BackendService);
// The DAST run was relinked from the old dast id to the repo (unified) id.
let run = db
.collection_named::<Document>("dast_scan_runs")
.find_one(doc! {})
.await
.unwrap()
.expect("run");
assert_eq!(run.get_str("target_id").unwrap(), repo_id.to_hex());
// --- Idempotent: re-run migrates nothing new ---
let again = onboarding::backfill_onboarded_targets(&db, false)
.await
.expect("backfill again");
assert_eq!(again.repos_migrated, 0);
assert_eq!(again.dast_targets_folded, 0);
assert_eq!(again.dast_targets_standalone, 0);
assert!(again.skipped_existing >= 2);
// --- Revert: onboarded targets gone, relink undone, marker cleared ---
onboarding::revert(&db).await.expect("revert");
assert_eq!(
db.onboarded_targets()
.count_documents(doc! {})
.await
.unwrap(),
0
);
let run_after = db
.collection_named::<Document>("dast_scan_runs")
.find_one(doc! {})
.await
.unwrap()
.expect("run");
assert_eq!(run_after.get_str("target_id").unwrap(), linked_id.to_hex());
assert!(!onboarding::already_applied(&db).await.unwrap());
cleanup(&pool, &prefix).await;
}
@@ -7,3 +7,4 @@
// Or nightly: (via CI with MongoDB service container)
mod api;
mod migration;