feat(onboarding): artifact ingest + classifier + native tramiton + suite seams (#138)

compliance-agent/src/ingest/blob.rs

@@ -0,0 +1,154 @@
+//! Content-addressed blob storage and archive extraction for ingest.
+//!
+//! Blobs are stored at `<base>/blobs/<sha[0:2]>/<sha>` and deduplicated by
+//! digest; per-run working directories live under `<base>/work/`.
+
+use std::fs::{self, File};
+use std::io::{self, Read};
+use std::path::{Path, PathBuf};
+
+use sha2::{Digest, Sha256};
+
+use crate::error::AgentError;
+
+/// Read buffer size for streaming hashes/copies (64 KiB).
+const BUF_LEN: usize = 64 * 1024;
+
+/// Stream-hash a file with SHA-256, returning the lowercase-hex digest and the
+/// byte length. Streams so large firmware images never load fully into memory.
+pub fn hash_file(path: &Path) -> Result<(String, u64), AgentError> {
+    let mut file = File::open(path)?;
+    let mut hasher = Sha256::new();
+    let mut buf = [0u8; BUF_LEN];
+    let mut total: u64 = 0;
+    loop {
+        let n = file.read(&mut buf)?;
+        if n == 0 {
+            break;
+        }
+        hasher.update(&buf[..n]);
+        total += n as u64;
+    }
+    Ok((hex::encode(hasher.finalize()), total))
+}
+
+/// Copy `src` into the content-addressed blob store under `base`, returning the
+/// stored path. Idempotent: an already-present blob is not rewritten.
+pub fn store_file(base: &Path, src: &Path, sha: &str) -> Result<PathBuf, AgentError> {
+    if sha.len() < 2 {
+        return Err(AgentError::Other(format!("invalid content hash '{sha}'")));
+    }
+    let dir = base.join("blobs").join(&sha[0..2]);
+    fs::create_dir_all(&dir)?;
+    let dest = dir.join(sha);
+    if !dest.exists() {
+        fs::copy(src, &dest)?;
+    }
+    Ok(dest)
+}
+
+/// Extract a zip archive into `dest` (created if needed). `enclosed_name`
+/// sanitizes each entry path, so this is safe against zip-slip traversal.
+pub fn extract_zip(archive: &Path, dest: &Path) -> Result<(), AgentError> {
+    let file = File::open(archive)?;
+    let mut zip =
+        zip::ZipArchive::new(file).map_err(|e| AgentError::Other(format!("open zip: {e}")))?;
+    fs::create_dir_all(dest)?;
+    for i in 0..zip.len() {
+        let mut entry = zip
+            .by_index(i)
+            .map_err(|e| AgentError::Other(format!("read zip entry: {e}")))?;
+        // `enclosed_name` returns `None` for traversal-unsafe paths — skip them.
+        let Some(rel) = entry.enclosed_name() else {
+            continue;
+        };
+        let out = dest.join(rel);
+        if entry.is_dir() {
+            fs::create_dir_all(&out)?;
+        } else {
+            if let Some(parent) = out.parent() {
+                fs::create_dir_all(parent)?;
+            }
+            let mut outfile = File::create(&out)?;
+            io::copy(&mut entry, &mut outfile)?;
+        }
+    }
+    Ok(())
+}
+
+/// The working directory for one artifact of a target: `<base>/work/<target>/<artifact>`.
+pub fn work_dir(base: &Path, target_id: &str, artifact_id: &str) -> PathBuf {
+    base.join("work").join(target_id).join(artifact_id)
+}
+
+#[cfg(test)]
+#[allow(clippy::expect_used, clippy::unwrap_used)]
+mod tests {
+    use super::*;
+
+    /// A unique scratch directory, removed on drop.
+    struct Scratch(PathBuf);
+    impl Scratch {
+        fn new() -> Self {
+            let p = std::env::temp_dir().join(format!("cs-ingest-{}", uuid::Uuid::new_v4()));
+            fs::create_dir_all(&p).expect("mkdir scratch");
+            Self(p)
+        }
+        fn path(&self) -> &Path {
+            &self.0
+        }
+    }
+    impl Drop for Scratch {
+        fn drop(&mut self) {
+            let _ = fs::remove_dir_all(&self.0);
+        }
+    }
+
+    #[test]
+    fn hash_is_stable_and_reports_size() {
+        let dir = Scratch::new();
+        let f = dir.path().join("a.bin");
+        fs::write(&f, b"hello world").expect("write");
+        let (sha, size) = hash_file(&f).expect("hash");
+        assert_eq!(size, 11);
+        // Known SHA-256 of "hello world".
+        assert_eq!(
+            sha,
+            "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"
+        );
+    }
+
+    #[test]
+    fn store_is_content_addressed_and_idempotent() {
+        let base = Scratch::new();
+        let src = base.path().join("src.bin");
+        fs::write(&src, b"payload").expect("write");
+        let (sha, _) = hash_file(&src).expect("hash");
+        let p1 = store_file(base.path(), &src, &sha).expect("store");
+        let p2 = store_file(base.path(), &src, &sha).expect("store again");
+        assert_eq!(p1, p2);
+        assert!(p1.ends_with(&sha));
+        assert!(p1.starts_with(base.path().join("blobs").join(&sha[0..2])));
+        assert_eq!(fs::read(&p1).expect("read"), b"payload");
+    }
+
+    #[test]
+    fn extract_zip_writes_entries() {
+        let base = Scratch::new();
+        let archive = base.path().join("a.zip");
+        {
+            let file = File::create(&archive).expect("create");
+            let mut w = zip::ZipWriter::new(file);
+            let opts: zip::write::SimpleFileOptions = Default::default();
+            w.start_file("dir/hello.txt", opts).expect("start");
+            io::Write::write_all(&mut w, b"hi").expect("write");
+            w.finish().expect("finish");
+        }
+        let dest = base.path().join("out");
+        extract_zip(&archive, &dest).expect("extract");
+        assert_eq!(
+            fs::read_to_string(dest.join("dir/hello.txt")).expect("read"),
+            "hi"
+        );
+    }
+}

compliance-agent/src/ingest/mod.rs

@@ -0,0 +1,334 @@
+//! Artifact ingest.
+//!
+//! Normalizes each [`Artifact`] on an [`OnboardedTarget`] into a local working
+//! path plus recorded metadata (content hash, size, discovered facts) that the
+//! classifier and scanners consume. Every blob is SHA-256 hashed — that digest
+//! is also the reconciliation key against sibling products (a firmware sha256
+//! matches tramiton's `Artifact.sha256`).
+
+mod blob;
+
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+
+use compliance_core::models::{Artifact, ArtifactKind, DetectedFact, OnboardedTarget};
+use compliance_core::AgentConfig;
+
+use crate::error::AgentError;
+use crate::pipeline::git::{GitOps, RepoCredentials};
+
+/// The paths and identifiers an ingest needs. Decoupled from the full
+/// [`AgentConfig`] so ingest is testable without a complete config.
+pub struct IngestContext<'a> {
+    /// Base directory for content-addressed blobs and working dirs.
+    pub artifact_store_base: &'a Path,
+    /// Base directory for git clones.
+    pub git_clone_base: &'a str,
+    /// Default SSH key path (used when an artifact provides none).
+    pub ssh_key_path: &'a str,
+    /// The id of the target these artifacts belong to (namespaces working dirs).
+    pub target_id: &'a str,
+}
+
+impl<'a> IngestContext<'a> {
+    /// Build an ingest context from the agent config for a given target.
+    pub fn from_config(config: &'a AgentConfig, target_id: &'a str) -> Self {
+        Self {
+            artifact_store_base: Path::new(&config.artifact_store_base_path),
+            git_clone_base: &config.git_clone_base_path,
+            ssh_key_path: &config.ssh_key_path,
+            target_id,
+        }
+    }
+}
+
+/// The result of ingesting one artifact.
+pub struct IngestedArtifact {
+    /// The artifact this corresponds to ([`Artifact::id`]).
+    pub artifact_id: String,
+    /// The artifact kind.
+    pub kind: ArtifactKind,
+    /// Local working path (clone dir, extracted dir, or blob file). `None` for
+    /// artifacts with no on-disk form (live URL, plaintext, container ref).
+    pub working_path: Option<PathBuf>,
+    /// SHA-256 of the content (blobs) or git head SHA (git repos).
+    pub content_hash: Option<String>,
+    /// Stored blob size in bytes, when applicable.
+    pub size_bytes: Option<u64>,
+    /// Facts discovered during ingest.
+    pub facts: Vec<DetectedFact>,
+}
+
+/// All ingested artifacts for a target, keyed by artifact id.
+pub struct IngestSet {
+    /// The ingested artifacts, keyed by [`Artifact::id`].
+    pub by_artifact: HashMap<String, IngestedArtifact>,
+}
+
+impl IngestSet {
+    /// The working paths of every ingested artifact that has one — the input the
+    /// classifier expects.
+    pub fn working_paths(&self) -> HashMap<String, PathBuf> {
+        self.by_artifact
+            .iter()
+            .filter_map(|(id, a)| a.working_path.clone().map(|p| (id.clone(), p)))
+            .collect()
+    }
+
+    /// The ingest result for a specific artifact.
+    pub fn get(&self, artifact_id: &str) -> Option<&IngestedArtifact> {
+        self.by_artifact.get(artifact_id)
+    }
+}
+
+/// Ingest every artifact on a target.
+pub fn ingest_all(
+    target: &OnboardedTarget,
+    ctx: &IngestContext<'_>,
+) -> Result<IngestSet, AgentError> {
+    let mut by_artifact = HashMap::new();
+    for artifact in &target.artifacts {
+        let ingested = ingest_artifact(artifact, ctx)?;
+        by_artifact.insert(artifact.id.clone(), ingested);
+    }
+    Ok(IngestSet { by_artifact })
+}
+
+/// Ingest a single artifact, dispatching on its kind.
+pub fn ingest_artifact(
+    artifact: &Artifact,
+    ctx: &IngestContext<'_>,
+) -> Result<IngestedArtifact, AgentError> {
+    match artifact.kind {
+        ArtifactKind::GitRepo => ingest_git(artifact, ctx),
+        ArtifactKind::SourceArchive | ArtifactKind::MobilePackage | ArtifactKind::PlcProject => {
+            ingest_blob(artifact, ctx, true)
+        }
+        ArtifactKind::FirmwareImage => ingest_blob(artifact, ctx, false),
+        ArtifactKind::ContainerImage => Ok(metadata_only(
+            artifact,
+            DetectedFact::new("container_ref", artifact.source_ref.as_str(), "ingest"),
+        )),
+        ArtifactKind::LiveUrl => Ok(metadata_only(
+            artifact,
+            DetectedFact::new("live_url", artifact.source_ref.as_str(), "ingest"),
+        )),
+        ArtifactKind::PlaintextDescription => Ok(metadata_only(
+            artifact,
+            DetectedFact::new(
+                "description_len",
+                artifact.source_ref.len().to_string(),
+                "ingest",
+            ),
+        )),
+    }
+}
+
+/// Clone (or fetch) a git artifact, recording the head SHA as the content hash.
+fn ingest_git(
+    artifact: &Artifact,
+    ctx: &IngestContext<'_>,
+) -> Result<IngestedArtifact, AgentError> {
+    let creds = credentials_for(artifact, ctx.ssh_key_path);
+    let git_ops = GitOps::new(ctx.git_clone_base, creds);
+    let repo_path = git_ops.clone_or_fetch(&artifact.source_ref, &artifact.id)?;
+    let head = GitOps::get_head_sha(&repo_path).ok();
+    Ok(IngestedArtifact {
+        artifact_id: artifact.id.clone(),
+        kind: artifact.kind,
+        working_path: Some(repo_path),
+        content_hash: head,
+        size_bytes: None,
+        facts: Vec::new(),
+    })
+}
+
+/// Store a blob artifact content-addressed. When `extract` is set and the blob
+/// is a zip container (source archive, APK/AAB/IPA), also unpack it into a
+/// working directory; otherwise the working path is the stored blob.
+fn ingest_blob(
+    artifact: &Artifact,
+    ctx: &IngestContext<'_>,
+    extract: bool,
+) -> Result<IngestedArtifact, AgentError> {
+    let base = ctx.artifact_store_base;
+    let src = local_source(artifact)?;
+    let (sha, size) = blob::hash_file(&src)?;
+    let stored = blob::store_file(base, &src, &sha)?;
+
+    let mut facts = Vec::new();
+    let working_path = if extract {
+        let dest = blob::work_dir(base, ctx.target_id, &artifact.id);
+        match blob::extract_zip(&stored, &dest) {
+            Ok(()) => dest,
+            Err(e) => {
+                // Not a zip (e.g. a tar.gz source archive) — keep the blob and
+                // note it so later stages can decide what to do.
+                facts.push(DetectedFact::new(
+                    "archive_unextracted",
+                    e.to_string(),
+                    "ingest",
+                ));
+                stored.clone()
+            }
+        }
+    } else {
+        stored.clone()
+    };
+
+    Ok(IngestedArtifact {
+        artifact_id: artifact.id.clone(),
+        kind: artifact.kind,
+        working_path: Some(working_path),
+        content_hash: Some(sha),
+        size_bytes: Some(size),
+        facts,
+    })
+}
+
+/// An artifact with no on-disk form: record a single fact, no hash/path.
+fn metadata_only(artifact: &Artifact, fact: DetectedFact) -> IngestedArtifact {
+    IngestedArtifact {
+        artifact_id: artifact.id.clone(),
+        kind: artifact.kind,
+        working_path: None,
+        content_hash: None,
+        size_bytes: None,
+        facts: vec![fact],
+    }
+}
+
+/// The local file backing a blob artifact: its `stored_path` if already
+/// uploaded, else its `source_ref` interpreted as a filesystem path.
+fn local_source(artifact: &Artifact) -> Result<PathBuf, AgentError> {
+    let path = artifact
+        .stored_path
+        .as_deref()
+        .unwrap_or(artifact.source_ref.as_str());
+    let path = PathBuf::from(path);
+    if !path.exists() {
+        return Err(AgentError::Other(format!(
+            "artifact {} source not found at {}",
+            artifact.id,
+            path.display()
+        )));
+    }
+    Ok(path)
+}
+
+/// Build git credentials from an artifact's auth plus a default SSH key path.
+fn credentials_for(artifact: &Artifact, default_ssh_key_path: &str) -> RepoCredentials {
+    let auth = artifact.auth.as_ref();
+    RepoCredentials {
+        ssh_key_path: auth
+            .and_then(|a| a.ssh_key_path.clone())
+            .or_else(|| Some(default_ssh_key_path.to_string())),
+        auth_token: auth.and_then(|a| a.secret.clone()),
+        auth_username: auth.and_then(|a| a.username.clone()),
+    }
+}
+
+#[cfg(test)]
+#[allow(clippy::expect_used, clippy::unwrap_used)]
+mod tests {
+    use super::*;
+    use compliance_core::models::{ArtifactAuth, TargetType};
+
+    /// A unique scratch directory, removed on drop.
+    struct Scratch(PathBuf);
+    impl Scratch {
+        fn new() -> Self {
+            let p = std::env::temp_dir().join(format!("cs-ingest-mod-{}", uuid::Uuid::new_v4()));
+            std::fs::create_dir_all(&p).expect("mkdir scratch");
+            Self(p)
+        }
+    }
+    impl Drop for Scratch {
+        fn drop(&mut self) {
+            let _ = std::fs::remove_dir_all(&self.0);
+        }
+    }
+
+    fn ctx_for<'a>(store: &'a Path, target_id: &'a str) -> IngestContext<'a> {
+        IngestContext {
+            artifact_store_base: store,
+            git_clone_base: "/tmp/cs-ingest-test-repos",
+            ssh_key_path: "/tmp/cs-ingest-test-ssh",
+            target_id,
+        }
+    }
+
+    #[test]
+    fn firmware_blob_is_hashed_and_stored() {
+        let scratch = Scratch::new();
+        let store = scratch.0.join("store");
+        let fw = scratch.0.join("fw.bin");
+        std::fs::write(&fw, b"firmware-bytes").expect("write");
+
+        let ctx = ctx_for(&store, "t1");
+        let artifact = Artifact::firmware_image(fw.to_string_lossy().to_string());
+        let out = ingest_artifact(&artifact, &ctx).expect("ingest");
+
+        assert_eq!(out.kind, ArtifactKind::FirmwareImage);
+        assert_eq!(out.size_bytes, Some(14));
+        let sha = out.content_hash.expect("hash");
+        assert_eq!(sha.len(), 64);
+        // working path is the content-addressed blob
+        let wp = out.working_path.expect("working path");
+        assert!(wp.starts_with(store.join("blobs")));
+    }
+
+    #[test]
+    fn live_url_has_no_blob() {
+        let scratch = Scratch::new();
+        let store = scratch.0.join("store");
+        let ctx = ctx_for(&store, "t1");
+        let artifact = Artifact::live_url("https://example.com");
+        let out = ingest_artifact(&artifact, &ctx).expect("ingest");
+        assert!(out.working_path.is_none());
+        assert!(out.content_hash.is_none());
+        assert!(out.facts.iter().any(|f| f.key == "live_url"));
+    }
+
+    #[test]
+    fn ingest_all_collects_working_paths() {
+        let scratch = Scratch::new();
+        let store = scratch.0.join("store");
+        let fw = scratch.0.join("fw.bin");
+        std::fs::write(&fw, b"abc").expect("write");
+        let ctx = ctx_for(&store, "t1");
+
+        let mut target = OnboardedTarget::new("t".to_string(), TargetType::FirmwareBareMetal);
+        target
+            .artifacts
+            .push(Artifact::firmware_image(fw.to_string_lossy().to_string()));
+        target.artifacts.push(Artifact::live_url("https://x"));
+
+        let set = ingest_all(&target, &ctx).expect("ingest all");
+        assert_eq!(set.by_artifact.len(), 2);
+        // Only the firmware artifact yields a working path.
+        assert_eq!(set.working_paths().len(), 1);
+    }
+
+    #[test]
+    fn credentials_prefer_artifact_auth() {
+        let mut artifact = Artifact::git_repo("https://git/x", "main");
+        artifact.auth = Some(ArtifactAuth {
+            method: "token".to_string(),
+            username: Some("bob".to_string()),
+            secret: Some("pat".to_string()),
+            ..Default::default()
+        });
+        let creds = credentials_for(&artifact, "/default/ssh/key");
+        assert_eq!(creds.auth_token.as_deref(), Some("pat"));
+        assert_eq!(creds.auth_username.as_deref(), Some("bob"));
+    }
+
+    #[test]
+    fn credentials_fall_back_to_default_ssh_key() {
+        let artifact = Artifact::git_repo("git@host:x.git", "main");
+        let creds = credentials_for(&artifact, "/default/ssh/key");
+        assert_eq!(creds.ssh_key_path.as_deref(), Some("/default/ssh/key"));
+        assert!(creds.auth_token.is_none());
+    }
+}