feat(onboarding): artifact ingest + classifier + native tramiton + suite seams (#138)

2026-07-10 16:00:51 +00:00
parent 675c4ef699
commit ef6ee3dcd1
18 changed files with 1661 additions and 6 deletions
@@ -0,0 +1,334 @@
+//! Artifact ingest.
+//!
+//! Normalizes each [`Artifact`] on an [`OnboardedTarget`] into a local working
+//! path plus recorded metadata (content hash, size, discovered facts) that the
+//! classifier and scanners consume. Every blob is SHA-256 hashed — that digest
+//! is also the reconciliation key against sibling products (a firmware sha256
+//! matches tramiton's `Artifact.sha256`).
+
+mod blob;
+
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+
+use compliance_core::models::{Artifact, ArtifactKind, DetectedFact, OnboardedTarget};
+use compliance_core::AgentConfig;
+
+use crate::error::AgentError;
+use crate::pipeline::git::{GitOps, RepoCredentials};
+
+/// The paths and identifiers an ingest needs. Decoupled from the full
+/// [`AgentConfig`] so ingest is testable without a complete config.
+pub struct IngestContext<'a> {
+    /// Base directory for content-addressed blobs and working dirs.
+    pub artifact_store_base: &'a Path,
+    /// Base directory for git clones.
+    pub git_clone_base: &'a str,
+    /// Default SSH key path (used when an artifact provides none).
+    pub ssh_key_path: &'a str,
+    /// The id of the target these artifacts belong to (namespaces working dirs).
+    pub target_id: &'a str,
+}
+
+impl<'a> IngestContext<'a> {
+    /// Build an ingest context from the agent config for a given target.
+    pub fn from_config(config: &'a AgentConfig, target_id: &'a str) -> Self {
+        Self {
+            artifact_store_base: Path::new(&config.artifact_store_base_path),
+            git_clone_base: &config.git_clone_base_path,
+            ssh_key_path: &config.ssh_key_path,
+            target_id,
+        }
+    }
+}
+
+/// The result of ingesting one artifact.
+pub struct IngestedArtifact {
+    /// The artifact this corresponds to ([`Artifact::id`]).
+    pub artifact_id: String,
+    /// The artifact kind.
+    pub kind: ArtifactKind,
+    /// Local working path (clone dir, extracted dir, or blob file). `None` for
+    /// artifacts with no on-disk form (live URL, plaintext, container ref).
+    pub working_path: Option<PathBuf>,
+    /// SHA-256 of the content (blobs) or git head SHA (git repos).
+    pub content_hash: Option<String>,
+    /// Stored blob size in bytes, when applicable.
+    pub size_bytes: Option<u64>,
+    /// Facts discovered during ingest.
+    pub facts: Vec<DetectedFact>,
+}
+
+/// All ingested artifacts for a target, keyed by artifact id.
+pub struct IngestSet {
+    /// The ingested artifacts, keyed by [`Artifact::id`].
+    pub by_artifact: HashMap<String, IngestedArtifact>,
+}
+
+impl IngestSet {
+    /// The working paths of every ingested artifact that has one — the input the
+    /// classifier expects.
+    pub fn working_paths(&self) -> HashMap<String, PathBuf> {
+        self.by_artifact
+            .iter()
+            .filter_map(|(id, a)| a.working_path.clone().map(|p| (id.clone(), p)))
+            .collect()
+    }
+
+    /// The ingest result for a specific artifact.
+    pub fn get(&self, artifact_id: &str) -> Option<&IngestedArtifact> {
+        self.by_artifact.get(artifact_id)
+    }
+}
+
+/// Ingest every artifact on a target.
+pub fn ingest_all(
+    target: &OnboardedTarget,
+    ctx: &IngestContext<'_>,
+) -> Result<IngestSet, AgentError> {
+    let mut by_artifact = HashMap::new();
+    for artifact in &target.artifacts {
+        let ingested = ingest_artifact(artifact, ctx)?;
+        by_artifact.insert(artifact.id.clone(), ingested);
+    }
+    Ok(IngestSet { by_artifact })
+}
+
+/// Ingest a single artifact, dispatching on its kind.
+pub fn ingest_artifact(
+    artifact: &Artifact,
+    ctx: &IngestContext<'_>,
+) -> Result<IngestedArtifact, AgentError> {
+    match artifact.kind {
+        ArtifactKind::GitRepo => ingest_git(artifact, ctx),
+        ArtifactKind::SourceArchive | ArtifactKind::MobilePackage | ArtifactKind::PlcProject => {
+            ingest_blob(artifact, ctx, true)
+        }
+        ArtifactKind::FirmwareImage => ingest_blob(artifact, ctx, false),
+        ArtifactKind::ContainerImage => Ok(metadata_only(
+            artifact,
+            DetectedFact::new("container_ref", artifact.source_ref.as_str(), "ingest"),
+        )),
+        ArtifactKind::LiveUrl => Ok(metadata_only(
+            artifact,
+            DetectedFact::new("live_url", artifact.source_ref.as_str(), "ingest"),
+        )),
+        ArtifactKind::PlaintextDescription => Ok(metadata_only(
+            artifact,
+            DetectedFact::new(
+                "description_len",
+                artifact.source_ref.len().to_string(),
+                "ingest",
+            ),
+        )),
+    }
+}
+
+/// Clone (or fetch) a git artifact, recording the head SHA as the content hash.
+fn ingest_git(
+    artifact: &Artifact,
+    ctx: &IngestContext<'_>,
+) -> Result<IngestedArtifact, AgentError> {
+    let creds = credentials_for(artifact, ctx.ssh_key_path);
+    let git_ops = GitOps::new(ctx.git_clone_base, creds);
+    let repo_path = git_ops.clone_or_fetch(&artifact.source_ref, &artifact.id)?;
+    let head = GitOps::get_head_sha(&repo_path).ok();
+    Ok(IngestedArtifact {
+        artifact_id: artifact.id.clone(),
+        kind: artifact.kind,
+        working_path: Some(repo_path),
+        content_hash: head,
+        size_bytes: None,
+        facts: Vec::new(),
+    })
+}
+
+/// Store a blob artifact content-addressed. When `extract` is set and the blob
+/// is a zip container (source archive, APK/AAB/IPA), also unpack it into a
+/// working directory; otherwise the working path is the stored blob.
+fn ingest_blob(
+    artifact: &Artifact,
+    ctx: &IngestContext<'_>,
+    extract: bool,
+) -> Result<IngestedArtifact, AgentError> {
+    let base = ctx.artifact_store_base;
+    let src = local_source(artifact)?;
+    let (sha, size) = blob::hash_file(&src)?;
+    let stored = blob::store_file(base, &src, &sha)?;
+
+    let mut facts = Vec::new();
+    let working_path = if extract {
+        let dest = blob::work_dir(base, ctx.target_id, &artifact.id);
+        match blob::extract_zip(&stored, &dest) {
+            Ok(()) => dest,
+            Err(e) => {
+                // Not a zip (e.g. a tar.gz source archive) — keep the blob and
+                // note it so later stages can decide what to do.
+                facts.push(DetectedFact::new(
+                    "archive_unextracted",
+                    e.to_string(),
+                    "ingest",
+                ));
+                stored.clone()
+            }
+        }
+    } else {
+        stored.clone()
+    };
+
+    Ok(IngestedArtifact {
+        artifact_id: artifact.id.clone(),
+        kind: artifact.kind,
+        working_path: Some(working_path),
+        content_hash: Some(sha),
+        size_bytes: Some(size),
+        facts,
+    })
+}
+
+/// An artifact with no on-disk form: record a single fact, no hash/path.
+fn metadata_only(artifact: &Artifact, fact: DetectedFact) -> IngestedArtifact {
+    IngestedArtifact {
+        artifact_id: artifact.id.clone(),
+        kind: artifact.kind,
+        working_path: None,
+        content_hash: None,
+        size_bytes: None,
+        facts: vec![fact],
+    }
+}
+
+/// The local file backing a blob artifact: its `stored_path` if already
+/// uploaded, else its `source_ref` interpreted as a filesystem path.
+fn local_source(artifact: &Artifact) -> Result<PathBuf, AgentError> {
+    let path = artifact
+        .stored_path
+        .as_deref()
+        .unwrap_or(artifact.source_ref.as_str());
+    let path = PathBuf::from(path);
+    if !path.exists() {
+        return Err(AgentError::Other(format!(
+            "artifact {} source not found at {}",
+            artifact.id,
+            path.display()
+        )));
+    }
+    Ok(path)
+}
+
+/// Build git credentials from an artifact's auth plus a default SSH key path.
+fn credentials_for(artifact: &Artifact, default_ssh_key_path: &str) -> RepoCredentials {
+    let auth = artifact.auth.as_ref();
+    RepoCredentials {
+        ssh_key_path: auth
+            .and_then(|a| a.ssh_key_path.clone())
+            .or_else(|| Some(default_ssh_key_path.to_string())),
+        auth_token: auth.and_then(|a| a.secret.clone()),
+        auth_username: auth.and_then(|a| a.username.clone()),
+    }
+}
+
+#[cfg(test)]
+#[allow(clippy::expect_used, clippy::unwrap_used)]
+mod tests {
+    use super::*;
+    use compliance_core::models::{ArtifactAuth, TargetType};
+
+    /// A unique scratch directory, removed on drop.
+    struct Scratch(PathBuf);
+    impl Scratch {
+        fn new() -> Self {
+            let p = std::env::temp_dir().join(format!("cs-ingest-mod-{}", uuid::Uuid::new_v4()));
+            std::fs::create_dir_all(&p).expect("mkdir scratch");
+            Self(p)
+        }
+    }
+    impl Drop for Scratch {
+        fn drop(&mut self) {
+            let _ = std::fs::remove_dir_all(&self.0);
+        }
+    }
+
+    fn ctx_for<'a>(store: &'a Path, target_id: &'a str) -> IngestContext<'a> {
+        IngestContext {
+            artifact_store_base: store,
+            git_clone_base: "/tmp/cs-ingest-test-repos",
+            ssh_key_path: "/tmp/cs-ingest-test-ssh",
+            target_id,
+        }
+    }
+
+    #[test]
+    fn firmware_blob_is_hashed_and_stored() {
+        let scratch = Scratch::new();
+        let store = scratch.0.join("store");
+        let fw = scratch.0.join("fw.bin");
+        std::fs::write(&fw, b"firmware-bytes").expect("write");
+
+        let ctx = ctx_for(&store, "t1");
+        let artifact = Artifact::firmware_image(fw.to_string_lossy().to_string());
+        let out = ingest_artifact(&artifact, &ctx).expect("ingest");
+
+        assert_eq!(out.kind, ArtifactKind::FirmwareImage);
+        assert_eq!(out.size_bytes, Some(14));
+        let sha = out.content_hash.expect("hash");
+        assert_eq!(sha.len(), 64);
+        // working path is the content-addressed blob
+        let wp = out.working_path.expect("working path");
+        assert!(wp.starts_with(store.join("blobs")));
+    }
+
+    #[test]
+    fn live_url_has_no_blob() {
+        let scratch = Scratch::new();
+        let store = scratch.0.join("store");
+        let ctx = ctx_for(&store, "t1");
+        let artifact = Artifact::live_url("https://example.com");
+        let out = ingest_artifact(&artifact, &ctx).expect("ingest");
+        assert!(out.working_path.is_none());
+        assert!(out.content_hash.is_none());
+        assert!(out.facts.iter().any(|f| f.key == "live_url"));
+    }
+
+    #[test]
+    fn ingest_all_collects_working_paths() {
+        let scratch = Scratch::new();
+        let store = scratch.0.join("store");
+        let fw = scratch.0.join("fw.bin");
+        std::fs::write(&fw, b"abc").expect("write");
+        let ctx = ctx_for(&store, "t1");
+
+        let mut target = OnboardedTarget::new("t".to_string(), TargetType::FirmwareBareMetal);
+        target
+            .artifacts
+            .push(Artifact::firmware_image(fw.to_string_lossy().to_string()));
+        target.artifacts.push(Artifact::live_url("https://x"));
+
+        let set = ingest_all(&target, &ctx).expect("ingest all");
+        assert_eq!(set.by_artifact.len(), 2);
+        // Only the firmware artifact yields a working path.
+        assert_eq!(set.working_paths().len(), 1);
+    }
+
+    #[test]
+    fn credentials_prefer_artifact_auth() {
+        let mut artifact = Artifact::git_repo("https://git/x", "main");
+        artifact.auth = Some(ArtifactAuth {
+            method: "token".to_string(),
+            username: Some("bob".to_string()),
+            secret: Some("pat".to_string()),
+            ..Default::default()
+        });
+        let creds = credentials_for(&artifact, "/default/ssh/key");
+        assert_eq!(creds.auth_token.as_deref(), Some("pat"));
+        assert_eq!(creds.auth_username.as_deref(), Some("bob"));
+    }
+
+    #[test]
+    fn credentials_fall_back_to_default_ssh_key() {
+        let artifact = Artifact::git_repo("git@host:x.git", "main");
+        let creds = credentials_for(&artifact, "/default/ssh/key");
+        assert_eq!(creds.ssh_key_path.as_deref(), Some("/default/ssh/key"));
+        assert!(creds.auth_token.is_none());
+    }
+}