//! Artifact ingest. //! //! Normalizes each [`Artifact`] on an [`OnboardedTarget`] into a local working //! path plus recorded metadata (content hash, size, discovered facts) that the //! classifier and scanners consume. Every blob is SHA-256 hashed — that digest //! is also the reconciliation key against sibling products (a firmware sha256 //! matches tramiton's `Artifact.sha256`). mod blob; use std::collections::HashMap; use std::path::{Path, PathBuf}; use compliance_core::models::{Artifact, ArtifactKind, DetectedFact, OnboardedTarget}; use compliance_core::AgentConfig; use crate::error::AgentError; use crate::pipeline::git::{GitOps, RepoCredentials}; /// The paths and identifiers an ingest needs. Decoupled from the full /// [`AgentConfig`] so ingest is testable without a complete config. pub struct IngestContext<'a> { /// Base directory for content-addressed blobs and working dirs. pub artifact_store_base: &'a Path, /// Base directory for git clones. pub git_clone_base: &'a str, /// Default SSH key path (used when an artifact provides none). pub ssh_key_path: &'a str, /// The id of the target these artifacts belong to (namespaces working dirs). pub target_id: &'a str, } impl<'a> IngestContext<'a> { /// Build an ingest context from the agent config for a given target. pub fn from_config(config: &'a AgentConfig, target_id: &'a str) -> Self { Self { artifact_store_base: Path::new(&config.artifact_store_base_path), git_clone_base: &config.git_clone_base_path, ssh_key_path: &config.ssh_key_path, target_id, } } } /// The result of ingesting one artifact. pub struct IngestedArtifact { /// The artifact this corresponds to ([`Artifact::id`]). pub artifact_id: String, /// The artifact kind. pub kind: ArtifactKind, /// Local working path (clone dir, extracted dir, or blob file). `None` for /// artifacts with no on-disk form (live URL, plaintext, container ref). pub working_path: Option, /// SHA-256 of the content (blobs) or git head SHA (git repos). pub content_hash: Option, /// Stored blob size in bytes, when applicable. pub size_bytes: Option, /// Facts discovered during ingest. pub facts: Vec, } /// All ingested artifacts for a target, keyed by artifact id. pub struct IngestSet { /// The ingested artifacts, keyed by [`Artifact::id`]. pub by_artifact: HashMap, } impl IngestSet { /// The working paths of every ingested artifact that has one — the input the /// classifier expects. pub fn working_paths(&self) -> HashMap { self.by_artifact .iter() .filter_map(|(id, a)| a.working_path.clone().map(|p| (id.clone(), p))) .collect() } /// The ingest result for a specific artifact. pub fn get(&self, artifact_id: &str) -> Option<&IngestedArtifact> { self.by_artifact.get(artifact_id) } } /// Ingest every artifact on a target. pub fn ingest_all( target: &OnboardedTarget, ctx: &IngestContext<'_>, ) -> Result { let mut by_artifact = HashMap::new(); for artifact in &target.artifacts { let ingested = ingest_artifact(artifact, ctx)?; by_artifact.insert(artifact.id.clone(), ingested); } Ok(IngestSet { by_artifact }) } /// Ingest a single artifact, dispatching on its kind. pub fn ingest_artifact( artifact: &Artifact, ctx: &IngestContext<'_>, ) -> Result { match artifact.kind { ArtifactKind::GitRepo => ingest_git(artifact, ctx), ArtifactKind::SourceArchive | ArtifactKind::MobilePackage | ArtifactKind::PlcProject => { ingest_blob(artifact, ctx, true) } ArtifactKind::FirmwareImage => ingest_blob(artifact, ctx, false), ArtifactKind::ContainerImage => Ok(metadata_only( artifact, DetectedFact::new("container_ref", artifact.source_ref.as_str(), "ingest"), )), ArtifactKind::LiveUrl => Ok(metadata_only( artifact, DetectedFact::new("live_url", artifact.source_ref.as_str(), "ingest"), )), ArtifactKind::PlaintextDescription => Ok(metadata_only( artifact, DetectedFact::new( "description_len", artifact.source_ref.len().to_string(), "ingest", ), )), } } /// Clone (or fetch) a git artifact, recording the head SHA as the content hash. fn ingest_git( artifact: &Artifact, ctx: &IngestContext<'_>, ) -> Result { let creds = credentials_for(artifact, ctx.ssh_key_path); let git_ops = GitOps::new(ctx.git_clone_base, creds); let repo_path = git_ops.clone_or_fetch(&artifact.source_ref, &artifact.id)?; let head = GitOps::get_head_sha(&repo_path).ok(); Ok(IngestedArtifact { artifact_id: artifact.id.clone(), kind: artifact.kind, working_path: Some(repo_path), content_hash: head, size_bytes: None, facts: Vec::new(), }) } /// Store a blob artifact content-addressed. When `extract` is set and the blob /// is a zip container (source archive, APK/AAB/IPA), also unpack it into a /// working directory; otherwise the working path is the stored blob. fn ingest_blob( artifact: &Artifact, ctx: &IngestContext<'_>, extract: bool, ) -> Result { let base = ctx.artifact_store_base; let src = local_source(artifact)?; let (sha, size) = blob::hash_file(&src)?; let stored = blob::store_file(base, &src, &sha)?; let mut facts = Vec::new(); let working_path = if extract { let dest = blob::work_dir(base, ctx.target_id, &artifact.id); match blob::extract_zip(&stored, &dest) { Ok(()) => dest, Err(e) => { // Not a zip (e.g. a tar.gz source archive) — keep the blob and // note it so later stages can decide what to do. facts.push(DetectedFact::new( "archive_unextracted", e.to_string(), "ingest", )); stored.clone() } } } else { stored.clone() }; Ok(IngestedArtifact { artifact_id: artifact.id.clone(), kind: artifact.kind, working_path: Some(working_path), content_hash: Some(sha), size_bytes: Some(size), facts, }) } /// An artifact with no on-disk form: record a single fact, no hash/path. fn metadata_only(artifact: &Artifact, fact: DetectedFact) -> IngestedArtifact { IngestedArtifact { artifact_id: artifact.id.clone(), kind: artifact.kind, working_path: None, content_hash: None, size_bytes: None, facts: vec![fact], } } /// The local file backing a blob artifact: its `stored_path` if already /// uploaded, else its `source_ref` interpreted as a filesystem path. fn local_source(artifact: &Artifact) -> Result { let path = artifact .stored_path .as_deref() .unwrap_or(artifact.source_ref.as_str()); let path = PathBuf::from(path); if !path.exists() { return Err(AgentError::Other(format!( "artifact {} source not found at {}", artifact.id, path.display() ))); } Ok(path) } /// Build git credentials from an artifact's auth plus a default SSH key path. fn credentials_for(artifact: &Artifact, default_ssh_key_path: &str) -> RepoCredentials { let auth = artifact.auth.as_ref(); RepoCredentials { ssh_key_path: auth .and_then(|a| a.ssh_key_path.clone()) .or_else(|| Some(default_ssh_key_path.to_string())), auth_token: auth.and_then(|a| a.secret.clone()), auth_username: auth.and_then(|a| a.username.clone()), } } #[cfg(test)] #[allow(clippy::expect_used, clippy::unwrap_used)] mod tests { use super::*; use compliance_core::models::{ArtifactAuth, TargetType}; /// A unique scratch directory, removed on drop. struct Scratch(PathBuf); impl Scratch { fn new() -> Self { let p = std::env::temp_dir().join(format!("cs-ingest-mod-{}", uuid::Uuid::new_v4())); std::fs::create_dir_all(&p).expect("mkdir scratch"); Self(p) } } impl Drop for Scratch { fn drop(&mut self) { let _ = std::fs::remove_dir_all(&self.0); } } fn ctx_for<'a>(store: &'a Path, target_id: &'a str) -> IngestContext<'a> { IngestContext { artifact_store_base: store, git_clone_base: "/tmp/cs-ingest-test-repos", ssh_key_path: "/tmp/cs-ingest-test-ssh", target_id, } } #[test] fn firmware_blob_is_hashed_and_stored() { let scratch = Scratch::new(); let store = scratch.0.join("store"); let fw = scratch.0.join("fw.bin"); std::fs::write(&fw, b"firmware-bytes").expect("write"); let ctx = ctx_for(&store, "t1"); let artifact = Artifact::firmware_image(fw.to_string_lossy().to_string()); let out = ingest_artifact(&artifact, &ctx).expect("ingest"); assert_eq!(out.kind, ArtifactKind::FirmwareImage); assert_eq!(out.size_bytes, Some(14)); let sha = out.content_hash.expect("hash"); assert_eq!(sha.len(), 64); // working path is the content-addressed blob let wp = out.working_path.expect("working path"); assert!(wp.starts_with(store.join("blobs"))); } #[test] fn live_url_has_no_blob() { let scratch = Scratch::new(); let store = scratch.0.join("store"); let ctx = ctx_for(&store, "t1"); let artifact = Artifact::live_url("https://example.com"); let out = ingest_artifact(&artifact, &ctx).expect("ingest"); assert!(out.working_path.is_none()); assert!(out.content_hash.is_none()); assert!(out.facts.iter().any(|f| f.key == "live_url")); } #[test] fn ingest_all_collects_working_paths() { let scratch = Scratch::new(); let store = scratch.0.join("store"); let fw = scratch.0.join("fw.bin"); std::fs::write(&fw, b"abc").expect("write"); let ctx = ctx_for(&store, "t1"); let mut target = OnboardedTarget::new("t".to_string(), TargetType::FirmwareBareMetal); target .artifacts .push(Artifact::firmware_image(fw.to_string_lossy().to_string())); target.artifacts.push(Artifact::live_url("https://x")); let set = ingest_all(&target, &ctx).expect("ingest all"); assert_eq!(set.by_artifact.len(), 2); // Only the firmware artifact yields a working path. assert_eq!(set.working_paths().len(), 1); } #[test] fn credentials_prefer_artifact_auth() { let mut artifact = Artifact::git_repo("https://git/x", "main"); artifact.auth = Some(ArtifactAuth { method: "token".to_string(), username: Some("bob".to_string()), secret: Some("pat".to_string()), ..Default::default() }); let creds = credentials_for(&artifact, "/default/ssh/key"); assert_eq!(creds.auth_token.as_deref(), Some("pat")); assert_eq!(creds.auth_username.as_deref(), Some("bob")); } #[test] fn credentials_fall_back_to_default_ssh_key() { let artifact = Artifact::git_repo("git@host:x.git", "main"); let creds = credentials_for(&artifact, "/default/ssh/key"); assert_eq!(creds.ssh_key_path.as_deref(), Some("/default/ssh/key")); assert!(creds.auth_token.is_none()); } }