//! Artifact ingest. //! //! Normalizes each [`Artifact`] on an [`OnboardedTarget`] into a local working //! path plus recorded metadata (content hash, size, discovered facts) that the //! classifier and scanners consume. Every blob is SHA-256 hashed — that digest //! is also the reconciliation key against sibling products (a firmware sha256 //! matches tramiton's `Artifact.sha256`). pub(crate) mod blob; use std::collections::HashMap; use std::path::{Path, PathBuf}; use compliance_core::models::{Artifact, ArtifactKind, DetectedFact, OnboardedTarget}; use compliance_core::AgentConfig; use crate::error::AgentError; use crate::pipeline::git::{GitOps, RepoCredentials}; /// The paths and identifiers an ingest needs. Decoupled from the full /// [`AgentConfig`] so ingest is testable without a complete config. pub struct IngestContext<'a> { /// Base directory for content-addressed blobs and working dirs. pub artifact_store_base: &'a Path, /// Base directory for git clones. pub git_clone_base: &'a str, /// Default SSH key path (used when an artifact provides none). pub ssh_key_path: &'a str, /// The id of the target these artifacts belong to (namespaces working dirs). pub target_id: &'a str, } impl<'a> IngestContext<'a> { /// Build an ingest context from the agent config for a given target. pub fn from_config(config: &'a AgentConfig, target_id: &'a str) -> Self { Self { artifact_store_base: Path::new(&config.artifact_store_base_path), git_clone_base: &config.git_clone_base_path, ssh_key_path: &config.ssh_key_path, target_id, } } } /// The result of ingesting one artifact. pub struct IngestedArtifact { /// The artifact this corresponds to ([`Artifact::id`]). pub artifact_id: String, /// The artifact kind. pub kind: ArtifactKind, /// Local working path (clone dir, extracted dir, or blob file). `None` for /// artifacts with no on-disk form (live URL, plaintext, container ref). pub working_path: Option, /// SHA-256 of the content (blobs) or git head SHA (git repos). pub content_hash: Option, /// Stored blob size in bytes, when applicable. pub size_bytes: Option, /// Facts discovered during ingest. pub facts: Vec, } /// All ingested artifacts for a target, keyed by artifact id. pub struct IngestSet { /// The ingested artifacts, keyed by [`Artifact::id`]. pub by_artifact: HashMap, } impl IngestSet { /// The working paths of every ingested artifact that has one — the input the /// classifier expects. pub fn working_paths(&self) -> HashMap { self.by_artifact .iter() .filter_map(|(id, a)| a.working_path.clone().map(|p| (id.clone(), p))) .collect() } /// The ingest result for a specific artifact. pub fn get(&self, artifact_id: &str) -> Option<&IngestedArtifact> { self.by_artifact.get(artifact_id) } } /// Ingest every artifact on a target. pub fn ingest_all( target: &OnboardedTarget, ctx: &IngestContext<'_>, ) -> Result { let mut by_artifact = HashMap::new(); for artifact in &target.artifacts { let ingested = ingest_artifact(artifact, ctx)?; by_artifact.insert(artifact.id.clone(), ingested); } Ok(IngestSet { by_artifact }) } /// Ingest a single artifact, dispatching on its kind. pub fn ingest_artifact( artifact: &Artifact, ctx: &IngestContext<'_>, ) -> Result { match artifact.kind { ArtifactKind::GitRepo => ingest_git(artifact, ctx), ArtifactKind::SourceArchive | ArtifactKind::MobilePackage | ArtifactKind::PlcProject => { ingest_blob(artifact, ctx, true) } ArtifactKind::FirmwareImage => ingest_blob(artifact, ctx, false), ArtifactKind::ContainerImage => Ok(metadata_only( artifact, DetectedFact::new("container_ref", artifact.source_ref.as_str(), "ingest"), )), ArtifactKind::LiveUrl => Ok(metadata_only( artifact, DetectedFact::new("live_url", artifact.source_ref.as_str(), "ingest"), )), ArtifactKind::PlaintextDescription => Ok(metadata_only( artifact, DetectedFact::new( "description_len", artifact.source_ref.len().to_string(), "ingest", ), )), } } /// Clone (or fetch) a git artifact, recording the head SHA as the content hash. fn ingest_git( artifact: &Artifact, ctx: &IngestContext<'_>, ) -> Result { let creds = credentials_for(artifact, ctx.ssh_key_path); let git_ops = GitOps::new(ctx.git_clone_base, creds); let repo_path = git_ops.clone_or_fetch(&artifact.source_ref, &artifact.id)?; let head = GitOps::get_head_sha(&repo_path).ok(); Ok(IngestedArtifact { artifact_id: artifact.id.clone(), kind: artifact.kind, working_path: Some(repo_path), content_hash: head, size_bytes: None, facts: Vec::new(), }) } /// Store a blob artifact content-addressed. When `extract` is set and the blob /// is a zip container (source archive, APK/AAB/IPA), also unpack it into a /// working directory; otherwise the working path is the stored blob. fn ingest_blob( artifact: &Artifact, ctx: &IngestContext<'_>, extract: bool, ) -> Result { let base = ctx.artifact_store_base; let src = local_source(artifact)?; let (sha, size) = blob::hash_file(&src)?; let stored = blob::store_file(base, &src, &sha)?; let mut facts = Vec::new(); let working_path = if extract { let dest = blob::work_dir(base, ctx.target_id, &artifact.id); match blob::extract_zip(&stored, &dest) { Ok(()) => dest, Err(e) => { // Not a zip container — this is a single uploaded file (e.g. a // `.st`/`.xml` PLC project or a `.tar.gz`). The content-addressed // blob has no extension, so materialize it into a working dir // under its original name; extension-based scanners (PLC) can then // discover it and report a readable path. facts.push(DetectedFact::new( "archive_unextracted", e.to_string(), "ingest", )); match materialize_single(&stored, &dest, &blob_file_name(artifact)) { Ok(dir) => dir, Err(copy_err) => { facts.push(DetectedFact::new( "materialize_failed", copy_err.to_string(), "ingest", )); stored.clone() } } } } } else { stored.clone() }; Ok(IngestedArtifact { artifact_id: artifact.id.clone(), kind: artifact.kind, working_path: Some(working_path), content_hash: Some(sha), size_bytes: Some(size), facts, }) } /// Copy a stored blob into `dest`/`name`, returning `dest`. Used when an /// "extractable" artifact turns out to be a single file rather than an archive. fn materialize_single(stored: &Path, dest: &Path, name: &str) -> Result { std::fs::create_dir_all(dest)?; std::fs::copy(stored, dest.join(name))?; Ok(dest.to_path_buf()) } /// A safe, single-segment file name for an artifact, preserving the original /// extension so scanners can identify it. Derives from `source_ref` (the /// uploaded/original file name); `file_name` strips any directory components, /// so this is traversal-safe. Falls back to the artifact id. fn blob_file_name(artifact: &Artifact) -> String { Path::new(&artifact.source_ref) .file_name() .and_then(|n| n.to_str()) .map(str::to_string) .filter(|s| !s.is_empty()) .unwrap_or_else(|| format!("artifact-{}", artifact.id)) } /// An artifact with no on-disk form: record a single fact, no hash/path. fn metadata_only(artifact: &Artifact, fact: DetectedFact) -> IngestedArtifact { IngestedArtifact { artifact_id: artifact.id.clone(), kind: artifact.kind, working_path: None, content_hash: None, size_bytes: None, facts: vec![fact], } } /// The local file backing a blob artifact: its `stored_path` if already /// uploaded, else its `source_ref` interpreted as a filesystem path. fn local_source(artifact: &Artifact) -> Result { let path = artifact .stored_path .as_deref() .unwrap_or(artifact.source_ref.as_str()); let path = PathBuf::from(path); if !path.exists() { return Err(AgentError::Other(format!( "artifact {} source not found at {}", artifact.id, path.display() ))); } Ok(path) } /// Build git credentials from an artifact's auth plus a default SSH key path. fn credentials_for(artifact: &Artifact, default_ssh_key_path: &str) -> RepoCredentials { let auth = artifact.auth.as_ref(); RepoCredentials { ssh_key_path: auth .and_then(|a| a.ssh_key_path.clone()) .or_else(|| Some(default_ssh_key_path.to_string())), auth_token: auth.and_then(|a| a.secret.clone()), auth_username: auth.and_then(|a| a.username.clone()), } } #[cfg(test)] #[allow(clippy::expect_used, clippy::unwrap_used)] mod tests { use super::*; use compliance_core::models::{ArtifactAuth, TargetType}; /// A unique scratch directory, removed on drop. struct Scratch(PathBuf); impl Scratch { fn new() -> Self { let p = std::env::temp_dir().join(format!("cs-ingest-mod-{}", uuid::Uuid::new_v4())); std::fs::create_dir_all(&p).expect("mkdir scratch"); Self(p) } } impl Drop for Scratch { fn drop(&mut self) { let _ = std::fs::remove_dir_all(&self.0); } } fn ctx_for<'a>(store: &'a Path, target_id: &'a str) -> IngestContext<'a> { IngestContext { artifact_store_base: store, git_clone_base: "/tmp/cs-ingest-test-repos", ssh_key_path: "/tmp/cs-ingest-test-ssh", target_id, } } #[test] fn firmware_blob_is_hashed_and_stored() { let scratch = Scratch::new(); let store = scratch.0.join("store"); let fw = scratch.0.join("fw.bin"); std::fs::write(&fw, b"firmware-bytes").expect("write"); let ctx = ctx_for(&store, "t1"); let artifact = Artifact::firmware_image(fw.to_string_lossy().to_string()); let out = ingest_artifact(&artifact, &ctx).expect("ingest"); assert_eq!(out.kind, ArtifactKind::FirmwareImage); assert_eq!(out.size_bytes, Some(14)); let sha = out.content_hash.expect("hash"); assert_eq!(sha.len(), 64); // working path is the content-addressed blob let wp = out.working_path.expect("working path"); assert!(wp.starts_with(store.join("blobs"))); } #[test] fn live_url_has_no_blob() { let scratch = Scratch::new(); let store = scratch.0.join("store"); let ctx = ctx_for(&store, "t1"); let artifact = Artifact::live_url("https://example.com"); let out = ingest_artifact(&artifact, &ctx).expect("ingest"); assert!(out.working_path.is_none()); assert!(out.content_hash.is_none()); assert!(out.facts.iter().any(|f| f.key == "live_url")); } #[test] fn ingest_all_collects_working_paths() { let scratch = Scratch::new(); let store = scratch.0.join("store"); let fw = scratch.0.join("fw.bin"); std::fs::write(&fw, b"abc").expect("write"); let ctx = ctx_for(&store, "t1"); let mut target = OnboardedTarget::new("t".to_string(), TargetType::FirmwareBareMetal); target .artifacts .push(Artifact::firmware_image(fw.to_string_lossy().to_string())); target.artifacts.push(Artifact::live_url("https://x")); let set = ingest_all(&target, &ctx).expect("ingest all"); assert_eq!(set.by_artifact.len(), 2); // Only the firmware artifact yields a working path. assert_eq!(set.working_paths().len(), 1); } #[test] fn credentials_prefer_artifact_auth() { let mut artifact = Artifact::git_repo("https://git/x", "main"); artifact.auth = Some(ArtifactAuth { method: "token".to_string(), username: Some("bob".to_string()), secret: Some("pat".to_string()), ..Default::default() }); let creds = credentials_for(&artifact, "/default/ssh/key"); assert_eq!(creds.auth_token.as_deref(), Some("pat")); assert_eq!(creds.auth_username.as_deref(), Some("bob")); } #[test] fn credentials_fall_back_to_default_ssh_key() { let artifact = Artifact::git_repo("git@host:x.git", "main"); let creds = credentials_for(&artifact, "/default/ssh/key"); assert_eq!(creds.ssh_key_path.as_deref(), Some("/default/ssh/key")); assert!(creds.auth_token.is_none()); } /// A single uploaded PLC file (not an archive) must land in a working dir /// under its original name so the PLC scanner can discover it by extension /// and report a readable path — the demo's upload → scan path. #[test] fn single_uploaded_plc_file_is_materialized_and_scannable() { use compliance_core::models::PlcFormat; let scratch = Scratch::new(); let store = scratch.0.join("store"); // Simulate the upload handler: bytes written to an `uploads/` path, // `source_ref` carrying the original (clean) file name. let uploads = scratch.0.join("uploads"); std::fs::create_dir_all(&uploads).expect("mkdir uploads"); let uploaded = uploads.join("a1b2c3_pump_station.st"); std::fs::write( &uploaded, "PROGRAM P\nVAR\n ApiKey : STRING := 'sk-live-1234';\nEND_VAR\nEND_PROGRAM\n", ) .expect("write st"); let mut artifact = Artifact::plc_project("pump_station.st", PlcFormat::StructuredText); artifact.stored_path = Some(uploaded.to_string_lossy().to_string()); let ctx = ctx_for(&store, "t-plc"); let out = ingest_artifact(&artifact, &ctx).expect("ingest"); // Working path is a directory (not the extensionless blob) holding the // file under its original name. let wp = out.working_path.expect("working path"); assert!(wp.is_dir(), "expected a working dir, got {wp:?}"); assert!(wp.join("pump_station.st").is_file()); // The PLC scanner finds the hardcoded credential and reports a clean path. let findings = crate::pipeline::plc::analyze_tree(&wp, "t-plc"); assert!( !findings.is_empty(), "scanner should flag the uploaded file" ); assert!(findings .iter() .any(|f| f.rule_id.as_deref() == Some("plc-hardcoded-credential"))); assert_eq!( findings[0].file_path.as_deref(), Some("pump_station.st"), "finding should reference the original file name" ); } }