Merge pull request #80 from nikstur/ensure-runtime-dependencies

Ensure runtime dependencies

nix/buildtime-dependencies.nix

@@ -5,8 +5,6 @@
 }:
 
 let
-  hello = "hwllo";
-
   drvOutputs = drv:
     if builtins.hasAttr "outputs" drv
     then map (output: drv.${output}) drv.outputs

nix/runtime-dependencies.nix

@@ -1,10 +1,10 @@
-# This is a wrapper around nixpkgs' closureInfo. It returns a newline 
+# This is a wrapper around nixpkgs' closureInfo. It returns a newline
 # separated list of the store paths of drv's runtime dependencies.
 { runCommand
 , closureInfo
 }:
 
 drv:
-runCommand "${drv.name}-runtime-dependencies.json" { } ''
+runCommand "${drv.name}-runtime-dependencies.txt" { } ''
   cat ${closureInfo { rootPaths = [ drv ]; }}/store-paths > $out
 ''

nix/tests/default.nix

@@ -30,17 +30,11 @@ let
     sha256 = "sha256-N9aEK2oYk3SoCczrRMt5ycdgXCPA5SHTKsS2CffFY14=";
   };
 
-  # To avoid network access, the base URL is replaced with a local URI to the above downloaded schemas
-  name = "bom-1.4.schema.json";
-  relativeReferencesSchema = pkgs.runCommand name { } ''
-    substitute "${cycloneDxSpec}/schema/${name}" "$out" \
-      --replace 'http://cyclonedx.org/schema/${name}' 'file://${cycloneDxSpec}/schema/'
-  '';
-
   buildBomAndValidate = drv: options:
     pkgs.runCommand "${drv.name}-bom-validation" { nativeBuildInputs = [ pkgs.check-jsonschema ]; } ''
       check-jsonschema \
-        --schemafile "${relativeReferencesSchema}" \
+        --schemafile "${cycloneDxSpec}/schema/bom-1.4.schema.json" \
+        --base-uri "${cycloneDxSpec}/schema/bom-1.4.schema.json" \
         "${buildBom drv options}"
       touch $out
     '';

rust/transformer/src/buildtime_input.rs

@@ -1,89 +1,24 @@
-use std::hash::{Hash, Hasher};
+use std::collections::HashMap;
+use std::fs;
+use std::path::Path;
 
-use serde::Deserialize;
+use anyhow::{Context, Result};
 
-#[derive(Deserialize, Debug)]
-pub struct BuildtimeInput(Vec<Derivation>);
+use crate::derivation::Derivation;
 
-impl BuildtimeInput {
-    pub fn remove_derivation(&mut self, derivation_path: &str) -> Derivation {
-        let index = self
-            .0
-            .iter()
-            .position(|derivation| derivation.path == derivation_path)
-            .expect("Unrecovereable error: buildtime input does not include target");
-        self.0.swap_remove(index)
-    }
-}
-
-impl IntoIterator for BuildtimeInput {
-    type Item = Derivation;
-    type IntoIter = std::vec::IntoIter<Self::Item>;
-
-    fn into_iter(self) -> Self::IntoIter {
-        self.0.into_iter()
-    }
-}
-
-#[derive(Deserialize, Clone, Debug)]
-pub struct Derivation {
-    pub path: String,
-    pub name: Option<String>,
-    pub pname: Option<String>,
-    pub version: Option<String>,
-    pub meta: Option<Meta>,
-}
-
-// Implement Eq and Hash so Itertools::unique can identify unique dependencies by path. The name
-// seems to be the best proxy to detect duplicates. Different outputs of the same derivation have
-// different paths. Thus, filtering by path alone doesn't adequately remove duplicates.
-impl PartialEq for Derivation {
-    fn eq(&self, other: &Self) -> bool {
-        self.name == other.name
-    }
-}
-impl Eq for Derivation {}
+#[derive(Clone)]
+pub struct BuildtimeInput(pub HashMap<String, Derivation>);
 
-impl Hash for Derivation {
-    fn hash<H: Hasher>(&self, state: &mut H) {
-        self.name.hash(state);
-    }
-}
-
-#[derive(Deserialize, Clone, Debug)]
-pub struct Meta {
-    pub license: Option<LicenseField>,
-    pub homepage: Option<String>,
-}
-
-#[derive(Deserialize, Clone, Debug)]
-#[serde(untagged)]
-pub enum LicenseField {
-    LicenseList(LicenseList),
-    License(License),
-    // In very rare cases the license is just a String.
-    // This mostly serves as a fallback so that serde doesn't panic.
-    String(String),
-}
-
-impl LicenseField {
-    pub fn into_vec(self) -> Vec<License> {
-        match self {
-            Self::LicenseList(license_list) => license_list.0,
-            Self::License(license) => vec![license],
-            // Fallback to handle very unusual license fields in Nix.
-            _ => vec![],
+impl BuildtimeInput {
+    pub fn from_file(path: &Path) -> Result<Self> {
+        let buildtime_input_json: Vec<Derivation> = serde_json::from_reader(
+            fs::File::open(path).with_context(|| format!("Failed to open {path:?}"))?,
+        )
+        .context("Failed to parse buildtime input")?;
+        let mut m = HashMap::new();
+        for derivation in buildtime_input_json {
+            m.insert(derivation.path.clone(), derivation);
         }
+        Ok(Self(m))
     }
 }
-
-#[derive(Deserialize, Clone, Debug)]
-pub struct LicenseList(Vec<License>);
-
-#[derive(Deserialize, Clone, Debug)]
-pub struct License {
-    #[serde(rename = "fullName")]
-    pub full_name: String,
-    #[serde(rename = "spdxId")]
-    pub spdx_id: Option<String>,
-}

rust/transformer/src/cli.rs

@@ -12,12 +12,12 @@ pub struct Cli {
     include_buildtime_dependencies: bool,
 
     /// Path to target derivation
-    target: PathBuf,
+    target: String,
 
-    /// Path to JSON containing buildtime input
+    /// Path to JSON containing the buildtime input
     buildtime_input: PathBuf,
 
-    /// Path to JSON containing runtime input
+    /// Path to a newline separated .txt file containing the runtime input
     runtime_input: PathBuf,
 }
 

rust/transformer/src/cyclonedx.rs

@@ -1,6 +1,6 @@
 use anyhow::Result;
 use cyclonedx_bom::external_models::uri::Purl;
-use cyclonedx_bom::models::bom::{Bom, UrnUuid};
+use cyclonedx_bom::models::bom::Bom;
 use cyclonedx_bom::models::component::{Classification, Component, Components};
 use cyclonedx_bom::models::external_reference::{
     ExternalReference, ExternalReferenceType, ExternalReferences,
@@ -9,7 +9,7 @@ use cyclonedx_bom::models::license::{License, LicenseChoice, Licenses};
 use cyclonedx_bom::models::metadata::Metadata;
 use cyclonedx_bom::models::tool::{Tool, Tools};
 
-use crate::buildtime_input::{self, Derivation, Meta};
+use crate::derivation::{self, Derivation, Meta};
 
 const VERSION: &str = env!("CARGO_PKG_VERSION");
 
@@ -67,7 +67,13 @@ impl CycloneDXComponent {
             Classification::Application,
             &name,
             &version,
-            Some(UrnUuid::generate().to_string()),
+            Some(
+                derivation
+                    .path
+                    .strip_prefix("/nix/store/")
+                    .unwrap_or(&derivation.path)
+                    .to_string(),
+            ),
         );
         component.purl = Purl::new("nix", &name, &version).ok();
         if let Some(meta) = derivation.meta {
@@ -96,7 +102,7 @@ fn convert_licenses(meta: &Meta) -> Option<Licenses> {
     }))
 }
 
-fn convert_license(license: buildtime_input::License) -> LicenseChoice {
+fn convert_license(license: derivation::License) -> LicenseChoice {
     match license.spdx_id {
         Some(spdx_id) => match License::license_id(&spdx_id) {
             Ok(license) => LicenseChoice::License(license),

rust/transformer/src/derivation.rs

@@ -0,0 +1,68 @@
+use serde::Deserialize;
+
+#[derive(Deserialize, Clone, Debug, Default)]
+pub struct Derivation {
+    pub path: String,
+    pub name: Option<String>,
+    pub pname: Option<String>,
+    pub version: Option<String>,
+    pub meta: Option<Meta>,
+}
+
+impl Derivation {
+    pub fn new(store_path: &str) -> Self {
+        // Because we only have the store path we have to derive the pname and version from it
+        let stripped = store_path.strip_prefix("/nix/store/");
+        let pname = stripped
+            .and_then(|s| s.split('-').nth(1))
+            .map(ToOwned::to_owned);
+        let version = stripped
+            .and_then(|s| s.split('-').last())
+            .map(ToOwned::to_owned);
+
+        Self {
+            path: store_path.to_string(),
+            pname,
+            version,
+            ..Self::default()
+        }
+    }
+}
+
+#[derive(Deserialize, Clone, Debug)]
+pub struct Meta {
+    pub license: Option<LicenseField>,
+    pub homepage: Option<String>,
+}
+
+#[derive(Deserialize, Clone, Debug)]
+#[serde(untagged)]
+pub enum LicenseField {
+    LicenseList(LicenseList),
+    License(License),
+    // In very rare cases the license is just a String.
+    // This mostly serves as a fallback so that serde doesn't panic.
+    String(String),
+}
+
+impl LicenseField {
+    pub fn into_vec(self) -> Vec<License> {
+        match self {
+            Self::LicenseList(license_list) => license_list.0,
+            Self::License(license) => vec![license],
+            // Fallback to handle very unusual license fields in Nix.
+            _ => vec![],
+        }
+    }
+}
+
+#[derive(Deserialize, Clone, Debug)]
+pub struct LicenseList(Vec<License>);
+
+#[derive(Deserialize, Clone, Debug)]
+pub struct License {
+    #[serde(rename = "fullName")]
+    pub full_name: String,
+    #[serde(rename = "spdxId")]
+    pub spdx_id: Option<String>,
+}

rust/transformer/src/main.rs

@@ -3,6 +3,7 @@
 mod buildtime_input;
 mod cli;
 mod cyclonedx;
+mod derivation;
 mod runtime_input;
 mod transform;
 

rust/transformer/src/runtime_input.rs

@@ -2,18 +2,15 @@ use std::collections::HashSet;
 use std::fs;
 use std::path::Path;
 
-use anyhow::Result;
+use anyhow::{Context, Result};
 
-pub struct RuntimeInput(HashSet<String>);
+pub struct RuntimeInput(pub HashSet<String>);
 
 impl RuntimeInput {
-    pub fn build(path: &Path) -> Result<Self> {
-        let file_content = fs::read_to_string(path)?;
+    pub fn from_file(path: &Path) -> Result<Self> {
+        let file_content =
+            fs::read_to_string(path).with_context(|| format!("Failed to read {path:?}"))?;
         let set = HashSet::from_iter(file_content.lines().map(|x| x.to_owned()));
         Ok(Self(set))
     }
-
-    pub fn contains(&self, value: &str) -> bool {
-        self.0.contains(value)
-    }
 }

rust/transformer/src/transform.rs

@@ -1,40 +1,51 @@
-use std::fs;
 use std::io::{self, Write};
 use std::path::PathBuf;
 
-use anyhow::{anyhow, Result};
+use anyhow::{Context, Result};
 use itertools::Itertools;
 
 use crate::buildtime_input::BuildtimeInput;
 use crate::cyclonedx::{CycloneDXBom, CycloneDXComponents};
+use crate::derivation::Derivation;
 use crate::runtime_input::RuntimeInput;
 
 pub fn transform(
     include_buildtime_dependencies: bool,
-    target_path: PathBuf,
+    target_path: String,
     buildtime_input_path: PathBuf,
     runtime_input_path: PathBuf,
 ) -> Result<()> {
-    let mut buildtime_input: BuildtimeInput =
-        serde_json::from_reader(fs::File::open(buildtime_input_path)?)?;
-
-    let target_derivation = buildtime_input.remove_derivation(
-        target_path
-            .to_str()
-            .ok_or_else(|| anyhow!("Failed to convert path to string: {:?}", target_path))?,
-    );
-
-    let runtime_input = RuntimeInput::build(&runtime_input_path)?;
+    let mut buildtime_input = BuildtimeInput::from_file(&buildtime_input_path)?;
+    let target_derivation = buildtime_input.0.remove(&target_path).with_context(|| {
+        format!("Buildtime input doesn't contain target derivation: {target_path}")
+    })?;
+
+    let mut runtime_input = RuntimeInput::from_file(&runtime_input_path)?;
+    runtime_input.0.remove(&target_path);
+
+    // Augment the runtime input with information from the buildtime input. The buildtime input,
+    // however, is not a strict superset of the runtime input. This has to do with how we query the
+    // buildinputs from Nix and how dependencies can "hide" in String Contexts.
+    let runtime_derivations = runtime_input.0.iter().map(|store_path| {
+        buildtime_input
+            .0
+            .get(store_path)
+            .map(ToOwned::to_owned)
+            .unwrap_or(Derivation::new(store_path))
+    });
+
+    let buildtime_derivations = buildtime_input
+        .0
+        .clone()
+        .into_values()
+        .filter(|derivation| !runtime_input.0.contains(&derivation.path))
+        .unique_by(|d| d.name.clone().unwrap_or(d.path.clone()));
 
     let components = if include_buildtime_dependencies {
-        CycloneDXComponents::new(buildtime_input.into_iter().unique())
+        let all_derivations = runtime_derivations.chain(buildtime_derivations);
+        CycloneDXComponents::new(all_derivations)
     } else {
-        CycloneDXComponents::new(
-            buildtime_input
-                .into_iter()
-                .unique()
-                .filter(|derivation| runtime_input.contains(&derivation.path)),
-        )
+        CycloneDXComponents::new(runtime_derivations)
     };
 
     let bom = CycloneDXBom::build(target_derivation, components)?;