Initial "chunking" code

This analyzes an ostree commit and splits it into chunks
suitable for output to separate layers in an OCI image.

lib/src/cli.rs

@@ -120,6 +120,10 @@ enum ContainerOpts {
         /// Corresponds to the Dockerfile `CMD` instruction.
         #[structopt(long)]
         cmd: Option<Vec<String>>,
+
+        #[structopt(long, hidden = true)]
+        /// Output in multiple blobs
+        ex_chunked: bool,
     },
 
     /// Commands for working with (possibly layered, non-encapsulated) container images.
@@ -238,6 +242,19 @@ enum TestingOpts {
     DetectEnv,
 }
 
+/// Experimental options
+#[derive(Debug, StructOpt)]
+enum ExperimentalOpts {
+    /// Print chunking
+    PrintChunks {
+        /// Path to the repository
+        #[structopt(long)]
+        repo: String,
+        /// The ostree ref or commt
+        rev: String,
+    },
+}
+
 /// Toplevel options for extended ostree functionality.
 #[derive(Debug, StructOpt)]
 #[structopt(name = "ostree-ext")]
@@ -252,6 +269,9 @@ enum Opt {
     ImaSign(ImaSignOpts),
     #[structopt(setting(structopt::clap::AppSettings::Hidden))]
     InternalOnlyForTesting(TestingOpts),
+    /// Experimental/debug CLI
+    #[structopt(setting(structopt::clap::AppSettings::Hidden))]
+    Experimental(ExperimentalOpts),
 }
 
 #[allow(clippy::from_over_into)]
@@ -357,13 +377,17 @@ async fn container_export(
     imgref: &ImageReference,
     labels: BTreeMap<String, String>,
     cmd: Option<Vec<String>>,
+    chunked: bool,
 ) -> Result<()> {
     let repo = &ostree::Repo::open_at(libc::AT_FDCWD, repo, gio::NONE_CANCELLABLE)?;
     let config = Config {
         labels: Some(labels),
         cmd,
     };
-    let opts = Some(Default::default());
+    let opts = Some(crate::container::ExportOpts {
+        chunked,
+        ..Default::default()
+    });
     let pushed = crate::container::encapsulate(repo, rev, &config, opts, imgref).await?;
     println!("{}", pushed);
     Ok(())
@@ -481,6 +505,7 @@ where
                 imgref,
                 labels,
                 cmd,
+                ex_chunked,
             } => {
                 let labels: Result<BTreeMap<_, _>> = labels
                     .into_iter()
@@ -491,7 +516,8 @@ where
                         Ok((k.to_string(), v.to_string()))
                     })
                     .collect();
-                container_export(&repo, &rev, &imgref, labels?, cmd).await
+
+                container_export(&repo, &rev, &imgref, labels?, cmd, ex_chunked).await
             }
             ContainerOpts::Image(opts) => match opts {
                 ContainerImageOpts::List { repo } => {
@@ -545,5 +571,14 @@ where
         },
         Opt::ImaSign(ref opts) => ima_sign(opts),
         Opt::InternalOnlyForTesting(ref opts) => testing(opts),
+        Opt::Experimental(ref opts) => match opts {
+            ExperimentalOpts::PrintChunks { repo, rev } => {
+                let repo = &ostree::Repo::open_at(libc::AT_FDCWD, &repo, gio::NONE_CANCELLABLE)?;
+                let mut chunks = crate::chunking::Chunking::new(repo, rev)?;
+                chunks.auto_chunk(repo)?;
+                crate::chunking::print(&chunks);
+                Ok(())
+            }
+        },
     }
 }

lib/src/container/encapsulate.rs

@@ -2,6 +2,7 @@
 
 use super::ociwriter::OciWriter;
 use super::*;
+use crate::chunking::Chunking;
 use crate::tar as ostree_tar;
 use anyhow::Context;
 use fn_error_context::context;
@@ -40,6 +41,42 @@ fn export_ostree_ref(
     w.complete()
 }
 
+/// Write an ostree commit to an OCI blob
+#[context("Writing ostree root to blob")]
+fn export_chunked(
+    repo: &ostree::Repo,
+    ociw: &mut OciWriter,
+    mut chunking: Chunking,
+    compression: Option<flate2::Compression>,
+    description: &str,
+) -> Result<()> {
+    let layers: Result<Vec<_>> = chunking
+        .take_chunks()
+        .into_iter()
+        .enumerate()
+        .map(|(i, chunk)| -> Result<_> {
+            let mut w = ociw.create_layer(compression)?;
+            ostree_tar::export_chunk(repo, &chunk, &mut w)
+                .with_context(|| format!("Exporting chunk {}", i))?;
+            let w = w.into_inner()?;
+            Ok((w.complete()?, chunk.name))
+        })
+        .collect();
+    for (layer, name) in layers? {
+        ociw.push_layer(layer, &name);
+    }
+    let mut w = ociw.create_layer(compression)?;
+    ostree_tar::export_final_chunk(repo, &chunking, &mut w)?;
+    let w = w.into_inner()?;
+    let final_layer = w.complete()?;
+    ociw.add_config_annotation(
+        crate::container::OSTREE_LAYER_LABEL,
+        &final_layer.blob.sha256,
+    );
+    ociw.push_layer(final_layer, description);
+    Ok(())
+}
+
 /// Generate an OCI image from a given ostree root
 #[context("Building oci")]
 fn build_oci(
@@ -67,6 +104,15 @@ fn build_oci(
     let commit_meta = &commit_v.child_value(0);
     let commit_meta = glib::VariantDict::new(Some(commit_meta));
 
+    let chunking = if opts.chunked {
+        // compression = Some(flate2::Compression::none());
+        let mut c = crate::chunking::Chunking::new(repo, commit)?;
+        c.auto_chunk(repo)?;
+        Some(c)
+    } else {
+        None
+    };
+
     if let Some(version) =
         commit_meta.lookup_value("version", Some(glib::VariantTy::new("s").unwrap()))
     {
@@ -91,15 +137,20 @@ fn build_oci(
         flate2::Compression::none()
     };
 
-    let rootfs_blob = export_ostree_ref(repo, commit, &mut writer, Some(compression))?;
+    let mut annos = HashMap::new();
+    annos.insert(BLOB_OSTREE_ANNOTATION.to_string(), "true".to_string());
     let description = if commit_subject.is_empty() {
         Cow::Owned(format!("ostree export of commit {}", commit))
     } else {
         Cow::Borrowed(commit_subject)
     };
-    let mut annos = HashMap::new();
-    annos.insert(BLOB_OSTREE_ANNOTATION.to_string(), "true".to_string());
-    writer.push_layer_annotated(rootfs_blob, Some(annos), &description);
+
+    if let Some(chunking) = chunking {
+        export_chunked(repo, &mut writer, chunking, Some(compression), &description)?;
+    } else {
+        let rootfs_blob = export_ostree_ref(repo, commit, &mut writer, Some(compression))?;
+        writer.push_layer_annotated(rootfs_blob, Some(annos), &description);
+    }
     writer.complete()?;
 
     Ok(ImageReference {
@@ -179,6 +230,8 @@ async fn build_impl(
 pub struct ExportOpts {
     /// If true, perform gzip compression of the tar layers.
     pub compress: bool,
+    /// Whether or not to generate multiple layers
+    pub chunked: bool,
 }
 
 /// Given an OSTree repository and ref, generate a container image.

lib/src/container/mod.rs

@@ -32,6 +32,8 @@ use std::ops::Deref;
 
 /// The label injected into a container image that contains the ostree commit SHA-256.
 pub const OSTREE_COMMIT_LABEL: &str = "ostree.commit";
+/// The label/annotation which contains the sha256 of the final commit.
+const OSTREE_LAYER_LABEL: &str = "ostree.layer";
 
 /// Our generic catchall fatal error, expected to be converted
 /// to a string to output to a terminal or logs.

lib/src/container/store.rs

@@ -143,7 +143,10 @@ pub struct PreparedImport {
 }
 
 // Given a manifest, compute its ostree ref name and cached ostree commit
-fn query_layer(repo: &ostree::Repo, layer: oci_image::Descriptor) -> Result<ManifestLayerState> {
+pub(crate) fn query_layer(
+    repo: &ostree::Repo,
+    layer: oci_image::Descriptor,
+) -> Result<ManifestLayerState> {
     let ostree_ref = ref_for_layer(&layer)?;
     let commit = repo.resolve_rev(&ostree_ref, true)?.map(|s| s.to_string());
     Ok(ManifestLayerState {
@@ -324,9 +327,9 @@ impl LayeredImageImporter {
                     base: Some(base_commit.clone()),
                     selinux: true,
                 };
-                let w =
+                let r =
                     crate::tar::write_tar(&self.repo, blob, layer.ostree_ref.as_str(), Some(opts));
-                let r = super::unencapsulate::join_fetch(w, driver)
+                let r = super::unencapsulate::join_fetch(r, driver)
                     .await
                     .with_context(|| format!("Parsing layer blob {}", layer.digest()))?;
                 layer_commits.push(r.commit);