push: add support for --force-compression

Adds support for --force-compression which allows end-users to force
push blobs with the selected compresison in --compression option, in
order to make sure that blobs of other compression on registry are not
reused.

Is equivalent to: force-compression here: https://docs.docker.com/build/exporters/#compression

Closes: containers#18660

Signed-off-by: Aditya R <[email protected]>

cmd/podman/images/push.go

@@ -102,6 +102,8 @@ func pushFlags(cmd *cobra.Command) {
 	flags.StringVar(&pushOptions.DigestFile, digestfileFlagName, "", "Write the digest of the pushed image to the specified file")
 	_ = cmd.RegisterFlagCompletionFunc(digestfileFlagName, completion.AutocompleteDefault)
 
+	flags.BoolVar(&pushOptions.ForceCompressionFormat, "force-compression", false, "Use the specified compression algorithm if the destination contains a differently-compressed variant already")
+
 	formatFlagName := "format"
 	flags.StringVarP(&pushOptions.Format, formatFlagName, "f", "", "Manifest type (oci, v2s2, or v2s1) to use in the destination (default is manifest type of source, with fallbacks)")
 	_ = cmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteManifestFormat)

docs/source/markdown/podman-manifest-push.1.md.in

@@ -42,6 +42,10 @@ the list or index itself. (Default true)
 
 @@option digestfile
 
+#### **--force-compression**
+
+Use the specified compression algorithm even if the destination contains a differently-compressed variant already.
+
 #### **--format**, **-f**=*format*
 
 Manifest list type (oci or v2s2) to use when pushing the list (default is oci).

docs/source/markdown/podman-push.1.md.in

@@ -70,6 +70,10 @@ Layer(s) to encrypt: 0-indexed layer indices with support for negative indexing
 
 The [protocol:keyfile] specifies the encryption protocol, which can be JWE (RFC7516), PGP (RFC4880), and PKCS7 (RFC2315) and the key material required for image encryption. For instance, jwe:/path/to/key.pem or pgp:[email protected] or pkcs7:/path/to/x509-file.
 
+### **--force-compression**
+
+Use the specified compression algorithm even if the destination contains a differently-compressed variant already.
+
 #### **--format**, **-f**=*format*
 
 Manifest Type (oci, v2s2, or v2s1) to use when pushing an image.

pkg/api/handlers/libpod/images_push.go

@@ -25,14 +25,15 @@ func PushImage(w http.ResponseWriter, r *http.Request) {
 	runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime)
 
 	query := struct {
-		All               bool   `schema:"all"`
-		CompressionFormat string `schema:"compressionFormat"`
-		CompressionLevel  *int   `schema:"compressionLevel"`
-		Destination       string `schema:"destination"`
-		Format            string `schema:"format"`
-		RemoveSignatures  bool   `schema:"removeSignatures"`
-		TLSVerify         bool   `schema:"tlsVerify"`
-		Quiet             bool   `schema:"quiet"`
+		All                    bool   `schema:"all"`
+		CompressionFormat      string `schema:"compressionFormat"`
+		CompressionLevel       *int   `schema:"compressionLevel"`
+		ForceCompressionFormat bool   `schema:"forceCompressionFormat"`
+		Destination            string `schema:"destination"`
+		Format                 string `schema:"format"`
+		RemoveSignatures       bool   `schema:"removeSignatures"`
+		TLSVerify              bool   `schema:"tlsVerify"`
+		Quiet                  bool   `schema:"quiet"`
 	}{
 		TLSVerify: true,
 		// #14971: older versions did not sent *any* data, so we need
@@ -73,15 +74,16 @@ func PushImage(w http.ResponseWriter, r *http.Request) {
 		password = authconf.Password
 	}
 	options := entities.ImagePushOptions{
-		All:               query.All,
-		Authfile:          authfile,
-		CompressionFormat: query.CompressionFormat,
-		CompressionLevel:  query.CompressionLevel,
-		Format:            query.Format,
-		Password:          password,
-		Quiet:             true,
-		RemoveSignatures:  query.RemoveSignatures,
-		Username:          username,
+		All:                    query.All,
+		Authfile:               authfile,
+		CompressionFormat:      query.CompressionFormat,
+		CompressionLevel:       query.CompressionLevel,
+		ForceCompressionFormat: query.ForceCompressionFormat,
+		Format:                 query.Format,
+		Password:               password,
+		Quiet:                  true,
+		RemoveSignatures:       query.RemoveSignatures,
+		Username:               username,
 	}
 
 	if _, found := r.URL.Query()["tlsVerify"]; found {

pkg/api/server/register_images.go

@@ -726,6 +726,11 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error {
 	//    type: string
 	//    description: Allows for pushing the image to a different destination than the image refers to.
 	//  - in: query
+	//    name: forceCompressionFormat
+	//    description: Use the specified compression algorithm if the destination contains a differently-compressed variant already.
+	//    type: boolean
+	//    default: false
+	//  - in: query
 	//    name: tlsVerify
 	//    description: Require TLS verification.
 	//    type: boolean

pkg/bindings/images/types.go

@@ -144,6 +144,10 @@ type PushOptions struct {
 	CompressionFormat *string
 	// CompressionLevel is the level to use for the compression of the blobs
 	CompressionLevel *int
+	// ForceCompressionFormat ensures that the compression algorithm set in
+	// CompressionFormat is used exclusively, and blobs of other compression
+	// algorithms are not reused.
+	ForceCompressionFormat *bool
 	// Add existing instances with requested compression algorithms to manifest list
 	AddCompression []string
 	// Manifest type of the pushed image

pkg/domain/entities/images.go

@@ -247,6 +247,10 @@ type ImagePushOptions struct {
 	//  If necessary, add clones of existing instances with requested compression algorithms to manifest list
 	// Note: Following option is only valid for `manifest push`
 	AddCompression []string
+	// ForceCompressionFormat ensures that the compression algorithm set in
+	// CompressionFormat is used exclusively, and blobs of other compression
+	// algorithms are not reused.
+	ForceCompressionFormat bool
 }
 
 // ImagePushReport is the response from pushing an image.

pkg/domain/infra/abi/images.go

@@ -317,6 +317,7 @@ func (ir *ImageEngine) Push(ctx context.Context, source string, destination stri
 	pushOptions.OciEncryptConfig = options.OciEncryptConfig
 	pushOptions.OciEncryptLayers = options.OciEncryptLayers
 	pushOptions.CompressionLevel = options.CompressionLevel
+	pushOptions.ForceCompressionFormat = options.ForceCompressionFormat
 
 	compressionFormat := options.CompressionFormat
 	if compressionFormat == "" {

pkg/domain/infra/tunnel/images.go

@@ -252,7 +252,7 @@ func (ir *ImageEngine) Push(ctx context.Context, source string, destination stri
 	}
 
 	options := new(images.PushOptions)
-	options.WithAll(opts.All).WithCompress(opts.Compress).WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithFormat(opts.Format).WithRemoveSignatures(opts.RemoveSignatures).WithQuiet(opts.Quiet).WithCompressionFormat(opts.CompressionFormat).WithProgressWriter(opts.Writer)
+	options.WithAll(opts.All).WithCompress(opts.Compress).WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithFormat(opts.Format).WithRemoveSignatures(opts.RemoveSignatures).WithQuiet(opts.Quiet).WithCompressionFormat(opts.CompressionFormat).WithProgressWriter(opts.Writer).WithForceCompressionFormat(opts.ForceCompressionFormat)
 
 	if opts.CompressionLevel != nil {
 		options.WithCompressionLevel(*opts.CompressionLevel)

test/e2e/push_test.go

@@ -84,6 +84,63 @@ var _ = Describe("Podman push", func() {
 		Expect(foundZstdFile).To(BeTrue(), "found zstd file")
 	})
 
+	It("push test --force-compression", func() {
+		if podmanTest.Host.Arch == "ppc64le" {
+			Skip("No registry image for ppc64le")
+		}
+		if isRootless() {
+			err := podmanTest.RestoreArtifact(REGISTRY_IMAGE)
+			Expect(err).ToNot(HaveOccurred())
+		}
+		lock := GetPortLock("5000")
+		defer lock.Unlock()
+		session := podmanTest.Podman([]string{"run", "-d", "--name", "registry", "-p", "5000:5000", REGISTRY_IMAGE, "/entrypoint.sh", "/etc/docker/registry/config.yml"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+
+		if !WaitContainerReady(podmanTest, "registry", "listening on", 20, 1) {
+			Skip("Cannot start docker registry.")
+		}
+
+		session = podmanTest.Podman([]string{"build", "-t", "imageone", "build/basicalpine"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+
+		push := podmanTest.Podman([]string{"push", "--tls-verify=false", "--remove-signatures", "imageone", "localhost:5000/image"})
+		push.WaitWithDefaultTimeout()
+		Expect(push).Should(Exit(0))
+
+		session = podmanTest.Podman([]string{"run", "--rm", "--net", "host", "quay.io/skopeo/stable", "inspect", "--tls-verify=false", "--raw", "docker://localhost:5000/image:latest"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		output := session.OutputToString()
+		// Default compression is gzip no traces of `zstd` should be there.
+		Expect(output).ToNot(ContainSubstring("zstd"))
+
+		push = podmanTest.Podman([]string{"push", "--tls-verify=false", "--compression-format", "zstd", "--remove-signatures", "imageone", "localhost:5000/image"})
+		push.WaitWithDefaultTimeout()
+		Expect(push).Should(Exit(0))
+
+		session = podmanTest.Podman([]string{"run", "--rm", "--net", "host", "quay.io/skopeo/stable", "inspect", "--tls-verify=false", "--raw", "docker://localhost:5000/image:latest"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		output = session.OutputToString()
+		// Although `--compression-format` is `zstd` but still no traces of `zstd` should be in image
+		// since blobs must be reused from last `gzip` image.
+		Expect(output).ToNot(ContainSubstring("zstd"))
+
+		push = podmanTest.Podman([]string{"push", "--tls-verify=false", "--compression-format", "zstd", "--force-compression", "--remove-signatures", "imageone", "localhost:5000/image"})
+		push.WaitWithDefaultTimeout()
+		Expect(push).Should(Exit(0))
+
+		session = podmanTest.Podman([]string{"run", "--rm", "--net", "host", "quay.io/skopeo/stable", "inspect", "--tls-verify=false", "--raw", "docker://localhost:5000/image:latest"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		output = session.OutputToString()
+		// Should contain `zstd` layer, substring `zstd` is enough to confirm in skopeo inspect output that `zstd` layer is present.
+		Expect(output).To(ContainSubstring("zstd"))
+	})
+
 	It("podman push to local registry", func() {
 		if podmanTest.Host.Arch == "ppc64le" {
 			Skip("No registry image for ppc64le")