nydusify: support --push-chunk-size option

contrib/ctr-remote/commands/rpull.go

@@ -24,7 +24,7 @@ import (
 	"github.com/containerd/containerd/cmd/ctr/commands"
 	"github.com/containerd/containerd/cmd/ctr/commands/content"
 	"github.com/containerd/containerd/images"
-	"github.com/containerd/containerd/log"
+	"github.com/containerd/log"
 	"github.com/containerd/nydus-snapshotter/pkg/label"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/urfave/cli"

contrib/ctr-remote/go.mod

@@ -4,6 +4,7 @@ go 1.20
 
 require (
 	github.com/containerd/containerd v1.7.6
+	github.com/containerd/log v0.1.0
 	github.com/containerd/nydus-snapshotter v0.13.2
 	github.com/opencontainers/image-spec v1.1.0-rc4
 	github.com/urfave/cli v1.22.12
@@ -54,7 +55,7 @@ require (
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/sirupsen/logrus v1.9.0 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/otel v1.14.0 // indirect
 	go.opentelemetry.io/otel/trace v1.14.0 // indirect

contrib/ctr-remote/go.sum

@@ -36,6 +36,8 @@ github.com/containerd/go-cni v1.1.9 h1:ORi7P1dYzCwVM6XPN4n3CbkuOx/NZ2DOqy+SHRdo9
 github.com/containerd/go-cni v1.1.9/go.mod h1:XYrZJ1d5W6E2VOvjffL3IZq0Dz6bsVlERHbekNK90PM=
 github.com/containerd/go-runc v1.0.0 h1:oU+lLv1ULm5taqgV/CJivypVODI4SUz1znWjv3nNYS0=
 github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok=
+github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
+github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
 github.com/containerd/nydus-snapshotter v0.13.2 h1:Wjc59CetOv9TxtQIvSBY1wrjHxwD48gHR/TxyEeRTmI=
 github.com/containerd/nydus-snapshotter v0.13.2/go.mod h1:XWAz9ytsjBuKPVXDKP3xoMlcSKNsGnjXlEup6DuzUIo=
 github.com/containerd/ttrpc v1.2.2 h1:9vqZr0pxwOF5koz6N0N3kJ0zDHokrcPxIR/ZR2YFtOs=
@@ -190,8 +192,8 @@ github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
-github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=

contrib/nydusify/cmd/nydusify.go

@@ -18,6 +18,7 @@ import (
 
 	"github.com/containerd/containerd/reference/docker"
 	"github.com/docker/distribution/reference"
+	"github.com/dustin/go-humanize"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/urfave/cli/v2"
@@ -1046,6 +1047,12 @@ func main() {
 					Usage: "Copy images for specific platforms, for example: 'linux/amd64,linux/arm64'",
 				},
 
+				&cli.StringFlag{
+					Name:  "push-chunk-size",
+					Value: "0MB",
+					Usage: "Chunk size for pushing a blob layer in chunked",
+				},
+
 				&cli.StringFlag{
 					Name:    "work-dir",
 					Value:   "./tmp",
@@ -1067,6 +1074,14 @@ func main() {
 					return err
 				}
 
+				pushChunkSize, err := humanize.ParseBytes(c.String("push-chunk-size"))
+				if err != nil {
+					return errors.Wrap(err, "invalid --push-chunk-size option")
+				}
+				if pushChunkSize > 0 {
+					logrus.Infof("will copy layer with chunk size %s", c.String("push-chunk-size"))
+				}
+
 				opt := copier.Opt{
 					WorkDir:        c.String("work-dir"),
 					NydusImagePath: c.String("nydus-image"),
@@ -1081,6 +1096,8 @@ func main() {
 
 					AllPlatforms: c.Bool("all-platforms"),
 					Platforms:    c.String("platform"),
+
+					PushChunkSize: int64(pushChunkSize),
 				}
 
 				return copier.Copy(context.Background(), opt)

contrib/nydusify/go.mod

@@ -35,7 +35,7 @@ require (
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
 	github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
-	github.com/Microsoft/hcsshim v0.11.0 // indirect
+	github.com/Microsoft/hcsshim v0.11.4 // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.24 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.30 // indirect
@@ -118,3 +118,5 @@ require (
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
+
+replace github.com/containerd/containerd => github.com/nydusaccelerator/containerd v0.0.0-20231121100328-6c4d1f35ac28

contrib/nydusify/go.sum

@@ -6,8 +6,8 @@ github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0/go.mod
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
-github.com/Microsoft/hcsshim v0.11.0 h1:7EFNIY4igHEXUdj1zXgAyU3fLc7QfOKHbkldRVTBdiM=
-github.com/Microsoft/hcsshim v0.11.0/go.mod h1:OEthFdQv/AD2RAdzR6Mm1N1KPCztGKDurW1Z8b8VGMM=
+github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8=
+github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w=
 github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible h1:KXeJoM1wo9I/6xPTyt6qCxoSZnmASiAjlrr0dyTUKt8=
 github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8=
 github.com/aws/aws-sdk-go-v2 v1.17.6 h1:Y773UK7OBqhzi5VDXMi1zVGsoj+CVHs2eaC2bDsLwi0=

contrib/nydusify/pkg/converter/converter.go

@@ -77,7 +77,7 @@ func Convert(ctx context.Context, opt Opt) error {
 	if err != nil {
 		return errors.Wrap(err, "create temp directory")
 	}
-	pvd, err := provider.New(tmpDir, hosts(opt), opt.CacheMaxRecords, opt.CacheVersion, platformMC)
+	pvd, err := provider.New(tmpDir, hosts(opt), opt.CacheMaxRecords, opt.CacheVersion, platformMC, 0)
 	if err != nil {
 		return err
 	}

contrib/nydusify/pkg/converter/provider/provider.go

@@ -6,15 +6,20 @@ package provider
 
 import (
 	"context"
+	"crypto/tls"
+	"net"
+	"net/http"
 	"os"
 	"path/filepath"
 	"sync"
+	"time"
 
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/platforms"
 	"github.com/containerd/containerd/remotes"
+	"github.com/containerd/containerd/remotes/docker"
 	"github.com/goharbor/acceleration-service/pkg/cache"
 	accelcontent "github.com/goharbor/acceleration-service/pkg/content"
 	"github.com/goharbor/acceleration-service/pkg/remote"
@@ -32,9 +37,10 @@ type Provider struct {
 	platformMC   platforms.MatchComparer
 	cacheSize    int
 	cacheVersion string
+	chunkSize    int64
 }
 
-func New(root string, hosts remote.HostFunc, cacheSize uint, cacheVersion string, platformMC platforms.MatchComparer) (*Provider, error) {
+func New(root string, hosts remote.HostFunc, cacheSize uint, cacheVersion string, platformMC platforms.MatchComparer, chunkSize int64) (*Provider, error) {
 	contentDir := filepath.Join(root, "content")
 	if err := os.MkdirAll(contentDir, 0755); err != nil {
 		return nil, err
@@ -51,9 +57,52 @@ func New(root string, hosts remote.HostFunc, cacheSize uint, cacheVersion string
 		cacheSize:    int(cacheSize),
 		platformMC:   platformMC,
 		cacheVersion: cacheVersion,
+		chunkSize:    chunkSize,
 	}, nil
 }
 
+func newDefaultClient(skipTLSVerify bool) *http.Client {
+	return &http.Client{
+		Transport: &http.Transport{
+			Proxy: http.ProxyFromEnvironment,
+			DialContext: (&net.Dialer{
+				Timeout:   30 * time.Second,
+				KeepAlive: 30 * time.Second,
+				DualStack: true,
+			}).DialContext,
+			MaxIdleConns:          10,
+			IdleConnTimeout:       30 * time.Second,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ExpectContinueTimeout: 5 * time.Second,
+			DisableKeepAlives:     true,
+			TLSNextProto:          make(map[string]func(authority string, c *tls.Conn) http.RoundTripper),
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: skipTLSVerify,
+			},
+		},
+	}
+}
+
+func newResolver(insecure, plainHTTP bool, credFunc remote.CredentialFunc, chunkSize int64) remotes.Resolver {
+	registryHosts := docker.ConfigureDefaultRegistries(
+		docker.WithAuthorizer(
+			docker.NewDockerAuthorizer(
+				docker.WithAuthClient(newDefaultClient(insecure)),
+				docker.WithAuthCreds(credFunc),
+			),
+		),
+		docker.WithClient(newDefaultClient(insecure)),
+		docker.WithPlainHTTP(func(host string) (bool, error) {
+			return plainHTTP, nil
+		}),
+		docker.WithChunkSize(chunkSize),
+	)
+
+	return docker.NewResolver(docker.ResolverOptions{
+		Hosts: registryHosts,
+	})
+}
+
 func (pvd *Provider) UsePlainHTTP() {
 	pvd.usePlainHTTP = true
 }
@@ -63,7 +112,7 @@ func (pvd *Provider) Resolver(ref string) (remotes.Resolver, error) {
 	if err != nil {
 		return nil, err
 	}
-	return remote.NewResolver(insecure, pvd.usePlainHTTP, credFunc), nil
+	return newResolver(insecure, pvd.usePlainHTTP, credFunc, pvd.chunkSize), nil
 }
 
 func (pvd *Provider) Pull(ctx context.Context, ref string) error {

contrib/nydusify/pkg/copier/copier.go

@@ -57,6 +57,8 @@ type Opt struct {
 
 	AllPlatforms bool
 	Platforms    string
+
+	PushChunkSize int64
 }
 
 type output struct {
@@ -277,7 +279,7 @@ func Copy(ctx context.Context, opt Opt) error {
 	if err != nil {
 		return errors.Wrap(err, "create temp directory")
 	}
-	pvd, err := provider.New(tmpDir, hosts(opt), 200, "v1", platformMC)
+	pvd, err := provider.New(tmpDir, hosts(opt), 200, "v1", platformMC, opt.PushChunkSize)
 	if err != nil {
 		return err
 	}

smoke/tests/image_test.go

@@ -127,7 +127,7 @@ func (i *ImageTestSuite) TestConvertAndCopyImage(t *testing.T, ctx tool.Context,
 	// Copy image
 	targetCopied := fmt.Sprintf("%s_copied", target)
 	copyCmd := fmt.Sprintf(
-		"%s %s copy --source %s --target %s --nydus-image %s --work-dir %s",
+		"%s %s copy --source %s --target %s --nydus-image %s --work-dir %s --push-chunk-size 1MB",
 		ctx.Binary.Nydusify, logLevel, target, targetCopied, ctx.Binary.Builder, ctx.Env.WorkDir,
 	)
 	tool.RunWithoutOutput(t, copyCmd)