diff --git a/contrib/ctr-remote/commands/rpull.go b/contrib/ctr-remote/commands/rpull.go index d5c0b11f7c1..87ad1dc47d6 100644 --- a/contrib/ctr-remote/commands/rpull.go +++ b/contrib/ctr-remote/commands/rpull.go @@ -24,7 +24,7 @@ import ( "github.com/containerd/containerd/cmd/ctr/commands" "github.com/containerd/containerd/cmd/ctr/commands/content" "github.com/containerd/containerd/images" - "github.com/containerd/containerd/log" + "github.com/containerd/log" "github.com/containerd/nydus-snapshotter/pkg/label" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/urfave/cli" diff --git a/contrib/ctr-remote/go.mod b/contrib/ctr-remote/go.mod index aebdc5e9ef6..a0526fc5e46 100644 --- a/contrib/ctr-remote/go.mod +++ b/contrib/ctr-remote/go.mod @@ -4,6 +4,7 @@ go 1.20 require ( github.com/containerd/containerd v1.7.6 + github.com/containerd/log v0.1.0 github.com/containerd/nydus-snapshotter v0.13.2 github.com/opencontainers/image-spec v1.1.0-rc4 github.com/urfave/cli v1.22.12 @@ -54,7 +55,7 @@ require ( github.com/pelletier/go-toml v1.9.5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/sirupsen/logrus v1.9.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel v1.14.0 // indirect go.opentelemetry.io/otel/trace v1.14.0 // indirect diff --git a/contrib/ctr-remote/go.sum b/contrib/ctr-remote/go.sum index c41029f5dc7..5c7069469c0 100644 --- a/contrib/ctr-remote/go.sum +++ b/contrib/ctr-remote/go.sum @@ -36,6 +36,8 @@ github.com/containerd/go-cni v1.1.9 h1:ORi7P1dYzCwVM6XPN4n3CbkuOx/NZ2DOqy+SHRdo9 github.com/containerd/go-cni v1.1.9/go.mod h1:XYrZJ1d5W6E2VOvjffL3IZq0Dz6bsVlERHbekNK90PM= github.com/containerd/go-runc v1.0.0 h1:oU+lLv1ULm5taqgV/CJivypVODI4SUz1znWjv3nNYS0= github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/nydus-snapshotter v0.13.2 h1:Wjc59CetOv9TxtQIvSBY1wrjHxwD48gHR/TxyEeRTmI= github.com/containerd/nydus-snapshotter v0.13.2/go.mod h1:XWAz9ytsjBuKPVXDKP3xoMlcSKNsGnjXlEup6DuzUIo= github.com/containerd/ttrpc v1.2.2 h1:9vqZr0pxwOF5koz6N0N3kJ0zDHokrcPxIR/ZR2YFtOs= @@ -190,8 +192,8 @@ github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= diff --git a/contrib/nydusify/cmd/nydusify.go b/contrib/nydusify/cmd/nydusify.go index 8ae38f434fb..9bec83641ff 100644 --- a/contrib/nydusify/cmd/nydusify.go +++ b/contrib/nydusify/cmd/nydusify.go @@ -18,6 +18,7 @@ import ( "github.com/containerd/containerd/reference/docker" "github.com/docker/distribution/reference" + "github.com/dustin/go-humanize" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/urfave/cli/v2" @@ -1046,6 +1047,12 @@ func main() { Usage: "Copy images for specific platforms, for example: 'linux/amd64,linux/arm64'", }, + &cli.StringFlag{ + Name: "push-chunk-size", + Value: "0MB", + Usage: "Chunk size for pushing a blob layer in chunked", + }, + &cli.StringFlag{ Name: "work-dir", Value: "./tmp", @@ -1067,6 +1074,14 @@ func main() { return err } + pushChunkSize, err := humanize.ParseBytes(c.String("push-chunk-size")) + if err != nil { + return errors.Wrap(err, "invalid --push-chunk-size option") + } + if pushChunkSize > 0 { + logrus.Infof("will copy layer with chunk size %s", c.String("push-chunk-size")) + } + opt := copier.Opt{ WorkDir: c.String("work-dir"), NydusImagePath: c.String("nydus-image"), @@ -1081,6 +1096,8 @@ func main() { AllPlatforms: c.Bool("all-platforms"), Platforms: c.String("platform"), + + PushChunkSize: int64(pushChunkSize), } return copier.Copy(context.Background(), opt) diff --git a/contrib/nydusify/go.mod b/contrib/nydusify/go.mod index d01c84fc145..3dabbb136f7 100644 --- a/contrib/nydusify/go.mod +++ b/contrib/nydusify/go.mod @@ -35,7 +35,7 @@ require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/Microsoft/hcsshim v0.11.0 // indirect + github.com/Microsoft/hcsshim v0.11.4 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.24 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.30 // indirect @@ -118,3 +118,5 @@ require ( gopkg.in/square/go-jose.v2 v2.5.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) + +replace github.com/containerd/containerd => github.com/nydusaccelerator/containerd v0.0.0-20231121100328-6c4d1f35ac28 diff --git a/contrib/nydusify/go.sum b/contrib/nydusify/go.sum index 74146c0ecd8..d2cdc1c3141 100644 --- a/contrib/nydusify/go.sum +++ b/contrib/nydusify/go.sum @@ -6,8 +6,8 @@ github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0/go.mod github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= -github.com/Microsoft/hcsshim v0.11.0 h1:7EFNIY4igHEXUdj1zXgAyU3fLc7QfOKHbkldRVTBdiM= -github.com/Microsoft/hcsshim v0.11.0/go.mod h1:OEthFdQv/AD2RAdzR6Mm1N1KPCztGKDurW1Z8b8VGMM= +github.com/Microsoft/hcsshim v0.11.4 h1:68vKo2VN8DE9AdN4tnkWnmdhqdbpUFM8OF3Airm7fz8= +github.com/Microsoft/hcsshim v0.11.4/go.mod h1:smjE4dvqPX9Zldna+t5FG3rnoHhaB7QYxPRqGcpAD9w= github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible h1:KXeJoM1wo9I/6xPTyt6qCxoSZnmASiAjlrr0dyTUKt8= github.com/aliyun/aliyun-oss-go-sdk v2.2.6+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= github.com/aws/aws-sdk-go-v2 v1.17.6 h1:Y773UK7OBqhzi5VDXMi1zVGsoj+CVHs2eaC2bDsLwi0= diff --git a/contrib/nydusify/pkg/converter/converter.go b/contrib/nydusify/pkg/converter/converter.go index aeae613fc93..198e581ffa9 100644 --- a/contrib/nydusify/pkg/converter/converter.go +++ b/contrib/nydusify/pkg/converter/converter.go @@ -77,7 +77,7 @@ func Convert(ctx context.Context, opt Opt) error { if err != nil { return errors.Wrap(err, "create temp directory") } - pvd, err := provider.New(tmpDir, hosts(opt), opt.CacheMaxRecords, opt.CacheVersion, platformMC) + pvd, err := provider.New(tmpDir, hosts(opt), opt.CacheMaxRecords, opt.CacheVersion, platformMC, 0) if err != nil { return err } diff --git a/contrib/nydusify/pkg/converter/provider/provider.go b/contrib/nydusify/pkg/converter/provider/provider.go index fd29d7424e8..1485778efe9 100644 --- a/contrib/nydusify/pkg/converter/provider/provider.go +++ b/contrib/nydusify/pkg/converter/provider/provider.go @@ -6,15 +6,20 @@ package provider import ( "context" + "crypto/tls" + "net" + "net/http" "os" "path/filepath" "sync" + "time" "github.com/containerd/containerd" "github.com/containerd/containerd/content" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/platforms" "github.com/containerd/containerd/remotes" + "github.com/containerd/containerd/remotes/docker" "github.com/goharbor/acceleration-service/pkg/cache" accelcontent "github.com/goharbor/acceleration-service/pkg/content" "github.com/goharbor/acceleration-service/pkg/remote" @@ -32,9 +37,10 @@ type Provider struct { platformMC platforms.MatchComparer cacheSize int cacheVersion string + chunkSize int64 } -func New(root string, hosts remote.HostFunc, cacheSize uint, cacheVersion string, platformMC platforms.MatchComparer) (*Provider, error) { +func New(root string, hosts remote.HostFunc, cacheSize uint, cacheVersion string, platformMC platforms.MatchComparer, chunkSize int64) (*Provider, error) { contentDir := filepath.Join(root, "content") if err := os.MkdirAll(contentDir, 0755); err != nil { return nil, err @@ -51,9 +57,52 @@ func New(root string, hosts remote.HostFunc, cacheSize uint, cacheVersion string cacheSize: int(cacheSize), platformMC: platformMC, cacheVersion: cacheVersion, + chunkSize: chunkSize, }, nil } +func newDefaultClient(skipTLSVerify bool) *http.Client { + return &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 10, + IdleConnTimeout: 30 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 5 * time.Second, + DisableKeepAlives: true, + TLSNextProto: make(map[string]func(authority string, c *tls.Conn) http.RoundTripper), + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: skipTLSVerify, + }, + }, + } +} + +func newResolver(insecure, plainHTTP bool, credFunc remote.CredentialFunc, chunkSize int64) remotes.Resolver { + registryHosts := docker.ConfigureDefaultRegistries( + docker.WithAuthorizer( + docker.NewDockerAuthorizer( + docker.WithAuthClient(newDefaultClient(insecure)), + docker.WithAuthCreds(credFunc), + ), + ), + docker.WithClient(newDefaultClient(insecure)), + docker.WithPlainHTTP(func(host string) (bool, error) { + return plainHTTP, nil + }), + docker.WithChunkSize(chunkSize), + ) + + return docker.NewResolver(docker.ResolverOptions{ + Hosts: registryHosts, + }) +} + func (pvd *Provider) UsePlainHTTP() { pvd.usePlainHTTP = true } @@ -63,7 +112,7 @@ func (pvd *Provider) Resolver(ref string) (remotes.Resolver, error) { if err != nil { return nil, err } - return remote.NewResolver(insecure, pvd.usePlainHTTP, credFunc), nil + return newResolver(insecure, pvd.usePlainHTTP, credFunc, pvd.chunkSize), nil } func (pvd *Provider) Pull(ctx context.Context, ref string) error { diff --git a/contrib/nydusify/pkg/copier/copier.go b/contrib/nydusify/pkg/copier/copier.go index 40ff90fcf87..4cec23ddea3 100644 --- a/contrib/nydusify/pkg/copier/copier.go +++ b/contrib/nydusify/pkg/copier/copier.go @@ -57,6 +57,8 @@ type Opt struct { AllPlatforms bool Platforms string + + PushChunkSize int64 } type output struct { @@ -277,7 +279,7 @@ func Copy(ctx context.Context, opt Opt) error { if err != nil { return errors.Wrap(err, "create temp directory") } - pvd, err := provider.New(tmpDir, hosts(opt), 200, "v1", platformMC) + pvd, err := provider.New(tmpDir, hosts(opt), 200, "v1", platformMC, opt.PushChunkSize) if err != nil { return err } diff --git a/smoke/tests/image_test.go b/smoke/tests/image_test.go index 6cea6d73a16..8d87bb86bd1 100644 --- a/smoke/tests/image_test.go +++ b/smoke/tests/image_test.go @@ -127,7 +127,7 @@ func (i *ImageTestSuite) TestConvertAndCopyImage(t *testing.T, ctx tool.Context, // Copy image targetCopied := fmt.Sprintf("%s_copied", target) copyCmd := fmt.Sprintf( - "%s %s copy --source %s --target %s --nydus-image %s --work-dir %s", + "%s %s copy --source %s --target %s --nydus-image %s --work-dir %s --push-chunk-size 1MB", ctx.Binary.Nydusify, logLevel, target, targetCopied, ctx.Binary.Builder, ctx.Env.WorkDir, ) tool.RunWithoutOutput(t, copyCmd)