Skip to content

Commit

Permalink
nydusify: support --push-chunk-size option
Browse files Browse the repository at this point in the history 
Reference: containerd/containerd#9405

Signed-off-by: Yan Song <[email protected]>
  • Loading branch information
@imeoer
imeoer committed Nov 21, 2023
1 parent 767adcf commit 845a72d
Show file tree
Hide file tree
Showing 8 changed files with 1,372 additions and 87 deletions.
17 changes: 17 additions & 0 deletions contrib/nydusify/cmd/nydusify.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (

"github.com/containerd/containerd/reference/docker"
"github.com/docker/distribution/reference"
"github.com/dustin/go-humanize"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
"github.com/urfave/cli/v2"
Expand Down Expand Up @@ -1046,6 +1047,12 @@ func main() {
Usage: "Copy images for specific platforms, for example: 'linux/amd64,linux/arm64'",
},

&cli.StringFlag{
Name: "push-chunk-size",
Value: "0MB",
Usage: "Chunk size for pushing a blob layer in chunked",
},

&cli.StringFlag{
Name: "work-dir",
Value: "./tmp",
Expand All @@ -1067,6 +1074,14 @@ func main() {
return err
}

pushChunkSize, err := humanize.ParseBytes(c.String("push-chunk-size"))
if err != nil {
return errors.Wrap(err, "invalid --push-chunk-size option")
}
if pushChunkSize > 0 {
logrus.Infof("will copy layer with chunk size %s", c.String("push-chunk-size"))
}

opt := copier.Opt{
WorkDir: c.String("work-dir"),
NydusImagePath: c.String("nydus-image"),
Expand All @@ -1081,6 +1096,8 @@ func main() {

AllPlatforms: c.Bool("all-platforms"),
Platforms: c.String("platform"),

PushChunkSize: int64(pushChunkSize),
}

return copier.Copy(context.Background(), opt)
Expand Down
36 changes: 20 additions & 16 deletions contrib/nydusify/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ require (
github.com/aws/aws-sdk-go-v2/credentials v1.13.16
github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.56
github.com/aws/aws-sdk-go-v2/service/s3 v1.30.6
github.com/containerd/containerd v1.7.6
github.com/containerd/containerd v1.7.9
github.com/containerd/nydus-snapshotter v0.13.2
github.com/docker/cli v24.0.6+incompatible
github.com/docker/distribution v2.8.2+incompatible
Expand All @@ -19,12 +19,12 @@ require (
github.com/hashicorp/go-hclog v1.3.1
github.com/hashicorp/go-plugin v1.4.5
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.1.0-rc4
github.com/opencontainers/image-spec v1.1.0-rc5
github.com/pkg/errors v0.9.1
github.com/pkg/xattr v0.4.9
github.com/prometheus/client_golang v1.16.0
github.com/sirupsen/logrus v1.9.0
github.com/stretchr/testify v1.8.3
github.com/sirupsen/logrus v1.9.3
github.com/stretchr/testify v1.8.4
github.com/urfave/cli/v2 v2.25.7
golang.org/x/sync v0.3.0
golang.org/x/sys v0.13.0
Expand All @@ -35,7 +35,7 @@ require (
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230811130428-ced1acdcaa24 // indirect
github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/Microsoft/hcsshim v0.11.0 // indirect
github.com/Microsoft/hcsshim v0.12.0-rc.0 // indirect
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.24 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.30 // indirect
Expand All @@ -52,17 +52,18 @@ require (
github.com/aws/smithy-go v1.13.5 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/containerd/cgroups v1.1.0 // indirect
github.com/containerd/cgroups/v3 v3.0.2 // indirect
github.com/containerd/continuity v0.4.2 // indirect
github.com/containerd/fifo v1.1.0 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/containerd/stargz-snapshotter v0.14.3 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect
github.com/containerd/ttrpc v1.2.2 // indirect
github.com/containerd/typeurl/v2 v2.1.1 // indirect
github.com/containers/ocicrypt v1.1.7 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/docker/docker v23.0.3+incompatible // indirect
github.com/docker/docker v23.0.5+incompatible // indirect
github.com/docker/docker-credential-helpers v0.7.0 // indirect
github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect
github.com/fatih/color v1.14.1 // indirect
Expand All @@ -75,7 +76,7 @@ require (
github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect
github.com/hashicorp/yamux v0.1.1 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/klauspost/compress v1.16.3 // indirect
github.com/klauspost/compress v1.17.2 // indirect
github.com/klauspost/cpuid v1.3.1 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.19 // indirect
Expand All @@ -87,34 +88,37 @@ require (
github.com/moby/sys/sequential v0.5.0 // indirect
github.com/moby/sys/signal v0.7.0 // indirect
github.com/oklog/run v1.1.0 // indirect
github.com/opencontainers/runc v1.1.5 // indirect
github.com/opencontainers/runtime-spec v1.1.0-rc.1 // indirect
github.com/opencontainers/runc v1.1.8 // indirect
github.com/opencontainers/runtime-spec v1.1.1-0.20230823135140-4fec88fd00a4 // indirect
github.com/opencontainers/selinux v1.11.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.10.1 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 // indirect
github.com/stretchr/objx v0.5.0 // indirect
github.com/vbatts/tar-split v0.11.2 // indirect
github.com/vbatts/tar-split v0.11.3 // indirect
github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
go.etcd.io/bbolt v1.3.7 // indirect
go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel v1.14.0 // indirect
go.opentelemetry.io/otel/trace v1.14.0 // indirect
go.opentelemetry.io/otel v1.19.0 // indirect
go.opentelemetry.io/otel/metric v1.19.0 // indirect
go.opentelemetry.io/otel/trace v1.19.0 // indirect
golang.org/x/crypto v0.14.0 // indirect
golang.org/x/mod v0.11.0 // indirect
golang.org/x/mod v0.12.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.10.0 // indirect
golang.org/x/tools v0.11.0 // indirect
google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect
google.golang.org/grpc v1.59.0 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/square/go-jose.v2 v2.5.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

replace github.com/containerd/containerd => github.com/nydusaccelerator/containerd v0.0.0-20231121100328-6c4d1f35ac28
Loading

0 comments on commit 845a72d

Please sign in to comment.