diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6ecd71320..4a5642fa7 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -35,5 +35,5 @@ jobs: name: Optimize steps: - uses: actions/checkout@v1 - - name: Run test for optimize + - name: Run test for optimize subcommand of ctr-remote run: ./script/make.sh test-optimize diff --git a/Makefile b/Makefile index 3b7dbbd23..b735a2c16 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ GO111MODULE_VALUE=off PREFIX ?= out/ PLUGINS=stargzfs-linux-amd64.so -CMD=rsnapshotd optimize ctr +CMD=rsnapshotd ctr-remote PLUGIN_BINARIES=$(addprefix $(PREFIX),$(PLUGINS)) CMD_BINARIES=$(addprefix $(PREFIX),$(CMD)) @@ -39,11 +39,8 @@ stargzfs-linux-amd64.so: FORCE rsnapshotd: FORCE GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ -v ./cmd/rsnapshotd -optimize: FORCE - GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ -v ./cmd/optimize - -ctr: FORCE - GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ -v ./cmd/ctr +ctr-remote: FORCE + GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ -v ./cmd/ctr-remote # TODO: git-validation check: diff --git a/README.md b/README.md index 90681791a..e0fb1e676 100644 --- a/README.md +++ b/README.md @@ -11,15 +11,15 @@ Related discussion of the snapshotter in containerd community: By using this snapshotter, images(even if they are huge) can be pulled in lightning speed because this skips pulling layers but fetches the contents on demand at runtime. ``` -# time ctr images rpull --plain-http registry2:5000/fedora:30 > /dev/null +# time ctr-remote images rpull --plain-http registry2:5000/fedora:30 > /dev/null real 0m0.447s user 0m0.081s sys 0m0.019s -# time ctr images rpull --plain-http registry2:5000/python:3.7 > /dev/null +# time ctr-remote images rpull --plain-http registry2:5000/python:3.7 > /dev/null real 0m1.041s user 0m0.073s sys 0m0.028s -# time ctr images rpull --plain-http registry2:5000/jenkins:2.60.3 > /dev/null +# time ctr-remote images rpull --plain-http registry2:5000/jenkins:2.60.3 > /dev/null real 0m1.231s user 0m0.112s sys 0m0.008s @@ -29,7 +29,7 @@ To achive that we supports following [filesystems](filesystems): ## Demo -You can test this snapshotter with the latest containerd. Though we still need patches on clients and we are working on, you can use [a customized version of ctr command](cmd/ctr) for a quick tasting. +You can test this snapshotter with the latest containerd. Though we still need patches on clients and we are working on, you can use [a customized version of ctr command](cmd/ctr-remote) for a quick tasting. __NOTICE:__ @@ -50,24 +50,24 @@ $ docker exec -it containerd_demo /bin/bash ### Prepare stargz-formatted image on a registry -Use optimize command to convert the image into stargz-formatted one as well as optimize the image for your workload. In this example, we optimize the image aming to speed up execution of `ls` command on `bash`. +Use `optimize` subcommand to convert the image into stargz-formatted one as well as optimize the image for your workload. In this example, we optimize the image aming to speed up execution of `ls` command on `bash`. ``` -# optimize -insecure -entrypoint='[ "/bin/bash", "-c" ]' -args='[ "ls" ]' \ - ubuntu:18.04 http://registry2:5000/ubuntu:18.04 +# ctr-remote image optimize --plain-http --entrypoint='[ "/bin/bash", "-c" ]' --args='[ "ls" ]' \ + ubuntu:18.04 http://registry2:5000/ubuntu:18.04 ``` The converted image is still __compatible with a normal docker image__ so you can still pull and run it with normal tools(e.g. docker). ### Run the container with remote snapshots Layer downloads don't occur. So this "pull" operation ends soon. ``` -# time /tmp/out/ctr images rpull --plain-http registry2:5000/ubuntu:18.04 +# time ctr-remote images rpull --plain-http registry2:5000/ubuntu:18.04 fetching sha256:728332a6... application/vnd.docker.distribution.manifest.v2+json fetching sha256:80026893... application/vnd.docker.container.image.v1+json real 0m0.176s user 0m0.025s sys 0m0.005s -# /tmp/out/ctr run --rm -t --snapshotter=remote registry2:5000/ubuntu:18.04 test /bin/bash +# ctr-remote run --rm -t --snapshotter=remote registry2:5000/ubuntu:18.04 test /bin/bash root@8dab301bd68d:/# ls bin boot dev etc home lib lib64 media mnt opt proc root run sbin srv sys tmp usr var ``` @@ -81,7 +81,7 @@ In the example showed above, you can pull images from your private repository on ``` # docker login (Enter username and password) -# /tmp/out/ctr image rpull --user : index.docker.io//ubuntu:18.04 +# ctr-remote image rpull --user : index.docker.io//ubuntu:18.04 ``` The `--user` option is just for containerd's side which doesn't recognize `~/.docker/config.json`. We doesn't use credentials specified by this option but uses `~/.docker/config.json` instead. diff --git a/cmd/optimize/main.go b/cmd/ctr-remote/commands/optimize.go similarity index 67% rename from cmd/optimize/main.go rename to cmd/ctr-remote/commands/optimize.go index 0b96b18f5..3bf5f2e76 100644 --- a/cmd/optimize/main.go +++ b/cmd/ctr-remote/commands/optimize.go @@ -14,19 +14,17 @@ limitations under the License. */ -package main +package commands import ( "bytes" "crypto/sha256" "encoding/hex" "encoding/json" - "flag" "fmt" "hash" "io" "io/ioutil" - "log" "os" "strings" "syscall" @@ -41,117 +39,137 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/stream" "github.com/google/go-containerregistry/pkg/v1/types" - "github.com/ktock/remote-snapshotter/cmd/optimize/logger" - "github.com/ktock/remote-snapshotter/cmd/optimize/sampler" - "github.com/ktock/remote-snapshotter/cmd/optimize/sorter" + "github.com/ktock/remote-snapshotter/cmd/ctr-remote/logger" + "github.com/ktock/remote-snapshotter/cmd/ctr-remote/sampler" + "github.com/ktock/remote-snapshotter/cmd/ctr-remote/sorter" imgpkg "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + "github.com/urfave/cli" "golang.org/x/sync/errgroup" ) -var ( - insecure = flag.Bool("insecure", false, "allow HTTP connections to the registry which has the prefix \"http://\"") - noopt = flag.Bool("noopt", false, "only stargzify and do not optimize layers") - period = flag.Int("period", 10, "time period to monitor access log") - username = flag.String("user", "", "user name to override image's default config") - cwd = flag.String("cwd", "", "working dir to override image's default config") - cArgs = flag.String("args", "", "command arguments to override image's default config(in JSON array)") - entrypoint = flag.String("entrypoint", "", "entrypoint to override image's default config(in JSON array)") - terminal = flag.Bool("t", false, "enable terminal for sample container") - cEnvs envs -) - -type envs struct { - list []string -} - -func (e *envs) String() string { - return strings.Join(e.list, ", ") -} - -func (e *envs) Set(value string) error { - e.list = append(e.list, value) - return nil -} - -func main() { - - // Set up logs package to get useful messages i.e. progress. - logs.Warn.SetOutput(os.Stderr) - logs.Progress.SetOutput(os.Stderr) - - // Parse arguments - flag.Var(&cEnvs, "env", "environment valulable to add or override to the image's default config") - flag.Parse() - if flag.NArg() < 2 { - fmt.Printf("usage: %s [OPTION]... INPUT_IMAGE OUTPUT_IMAGE\n", os.Args[0]) - os.Exit(1) - } - args := flag.Args() - src, dst, opts, err := parseArgs(args) - if err != nil { - log.Fatal(err) - } +const defaultPeriod = 10 + +var OptimizeCommand = cli.Command{ + Name: "optimize", + Usage: "optimize an image with user-specified workload", + ArgsUsage: "[flags] ", + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "plain-http", + Usage: "allow HTTP connections to the registry which has the prefix \"http://\"", + }, + cli.BoolFlag{ + Name: "stargz-only", + Usage: "only stargzify and do not optimize layers", + }, + cli.BoolFlag{ + Name: "t", + Usage: "only stargzify and do not optimize layers", + }, + cli.IntFlag{ + Name: "period", + Usage: "time period to monitor access log", + Value: defaultPeriod, + }, + cli.StringFlag{ + Name: "user", + Usage: "user name to override image's default config", + }, + cli.StringFlag{ + Name: "cwd", + Usage: "working dir to override image's default config", + }, + cli.StringFlag{ + Name: "args", + Usage: "command arguments to override image's default config(in JSON array)", + }, + cli.StringFlag{ + Name: "entrypoint", + Usage: "entrypoint to override image's default config(in JSON array)", + }, + cli.StringSliceFlag{ + Name: "env", + Usage: "environment valulable to add or override to the image's default config", + }, + }, + Action: func(context *cli.Context) error { + + // Set up logs package to get useful messages e.g. progress. + logs.Warn.SetOutput(os.Stderr) + logs.Progress.SetOutput(os.Stderr) + + // Parse arguments + var ( + src = context.Args().Get(0) + dst = context.Args().Get(1) + ) + if src == "" || dst == "" { + return fmt.Errorf("source and destination of the target image must be specified") + } + opts, err := parseArgs(context) + if err != nil { + return err + } - // Convert and push image - srcRef, err := parseReference(src) - if err != nil { - log.Fatal(err) - } - dstRef, err := parseReference(dst) - if err != nil { - log.Fatal(err) - } - err = convert(srcRef, dstRef, opts...) - if err != nil { - log.Fatal(err) - } + // Convert and push image + srcRef, err := parseReference(src, context) + if err != nil { + return err + } + dstRef, err := parseReference(dst, context) + if err != nil { + return err + } + err = convert(context, srcRef, dstRef, opts...) + if err != nil { + return err + } + return nil + }, } -func parseArgs(args []string) (src string, dst string, opts []sampler.Option, err error) { - src = args[0] - dst = args[1] - - if len(cEnvs.list) > 0 { - opts = append(opts, sampler.WithEnvs(cEnvs.list)) +func parseArgs(clicontext *cli.Context) (opts []sampler.Option, err error) { + if env := clicontext.StringSlice("env"); len(env) > 0 { + opts = append(opts, sampler.WithEnvs(env)) } - if *cArgs != "" { + if args := clicontext.String("args"); args != "" { var as []string - err = json.Unmarshal([]byte(*cArgs), &as) + err = json.Unmarshal([]byte(args), &as) if err != nil { - return "", "", nil, errors.Wrapf(err, "invalid option \"args\"") + return nil, errors.Wrapf(err, "invalid option \"args\"") } opts = append(opts, sampler.WithArgs(as)) } - if *entrypoint != "" { + if entrypoint := clicontext.String("entrypoint"); entrypoint != "" { var es []string - err = json.Unmarshal([]byte(*entrypoint), &es) + err = json.Unmarshal([]byte(entrypoint), &es) if err != nil { - return "", "", nil, errors.Wrapf(err, "invalid option \"entrypoint\"") + return nil, errors.Wrapf(err, "invalid option \"entrypoint\"") } opts = append(opts, sampler.WithEntrypoint(es)) } - if *username != "" { - opts = append(opts, sampler.WithUser(*username)) + if username := clicontext.String("user"); username != "" { + opts = append(opts, sampler.WithUser(username)) } - if *cwd != "" { - opts = append(opts, sampler.WithWorkingDir(*cwd)) + if cwd := clicontext.String("cwd"); cwd != "" { + opts = append(opts, sampler.WithWorkingDir(cwd)) } - if *terminal { + if clicontext.Bool("t") { opts = append(opts, sampler.WithTerminal()) } return } -func parseReference(path string) (name.Reference, error) { +func parseReference(path string, clicontext *cli.Context) (name.Reference, error) { var opts []name.Option if strings.HasPrefix(path, "http://") { path = strings.TrimPrefix(path, "http://") - if *insecure { + if clicontext.Bool("plain-http") { opts = append(opts, name.Insecure) } else { - return nil, fmt.Errorf("\"-insecure\" option must be specified to connect to %q using HTTP", path) + return nil, fmt.Errorf("\"--plain-http\" option must be specified to connect to %q using HTTP", path) } } ref, err := name.ParseReference(path, opts...) @@ -162,7 +180,7 @@ func parseReference(path string) (name.Reference, error) { return ref, nil } -func convert(srcRef, dstRef name.Reference, runopts ...sampler.Option) error { +func convert(clicontext *cli.Context, srcRef, dstRef name.Reference, runopts ...sampler.Option) error { // Pull source image srcImg, err := remote.Image(srcRef, remote.WithAuthFromKeychain(authn.DefaultKeychain)) if err != nil { @@ -175,7 +193,7 @@ func convert(srcRef, dstRef name.Reference, runopts ...sampler.Option) error { if err != nil { return err } - if !*noopt { + if !clicontext.Bool("stargz-only") { configData, err := srcImg.RawConfigFile() if err != nil { return err @@ -184,7 +202,7 @@ func convert(srcRef, dstRef name.Reference, runopts ...sampler.Option) error { if err := json.Unmarshal(configData, &config); err != nil { return fmt.Errorf("failed to parse image config file: %v", err) } - layers, err = optimize(layers, config, runopts...) + layers, err = optimize(clicontext, layers, config, runopts...) if err != nil { return err } @@ -221,7 +239,7 @@ func convert(srcRef, dstRef name.Reference, runopts ...sampler.Option) error { } // The order of the "in" list must be base layer first, top layer last. -func optimize(in []regpkg.Layer, config imgpkg.Image, opts ...sampler.Option) (out []regpkg.Layer, err error) { +func optimize(clicontext *cli.Context, in []regpkg.Layer, config imgpkg.Image, opts ...sampler.Option) (out []regpkg.Layer, err error) { root := "" mktemp := func() (path string, err error) { if path, err = ioutil.TempDir(root, "optimize"); err != nil { @@ -313,7 +331,7 @@ func optimize(in []regpkg.Layer, config imgpkg.Image, opts ...sampler.Option) (o defer syscall.Unmount(rootfs, syscall.MNT_FORCE) // run workload - if err = sampler.Run(bundle, config, *period, opts...); err != nil { + if err = sampler.Run(bundle, config, clicontext.Int("period"), opts...); err != nil { return nil, errors.Wrap(err, "failed to run the sampler") } diff --git a/cmd/ctr/main.go b/cmd/ctr-remote/commands/rpull.go similarity index 85% rename from cmd/ctr/main.go rename to cmd/ctr-remote/commands/rpull.go index ad2355bad..965519976 100644 --- a/cmd/ctr/main.go +++ b/cmd/ctr-remote/commands/rpull.go @@ -14,20 +14,17 @@ limitations under the License. */ -package main +package commands import ( "context" "fmt" - "os" "github.com/containerd/containerd" - "github.com/containerd/containerd/cmd/ctr/app" "github.com/containerd/containerd/cmd/ctr/commands" "github.com/containerd/containerd/cmd/ctr/commands/content" "github.com/containerd/containerd/images" "github.com/containerd/containerd/log" - "github.com/containerd/containerd/pkg/seed" stargz "github.com/ktock/remote-snapshotter/filesystems/stargz/handler" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/urfave/cli" @@ -37,25 +34,7 @@ const ( remoteSnapshotterName = "remote" ) -func init() { - seed.WithTimeAndRand() -} - -func main() { - app := app.New() - for i := range app.Commands { - if app.Commands[i].Name == "images" { - app.Commands[i].Subcommands = append(app.Commands[i].Subcommands, rpullCommand) - break - } - } - if err := app.Run(os.Args); err != nil { - fmt.Fprintf(os.Stderr, "ctr: %s\n", err) - os.Exit(1) - } -} - -var rpullCommand = cli.Command{ +var RpullCommand = cli.Command{ Name: "rpull", Usage: "pull an image from a remote levaraging remote snapshotter", ArgsUsage: "[flags] ", diff --git a/cmd/optimize/logger/logger.go b/cmd/ctr-remote/logger/logger.go similarity index 99% rename from cmd/optimize/logger/logger.go rename to cmd/ctr-remote/logger/logger.go index 432b05fd5..239329b94 100644 --- a/cmd/optimize/logger/logger.go +++ b/cmd/ctr-remote/logger/logger.go @@ -28,7 +28,7 @@ import ( "github.com/hanwen/go-fuse/fuse" "github.com/hanwen/go-fuse/fuse/nodefs" - "github.com/ktock/remote-snapshotter/cmd/optimize/util" + "github.com/ktock/remote-snapshotter/cmd/ctr-remote/util" "golang.org/x/sys/unix" ) diff --git a/cmd/optimize/logger/logger_test.go b/cmd/ctr-remote/logger/logger_test.go similarity index 100% rename from cmd/optimize/logger/logger_test.go rename to cmd/ctr-remote/logger/logger_test.go diff --git a/cmd/ctr-remote/main.go b/cmd/ctr-remote/main.go new file mode 100644 index 000000000..b7777ad4c --- /dev/null +++ b/cmd/ctr-remote/main.go @@ -0,0 +1,46 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + + "github.com/containerd/containerd/cmd/ctr/app" + "github.com/containerd/containerd/pkg/seed" + "github.com/ktock/remote-snapshotter/cmd/ctr-remote/commands" + "github.com/urfave/cli" +) + +func init() { + seed.WithTimeAndRand() +} + +func main() { + customCommands := []cli.Command{commands.RpullCommand, commands.OptimizeCommand} + app := app.New() + for i := range app.Commands { + if app.Commands[i].Name == "images" { + app.Commands[i].Subcommands = append(app.Commands[i].Subcommands, customCommands...) + break + } + } + if err := app.Run(os.Args); err != nil { + fmt.Fprintf(os.Stderr, "ctr: %s\n", err) + os.Exit(1) + } +} diff --git a/cmd/optimize/sampler/options.go b/cmd/ctr-remote/sampler/options.go similarity index 100% rename from cmd/optimize/sampler/options.go rename to cmd/ctr-remote/sampler/options.go diff --git a/cmd/optimize/sampler/sampler.go b/cmd/ctr-remote/sampler/sampler.go similarity index 100% rename from cmd/optimize/sampler/sampler.go rename to cmd/ctr-remote/sampler/sampler.go diff --git a/cmd/optimize/sorter/sorter.go b/cmd/ctr-remote/sorter/sorter.go similarity index 98% rename from cmd/optimize/sorter/sorter.go rename to cmd/ctr-remote/sorter/sorter.go index 572484e59..15dbe9e94 100644 --- a/cmd/optimize/sorter/sorter.go +++ b/cmd/ctr-remote/sorter/sorter.go @@ -24,7 +24,7 @@ import ( "path" "strings" - "github.com/ktock/remote-snapshotter/cmd/optimize/util" + "github.com/ktock/remote-snapshotter/cmd/ctr-remote/util" ) const ( diff --git a/cmd/optimize/sorter/sorter_test.go b/cmd/ctr-remote/sorter/sorter_test.go similarity index 100% rename from cmd/optimize/sorter/sorter_test.go rename to cmd/ctr-remote/sorter/sorter_test.go diff --git a/cmd/optimize/util/util.go b/cmd/ctr-remote/util/util.go similarity index 100% rename from cmd/optimize/util/util.go rename to cmd/ctr-remote/util/util.go diff --git a/cmd/optimize/util/util_test.go b/cmd/ctr-remote/util/util_test.go similarity index 100% rename from cmd/optimize/util/util_test.go rename to cmd/ctr-remote/util/util_test.go diff --git a/overview.md b/overview.md index 3ca1d585b..2174f9801 100644 --- a/overview.md +++ b/overview.md @@ -17,7 +17,7 @@ By default, Remote Snapshotter supports standard compatible remote snapshots fun The image format that achieves it is _stargz_ by [CRFS](https://github.com/google/crfs). Stargz format is backwards-compatible to container standards so you can push stargz-formatted images to container registries and run them using container runtimes including Docker. When you run a container image and it is formatted as stargz image, Remote Snapshotter automatically prepares container's rootfs layers as remote snapshots. -As an image converter command, you can use CRFS-official `stargzify` or our `optimize` which has additional optimization functionality. +As an image converter command, you can use CRFS-official `stargzify` or our `ctr-remote` which has additional optimization functionality. Furthermore, Remote Snapshotter has a pluggable architecture so you can use any image formats and filesystems you want not only stargz. @@ -41,12 +41,12 @@ You can also use `~/.docker/config.json`-based authentication for your private r Because file contents are fetched via NW on each access, read performance would be one of the major concerns. To mitigate it, Remote Snapshotter provides additional workload-oriented optimization. -When you convert an image to stargz format using `optimize`, you can specify some options which describe your workload (i.e. entrypoint commands, environment variables, etc.). +When you convert an image to stargz format using `ctr-remote`, you can specify some options which describe your workload (i.e. entrypoint commands, environment variables, etc.). For example, we can optimize `ubuntu:18.04` image for execution of `ls` command on `bash` as following, ``` -# optimize -insecure -entrypoint='[ "/bin/bash", "-c" ]' -args='[ "ls" ]' \ - ubuntu:18.04 http://registry2:5000/ubuntu:18.04 +# ctr-remote image optimize --plain-http --entrypoint='[ "/bin/bash", "-c" ]' --args='[ "ls" ]' \ + ubuntu:18.04 http://registry2:5000/ubuntu:18.04 ``` Then it runs the workload in an isolated environment, monitor all file events, and marks accessed files, which are also likely accessed in your production environment too). diff --git a/script/integration/containerd/entrypoint.sh b/script/integration/containerd/entrypoint.sh index f5fa8cf4f..4e6f861b2 100755 --- a/script/integration/containerd/entrypoint.sh +++ b/script/integration/containerd/entrypoint.sh @@ -18,7 +18,6 @@ PLUGINS=(remote) REGISTRY_HOST=registry_integration DUMMYUSER=dummyuser DUMMYPASS=dummypass -CTR_PREFIX=/tmp/out/ RETRYNUM=100 RETRYINTERVAL=1 @@ -97,14 +96,12 @@ check "Login to the registry" # Prepare images gcrane cp ubuntu:18.04 "${REGISTRY_HOST}:5000/ubuntu:18.04" -GO111MODULE=off PREFIX=/tmp/out/ make clean && \ - GO111MODULE=off PREFIX=/tmp/out/ make optimize && \ - /tmp/out/optimize -noopt "${REGISTRY_HOST}:5000/ubuntu:18.04" "${REGISTRY_HOST}:5000/ubuntu:stargz" +GO111MODULE=off PREFIX=/tmp/ctr/ make clean && \ + GO111MODULE=off PREFIX=/tmp/ctr/ make ctr-remote && \ + install /tmp/ctr/ctr-remote /usr/local/bin && \ + ctr-remote image optimize --stargz-only "${REGISTRY_HOST}:5000/ubuntu:18.04" "${REGISTRY_HOST}:5000/ubuntu:stargz" check "Stargzifying images" -# Make customized ctr -PREFIX="${CTR_PREFIX}" make ctr -j2 - # Wait for booting remote snapshotter RETRYNUM=600 retry ls /run/rsnapshotd/rsnapshotd.sock mkdir -p /etc/containerd && \ @@ -115,7 +112,7 @@ mkdir -p /etc/containerd && \ reboot_containerd --log-level debug --config=/etc/containerd/config.toml NOTFOUND=false for PLUGIN in ${PLUGINS[@]}; do - OK=$("${CTR_PREFIX}ctr" plugins ls \ + OK=$(ctr-remote plugins ls \ | grep io.containerd.snapshotter \ | sed -E 's/ +/ /g' \ | cut -d ' ' -f 2,4 \ @@ -134,31 +131,31 @@ fi ############ # Tests for stargz filesystem reboot_containerd --log-level debug --config=/etc/containerd/config.toml -"${CTR_PREFIX}ctr" images pull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}:5000/ubuntu:18.04" +ctr-remote images pull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}:5000/ubuntu:18.04" check "Getting normal image with normal snapshotter" -"${CTR_PREFIX}ctr" run --rm "${REGISTRY_HOST}:5000/ubuntu:18.04" test tar -c /usr > /usr_normal_unstargz.tar +ctr-remote run --rm "${REGISTRY_HOST}:5000/ubuntu:18.04" test tar -c /usr > /usr_normal_unstargz.tar reboot_containerd --log-level debug --config=/etc/containerd/config.toml -"${CTR_PREFIX}ctr" images rpull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}:5000/ubuntu:18.04" +ctr-remote images rpull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}:5000/ubuntu:18.04" check "Getting normal image with remote snapshotter" -"${CTR_PREFIX}ctr" run --rm --snapshotter=remote "${REGISTRY_HOST}:5000/ubuntu:18.04" test tar -c /usr > /usr_remote_unstargz.tar +ctr-remote run --rm --snapshotter=remote "${REGISTRY_HOST}:5000/ubuntu:18.04" test tar -c /usr > /usr_remote_unstargz.tar reboot_containerd --log-level debug --config=/etc/containerd/config.toml -"${CTR_PREFIX}ctr" images pull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}:5000/ubuntu:stargz" +ctr-remote images pull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}:5000/ubuntu:stargz" check "Getting stargz image with normal snapshotter" -"${CTR_PREFIX}ctr" run --rm "${REGISTRY_HOST}:5000/ubuntu:stargz" test tar -c /usr > /usr_normal_stargz.tar +ctr-remote run --rm "${REGISTRY_HOST}:5000/ubuntu:stargz" test tar -c /usr > /usr_normal_stargz.tar PULL_LOG=$(mktemp) check "Preparing log file" reboot_containerd --log-level debug --config=/etc/containerd/config.toml -"${CTR_PREFIX}ctr" images rpull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}:5000/ubuntu:stargz" | tee "${PULL_LOG}" +ctr-remote images rpull --user "${DUMMYUSER}:${DUMMYPASS}" "${REGISTRY_HOST}:5000/ubuntu:stargz" | tee "${PULL_LOG}" check "Getting stargz image with remote snapshotter" if ! isServedAsRemoteSnapshot "${PULL_LOG}" ; then echo "Failed to serve all layers as remote snapshots: ${LAYER_LOG}" exit 1 fi rm "${PULL_LOG}" -"${CTR_PREFIX}ctr" run --rm --snapshotter=remote "${REGISTRY_HOST}:5000/ubuntu:stargz" test tar -c /usr > /usr_remote_stargz.tar +ctr-remote run --rm --snapshotter=remote "${REGISTRY_HOST}:5000/ubuntu:stargz" test tar -c /usr > /usr_remote_stargz.tar mkdir /usr_normal_unstargz /usr_remote_unstargz /usr_normal_stargz /usr_remote_stargz tar -xf /usr_normal_unstargz.tar -C /usr_normal_unstargz diff --git a/script/optimize/optimize/entrypoint.sh b/script/optimize/optimize/entrypoint.sh index 6ea228342..ac547c438 100755 --- a/script/optimize/optimize/entrypoint.sh +++ b/script/optimize/optimize/entrypoint.sh @@ -115,8 +115,8 @@ check "Build and push original image" WORKING_DIR=$(mktemp -d) check "Prepare tempdir for workspace" GO111MODULE=off PREFIX=/tmp/out/ make clean && \ - GO111MODULE=off PREFIX=/tmp/out/ make optimize && \ - /tmp/out/optimize -entrypoint='[ "/accessor" ]' "${ORG_IMAGE_TAG}" "${OPT_IMAGE_TAG}" + GO111MODULE=off PREFIX=/tmp/out/ make ctr-remote && \ + /tmp/out/ctr-remote image optimize -entrypoint='[ "/accessor" ]' "${ORG_IMAGE_TAG}" "${OPT_IMAGE_TAG}" check "Optimize original image" # Validate optimized image diff --git a/vendor.conf b/vendor.conf index cfef40f86..a4b4db303 100644 --- a/vendor.conf +++ b/vendor.conf @@ -14,7 +14,7 @@ github.com/google/go-containerregistry 2ce3ea99b462cbf187a16bb5d0fb1d5820694538 github.com/hanwen/go-fuse v1.0.0 golang.org/x/sys c990c680b611ac1aeb7d8f2af94a825f98d69720 https://github.com/golang/sys -# Depenedencies for optimize +# Depenedencies for optimize subcommand github.com/containerd/go-runc a5c2862aed5e6358b305b0e16bfce58e0549b1cd github.com/moby/buildkit v0.6.3 github.com/opencontainers/runtime-spec 29686dbc5559d93fb1ef402eeda3e35c38d75af4 # v1.0.1-59-g29686db