From f01c6a02a84e20c3f17df41796bd59d072d6e2f5 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Thu, 16 Jun 2016 22:50:44 -0700 Subject: [PATCH 01/13] image/cas: Add a generic CAS interface And implement that interface for tarballs based on the specs image-layout. I plan on adding other backends later, but this is enough for a proof of concept. Also add a new 'cas get' subcommand to oci-image-tool so folks can access the new read functionality from the command line. In a subsequent commit, I'll replace oci-image-tools walker functionality with this new API. The Context interface follows the pattern recommended in [1], allowing callers to cancel long running actions (e.g. push/pull over the network for engine implementations that communicate with a remote store). [1]: https://blog.golang.org/context Signed-off-by: W. Trevor King --- cmd/oci-image-tool/cas.go | 33 +++++++++++ cmd/oci-image-tool/cas_get.go | 100 ++++++++++++++++++++++++++++++++++ cmd/oci-image-tool/main.go | 1 + image/cas/interface.go | 43 +++++++++++++++ image/cas/layout/interface.go | 25 +++++++++ image/cas/layout/main.go | 36 ++++++++++++ image/cas/layout/tar.go | 96 ++++++++++++++++++++++++++++++++ 7 files changed, 334 insertions(+) create mode 100644 cmd/oci-image-tool/cas.go create mode 100644 cmd/oci-image-tool/cas_get.go create mode 100644 image/cas/interface.go create mode 100644 image/cas/layout/interface.go create mode 100644 image/cas/layout/main.go create mode 100644 image/cas/layout/tar.go diff --git a/cmd/oci-image-tool/cas.go b/cmd/oci-image-tool/cas.go new file mode 100644 index 000000000..1411d4c61 --- /dev/null +++ b/cmd/oci-image-tool/cas.go @@ -0,0 +1,33 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "io" + "log" + + "github.com/spf13/cobra" +) + +func newCASCmd(stdout io.Writer, stderr *log.Logger) *cobra.Command { + cmd := &cobra.Command{ + Use: "cas", + Short: "Content-addressable storage manipulation", + } + + cmd.AddCommand(newCASGetCmd(stdout, stderr)) + + return cmd +} diff --git a/cmd/oci-image-tool/cas_get.go b/cmd/oci-image-tool/cas_get.go new file mode 100644 index 000000000..1599993d8 --- /dev/null +++ b/cmd/oci-image-tool/cas_get.go @@ -0,0 +1,100 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "io" + "io/ioutil" + "log" + "os" + + "github.com/opencontainers/image-spec/image/cas/layout" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type casGetCmd struct { + stdout io.Writer + stderr *log.Logger + path string + digest string +} + +func newCASGetCmd(stdout io.Writer, stderr *log.Logger) *cobra.Command { + state := &casGetCmd{ + stdout: stdout, + stderr: stderr, + } + + return &cobra.Command{ + Use: "get PATH DIGEST", + Short: "Retrieve a blob from the store", + Long: "Retrieve a blob from the store and write it to stdout.", + Run: state.Run, + } +} + +func (state *casGetCmd) Run(cmd *cobra.Command, args []string) { + if len(args) != 2 { + state.stderr.Print("both PATH and DIGEST must be provided") + if err := cmd.Usage(); err != nil { + state.stderr.Println(err) + } + os.Exit(1) + } + + state.path = args[0] + state.digest = args[1] + + err := state.run() + if err != nil { + state.stderr.Println(err) + os.Exit(1) + } + + os.Exit(0) +} + +func (state *casGetCmd) run() (err error) { + ctx := context.Background() + + engine, err := layout.NewEngine(state.path) + if err != nil { + return err + } + defer engine.Close() + + reader, err := engine.Get(ctx, state.digest) + if err != nil { + return err + } + defer reader.Close() + + bytes, err := ioutil.ReadAll(reader) + if err != nil { + return err + } + + n, err := state.stdout.Write(bytes) + if err != nil { + return err + } + if n < len(bytes) { + return fmt.Errorf("wrote %d of %d bytes", n, len(bytes)) + } + + return nil +} diff --git a/cmd/oci-image-tool/main.go b/cmd/oci-image-tool/main.go index 7cd350e88..d1a873e65 100644 --- a/cmd/oci-image-tool/main.go +++ b/cmd/oci-image-tool/main.go @@ -33,6 +33,7 @@ func main() { cmd.AddCommand(newValidateCmd(stdout, stderr)) cmd.AddCommand(newUnpackCmd(stdout, stderr)) cmd.AddCommand(newBundleCmd(stdout, stderr)) + cmd.AddCommand(newCASCmd(os.Stdout, stderr)) if err := cmd.Execute(); err != nil { stderr.Println(err) diff --git a/image/cas/interface.go b/image/cas/interface.go new file mode 100644 index 000000000..4070d7a23 --- /dev/null +++ b/image/cas/interface.go @@ -0,0 +1,43 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package cas implements generic content-addressable storage. +package cas + +import ( + "io" + + "golang.org/x/net/context" +) + +// Engine represents a content-addressable storage engine. +type Engine interface { + + // Put adds a new blob to the store. The action is idempotent; a + // nil return means "that content is stored at DIGEST" without + // implying "because of your Put()". + Put(ctx context.Context, reader io.Reader) (digest string, err error) + + // Get returns a reader for retrieving a blob from the store. + // Returns os.ErrNotExist if the digest is not found. + Get(ctx context.Context, digest string) (reader io.ReadCloser, err error) + + // Delete removes a blob from the store. Returns os.ErrNotExist if + // the digest is not found. + Delete(ctx context.Context, digest string) (err error) + + // Close releases resources held by the engine. Subsequent engine + // method calls will fail. + Close() (err error) +} diff --git a/image/cas/layout/interface.go b/image/cas/layout/interface.go new file mode 100644 index 000000000..dda72d6bd --- /dev/null +++ b/image/cas/layout/interface.go @@ -0,0 +1,25 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "io" +) + +// ReadWriteSeekCloser wraps the Read, Write, Seek, and Close methods. +type ReadWriteSeekCloser interface { + io.ReadWriteSeeker + io.Closer +} diff --git a/image/cas/layout/main.go b/image/cas/layout/main.go new file mode 100644 index 000000000..56adfffa6 --- /dev/null +++ b/image/cas/layout/main.go @@ -0,0 +1,36 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package layout implements the cas interface using the image-spec's +// image-layout [1]. +// +// [1]: https://github.com/opencontainers/image-spec/blob/master/image-layout.md +package layout + +import ( + "os" + + "github.com/opencontainers/image-spec/image/cas" +) + +// NewEngine instantiates an engine with the appropriate backend (tar, +// HTTP, ...). +func NewEngine(path string) (engine cas.Engine, err error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + + return NewTarEngine(file) +} diff --git a/image/cas/layout/tar.go b/image/cas/layout/tar.go new file mode 100644 index 000000000..62a3d9360 --- /dev/null +++ b/image/cas/layout/tar.go @@ -0,0 +1,96 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "archive/tar" + "errors" + "fmt" + "io" + "io/ioutil" + "os" + "strings" + + "github.com/opencontainers/image-spec/image/cas" + "golang.org/x/net/context" +) + +// TarEngine is a cas.Engine backed by a tar file. +type TarEngine struct { + file ReadWriteSeekCloser +} + +// NewTarEngine returns a new TarEngine. +func NewTarEngine(file ReadWriteSeekCloser) (engine cas.Engine, err error) { + engine = &TarEngine{ + file: file, + } + + return engine, nil +} + +// Put adds a new blob to the store. +func (engine *TarEngine) Put(ctx context.Context, reader io.Reader) (digest string, err error) { + // FIXME + return "", errors.New("TarEngine.Put is not supported yet") +} + +// Get returns a reader for retrieving a blob from the store. +func (engine *TarEngine) Get(ctx context.Context, digest string) (reader io.ReadCloser, err error) { + fields := strings.SplitN(digest, ":", 2) + if len(fields) != 2 { + return nil, fmt.Errorf("invalid digest: %q, %v", digest, fields) + } + algorithm := fields[0] + hash := fields[1] + + targetName := fmt.Sprintf("./blobs/%s/%s", algorithm, hash) + + _, err = engine.file.Seek(0, os.SEEK_SET) + if err != nil { + return nil, err + } + + tarReader := tar.NewReader(engine.file) + for { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + header, err := tarReader.Next() + if err == io.EOF { + return nil, os.ErrNotExist + } else if err != nil { + return nil, err + } + + if header.Name == targetName { + return ioutil.NopCloser(tarReader), nil + } + } +} + +// Delete removes a blob from the store. +func (engine *TarEngine) Delete(ctx context.Context, digest string) (err error) { + // FIXME + return errors.New("TarEngine.Delete is not supported yet") +} + +// Close releases resources held by the engine. +func (engine *TarEngine) Close() (err error) { + return engine.file.Close() +} From 2d555835b84d5cf7deaa9b108efac43e683a372e Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 17 Jun 2016 00:38:03 -0700 Subject: [PATCH 02/13] image/refs: Add a generic name-based reference interface And implement that interface for tarballs based on the specs image-layout. I plan on adding other backends later, but this is enough for a proof of concept. Also add new 'refs list' and 'refs get' subcommands to oci-image-tool so folks can access the new read functionality from the command line. The Engine.List interface uses a callback instead of returning channels or a slice. Benefits vs. returning a slice of names: * There's no need to allocate a slice for the results, so calls with large (or negative) 'size' values can be made without consuming large amounts of memory. * The name collection and processing can happen concurrently, so: * We don't waste cycles collecting names we won't use. * Slow collection can happen in the background if/when the consumer is blocked on something else. The benefit of using callbacks vs. returning name and error channels (as discussed in [1]) is more of a trade-off. Stephen Day [2] and JT Olds [3] don't like channel's internal locks. Dave Cheney doesn't have a problem with them [4]. Which approach is more efficient for a given situation depends on how expensive it is for the engine to find the next key and how expensive it is to act on a returned name. If both are expensive, you want goroutines in there somewhere to get concurrent execution, and channels will help those goroutines communicate. When either action is fast (or both are fast), channels are unnecessary overhead. By using a callback in the interface, we avoid baking in the overhead. Folks who want concurrent execution can initialize their own channel, launch List in a goroutine, and use the callback to inject names into their channel. In a subsequent commit, I'll replace oci-image-tools walker functionality with this new API. I'd prefer casLayout for the imported package, but Stephen doesn't want camelCase for package names [5]. [1]: https://blog.golang.org/pipelines [2]: https://github.com/opencontainers/image-spec/pull/159#discussion_r76874690 [3]: http://www.jtolds.com/writing/2016/03/go-channels-are-bad-and-you-should-feel-bad/ [4]: https://groups.google.com/d/msg/golang-nuts/LM648yrPpck/idyupwodAwAJ Subject: Re: [go-nuts] Re: "Go channels are bad and you should feel bad" Date: Wed, 2 Mar 2016 16:04:13 -0800 (PST) Message-Id: [5]: https://github.com/opencontainers/image-spec/pull/159#discussion_r76720225 Signed-off-by: W. Trevor King --- cmd/oci-image-tool/main.go | 1 + cmd/oci-image-tool/refs.go | 34 ++++++++ cmd/oci-image-tool/refs_get.go | 84 +++++++++++++++++++ cmd/oci-image-tool/refs_list.go | 89 ++++++++++++++++++++ image/refs/interface.go | 78 ++++++++++++++++++ image/refs/layout/main.go | 36 +++++++++ image/refs/layout/tar.go | 139 ++++++++++++++++++++++++++++++++ 7 files changed, 461 insertions(+) create mode 100644 cmd/oci-image-tool/refs.go create mode 100644 cmd/oci-image-tool/refs_get.go create mode 100644 cmd/oci-image-tool/refs_list.go create mode 100644 image/refs/interface.go create mode 100644 image/refs/layout/main.go create mode 100644 image/refs/layout/tar.go diff --git a/cmd/oci-image-tool/main.go b/cmd/oci-image-tool/main.go index d1a873e65..fcd4c7aef 100644 --- a/cmd/oci-image-tool/main.go +++ b/cmd/oci-image-tool/main.go @@ -33,6 +33,7 @@ func main() { cmd.AddCommand(newValidateCmd(stdout, stderr)) cmd.AddCommand(newUnpackCmd(stdout, stderr)) cmd.AddCommand(newBundleCmd(stdout, stderr)) + cmd.AddCommand(newRefsCmd(os.Stdout, stderr)) cmd.AddCommand(newCASCmd(os.Stdout, stderr)) if err := cmd.Execute(); err != nil { diff --git a/cmd/oci-image-tool/refs.go b/cmd/oci-image-tool/refs.go new file mode 100644 index 000000000..5786d1d63 --- /dev/null +++ b/cmd/oci-image-tool/refs.go @@ -0,0 +1,34 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "io" + "log" + + "github.com/spf13/cobra" +) + +func newRefsCmd(stdout io.Writer, stderr *log.Logger) *cobra.Command { + cmd := &cobra.Command{ + Use: "refs", + Short: "Name-based reference manipulation", + } + + cmd.AddCommand(newRefsGetCmd(stdout, stderr)) + cmd.AddCommand(newRefsListCmd(stdout, stderr)) + + return cmd +} diff --git a/cmd/oci-image-tool/refs_get.go b/cmd/oci-image-tool/refs_get.go new file mode 100644 index 000000000..0f9582fc5 --- /dev/null +++ b/cmd/oci-image-tool/refs_get.go @@ -0,0 +1,84 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "io" + "log" + "os" + + "github.com/opencontainers/image-spec/image/refs/layout" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type refsGetCmd struct { + stdout io.Writer + stderr *log.Logger + path string + name string +} + +func newRefsGetCmd(stdout io.Writer, stderr *log.Logger) *cobra.Command { + state := &refsGetCmd{ + stdout: stdout, + stderr: stderr, + } + + return &cobra.Command{ + Use: "get PATH NAME", + Short: "Retrieve a reference from the store", + Run: state.Run, + } +} + +func (state *refsGetCmd) Run(cmd *cobra.Command, args []string) { + if len(args) != 2 { + state.stderr.Print("both PATH and NAME must be provided") + if err := cmd.Usage(); err != nil { + state.stderr.Println(err) + } + os.Exit(1) + } + + state.path = args[0] + state.name = args[1] + + err := state.run() + if err != nil { + state.stderr.Println(err) + os.Exit(1) + } + + os.Exit(0) +} + +func (state *refsGetCmd) run() (err error) { + ctx := context.Background() + + engine, err := layout.NewEngine(state.path) + if err != nil { + return err + } + defer engine.Close() + + descriptor, err := engine.Get(ctx, state.name) + if err != nil { + return err + } + + return json.NewEncoder(state.stdout).Encode(&descriptor) +} diff --git a/cmd/oci-image-tool/refs_list.go b/cmd/oci-image-tool/refs_list.go new file mode 100644 index 000000000..ef5bf9604 --- /dev/null +++ b/cmd/oci-image-tool/refs_list.go @@ -0,0 +1,89 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "io" + "log" + "os" + + "github.com/opencontainers/image-spec/image/refs/layout" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type refsListCmd struct { + stdout io.Writer + stderr *log.Logger + path string +} + +func newRefsListCmd(stdout io.Writer, stderr *log.Logger) *cobra.Command { + state := &refsListCmd{ + stdout: stdout, + stderr: stderr, + } + + return &cobra.Command{ + Use: "list PATH", + Short: "Return available names from the store.", + Run: state.Run, + } +} + +func (state *refsListCmd) Run(cmd *cobra.Command, args []string) { + if len(args) != 1 { + state.stderr.Print("PATH must be provided") + if err := cmd.Usage(); err != nil { + state.stderr.Println(err) + } + os.Exit(1) + } + + state.path = args[0] + + err := state.run() + if err != nil { + state.stderr.Println(err) + os.Exit(1) + } + + os.Exit(0) +} + +func (state *refsListCmd) run() (err error) { + ctx := context.Background() + + engine, err := layout.NewEngine(state.path) + if err != nil { + return err + } + defer engine.Close() + + return engine.List(ctx, "", -1, 0, state.printName) +} + +func (state *refsListCmd) printName(ctx context.Context, name string) (err error) { + n, err := io.WriteString(state.stdout, fmt.Sprintf("%s\n", name)) + if err != nil { + return err + } + if n < len(name)+1 { + err = fmt.Errorf("wrote %d of %d characters", n, len(name)+1) + return err + } + return nil +} diff --git a/image/refs/interface.go b/image/refs/interface.go new file mode 100644 index 000000000..e71161582 --- /dev/null +++ b/image/refs/interface.go @@ -0,0 +1,78 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package refs implements generic name-based reference access. +package refs + +import ( + "github.com/opencontainers/image-spec/specs-go" + "golang.org/x/net/context" +) + +// ListNameCallback templates an Engine.List callback used for +// processing names. See Engine.List for more details. +type ListNameCallback func(ctx context.Context, name string) (err error) + +// Engine represents a name-based reference storage engine. +type Engine interface { + + // Put adds a new reference to the store. The action is idempotent; + // a nil return means "that descriptor is stored at NAME" without + // implying "because of your Put()". + Put(ctx context.Context, name string, descriptor *specs.Descriptor) (err error) + + // Get returns a reference from the store. Returns os.ErrNotExist + // if the name is not found. + Get(ctx context.Context, name string) (descriptor *specs.Descriptor, err error) + + // List returns available names from the store. + // + // Results are sorted alphabetically. + // + // Arguments: + // + // * ctx: gives callers a way to gracefully cancel a long-running + // list. + // * prefix: limits the result set to names starting with that + // value. + // * size: limits the length of the result set to the first 'size' + // matches. A value of -1 means "all results". + // * from: shifts the result set to start from the 'from'th match. + // * nameCallback: called for every matching name. List returns any + // errors returned by nameCallback and aborts further listing. + // + // For example, a store with names like: + // + // * 123 + // * abcd + // * abce + // * abcf + // * abcg + // + // will have the following call/result pairs: + // + // * List(ctx, "", -1, 0, printName) -> "123", "abcd", "abce", "abcf", "abcg" + // * List(ctx, "", 2, 0, printName) -> "123", "abcd" + // * List(ctx, "", 2, 1, printName) -> "abcd", "abce" + // * List(ctx,"abc", 2, 1, printName) -> "abce", "abcf" + List(ctx context.Context, prefix string, size int, from int, nameCallback ListNameCallback) (err error) + + // Delete removes a reference from the store. Returns + // os.ErrNotExist if the name is not found. + Delete(ctx context.Context, name string) (err error) + + // Close releases resources held by the engine. Subsequent engine + // method calls will fail. + Close() (err error) +} diff --git a/image/refs/layout/main.go b/image/refs/layout/main.go new file mode 100644 index 000000000..0c3568d8c --- /dev/null +++ b/image/refs/layout/main.go @@ -0,0 +1,36 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package layout implements the refs interface using the image-spec's +// image-layout [1]. +// +// [1]: https://github.com/opencontainers/image-spec/blob/master/image-layout.md +package layout + +import ( + "os" + + "github.com/opencontainers/image-spec/image/refs" +) + +// NewEngine instantiates an engine with the appropriate backend (tar, +// HTTP, ...). +func NewEngine(path string) (engine refs.Engine, err error) { + file, err := os.Open(path) + if err != nil { + return nil, err + } + + return NewTarEngine(file) +} diff --git a/image/refs/layout/tar.go b/image/refs/layout/tar.go new file mode 100644 index 000000000..0a5db5f10 --- /dev/null +++ b/image/refs/layout/tar.go @@ -0,0 +1,139 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "archive/tar" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + + caslayout "github.com/opencontainers/image-spec/image/cas/layout" + "github.com/opencontainers/image-spec/image/refs" + "github.com/opencontainers/image-spec/specs-go" + "golang.org/x/net/context" +) + +// TarEngine is a refs.Engine backed by a tar file. +type TarEngine struct { + file caslayout.ReadWriteSeekCloser +} + +// NewTarEngine returns a new TarEngine. +func NewTarEngine(file caslayout.ReadSeekCloser) (engine refs.Engine, err error) { + engine = &TarEngine{ + file: file, + } + + return engine, nil +} + +// Put adds a new reference to the store. +func (engine *TarEngine) Put(ctx context.Context, name string, descriptor *specs.Descriptor) (err error) { + // FIXME + return errors.New("TarEngine.Put is not supported yet") +} + +// Get returns a reference from the store. +func (engine *TarEngine) Get(ctx context.Context, name string) (descriptor *specs.Descriptor, err error) { + targetName := fmt.Sprintf("./refs/%s", name) + + _, err = engine.file.Seek(0, os.SEEK_SET) + if err != nil { + return nil, err + } + + tarReader := tar.NewReader(engine.file) + for { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + header, err := tarReader.Next() + if err == io.EOF { + return nil, os.ErrNotExist + } else if err != nil { + return nil, err + } + + if header.Name == targetName { + decoder := json.NewDecoder(tarReader) + var desc specs.Descriptor + err = decoder.Decode(&desc) + if err != nil { + return nil, err + } + return &desc, nil + } + } +} + +// List returns available names from the store. +func (engine *TarEngine) List(ctx context.Context, prefix string, size int, from int, nameCallback refs.ListNameCallback) (err error) { + var i = 0 + + _, err = engine.file.Seek(0, os.SEEK_SET) + if err != nil { + return nil + } + + pathPrefix := fmt.Sprintf("./refs/%s", prefix) + + tarReader := tar.NewReader(engine.file) + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + var header *tar.Header + header, err = tarReader.Next() + if err == io.EOF { + return nil + } else if err != nil { + return err + } + + if strings.HasPrefix(header.Name, pathPrefix) && len(header.Name) > 7 { + i++ + if i > from { + err = nameCallback(ctx, header.Name[7:]) + if err != nil { + return err + } + if i-from == size { + return nil + } + } + } + } +} + +// Delete removes a reference from the store. +func (engine *TarEngine) Delete(ctx context.Context, name string) (err error) { + // FIXME + return errors.New("TarEngine.Delete is not supported yet") +} + +// Close releases resources held by the engine. +func (engine *TarEngine) Close() (err error) { + return engine.file.Close() +} From f2dc0e2d92eb0b39adff14601d84606d894bed16 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 17 Jun 2016 07:04:08 -0700 Subject: [PATCH 03/13] image: Refactor to use cas/ref engines instead of walkers The validation/unpacking code doesn't really care what the reference and CAS implemenations are. And the new generic interfaces in image/refs and image/cas will scale better as we add new backends than the walker interface. The old tar/directory distinction between image and imageLayout is gone. The new CAS/refs engines don't support directory backends yet (I plan on adding them once the engine framework lands), but the new framework will handle tar/directory/... detection inside layout.NewEngine (and possibly inside a new (cas|refs).NewEngine when we grow engine types that aren't based on image-layout). I'd prefer casLayout and refsLayout for the imported packages, but Stephen doesn't want camelCase for package names [1]. [1]: https://github.com/opencontainers/image-spec/pull/159#discussion_r76720225 Signed-off-by: W. Trevor King --- cmd/oci-image-tool/autodetect.go | 3 +- cmd/oci-image-tool/create_runtime_bundle.go | 9 +- .../oci-image-tool-create-runtime-bundle.1.md | 2 +- .../man/oci-image-tool-unpack.1.md | 2 +- .../man/oci-image-tool-validate.1.md | 6 +- cmd/oci-image-tool/unpack.go | 13 +- cmd/oci-image-tool/validate.go | 14 +- image/config.go | 60 ++++----- image/descriptor.go | 88 ++----------- image/image.go | 123 +++++++++--------- image/manifest.go | 92 +++++-------- image/walker.go | 112 ---------------- 12 files changed, 157 insertions(+), 367 deletions(-) delete mode 100644 image/walker.go diff --git a/cmd/oci-image-tool/autodetect.go b/cmd/oci-image-tool/autodetect.go index 094e7b9ca..d71b657d0 100644 --- a/cmd/oci-image-tool/autodetect.go +++ b/cmd/oci-image-tool/autodetect.go @@ -27,7 +27,6 @@ import ( // supported autodetection types const ( - typeImageLayout = "imageLayout" typeImage = "image" typeManifest = "manifest" typeManifestList = "manifestList" @@ -43,7 +42,7 @@ func autodetect(path string) (string, error) { } if fi.IsDir() { - return typeImageLayout, nil + return typeImage, nil } f, err := os.Open(path) diff --git a/cmd/oci-image-tool/create_runtime_bundle.go b/cmd/oci-image-tool/create_runtime_bundle.go index 57b6745bf..ec07c0acf 100644 --- a/cmd/oci-image-tool/create_runtime_bundle.go +++ b/cmd/oci-image-tool/create_runtime_bundle.go @@ -22,11 +22,11 @@ import ( "github.com/opencontainers/image-spec/image" "github.com/spf13/cobra" + "golang.org/x/net/context" ) // supported bundle types var bundleTypes = []string{ - typeImageLayout, typeImage, } @@ -82,6 +82,8 @@ func (v *bundleCmd) Run(cmd *cobra.Command, args []string) { os.Exit(1) } + ctx := context.Background() + if _, err := os.Stat(args[1]); os.IsNotExist(err) { v.stderr.Printf("destination path %s does not exist", args[1]) os.Exit(1) @@ -98,11 +100,8 @@ func (v *bundleCmd) Run(cmd *cobra.Command, args []string) { var err error switch v.typ { - case typeImageLayout: - err = image.CreateRuntimeBundleLayout(args[0], args[1], v.ref, v.root) - case typeImage: - err = image.CreateRuntimeBundle(args[0], args[1], v.ref, v.root) + err = image.CreateRuntimeBundle(ctx, args[0], args[1], v.ref, v.root) } if err != nil { diff --git a/cmd/oci-image-tool/man/oci-image-tool-create-runtime-bundle.1.md b/cmd/oci-image-tool/man/oci-image-tool-create-runtime-bundle.1.md index d033bfb06..41f40751b 100644 --- a/cmd/oci-image-tool/man/oci-image-tool-create-runtime-bundle.1.md +++ b/cmd/oci-image-tool/man/oci-image-tool-create-runtime-bundle.1.md @@ -22,7 +22,7 @@ oci-image-tool-create-runtime-bundle \- Create an OCI image runtime bundle A directory representing the root filesystem of the container in the OCI runtime bundle. It is strongly recommended to keep the default value. (default "rootfs") **--type** - Type of the file to unpack. If unset, oci-image-tool will try to auto-detect the type. One of "imageLayout,image" + Type of the file to unpack. If unset, oci-image-tool will try to auto-detect the type. One of "image" # EXAMPLES ``` diff --git a/cmd/oci-image-tool/man/oci-image-tool-unpack.1.md b/cmd/oci-image-tool/man/oci-image-tool-unpack.1.md index b73e21829..ac6134ebd 100644 --- a/cmd/oci-image-tool/man/oci-image-tool-unpack.1.md +++ b/cmd/oci-image-tool/man/oci-image-tool-unpack.1.md @@ -19,7 +19,7 @@ oci-image-tool-unpack \- Unpack an image or image source layout The ref pointing to the manifest to be unpacked. This must be present in the "refs" subdirectory of the image. (default "v1.0") **--type** - Type of the file to unpack. If unset, oci-image-tool will try to auto-detect the type. One of "imageLayout,image" + Type of the file to unpack. If unset, oci-image-tool will try to auto-detect the type. One of "image" # EXAMPLES ``` diff --git a/cmd/oci-image-tool/man/oci-image-tool-validate.1.md b/cmd/oci-image-tool/man/oci-image-tool-validate.1.md index 117ef9b2d..832cefbd5 100644 --- a/cmd/oci-image-tool/man/oci-image-tool-validate.1.md +++ b/cmd/oci-image-tool/man/oci-image-tool-validate.1.md @@ -16,15 +16,15 @@ oci-image-tool-validate \- Validate one or more image files Print usage statement **--ref** - The ref pointing to the manifest to be validated. This must be present in the "refs" subdirectory of the image. Only applicable if type is image or imageLayout. (default "v1.0") + The ref pointing to the manifest to be validated. This must be present in the "refs" subdirectory of the image. Only applicable if type is image. (default "v1.0") **--type** - Type of the file to validate. If unset, oci-image-tool will try to auto-detect the type. One of "imageLayout,image,manifest,manifestList,config" + Type of the file to validate. If unset, oci-image-tool will try to auto-detect the type. One of "image,manifest,manifestList,config" # EXAMPLES ``` $ skopeo copy docker://busybox oci:busybox-oci -$ oci-image-tool validate --type imageLayout --ref latest busybox-oci +$ oci-image-tool validate --type image --ref latest busybox-oci busybox-oci: OK ``` diff --git a/cmd/oci-image-tool/unpack.go b/cmd/oci-image-tool/unpack.go index e02ad706d..d2d3c7f67 100644 --- a/cmd/oci-image-tool/unpack.go +++ b/cmd/oci-image-tool/unpack.go @@ -22,11 +22,11 @@ import ( "github.com/opencontainers/image-spec/image" "github.com/spf13/cobra" + "golang.org/x/net/context" ) // supported unpack types var unpackTypes = []string{ - typeImageLayout, typeImage, } @@ -45,8 +45,8 @@ func newUnpackCmd(stdout, stderr *log.Logger) *cobra.Command { cmd := &cobra.Command{ Use: "unpack [src] [dest]", - Short: "Unpack an image or image source layout", - Long: `Unpack the OCI image .tar file or OCI image layout directory present at [src] to the destination directory [dest].`, + Short: "Unpack an image", + Long: `Unpack the OCI image present at [src] to the destination directory [dest].`, Run: v.Run, } @@ -75,6 +75,8 @@ func (v *unpackCmd) Run(cmd *cobra.Command, args []string) { os.Exit(1) } + ctx := context.Background() + if v.typ == "" { typ, err := autodetect(args[0]) if err != nil { @@ -86,11 +88,8 @@ func (v *unpackCmd) Run(cmd *cobra.Command, args []string) { var err error switch v.typ { - case typeImageLayout: - err = image.UnpackLayout(args[0], args[1], v.ref) - case typeImage: - err = image.Unpack(args[0], args[1], v.ref) + err = image.Unpack(ctx, args[0], args[1], v.ref) } if err != nil { diff --git a/cmd/oci-image-tool/validate.go b/cmd/oci-image-tool/validate.go index c2b42e2ef..8171e4e60 100644 --- a/cmd/oci-image-tool/validate.go +++ b/cmd/oci-image-tool/validate.go @@ -24,11 +24,11 @@ import ( "github.com/opencontainers/image-spec/schema" "github.com/pkg/errors" "github.com/spf13/cobra" + "golang.org/x/net/context" ) // supported validation types var validateTypes = []string{ - typeImageLayout, typeImage, typeManifest, typeManifestList, @@ -64,7 +64,7 @@ func newValidateCmd(stdout, stderr *log.Logger) *cobra.Command { cmd.Flags().StringVar( &v.ref, "ref", "v1.0", - `The ref pointing to the manifest to be validated. This must be present in the "refs" subdirectory of the image. Only applicable if type is image or imageLayout.`, + `The ref pointing to the manifest to be validated. This must be present in the "refs" subdirectory of the image. Only applicable if type is image.`, ) return cmd @@ -79,9 +79,11 @@ func (v *validateCmd) Run(cmd *cobra.Command, args []string) { os.Exit(1) } + ctx := context.Background() + var exitcode int for _, arg := range args { - err := v.validatePath(arg) + err := v.validatePath(ctx, arg) if err == nil { v.stdout.Printf("%s: OK", arg) @@ -111,7 +113,7 @@ func (v *validateCmd) Run(cmd *cobra.Command, args []string) { os.Exit(exitcode) } -func (v *validateCmd) validatePath(name string) error { +func (v *validateCmd) validatePath(ctx context.Context, name string) error { var err error typ := v.typ @@ -122,10 +124,8 @@ func (v *validateCmd) validatePath(name string) error { } switch typ { - case typeImageLayout: - return image.ValidateLayout(name, v.ref) case typeImage: - return image.Validate(name, v.ref) + return image.Validate(ctx, name, v.ref) } f, err := os.Open(name) diff --git a/image/config.go b/image/config.go index 67001ce40..885c88211 100644 --- a/image/config.go +++ b/image/config.go @@ -18,16 +18,16 @@ import ( "bytes" "encoding/json" "fmt" - "io" "io/ioutil" - "os" - "path/filepath" "strconv" "strings" + "github.com/opencontainers/image-spec/image/cas" "github.com/opencontainers/image-spec/schema" - "github.com/opencontainers/runtime-spec/specs-go" + imageSpecs "github.com/opencontainers/image-spec/specs-go" + runtimeSpecs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" + "golang.org/x/net/context" ) type cfg struct { @@ -49,43 +49,35 @@ type config struct { Config cfg `json:"config"` } -func findConfig(w walker, d *descriptor) (*config, error) { - var c config - cpath := filepath.Join("blobs", d.algo(), d.hash()) +func findConfig(ctx context.Context, engine cas.Engine, descriptor *imageSpecs.Descriptor) (*config, error) { + reader, err := engine.Get(ctx, descriptor.Digest) + if err != nil { + return nil, err + } - switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() || filepath.Clean(path) != cpath { - return nil - } - buf, err := ioutil.ReadAll(r) - if err != nil { - return errors.Wrapf(err, "%s: error reading config", path) - } + buf, err := ioutil.ReadAll(reader) + if err != nil { + return nil, errors.Wrapf(err, "%s: error reading manifest", descriptor.Digest) + } - if err := schema.MediaTypeImageConfig.Validate(bytes.NewReader(buf)); err != nil { - return errors.Wrapf(err, "%s: config validation failed", path) - } + if err := schema.MediaTypeImageConfig.Validate(bytes.NewReader(buf)); err != nil { + return nil, errors.Wrapf(err, "%s: config validation failed", descriptor.Digest) + } - if err := json.Unmarshal(buf, &c); err != nil { - return err - } - return errEOW - }); err { - case nil: - return nil, fmt.Errorf("%s: config not found", cpath) - case errEOW: - return &c, nil - default: + var c config + if err := json.Unmarshal(buf, &c); err != nil { return nil, err } + + return &c, nil } -func (c *config) runtimeSpec(rootfs string) (*specs.Spec, error) { +func (c *config) runtimeSpec(rootfs string) (*runtimeSpecs.Spec, error) { if c.OS != "linux" { return nil, fmt.Errorf("%s: unsupported OS", c.OS) } - var s specs.Spec + var s runtimeSpecs.Spec s.Version = "0.5.0" // we should at least apply the default spec, otherwise this is totally useless s.Process.Terminal = true @@ -128,12 +120,12 @@ func (c *config) runtimeSpec(rootfs string) (*specs.Spec, error) { swap := uint64(c.Config.MemorySwap) shares := uint64(c.Config.CPUShares) - s.Linux.Resources = &specs.Resources{ - CPU: &specs.CPU{ + s.Linux.Resources = &runtimeSpecs.Resources{ + CPU: &runtimeSpecs.CPU{ Shares: &shares, }, - Memory: &specs.Memory{ + Memory: &runtimeSpecs.Memory{ Limit: &mem, Reservation: &mem, Swap: &swap, @@ -143,7 +135,7 @@ func (c *config) runtimeSpec(rootfs string) (*specs.Spec, error) { for vol := range c.Config.Volumes { s.Mounts = append( s.Mounts, - specs.Mount{ + runtimeSpecs.Mount{ Destination: vol, Type: "bind", Options: []string{"rbind"}, diff --git a/image/descriptor.go b/image/descriptor.go index 106ab7fd9..1494af8d4 100644 --- a/image/descriptor.go +++ b/image/descriptor.go @@ -17,88 +17,24 @@ package image import ( "crypto/sha256" "encoding/hex" - "encoding/json" - "fmt" "io" - "os" - "path/filepath" - "strings" + "github.com/opencontainers/image-spec/image/cas" + "github.com/opencontainers/image-spec/specs-go" "github.com/pkg/errors" + "golang.org/x/net/context" ) -type descriptor struct { - MediaType string `json:"mediaType"` - Digest string `json:"digest"` - Size int64 `json:"size"` -} - -func (d *descriptor) algo() string { - pts := strings.SplitN(d.Digest, ":", 2) - if len(pts) != 2 { - return "" - } - return pts[0] -} - -func (d *descriptor) hash() string { - pts := strings.SplitN(d.Digest, ":", 2) - if len(pts) != 2 { - return "" - } - return pts[1] -} - -func findDescriptor(w walker, name string) (*descriptor, error) { - var d descriptor - dpath := filepath.Join("refs", name) - - switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() || filepath.Clean(path) != dpath { - return nil - } - - if err := json.NewDecoder(r).Decode(&d); err != nil { - return err - } - - return errEOW - }); err { - case nil: - return nil, fmt.Errorf("%s: descriptor not found", dpath) - case errEOW: - return &d, nil - default: - return nil, err +func validateDescriptor(ctx context.Context, engine cas.Engine, descriptor *specs.Descriptor) error { + reader, err := engine.Get(ctx, descriptor.Digest) + if err != nil { + return err } -} -func (d *descriptor) validate(w walker) error { - switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() { - return nil - } - - filename, err := filepath.Rel(filepath.Join("blobs", d.algo()), filepath.Clean(path)) - if err != nil || d.hash() != filename { - return nil - } - - if err := d.validateContent(r); err != nil { - return err - } - return errEOW - }); err { - case nil: - return fmt.Errorf("%s: not found", d.Digest) - case errEOW: - return nil - default: - return errors.Wrapf(err, "%s: validation failed", d.Digest) - } + return validateContent(ctx, descriptor, reader) } -func (d *descriptor) validateContent(r io.Reader) error { +func validateContent(ctx context.Context, descriptor *specs.Descriptor, r io.Reader) error { h := sha256.New() n, err := io.Copy(h, r) if err != nil { @@ -107,13 +43,15 @@ func (d *descriptor) validateContent(r io.Reader) error { digest := "sha256:" + hex.EncodeToString(h.Sum(nil)) - if digest != d.Digest { + if digest != descriptor.Digest { return errors.New("digest mismatch") } - if n != d.Size { + if n != descriptor.Size { return errors.New("size mismatch") } + // FIXME: check descriptor.MediaType, when possible + return nil } diff --git a/image/image.go b/image/image.go index 04ce278ee..18723ad09 100644 --- a/image/image.go +++ b/image/image.go @@ -19,136 +19,133 @@ import ( "os" "path/filepath" - "github.com/pkg/errors" + "github.com/opencontainers/image-spec/image/cas" + caslayout "github.com/opencontainers/image-spec/image/cas/layout" + "github.com/opencontainers/image-spec/image/refs" + refslayout "github.com/opencontainers/image-spec/image/refs/layout" + "golang.org/x/net/context" ) -// ValidateLayout walks through the file tree given by src and -// validates the manifest pointed to by the given ref -// or returns an error if the validation failed. -func ValidateLayout(src, ref string) error { - return validate(newPathWalker(src), ref) -} +// Validate validates the given reference. +func Validate(ctx context.Context, path, ref string) error { + refEngine, err := refslayout.NewEngine(path) + if err != nil { + return err + } + defer refEngine.Close() -// Validate walks through the given .tar file and -// validates the manifest pointed to by the given ref -// or returns an error if the validation failed. -func Validate(tarFile, ref string) error { - f, err := os.Open(tarFile) + casEngine, err := caslayout.NewEngine(path) if err != nil { - return errors.Wrap(err, "unable to open file") + return err } - defer f.Close() + defer casEngine.Close() - return validate(newTarWalker(f), ref) + return validate(ctx, refEngine, casEngine, ref) } -func validate(w walker, refName string) error { - ref, err := findDescriptor(w, refName) +func validate(ctx context.Context, refEngine refs.Engine, casEngine cas.Engine, ref string) error { + descriptor, err := refEngine.Get(ctx, ref) if err != nil { return err } - if err = ref.validate(w); err != nil { + err = validateDescriptor(ctx, casEngine, descriptor) + if err != nil { return err } - m, err := findManifest(w, ref) + m, err := findManifest(ctx, casEngine, descriptor) if err != nil { return err } - return m.validate(w) + return m.validate(ctx, casEngine) } -// UnpackLayout walks through the file tree given by src and -// using the layers specified in the manifest pointed to by the given ref -// and unpacks all layers in the given destination directory -// or returns an error if the unpacking failed. -func UnpackLayout(src, dest, ref string) error { - return unpack(newPathWalker(src), dest, ref) -} +// Unpack unpacks the given reference to a destination directory. +func Unpack(ctx context.Context, path, dest, ref string) error { + refEngine, err := refslayout.NewEngine(path) + if err != nil { + return err + } + defer refEngine.Close() -// Unpack walks through the given .tar file and -// using the layers specified in the manifest pointed to by the given ref -// and unpacks all layers in the given destination directory -// or returns an error if the unpacking failed. -func Unpack(tarFile, dest, ref string) error { - f, err := os.Open(tarFile) + casEngine, err := caslayout.NewEngine(path) if err != nil { - return errors.Wrap(err, "unable to open file") + return err } - defer f.Close() + defer casEngine.Close() - return unpack(newTarWalker(f), dest, ref) + return unpack(ctx, refEngine, casEngine, dest, ref) } -func unpack(w walker, dest, refName string) error { - ref, err := findDescriptor(w, refName) +func unpack(ctx context.Context, refEngine refs.Engine, casEngine cas.Engine, dest, ref string) error { + descriptor, err := refEngine.Get(ctx, ref) if err != nil { return err } - if err = ref.validate(w); err != nil { + err = validateDescriptor(ctx, casEngine, descriptor) + if err != nil { return err } - m, err := findManifest(w, ref) + m, err := findManifest(ctx, casEngine, descriptor) if err != nil { return err } - if err = m.validate(w); err != nil { + if err = m.validate(ctx, casEngine); err != nil { return err } - return m.unpack(w, dest) + return m.unpack(ctx, casEngine, dest) } -// CreateRuntimeBundleLayout walks through the file tree given by src and -// creates an OCI runtime bundle in the given destination dest -// or returns an error if the unpacking failed. -func CreateRuntimeBundleLayout(src, dest, ref, root string) error { - return createRuntimeBundle(newPathWalker(src), dest, ref, root) -} +// CreateRuntimeBundle creates an OCI runtime bundle in the given +// destination. +func CreateRuntimeBundle(ctx context.Context, path, dest, ref, rootfs string) error { + refEngine, err := refslayout.NewEngine(path) + if err != nil { + return err + } + defer refEngine.Close() -// CreateRuntimeBundle walks through the given .tar file and -// creates an OCI runtime bundle in the given destination dest -// or returns an error if the unpacking failed. -func CreateRuntimeBundle(tarFile, dest, ref, root string) error { - f, err := os.Open(tarFile) + casEngine, err := caslayout.NewEngine(path) if err != nil { - return errors.Wrap(err, "unable to open file") + return err } - defer f.Close() + defer casEngine.Close() - return createRuntimeBundle(newTarWalker(f), dest, ref, root) + return createRuntimeBundle(ctx, refEngine, casEngine, dest, ref, rootfs) } -func createRuntimeBundle(w walker, dest, refName, rootfs string) error { - ref, err := findDescriptor(w, refName) +func createRuntimeBundle(ctx context.Context, refEngine refs.Engine, casEngine cas.Engine, dest, ref, rootfs string) error { + descriptor, err := refEngine.Get(ctx, ref) if err != nil { return err } - if err = ref.validate(w); err != nil { + err = validateDescriptor(ctx, casEngine, descriptor) + if err != nil { return err } - m, err := findManifest(w, ref) + m, err := findManifest(ctx, casEngine, descriptor) if err != nil { return err } - if err = m.validate(w); err != nil { + if err = m.validate(ctx, casEngine); err != nil { return err } - c, err := findConfig(w, &m.Config) + c, err := findConfig(ctx, casEngine, m.Config) if err != nil { return err } - err = m.unpack(w, filepath.Join(dest, rootfs)) + err = m.unpack(ctx, casEngine, filepath.Join(dest, rootfs)) if err != nil { return err } diff --git a/image/manifest.go b/image/manifest.go index 8bac949c8..703c15641 100644 --- a/image/manifest.go +++ b/image/manifest.go @@ -27,59 +27,52 @@ import ( "strings" "time" + "github.com/opencontainers/image-spec/image/cas" "github.com/opencontainers/image-spec/schema" + "github.com/opencontainers/image-spec/specs-go" "github.com/pkg/errors" + "golang.org/x/net/context" ) type manifest struct { - Config descriptor `json:"config"` - Layers []descriptor `json:"layers"` + Config *specs.Descriptor `json:"config"` + Layers []specs.Descriptor `json:"layers"` } -func findManifest(w walker, d *descriptor) (*manifest, error) { - var m manifest - mpath := filepath.Join("blobs", d.algo(), d.hash()) - - switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() || filepath.Clean(path) != mpath { - return nil - } - - buf, err := ioutil.ReadAll(r) - if err != nil { - return errors.Wrapf(err, "%s: error reading manifest", path) - } - - if err := schema.MediaTypeManifest.Validate(bytes.NewReader(buf)); err != nil { - return errors.Wrapf(err, "%s: manifest validation failed", path) - } +func findManifest(ctx context.Context, engine cas.Engine, descriptor *specs.Descriptor) (*manifest, error) { + reader, err := engine.Get(ctx, descriptor.Digest) + if err != nil { + return nil, err + } - if err := json.Unmarshal(buf, &m); err != nil { - return err - } + buf, err := ioutil.ReadAll(reader) + if err != nil { + return nil, errors.Wrapf(err, "%s: error reading manifest", descriptor.Digest) + } - if len(m.Layers) == 0 { - return fmt.Errorf("%s: no layers found", path) - } + if err := schema.MediaTypeManifest.Validate(bytes.NewReader(buf)); err != nil { + return nil, errors.Wrapf(err, "%s: manifest validation failed", descriptor.Digest) + } - return errEOW - }); err { - case nil: - return nil, fmt.Errorf("%s: manifest not found", mpath) - case errEOW: - return &m, nil - default: + var m manifest + if err := json.Unmarshal(buf, &m); err != nil { return nil, err } + + if len(m.Layers) == 0 { + return nil, fmt.Errorf("%s: no layers found", descriptor.Digest) + } + + return &m, nil } -func (m *manifest) validate(w walker) error { - if err := m.Config.validate(w); err != nil { +func (m *manifest) validate(ctx context.Context, engine cas.Engine) error { + if err := validateDescriptor(ctx, engine, m.Config); err != nil { return errors.Wrap(err, "config validation failed") } for _, d := range m.Layers { - if err := d.validate(w); err != nil { + if err := validateDescriptor(ctx, engine, &d); err != nil { return errors.Wrap(err, "layer validation failed") } } @@ -87,35 +80,20 @@ func (m *manifest) validate(w walker) error { return nil } -func (m *manifest) unpack(w walker, dest string) error { +func (m *manifest) unpack(ctx context.Context, engine cas.Engine, dest string) error { for _, d := range m.Layers { if d.MediaType != string(schema.MediaTypeImageConfig) { continue } - switch err := w.walk(func(path string, info os.FileInfo, r io.Reader) error { - if info.IsDir() { - return nil - } - - dd, err := filepath.Rel(filepath.Join("blobs", d.algo()), filepath.Clean(path)) - if err != nil || d.hash() != dd { - return nil - } - - if err := unpackLayer(dest, r); err != nil { - return errors.Wrap(err, "error extracting layer") - } - - return errEOW - }); err { - case nil: - return fmt.Errorf("%s: layer not found", dest) - case errEOW: - return nil - default: + reader, err := engine.Get(ctx, d.Digest) + if err != nil { return err } + + if err := unpackLayer(dest, reader); err != nil { + return errors.Wrap(err, "error extracting layer") + } } return nil } diff --git a/image/walker.go b/image/walker.go deleted file mode 100644 index 777bce7d5..000000000 --- a/image/walker.go +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright 2016 The Linux Foundation -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package image - -import ( - "archive/tar" - "fmt" - "io" - "os" - "path/filepath" - - "github.com/pkg/errors" -) - -var ( - errEOW = fmt.Errorf("end of walk") // error to signal stop walking -) - -// walkFunc is a function type that gets called for each file or directory visited by the Walker. -type walkFunc func(path string, _ os.FileInfo, _ io.Reader) error - -// walker is the interface that walks through a file tree, -// calling walk for each file or directory in the tree. -type walker interface { - walk(walkFunc) error -} - -type tarWalker struct { - r io.ReadSeeker -} - -// newTarWalker returns a Walker that walks through .tar files. -func newTarWalker(r io.ReadSeeker) walker { - return &tarWalker{r} -} - -func (w *tarWalker) walk(f walkFunc) error { - if _, err := w.r.Seek(0, os.SEEK_SET); err != nil { - return errors.Wrapf(err, "unable to reset") - } - - tr := tar.NewReader(w.r) - -loop: - for { - hdr, err := tr.Next() - switch err { - case io.EOF: - break loop - case nil: - // success, continue below - default: - return errors.Wrapf(err, "error advancing tar stream") - } - - info := hdr.FileInfo() - if err := f(hdr.Name, info, tr); err != nil { - return err - } - } - - return nil -} - -type eofReader struct{} - -func (eofReader) Read(_ []byte) (int, error) { - return 0, io.EOF -} - -type pathWalker struct { - root string -} - -// newPathWalker returns a Walker that walks through directories -// starting at the given root path. It does not follow symlinks. -func newPathWalker(root string) walker { - return &pathWalker{root} -} - -func (w *pathWalker) walk(f walkFunc) error { - return filepath.Walk(w.root, func(path string, info os.FileInfo, err error) error { - rel, err := filepath.Rel(w.root, path) - if err != nil { - return errors.Wrap(err, "error walking path") // err from filepath.Walk includes path name - } - - if info.IsDir() { // behave like a tar reader for directories - return f(rel, info, eofReader{}) - } - - file, err := os.Open(path) - if err != nil { - return errors.Wrap(err, "unable to open file") // os.Open includes the path - } - defer file.Close() - - return f(rel, info, file) - }) -} From 4c68e7632bfe85be34cf7bcab4dbfc1307a24032 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Fri, 17 Jun 2016 10:43:20 -0700 Subject: [PATCH 04/13] specs-go: Add ImageLayoutVersion and check oci-layout in tar engines Collect the shared stuff in the image/layout utility package. Signed-off-by: W. Trevor King --- cmd/oci-image-tool/cas_get.go | 2 +- cmd/oci-image-tool/refs_get.go | 2 +- cmd/oci-image-tool/refs_list.go | 2 +- image/cas/layout/main.go | 5 ++- image/cas/layout/tar.go | 10 ++++- image/image.go | 12 +++--- image/layout/doc.go | 16 ++++++++ image/layout/tar.go | 67 +++++++++++++++++++++++++++++++++ image/refs/layout/main.go | 5 ++- image/refs/layout/tar.go | 10 ++++- specs-go/layout.go | 21 +++++++++++ 11 files changed, 135 insertions(+), 17 deletions(-) create mode 100644 image/layout/doc.go create mode 100644 image/layout/tar.go create mode 100644 specs-go/layout.go diff --git a/cmd/oci-image-tool/cas_get.go b/cmd/oci-image-tool/cas_get.go index 1599993d8..46d998a90 100644 --- a/cmd/oci-image-tool/cas_get.go +++ b/cmd/oci-image-tool/cas_get.go @@ -71,7 +71,7 @@ func (state *casGetCmd) Run(cmd *cobra.Command, args []string) { func (state *casGetCmd) run() (err error) { ctx := context.Background() - engine, err := layout.NewEngine(state.path) + engine, err := layout.NewEngine(ctx, state.path) if err != nil { return err } diff --git a/cmd/oci-image-tool/refs_get.go b/cmd/oci-image-tool/refs_get.go index 0f9582fc5..b1427489d 100644 --- a/cmd/oci-image-tool/refs_get.go +++ b/cmd/oci-image-tool/refs_get.go @@ -69,7 +69,7 @@ func (state *refsGetCmd) Run(cmd *cobra.Command, args []string) { func (state *refsGetCmd) run() (err error) { ctx := context.Background() - engine, err := layout.NewEngine(state.path) + engine, err := layout.NewEngine(ctx, state.path) if err != nil { return err } diff --git a/cmd/oci-image-tool/refs_list.go b/cmd/oci-image-tool/refs_list.go index ef5bf9604..1ebba82d2 100644 --- a/cmd/oci-image-tool/refs_list.go +++ b/cmd/oci-image-tool/refs_list.go @@ -67,7 +67,7 @@ func (state *refsListCmd) Run(cmd *cobra.Command, args []string) { func (state *refsListCmd) run() (err error) { ctx := context.Background() - engine, err := layout.NewEngine(state.path) + engine, err := layout.NewEngine(ctx, state.path) if err != nil { return err } diff --git a/image/cas/layout/main.go b/image/cas/layout/main.go index 56adfffa6..7bc4491ad 100644 --- a/image/cas/layout/main.go +++ b/image/cas/layout/main.go @@ -22,15 +22,16 @@ import ( "os" "github.com/opencontainers/image-spec/image/cas" + "golang.org/x/net/context" ) // NewEngine instantiates an engine with the appropriate backend (tar, // HTTP, ...). -func NewEngine(path string) (engine cas.Engine, err error) { +func NewEngine(ctx context.Context, path string) (engine cas.Engine, err error) { file, err := os.Open(path) if err != nil { return nil, err } - return NewTarEngine(file) + return NewTarEngine(ctx, file) } diff --git a/image/cas/layout/tar.go b/image/cas/layout/tar.go index 62a3d9360..c218e0db0 100644 --- a/image/cas/layout/tar.go +++ b/image/cas/layout/tar.go @@ -24,6 +24,7 @@ import ( "strings" "github.com/opencontainers/image-spec/image/cas" + "github.com/opencontainers/image-spec/image/layout" "golang.org/x/net/context" ) @@ -33,11 +34,16 @@ type TarEngine struct { } // NewTarEngine returns a new TarEngine. -func NewTarEngine(file ReadWriteSeekCloser) (engine cas.Engine, err error) { - engine = &TarEngine{ +func NewTarEngine(ctx context.Context, file ReadWriteSeekCloser) (eng cas.Engine, err error) { + engine := &TarEngine{ file: file, } + err = layout.CheckTarVersion(ctx, engine.file) + if err != nil { + return nil, err + } + return engine, nil } diff --git a/image/image.go b/image/image.go index 18723ad09..da97f7579 100644 --- a/image/image.go +++ b/image/image.go @@ -28,13 +28,13 @@ import ( // Validate validates the given reference. func Validate(ctx context.Context, path, ref string) error { - refEngine, err := refslayout.NewEngine(path) + refEngine, err := refslayout.NewEngine(ctx, path) if err != nil { return err } defer refEngine.Close() - casEngine, err := caslayout.NewEngine(path) + casEngine, err := caslayout.NewEngine(ctx, path) if err != nil { return err } @@ -64,13 +64,13 @@ func validate(ctx context.Context, refEngine refs.Engine, casEngine cas.Engine, // Unpack unpacks the given reference to a destination directory. func Unpack(ctx context.Context, path, dest, ref string) error { - refEngine, err := refslayout.NewEngine(path) + refEngine, err := refslayout.NewEngine(ctx, path) if err != nil { return err } defer refEngine.Close() - casEngine, err := caslayout.NewEngine(path) + casEngine, err := caslayout.NewEngine(ctx, path) if err != nil { return err } @@ -105,13 +105,13 @@ func unpack(ctx context.Context, refEngine refs.Engine, casEngine cas.Engine, de // CreateRuntimeBundle creates an OCI runtime bundle in the given // destination. func CreateRuntimeBundle(ctx context.Context, path, dest, ref, rootfs string) error { - refEngine, err := refslayout.NewEngine(path) + refEngine, err := refslayout.NewEngine(ctx, path) if err != nil { return err } defer refEngine.Close() - casEngine, err := caslayout.NewEngine(path) + casEngine, err := caslayout.NewEngine(ctx, path) if err != nil { return err } diff --git a/image/layout/doc.go b/image/layout/doc.go new file mode 100644 index 000000000..94dd3f42e --- /dev/null +++ b/image/layout/doc.go @@ -0,0 +1,16 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package layout defines utility code shared by refs/layout and cas/layout. +package layout diff --git a/image/layout/tar.go b/image/layout/tar.go new file mode 100644 index 000000000..5eddc4140 --- /dev/null +++ b/image/layout/tar.go @@ -0,0 +1,67 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package layout + +import ( + "archive/tar" + "encoding/json" + "errors" + "fmt" + "io" + "os" + + "github.com/opencontainers/image-spec/specs-go" + "golang.org/x/net/context" +) + +// CheckTarVersion walks a tarball pointed to by reader and returns an +// error if oci-layout is missing or has unrecognized content. +func CheckTarVersion(ctx context.Context, reader io.ReadSeeker) (err error) { + _, err = reader.Seek(0, os.SEEK_SET) + if err != nil { + return err + } + + tarReader := tar.NewReader(reader) + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + header, err := tarReader.Next() + if err == io.EOF { + return errors.New("oci-layout not found") + } + if err != nil { + return err + } + + if header.Name == "./oci-layout" { + decoder := json.NewDecoder(tarReader) + var version specs.ImageLayoutVersion + err = decoder.Decode(&version) + if err != nil { + return err + } + if version.Version != "1.0.0" { + return fmt.Errorf("unrecognized imageLayoutVersion: %q", version.Version) + } + + return nil + } + } +} diff --git a/image/refs/layout/main.go b/image/refs/layout/main.go index 0c3568d8c..94b9e5a38 100644 --- a/image/refs/layout/main.go +++ b/image/refs/layout/main.go @@ -22,15 +22,16 @@ import ( "os" "github.com/opencontainers/image-spec/image/refs" + "golang.org/x/net/context" ) // NewEngine instantiates an engine with the appropriate backend (tar, // HTTP, ...). -func NewEngine(path string) (engine refs.Engine, err error) { +func NewEngine(ctx context.Context, path string) (engine refs.Engine, err error) { file, err := os.Open(path) if err != nil { return nil, err } - return NewTarEngine(file) + return NewTarEngine(ctx, file) } diff --git a/image/refs/layout/tar.go b/image/refs/layout/tar.go index 0a5db5f10..0188863a6 100644 --- a/image/refs/layout/tar.go +++ b/image/refs/layout/tar.go @@ -24,6 +24,7 @@ import ( "strings" caslayout "github.com/opencontainers/image-spec/image/cas/layout" + imagelayout "github.com/opencontainers/image-spec/image/layout" "github.com/opencontainers/image-spec/image/refs" "github.com/opencontainers/image-spec/specs-go" "golang.org/x/net/context" @@ -35,11 +36,16 @@ type TarEngine struct { } // NewTarEngine returns a new TarEngine. -func NewTarEngine(file caslayout.ReadSeekCloser) (engine refs.Engine, err error) { - engine = &TarEngine{ +func NewTarEngine(ctx context.Context, file caslayout.ReadWriteSeekCloser) (eng refs.Engine, err error) { + engine := &TarEngine{ file: file, } + err = imagelayout.CheckTarVersion(ctx, engine.file) + if err != nil { + return nil, err + } + return engine, nil } diff --git a/specs-go/layout.go b/specs-go/layout.go new file mode 100644 index 000000000..1d3263fd4 --- /dev/null +++ b/specs-go/layout.go @@ -0,0 +1,21 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package specs + +// ImageLayoutVersion represents the oci-version content for the image +// layout format. +type ImageLayoutVersion struct { + Version string `json:"imageLayoutVersion"` +} From 002f7086fa2a281d220f7c9a1330a00eade757eb Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Sat, 9 Jul 2016 21:34:44 -0700 Subject: [PATCH 05/13] image/layout/tar.go: Add TarEntryByName Making the CAS/refs Get implementations more DRY. Signed-off-by: W. Trevor King --- image/cas/layout/tar.go | 24 ++---------------- image/layout/tar.go | 55 +++++++++++++++++++++++++--------------- image/refs/layout/tar.go | 32 +++++------------------ 3 files changed, 44 insertions(+), 67 deletions(-) diff --git a/image/cas/layout/tar.go b/image/cas/layout/tar.go index c218e0db0..25fc398c2 100644 --- a/image/cas/layout/tar.go +++ b/image/cas/layout/tar.go @@ -15,12 +15,10 @@ package layout import ( - "archive/tar" "errors" "fmt" "io" "io/ioutil" - "os" "strings" "github.com/opencontainers/image-spec/image/cas" @@ -64,30 +62,12 @@ func (engine *TarEngine) Get(ctx context.Context, digest string) (reader io.Read targetName := fmt.Sprintf("./blobs/%s/%s", algorithm, hash) - _, err = engine.file.Seek(0, os.SEEK_SET) + _, tarReader, err := layout.TarEntryByName(ctx, engine.file, targetName) if err != nil { return nil, err } - tarReader := tar.NewReader(engine.file) - for { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - header, err := tarReader.Next() - if err == io.EOF { - return nil, os.ErrNotExist - } else if err != nil { - return nil, err - } - - if header.Name == targetName { - return ioutil.NopCloser(tarReader), nil - } - } + return ioutil.NopCloser(tarReader), nil } // Delete removes a blob from the store. diff --git a/image/layout/tar.go b/image/layout/tar.go index 5eddc4140..6c9051c60 100644 --- a/image/layout/tar.go +++ b/image/layout/tar.go @@ -26,42 +26,57 @@ import ( "golang.org/x/net/context" ) -// CheckTarVersion walks a tarball pointed to by reader and returns an -// error if oci-layout is missing or has unrecognized content. -func CheckTarVersion(ctx context.Context, reader io.ReadSeeker) (err error) { +// TarEntryByName walks a tarball pointed to by reader, finds an +// entry matching the given name, and returns the header and reader +// for that entry. Returns os.ErrNotExist if the path is not found. +func TarEntryByName(ctx context.Context, reader io.ReadSeeker, name string) (header *tar.Header, tarReader *tar.Reader, err error) { _, err = reader.Seek(0, os.SEEK_SET) if err != nil { - return err + return nil, nil, err } - tarReader := tar.NewReader(reader) + tarReader = tar.NewReader(reader) for { select { case <-ctx.Done(): - return ctx.Err() + return nil, nil, ctx.Err() default: } header, err := tarReader.Next() if err == io.EOF { - return errors.New("oci-layout not found") + return nil, nil, os.ErrNotExist } if err != nil { - return err + return nil, nil, err } - if header.Name == "./oci-layout" { - decoder := json.NewDecoder(tarReader) - var version specs.ImageLayoutVersion - err = decoder.Decode(&version) - if err != nil { - return err - } - if version.Version != "1.0.0" { - return fmt.Errorf("unrecognized imageLayoutVersion: %q", version.Version) - } - - return nil + if header.Name == name { + return header, tarReader, nil } } } + +// CheckTarVersion walks a tarball pointed to by reader and returns an +// error if oci-layout is missing or has unrecognized content. +func CheckTarVersion(ctx context.Context, reader io.ReadSeeker) (err error) { + _, tarReader, err := TarEntryByName(ctx, reader, "./oci-layout") + if err == os.ErrNotExist { + return errors.New("oci-layout not found") + } + if err != nil { + return err + } + + decoder := json.NewDecoder(tarReader) + var version specs.ImageLayoutVersion + err = decoder.Decode(&version) + if err != nil { + return err + } + if version.Version != "1.0.0" { + return fmt.Errorf("unrecognized imageLayoutVersion: %q", version.Version) + } + + return nil +} diff --git a/image/refs/layout/tar.go b/image/refs/layout/tar.go index 0188863a6..beded71a9 100644 --- a/image/refs/layout/tar.go +++ b/image/refs/layout/tar.go @@ -59,36 +59,18 @@ func (engine *TarEngine) Put(ctx context.Context, name string, descriptor *specs func (engine *TarEngine) Get(ctx context.Context, name string) (descriptor *specs.Descriptor, err error) { targetName := fmt.Sprintf("./refs/%s", name) - _, err = engine.file.Seek(0, os.SEEK_SET) + _, tarReader, err := imagelayout.TarEntryByName(ctx, engine.file, targetName) if err != nil { return nil, err } - tarReader := tar.NewReader(engine.file) - for { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - header, err := tarReader.Next() - if err == io.EOF { - return nil, os.ErrNotExist - } else if err != nil { - return nil, err - } - - if header.Name == targetName { - decoder := json.NewDecoder(tarReader) - var desc specs.Descriptor - err = decoder.Decode(&desc) - if err != nil { - return nil, err - } - return &desc, nil - } + decoder := json.NewDecoder(tarReader) + var desc specs.Descriptor + err = decoder.Decode(&desc) + if err != nil { + return nil, err } + return &desc, nil } // List returns available names from the store. From 89a36d8af3d17ae72f5f5a8b56fb2562d360bce0 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Tue, 26 Jul 2016 21:25:12 -0700 Subject: [PATCH 06/13] image/cas/put: Add a PutJSON helper A fair amount of image setup involves pushing JSON objects to CAS, so provide a convenient wrapper around that. This implementation could be improved, with at least: * Consistent key sorts, etc. to increase the chances of matching an existing CAS blob. * Streaming the marshaled JSON into the engine to avoid serializing it in memory before passing it into Engine.Put. But the API is fine, and we can improve the implementation as we go. Signed-off-by: W. Trevor King --- image/cas/put.go | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 image/cas/put.go diff --git a/image/cas/put.go b/image/cas/put.go new file mode 100644 index 000000000..0f6188181 --- /dev/null +++ b/image/cas/put.go @@ -0,0 +1,47 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cas + +import ( + "bytes" + "encoding/json" + + "github.com/opencontainers/image-spec/specs-go" + "golang.org/x/net/context" +) + +// PutJSON writes a generic JSON object to content-addressable storage +// and returns a Descriptor referencing it. +func PutJSON(ctx context.Context, engine Engine, data interface{}, mediaType string) (descriptor *specs.Descriptor, err error) { + jsonBytes, err := json.Marshal(data) + if err != nil { + return nil, err + } + size := len(jsonBytes) + size64 := int64(size) // panics on overflow + + reader := bytes.NewReader(jsonBytes) + digest, err := engine.Put(ctx, reader) + if err != nil { + return nil, err + } + + descriptor = &specs.Descriptor{ + MediaType: mediaType, + Digest: digest, + Size: size64, + } + return descriptor, nil +} From 1696995a1a6816dcef10cf1dc5f999024e9b421d Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Wed, 27 Jul 2016 15:28:44 -0700 Subject: [PATCH 07/13] vendor: Bundle golang.org/x/net/context Generated with: $ make install.tools $ make update-deps Signed-off-by: W. Trevor King --- glide.lock | 8 +- vendor/golang.org/x/net/LICENSE | 27 ++ vendor/golang.org/x/net/PATENTS | 22 ++ vendor/golang.org/x/net/context/context.go | 156 ++++++++++ vendor/golang.org/x/net/context/go17.go | 72 +++++ vendor/golang.org/x/net/context/pre_go17.go | 300 ++++++++++++++++++++ 6 files changed, 583 insertions(+), 2 deletions(-) create mode 100644 vendor/golang.org/x/net/LICENSE create mode 100644 vendor/golang.org/x/net/PATENTS create mode 100644 vendor/golang.org/x/net/context/context.go create mode 100644 vendor/golang.org/x/net/context/go17.go create mode 100644 vendor/golang.org/x/net/context/pre_go17.go diff --git a/glide.lock b/glide.lock index 57280d2f1..4d9447fc5 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ hash: 223985f204597c6ed49657a4fc38273f683d40c39e8d48d13ed0dbf632107427 -updated: 2016-07-22T16:40:50.020731917+02:00 +updated: 2016-07-27T15:27:46.674460763-07:00 imports: - name: github.com/inconshreveable/mousetrap version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 @@ -24,7 +24,11 @@ imports: - name: github.com/xeipuuv/gojsonschema version: d5336c75940ef31c9ceeb0ae64cf92944bccb4ee - name: go4.org - version: 85455cb60c902182109ca27131042a41bc4cb85d + version: 401618586120d672bfd8ddf033bafd1c96c31241 subpackages: - errorutil +- name: golang.org/x/net + version: 6a513affb38dc9788b449d59ffed099b8de18fa0 + subpackages: + - context testImports: [] diff --git a/vendor/golang.org/x/net/LICENSE b/vendor/golang.org/x/net/LICENSE new file mode 100644 index 000000000..6a66aea5e --- /dev/null +++ b/vendor/golang.org/x/net/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/golang.org/x/net/PATENTS b/vendor/golang.org/x/net/PATENTS new file mode 100644 index 000000000..733099041 --- /dev/null +++ b/vendor/golang.org/x/net/PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the Go project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of Go, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of Go. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of Go or any code incorporated within this +implementation of Go constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of Go +shall terminate as of the date such litigation is filed. diff --git a/vendor/golang.org/x/net/context/context.go b/vendor/golang.org/x/net/context/context.go new file mode 100644 index 000000000..134654cf7 --- /dev/null +++ b/vendor/golang.org/x/net/context/context.go @@ -0,0 +1,156 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package context defines the Context type, which carries deadlines, +// cancelation signals, and other request-scoped values across API boundaries +// and between processes. +// +// Incoming requests to a server should create a Context, and outgoing calls to +// servers should accept a Context. The chain of function calls between must +// propagate the Context, optionally replacing it with a modified copy created +// using WithDeadline, WithTimeout, WithCancel, or WithValue. +// +// Programs that use Contexts should follow these rules to keep interfaces +// consistent across packages and enable static analysis tools to check context +// propagation: +// +// Do not store Contexts inside a struct type; instead, pass a Context +// explicitly to each function that needs it. The Context should be the first +// parameter, typically named ctx: +// +// func DoSomething(ctx context.Context, arg Arg) error { +// // ... use ctx ... +// } +// +// Do not pass a nil Context, even if a function permits it. Pass context.TODO +// if you are unsure about which Context to use. +// +// Use context Values only for request-scoped data that transits processes and +// APIs, not for passing optional parameters to functions. +// +// The same Context may be passed to functions running in different goroutines; +// Contexts are safe for simultaneous use by multiple goroutines. +// +// See http://blog.golang.org/context for example code for a server that uses +// Contexts. +package context // import "golang.org/x/net/context" + +import "time" + +// A Context carries a deadline, a cancelation signal, and other values across +// API boundaries. +// +// Context's methods may be called by multiple goroutines simultaneously. +type Context interface { + // Deadline returns the time when work done on behalf of this context + // should be canceled. Deadline returns ok==false when no deadline is + // set. Successive calls to Deadline return the same results. + Deadline() (deadline time.Time, ok bool) + + // Done returns a channel that's closed when work done on behalf of this + // context should be canceled. Done may return nil if this context can + // never be canceled. Successive calls to Done return the same value. + // + // WithCancel arranges for Done to be closed when cancel is called; + // WithDeadline arranges for Done to be closed when the deadline + // expires; WithTimeout arranges for Done to be closed when the timeout + // elapses. + // + // Done is provided for use in select statements: + // + // // Stream generates values with DoSomething and sends them to out + // // until DoSomething returns an error or ctx.Done is closed. + // func Stream(ctx context.Context, out chan<- Value) error { + // for { + // v, err := DoSomething(ctx) + // if err != nil { + // return err + // } + // select { + // case <-ctx.Done(): + // return ctx.Err() + // case out <- v: + // } + // } + // } + // + // See http://blog.golang.org/pipelines for more examples of how to use + // a Done channel for cancelation. + Done() <-chan struct{} + + // Err returns a non-nil error value after Done is closed. Err returns + // Canceled if the context was canceled or DeadlineExceeded if the + // context's deadline passed. No other values for Err are defined. + // After Done is closed, successive calls to Err return the same value. + Err() error + + // Value returns the value associated with this context for key, or nil + // if no value is associated with key. Successive calls to Value with + // the same key returns the same result. + // + // Use context values only for request-scoped data that transits + // processes and API boundaries, not for passing optional parameters to + // functions. + // + // A key identifies a specific value in a Context. Functions that wish + // to store values in Context typically allocate a key in a global + // variable then use that key as the argument to context.WithValue and + // Context.Value. A key can be any type that supports equality; + // packages should define keys as an unexported type to avoid + // collisions. + // + // Packages that define a Context key should provide type-safe accessors + // for the values stores using that key: + // + // // Package user defines a User type that's stored in Contexts. + // package user + // + // import "golang.org/x/net/context" + // + // // User is the type of value stored in the Contexts. + // type User struct {...} + // + // // key is an unexported type for keys defined in this package. + // // This prevents collisions with keys defined in other packages. + // type key int + // + // // userKey is the key for user.User values in Contexts. It is + // // unexported; clients use user.NewContext and user.FromContext + // // instead of using this key directly. + // var userKey key = 0 + // + // // NewContext returns a new Context that carries value u. + // func NewContext(ctx context.Context, u *User) context.Context { + // return context.WithValue(ctx, userKey, u) + // } + // + // // FromContext returns the User value stored in ctx, if any. + // func FromContext(ctx context.Context) (*User, bool) { + // u, ok := ctx.Value(userKey).(*User) + // return u, ok + // } + Value(key interface{}) interface{} +} + +// Background returns a non-nil, empty Context. It is never canceled, has no +// values, and has no deadline. It is typically used by the main function, +// initialization, and tests, and as the top-level Context for incoming +// requests. +func Background() Context { + return background +} + +// TODO returns a non-nil, empty Context. Code should use context.TODO when +// it's unclear which Context to use or it is not yet available (because the +// surrounding function has not yet been extended to accept a Context +// parameter). TODO is recognized by static analysis tools that determine +// whether Contexts are propagated correctly in a program. +func TODO() Context { + return todo +} + +// A CancelFunc tells an operation to abandon its work. +// A CancelFunc does not wait for the work to stop. +// After the first call, subsequent calls to a CancelFunc do nothing. +type CancelFunc func() diff --git a/vendor/golang.org/x/net/context/go17.go b/vendor/golang.org/x/net/context/go17.go new file mode 100644 index 000000000..f8cda19ad --- /dev/null +++ b/vendor/golang.org/x/net/context/go17.go @@ -0,0 +1,72 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.7 + +package context + +import ( + "context" // standard library's context, as of Go 1.7 + "time" +) + +var ( + todo = context.TODO() + background = context.Background() +) + +// Canceled is the error returned by Context.Err when the context is canceled. +var Canceled = context.Canceled + +// DeadlineExceeded is the error returned by Context.Err when the context's +// deadline passes. +var DeadlineExceeded = context.DeadlineExceeded + +// WithCancel returns a copy of parent with a new Done channel. The returned +// context's Done channel is closed when the returned cancel function is called +// or when the parent context's Done channel is closed, whichever happens first. +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete. +func WithCancel(parent Context) (ctx Context, cancel CancelFunc) { + ctx, f := context.WithCancel(parent) + return ctx, CancelFunc(f) +} + +// WithDeadline returns a copy of the parent context with the deadline adjusted +// to be no later than d. If the parent's deadline is already earlier than d, +// WithDeadline(parent, d) is semantically equivalent to parent. The returned +// context's Done channel is closed when the deadline expires, when the returned +// cancel function is called, or when the parent context's Done channel is +// closed, whichever happens first. +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete. +func WithDeadline(parent Context, deadline time.Time) (Context, CancelFunc) { + ctx, f := context.WithDeadline(parent, deadline) + return ctx, CancelFunc(f) +} + +// WithTimeout returns WithDeadline(parent, time.Now().Add(timeout)). +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete: +// +// func slowOperationWithTimeout(ctx context.Context) (Result, error) { +// ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) +// defer cancel() // releases resources if slowOperation completes before timeout elapses +// return slowOperation(ctx) +// } +func WithTimeout(parent Context, timeout time.Duration) (Context, CancelFunc) { + return WithDeadline(parent, time.Now().Add(timeout)) +} + +// WithValue returns a copy of parent in which the value associated with key is +// val. +// +// Use context Values only for request-scoped data that transits processes and +// APIs, not for passing optional parameters to functions. +func WithValue(parent Context, key interface{}, val interface{}) Context { + return context.WithValue(parent, key, val) +} diff --git a/vendor/golang.org/x/net/context/pre_go17.go b/vendor/golang.org/x/net/context/pre_go17.go new file mode 100644 index 000000000..5a30acabd --- /dev/null +++ b/vendor/golang.org/x/net/context/pre_go17.go @@ -0,0 +1,300 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.7 + +package context + +import ( + "errors" + "fmt" + "sync" + "time" +) + +// An emptyCtx is never canceled, has no values, and has no deadline. It is not +// struct{}, since vars of this type must have distinct addresses. +type emptyCtx int + +func (*emptyCtx) Deadline() (deadline time.Time, ok bool) { + return +} + +func (*emptyCtx) Done() <-chan struct{} { + return nil +} + +func (*emptyCtx) Err() error { + return nil +} + +func (*emptyCtx) Value(key interface{}) interface{} { + return nil +} + +func (e *emptyCtx) String() string { + switch e { + case background: + return "context.Background" + case todo: + return "context.TODO" + } + return "unknown empty Context" +} + +var ( + background = new(emptyCtx) + todo = new(emptyCtx) +) + +// Canceled is the error returned by Context.Err when the context is canceled. +var Canceled = errors.New("context canceled") + +// DeadlineExceeded is the error returned by Context.Err when the context's +// deadline passes. +var DeadlineExceeded = errors.New("context deadline exceeded") + +// WithCancel returns a copy of parent with a new Done channel. The returned +// context's Done channel is closed when the returned cancel function is called +// or when the parent context's Done channel is closed, whichever happens first. +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete. +func WithCancel(parent Context) (ctx Context, cancel CancelFunc) { + c := newCancelCtx(parent) + propagateCancel(parent, c) + return c, func() { c.cancel(true, Canceled) } +} + +// newCancelCtx returns an initialized cancelCtx. +func newCancelCtx(parent Context) *cancelCtx { + return &cancelCtx{ + Context: parent, + done: make(chan struct{}), + } +} + +// propagateCancel arranges for child to be canceled when parent is. +func propagateCancel(parent Context, child canceler) { + if parent.Done() == nil { + return // parent is never canceled + } + if p, ok := parentCancelCtx(parent); ok { + p.mu.Lock() + if p.err != nil { + // parent has already been canceled + child.cancel(false, p.err) + } else { + if p.children == nil { + p.children = make(map[canceler]bool) + } + p.children[child] = true + } + p.mu.Unlock() + } else { + go func() { + select { + case <-parent.Done(): + child.cancel(false, parent.Err()) + case <-child.Done(): + } + }() + } +} + +// parentCancelCtx follows a chain of parent references until it finds a +// *cancelCtx. This function understands how each of the concrete types in this +// package represents its parent. +func parentCancelCtx(parent Context) (*cancelCtx, bool) { + for { + switch c := parent.(type) { + case *cancelCtx: + return c, true + case *timerCtx: + return c.cancelCtx, true + case *valueCtx: + parent = c.Context + default: + return nil, false + } + } +} + +// removeChild removes a context from its parent. +func removeChild(parent Context, child canceler) { + p, ok := parentCancelCtx(parent) + if !ok { + return + } + p.mu.Lock() + if p.children != nil { + delete(p.children, child) + } + p.mu.Unlock() +} + +// A canceler is a context type that can be canceled directly. The +// implementations are *cancelCtx and *timerCtx. +type canceler interface { + cancel(removeFromParent bool, err error) + Done() <-chan struct{} +} + +// A cancelCtx can be canceled. When canceled, it also cancels any children +// that implement canceler. +type cancelCtx struct { + Context + + done chan struct{} // closed by the first cancel call. + + mu sync.Mutex + children map[canceler]bool // set to nil by the first cancel call + err error // set to non-nil by the first cancel call +} + +func (c *cancelCtx) Done() <-chan struct{} { + return c.done +} + +func (c *cancelCtx) Err() error { + c.mu.Lock() + defer c.mu.Unlock() + return c.err +} + +func (c *cancelCtx) String() string { + return fmt.Sprintf("%v.WithCancel", c.Context) +} + +// cancel closes c.done, cancels each of c's children, and, if +// removeFromParent is true, removes c from its parent's children. +func (c *cancelCtx) cancel(removeFromParent bool, err error) { + if err == nil { + panic("context: internal error: missing cancel error") + } + c.mu.Lock() + if c.err != nil { + c.mu.Unlock() + return // already canceled + } + c.err = err + close(c.done) + for child := range c.children { + // NOTE: acquiring the child's lock while holding parent's lock. + child.cancel(false, err) + } + c.children = nil + c.mu.Unlock() + + if removeFromParent { + removeChild(c.Context, c) + } +} + +// WithDeadline returns a copy of the parent context with the deadline adjusted +// to be no later than d. If the parent's deadline is already earlier than d, +// WithDeadline(parent, d) is semantically equivalent to parent. The returned +// context's Done channel is closed when the deadline expires, when the returned +// cancel function is called, or when the parent context's Done channel is +// closed, whichever happens first. +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete. +func WithDeadline(parent Context, deadline time.Time) (Context, CancelFunc) { + if cur, ok := parent.Deadline(); ok && cur.Before(deadline) { + // The current deadline is already sooner than the new one. + return WithCancel(parent) + } + c := &timerCtx{ + cancelCtx: newCancelCtx(parent), + deadline: deadline, + } + propagateCancel(parent, c) + d := deadline.Sub(time.Now()) + if d <= 0 { + c.cancel(true, DeadlineExceeded) // deadline has already passed + return c, func() { c.cancel(true, Canceled) } + } + c.mu.Lock() + defer c.mu.Unlock() + if c.err == nil { + c.timer = time.AfterFunc(d, func() { + c.cancel(true, DeadlineExceeded) + }) + } + return c, func() { c.cancel(true, Canceled) } +} + +// A timerCtx carries a timer and a deadline. It embeds a cancelCtx to +// implement Done and Err. It implements cancel by stopping its timer then +// delegating to cancelCtx.cancel. +type timerCtx struct { + *cancelCtx + timer *time.Timer // Under cancelCtx.mu. + + deadline time.Time +} + +func (c *timerCtx) Deadline() (deadline time.Time, ok bool) { + return c.deadline, true +} + +func (c *timerCtx) String() string { + return fmt.Sprintf("%v.WithDeadline(%s [%s])", c.cancelCtx.Context, c.deadline, c.deadline.Sub(time.Now())) +} + +func (c *timerCtx) cancel(removeFromParent bool, err error) { + c.cancelCtx.cancel(false, err) + if removeFromParent { + // Remove this timerCtx from its parent cancelCtx's children. + removeChild(c.cancelCtx.Context, c) + } + c.mu.Lock() + if c.timer != nil { + c.timer.Stop() + c.timer = nil + } + c.mu.Unlock() +} + +// WithTimeout returns WithDeadline(parent, time.Now().Add(timeout)). +// +// Canceling this context releases resources associated with it, so code should +// call cancel as soon as the operations running in this Context complete: +// +// func slowOperationWithTimeout(ctx context.Context) (Result, error) { +// ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) +// defer cancel() // releases resources if slowOperation completes before timeout elapses +// return slowOperation(ctx) +// } +func WithTimeout(parent Context, timeout time.Duration) (Context, CancelFunc) { + return WithDeadline(parent, time.Now().Add(timeout)) +} + +// WithValue returns a copy of parent in which the value associated with key is +// val. +// +// Use context Values only for request-scoped data that transits processes and +// APIs, not for passing optional parameters to functions. +func WithValue(parent Context, key interface{}, val interface{}) Context { + return &valueCtx{parent, key, val} +} + +// A valueCtx carries a key-value pair. It implements Value for that key and +// delegates all other calls to the embedded Context. +type valueCtx struct { + Context + key, val interface{} +} + +func (c *valueCtx) String() string { + return fmt.Sprintf("%v.WithValue(%#v, %#v)", c.Context, c.key, c.val) +} + +func (c *valueCtx) Value(key interface{}) interface{} { + if c.key == key { + return c.val + } + return c.Context.Value(key) +} From c92ec50c74d0f2a728da0847f3b4592535752fda Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Wed, 27 Jul 2016 15:31:04 -0700 Subject: [PATCH 08/13] image/*/interface: Add unstable warnings to Engines As called for during today's meeting [1]. The wording is from Brandon [2]. [1]: http://ircbot.wl.linuxfoundation.org/meetings/opencontainers/2016/opencontainers.2016-07-27-21.00.log.html#l-54 [2]: https://github.com/opencontainers/image-spec/pull/159/files/9f8a18eb1cb2a1cbccd6fe137d743dccc2142901#r72525379 Signed-off-by: W. Trevor King --- image/cas/interface.go | 4 ++++ image/refs/interface.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/image/cas/interface.go b/image/cas/interface.go index 4070d7a23..7cc6e00f5 100644 --- a/image/cas/interface.go +++ b/image/cas/interface.go @@ -22,6 +22,10 @@ import ( ) // Engine represents a content-addressable storage engine. +// +// This interface is for internal use of oci-image-tool for the time +// being. It is subject to change. This notice will be removed when +// and if the interface becomes stable. type Engine interface { // Put adds a new blob to the store. The action is idempotent; a diff --git a/image/refs/interface.go b/image/refs/interface.go index e71161582..230aacba0 100644 --- a/image/refs/interface.go +++ b/image/refs/interface.go @@ -25,6 +25,10 @@ import ( type ListNameCallback func(ctx context.Context, name string) (err error) // Engine represents a name-based reference storage engine. +// +// This interface is for internal use of oci-image-tool for the time +// being. It is subject to change. This notice will be removed when +// and if the interface becomes stable. type Engine interface { // Put adds a new reference to the store. The action is idempotent; From 469bbb6ed94a9d9a056b7707bc88c011ed292808 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Wed, 27 Jul 2016 15:16:10 -0700 Subject: [PATCH 09/13] image/cas: Implement Engine.Put This is a bit awkward. For writing a tar entry, we need to know both the name and size of the file ahead of time. The implementation in this commit accomplishes that by reading the Put content into a buffer, hashing and sizing the buffer, and then calling WriteTarEntryByName to create the entry. With a filesystem-backed CAS engine, we could avoid the buffer by writing the file to a temporary location with rolling hash and size tracking and then renaming the temporary file to the appropriate path. WriteTarEntryByName itself has awkward buffering to avoid dropping anything onto disk. It reads through its current file and writes the new tar into a buffer, and then writes that buffer back back over its current file. There are a few issues with this: * It's a lot more work than you need if you're just appending a new entry to the end of the tarball. But writing the whole file into a buffer means we don't have to worry about the trailing blocks that mark the end of the tarball; that's all handled transparently for us by the Go implementation. And this implementation doesn't have to be performant (folks should not be using tarballs to back write-heavy engines). * It could leave you with a corrupted tarball if the caller dies mid-overwrite. Again, I expect folks will only ever write to a tarball when building a tarball for publishing. If the caller dies, you can just start over. Folks looking for a more reliable implementation should use a filesystem-backed engine. * It could leave you with dangling bytes at the end of the tarball. I couldn't find a Go invocation to truncate the file. Go does have an ftruncate(2) wrapper [1], but it doesn't seem to be exposed at the io.Reader/io.Writer/... level. So if you write a shorter file with the same name as the original, you may end up with some dangling bytes. cas.Engine.Put protects against excessive writes with a Get guard; after hashing the new data, Put trys to Get it from the tarball and only writes a new entry if it can't find an existing entry. This also protects the CAS engine from the dangling-bytes issue. The 0666 file modes and 0777 directory modes rely on the caller's umask to appropriately limit user/group/other permissions for the tarball itself and any content extracted to the filesystem from the tarball. The trailing slash manipulation (stripping before comparison and injecting before creation) is based on part of libarchive's description of old-style archives [2]: name Pathname, stored as a null-terminated string. Early tar implementations only stored regular files (including hardlinks to those files). One common early convention used a trailing "/" character to indicate a directory name, allowing directory permissions and owner information to be archived and restored. and POSIX ustar archives [3]: name, prefix ... The standard does not require a trailing / character on directory names, though most implementations still include this for compatibility reasons. [1]: https://golang.org/pkg/syscall/#Ftruncate [2]: https://github.com/libarchive/libarchive/wiki/ManPageTar5#old-style-archive-format [3]: https://github.com/libarchive/libarchive/wiki/ManPageTar5#posix-ustar-archives Signed-off-by: W. Trevor King --- cmd/oci-image-tool/cas.go | 1 + cmd/oci-image-tool/cas_put.go | 90 ++++++++++++++++++++++ image/cas/layout/main.go | 2 +- image/cas/layout/tar.go | 30 +++++++- image/layout/tar.go | 139 ++++++++++++++++++++++++++++++++++ 5 files changed, 259 insertions(+), 3 deletions(-) create mode 100644 cmd/oci-image-tool/cas_put.go diff --git a/cmd/oci-image-tool/cas.go b/cmd/oci-image-tool/cas.go index 1411d4c61..253950d8e 100644 --- a/cmd/oci-image-tool/cas.go +++ b/cmd/oci-image-tool/cas.go @@ -28,6 +28,7 @@ func newCASCmd(stdout io.Writer, stderr *log.Logger) *cobra.Command { } cmd.AddCommand(newCASGetCmd(stdout, stderr)) + cmd.AddCommand(newCASPutCmd(stdout, stderr)) return cmd } diff --git a/cmd/oci-image-tool/cas_put.go b/cmd/oci-image-tool/cas_put.go new file mode 100644 index 000000000..28b6633db --- /dev/null +++ b/cmd/oci-image-tool/cas_put.go @@ -0,0 +1,90 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "io" + "log" + "os" + + "github.com/opencontainers/image-spec/image/cas/layout" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type casPutCmd struct { + stdout io.Writer + stderr *log.Logger + path string +} + +func newCASPutCmd(stdout io.Writer, stderr *log.Logger) *cobra.Command { + state := &casPutCmd{ + stdout: stdout, + stderr: stderr, + } + + return &cobra.Command{ + Use: "put PATH", + Short: "Write a blob to the store", + Long: "Read a blob from stdin, write it to the store, and print the digest to stdout.", + Run: state.Run, + } +} + +func (state *casPutCmd) Run(cmd *cobra.Command, args []string) { + if len(args) != 1 { + if err := cmd.Usage(); err != nil { + state.stderr.Println(err) + } + os.Exit(1) + } + + state.path = args[0] + + err := state.run() + if err != nil { + state.stderr.Println(err) + os.Exit(1) + } + + os.Exit(0) +} + +func (state *casPutCmd) run() (err error) { + ctx := context.Background() + + engine, err := layout.NewEngine(ctx, state.path) + if err != nil { + return err + } + defer engine.Close() + + digest, err := engine.Put(ctx, os.Stdin) + if err != nil { + return err + } + + n, err := fmt.Fprintln(state.stdout, digest) + if err != nil { + return err + } + if n < len(digest) { + return fmt.Errorf("wrote %d of %d bytes", n, len(digest)) + } + + return nil +} diff --git a/image/cas/layout/main.go b/image/cas/layout/main.go index 7bc4491ad..79d991ee4 100644 --- a/image/cas/layout/main.go +++ b/image/cas/layout/main.go @@ -28,7 +28,7 @@ import ( // NewEngine instantiates an engine with the appropriate backend (tar, // HTTP, ...). func NewEngine(ctx context.Context, path string) (engine cas.Engine, err error) { - file, err := os.Open(path) + file, err := os.OpenFile(path, os.O_RDWR, 0) if err != nil { return nil, err } diff --git a/image/cas/layout/tar.go b/image/cas/layout/tar.go index 25fc398c2..1ece253fa 100644 --- a/image/cas/layout/tar.go +++ b/image/cas/layout/tar.go @@ -15,10 +15,14 @@ package layout import ( + "bytes" + "crypto/sha256" + "encoding/hex" "errors" "fmt" "io" "io/ioutil" + "os" "strings" "github.com/opencontainers/image-spec/image/cas" @@ -47,8 +51,30 @@ func NewTarEngine(ctx context.Context, file ReadWriteSeekCloser) (eng cas.Engine // Put adds a new blob to the store. func (engine *TarEngine) Put(ctx context.Context, reader io.Reader) (digest string, err error) { - // FIXME - return "", errors.New("TarEngine.Put is not supported yet") + data, err := ioutil.ReadAll(reader) + if err != nil { + return "", err + } + + size := int64(len(data)) + hash := sha256.Sum256(data) + hexHash := hex.EncodeToString(hash[:]) + algorithm := "sha256" + digest = fmt.Sprintf("%s:%s", algorithm, hexHash) + + _, err = engine.Get(ctx, digest) + if err == os.ErrNotExist { + targetName := fmt.Sprintf("./blobs/%s/%s", algorithm, hexHash) + reader = bytes.NewReader(data) + err = layout.WriteTarEntryByName(ctx, engine.file, targetName, reader, &size) + if err != nil { + return "", err + } + } else if err != nil { + return "", err + } + + return digest, nil } // Get returns a reader for retrieving a blob from the store. diff --git a/image/layout/tar.go b/image/layout/tar.go index 6c9051c60..c7786cec7 100644 --- a/image/layout/tar.go +++ b/image/layout/tar.go @@ -16,11 +16,15 @@ package layout import ( "archive/tar" + "bytes" "encoding/json" "errors" "fmt" "io" + "io/ioutil" "os" + "strings" + "time" "github.com/opencontainers/image-spec/specs-go" "golang.org/x/net/context" @@ -57,6 +61,141 @@ func TarEntryByName(ctx context.Context, reader io.ReadSeeker, name string) (hea } } +// WriteTarEntryByName reads content from reader into an entry at name +// in the tarball at file, replacing a previous entry with that name +// (if any). The current implementation avoids writing a temporary +// file to disk, but risks leaving a corrupted tarball if the program +// crashes mid-write. +// +// To add an entry to a tarball (with Go's interface) you need to know +// the size ahead of time. If you set the size argument, +// WriteTarEntryByName will use that size in the entry header (and +// Go's implementation will check to make sure it matches the length +// of content read from reader). If unset, WriteTarEntryByName will +// copy reader into a local buffer, measure its size, and then write +// the entry header and content. +func WriteTarEntryByName(ctx context.Context, file io.ReadWriteSeeker, name string, reader io.Reader, size *int64) (err error) { + var buffer bytes.Buffer + tarWriter := tar.NewWriter(&buffer) + + components := strings.Split(name, "/") + if components[0] != "." { + return fmt.Errorf("tar name entry does not start with './': %q", name) + } + + var parents []string + for i := 2; i < len(components); i++ { + parents = append(parents, strings.Join(components[:i], "/")) + } + + _, err = file.Seek(0, os.SEEK_SET) + if err != nil { + return err + } + + tarReader := tar.NewReader(file) + found := false + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + var header *tar.Header + header, err = tarReader.Next() + if err == io.EOF { + break + } else if err != nil { + return err + } + + dirName := strings.TrimRight(header.Name, "/") + for i, parent := range parents { + if dirName == parent { + parents = append(parents[:i], parents[i+1:]...) + break + } + } + + if header.Name == name { + found = true + err = writeTarEntry(ctx, tarWriter, name, reader, size) + } else { + err = tarWriter.WriteHeader(header) + if err != nil { + return err + } + _, err = io.Copy(tarWriter, tarReader) + } + if err != nil { + return err + } + } + + if !found { + now := time.Now() + for _, parent := range parents { + header := &tar.Header{ + Name: parent + "/", + Mode: 0777, + ModTime: now, + Typeflag: tar.TypeDir, + } + err = tarWriter.WriteHeader(header) + if err != nil { + return err + } + } + err = writeTarEntry(ctx, tarWriter, name, reader, size) + if err != nil { + return err + } + } + + err = tarWriter.Close() + if err != nil { + return err + } + + _, err = file.Seek(0, os.SEEK_SET) + if err != nil { + return err + } + // FIXME: truncate file + + _, err = buffer.WriteTo(file) + return err +} + +func writeTarEntry(ctx context.Context, writer *tar.Writer, name string, reader io.Reader, size *int64) (err error) { + if size == nil { + var data []byte + data, err = ioutil.ReadAll(reader) + if err != nil { + return err + } + reader = bytes.NewReader(data) + _size := int64(len(data)) + size = &_size + } + now := time.Now() + header := &tar.Header{ + Name: name, + Mode: 0666, + Size: *size, + ModTime: now, + Typeflag: tar.TypeReg, + } + err = writer.WriteHeader(header) + if err != nil { + return err + } + + _, err = io.Copy(writer, reader) + return err +} + // CheckTarVersion walks a tarball pointed to by reader and returns an // error if oci-layout is missing or has unrecognized content. func CheckTarVersion(ctx context.Context, reader io.ReadSeeker) (err error) { From be239fda273220cee725201b622f5404bfc2a057 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Thu, 28 Jul 2016 21:50:06 -0700 Subject: [PATCH 10/13] image/refs: Implement Engine.Put This is pretty straightforward with the new WriteTarEntryByName helper. I considered pulling the ref name -> path conversion (%s -> ./refs/%s) out into a helper function to stay DRY, but the logic is simple enough that it seemed more intuitive to leave it inline. Signed-off-by: W. Trevor King --- cmd/oci-image-tool/refs.go | 1 + cmd/oci-image-tool/refs_put.go | 87 ++++++++++++++++++++++++++++++++++ image/refs/layout/main.go | 2 +- image/refs/layout/tar.go | 12 ++++- 4 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 cmd/oci-image-tool/refs_put.go diff --git a/cmd/oci-image-tool/refs.go b/cmd/oci-image-tool/refs.go index 5786d1d63..e5480f289 100644 --- a/cmd/oci-image-tool/refs.go +++ b/cmd/oci-image-tool/refs.go @@ -27,6 +27,7 @@ func newRefsCmd(stdout io.Writer, stderr *log.Logger) *cobra.Command { Short: "Name-based reference manipulation", } + cmd.AddCommand(newRefsPutCmd(stdout, stderr)) cmd.AddCommand(newRefsGetCmd(stdout, stderr)) cmd.AddCommand(newRefsListCmd(stdout, stderr)) diff --git a/cmd/oci-image-tool/refs_put.go b/cmd/oci-image-tool/refs_put.go new file mode 100644 index 000000000..ef90228f5 --- /dev/null +++ b/cmd/oci-image-tool/refs_put.go @@ -0,0 +1,87 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "io" + "log" + "os" + + "github.com/opencontainers/image-spec/image/refs/layout" + "github.com/opencontainers/image-spec/specs-go" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type refsPutCmd struct { + stdout io.Writer + stderr *log.Logger + path string + name string +} + +func newRefsPutCmd(stdout io.Writer, stderr *log.Logger) *cobra.Command { + state := &refsPutCmd{ + stdout: stdout, + stderr: stderr, + } + + return &cobra.Command{ + Use: "put PATH NAME", + Short: "Write a reference to the store", + Long: "Read descriptor JSON from stdin and write it to the store.", + Run: state.Run, + } +} + +func (state *refsPutCmd) Run(cmd *cobra.Command, args []string) { + if len(args) != 2 { + if err := cmd.Usage(); err != nil { + state.stderr.Println(err) + } + os.Exit(1) + } + + state.path = args[0] + state.name = args[1] + + err := state.run() + if err != nil { + state.stderr.Println(err) + os.Exit(1) + } + + os.Exit(0) +} + +func (state *refsPutCmd) run() (err error) { + ctx := context.Background() + + engine, err := layout.NewEngine(ctx, state.path) + if err != nil { + return err + } + defer engine.Close() + + decoder := json.NewDecoder(os.Stdin) + var descriptor specs.Descriptor + err = decoder.Decode(&descriptor) + if err != nil { + return err + } + + return engine.Put(ctx, state.name, &descriptor) +} diff --git a/image/refs/layout/main.go b/image/refs/layout/main.go index 94b9e5a38..98fe86e84 100644 --- a/image/refs/layout/main.go +++ b/image/refs/layout/main.go @@ -28,7 +28,7 @@ import ( // NewEngine instantiates an engine with the appropriate backend (tar, // HTTP, ...). func NewEngine(ctx context.Context, path string) (engine refs.Engine, err error) { - file, err := os.Open(path) + file, err := os.OpenFile(path, os.O_RDWR, 0) if err != nil { return nil, err } diff --git a/image/refs/layout/tar.go b/image/refs/layout/tar.go index beded71a9..d5199c003 100644 --- a/image/refs/layout/tar.go +++ b/image/refs/layout/tar.go @@ -16,6 +16,7 @@ package layout import ( "archive/tar" + "bytes" "encoding/json" "errors" "fmt" @@ -51,8 +52,15 @@ func NewTarEngine(ctx context.Context, file caslayout.ReadWriteSeekCloser) (eng // Put adds a new reference to the store. func (engine *TarEngine) Put(ctx context.Context, name string, descriptor *specs.Descriptor) (err error) { - // FIXME - return errors.New("TarEngine.Put is not supported yet") + data, err := json.Marshal(descriptor) + if err != nil { + return err + } + + size := int64(len(data)) + reader := bytes.NewReader(data) + targetName := fmt.Sprintf("./refs/%s", name) + return imagelayout.WriteTarEntryByName(ctx, engine.file, targetName, reader, &size) } // Get returns a reference from the store. From 87d6a5351fdcc738b60f5b1612ff528307f8dbe8 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Thu, 28 Jul 2016 21:34:02 -0700 Subject: [PATCH 11/13] image/layout/tar: Add a CreateTarFile helper The NewEngine commands for the tar-backed image-layout engines (both the CAS and refs engines) open files O_RDWR and expect image-layout compatible content in the tarball. That makes sense, but for folks who *don't* have such a tarball, a helper like CreateTarFile makes it easy to explicitly create an empty one. The 0666 file modes and 0777 directory modes rely on the caller's umask to appropriately limit user/group/other permissions for the tarball itself and any content extracted to the filesystem from the tarball. The trailing slashes are based on part of libarchive's description of old-style archives [1]: name Pathname, stored as a null-terminated string. Early tar implementations only stored regular files (including hardlinks to those files). One common early convention used a trailing "/" character to indicate a directory name, allowing directory permissions and owner information to be archived and restored. and POSIX ustar archives [2]: name, prefix ... The standard does not require a trailing / character on directory names, though most implementations still include this for compatibility reasons. Expose this new functionality on the command line as: $ oci-image-tool init image-layout PATH where 'init' and 'image-layout' are two separate levels in case we support initializing additional types of repositories in the future. [1]: https://github.com/libarchive/libarchive/wiki/ManPageTar5#old-style-archive-format [2]: https://github.com/libarchive/libarchive/wiki/ManPageTar5#posix-ustar-archives Signed-off-by: W. Trevor King --- cmd/oci-image-tool/init.go | 33 +++++++++++++ cmd/oci-image-tool/init_image_layout.go | 62 +++++++++++++++++++++++++ cmd/oci-image-tool/main.go | 1 + image/layout/tar.go | 47 +++++++++++++++++++ 4 files changed, 143 insertions(+) create mode 100644 cmd/oci-image-tool/init.go create mode 100644 cmd/oci-image-tool/init_image_layout.go diff --git a/cmd/oci-image-tool/init.go b/cmd/oci-image-tool/init.go new file mode 100644 index 000000000..2c0b2c979 --- /dev/null +++ b/cmd/oci-image-tool/init.go @@ -0,0 +1,33 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "io" + "log" + + "github.com/spf13/cobra" +) + +func newInitCmd(stdout io.Writer, stderr *log.Logger) *cobra.Command { + cmd := &cobra.Command{ + Use: "init", + Short: "Initialize an OCI image", + } + + cmd.AddCommand(newInitImageLayoutCmd(stdout, stderr)) + + return cmd +} diff --git a/cmd/oci-image-tool/init_image_layout.go b/cmd/oci-image-tool/init_image_layout.go new file mode 100644 index 000000000..ac91ff435 --- /dev/null +++ b/cmd/oci-image-tool/init_image_layout.go @@ -0,0 +1,62 @@ +// Copyright 2016 The Linux Foundation +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "io" + "log" + "os" + + "github.com/opencontainers/image-spec/image/layout" + "github.com/spf13/cobra" + "golang.org/x/net/context" +) + +type initImageLayout struct { + stderr *log.Logger +} + +func newInitImageLayoutCmd(stdout io.Writer, stderr *log.Logger) *cobra.Command { + state := &initImageLayout{ + stderr: stderr, + } + + return &cobra.Command{ + Use: "image-layout PATH", + Short: "Initialize an OCI image-layout repository", + Run: state.Run, + } +} + +func (state *initImageLayout) Run(cmd *cobra.Command, args []string) { + if len(args) != 1 { + if err := cmd.Usage(); err != nil { + state.stderr.Println(err) + } + os.Exit(1) + } + + path := args[0] + + ctx := context.Background() + + err := layout.CreateTarFile(ctx, path) + if err != nil { + state.stderr.Println(err) + os.Exit(1) + } + + os.Exit(0) +} diff --git a/cmd/oci-image-tool/main.go b/cmd/oci-image-tool/main.go index fcd4c7aef..18d12aed5 100644 --- a/cmd/oci-image-tool/main.go +++ b/cmd/oci-image-tool/main.go @@ -30,6 +30,7 @@ func main() { stdout := log.New(os.Stdout, "", 0) stderr := log.New(os.Stderr, "", 0) + cmd.AddCommand(newInitCmd(os.Stdout, stderr)) cmd.AddCommand(newValidateCmd(stdout, stderr)) cmd.AddCommand(newUnpackCmd(stdout, stderr)) cmd.AddCommand(newBundleCmd(stdout, stderr)) diff --git a/image/layout/tar.go b/image/layout/tar.go index c7786cec7..f67c20b3f 100644 --- a/image/layout/tar.go +++ b/image/layout/tar.go @@ -219,3 +219,50 @@ func CheckTarVersion(ctx context.Context, reader io.ReadSeeker) (err error) { return nil } + +// CreateTarFile creates a new image-layout tar file at the given path. +func CreateTarFile(ctx context.Context, path string) (err error) { + file, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) + if err != nil { + return err + } + defer file.Close() + + tarWriter := tar.NewWriter(file) + defer tarWriter.Close() + + now := time.Now() + for _, name := range []string{"./blobs/", "./refs/"} { + header := &tar.Header{ + Name: name, + Mode: 0777, + ModTime: now, + Typeflag: tar.TypeDir, + } + err = tarWriter.WriteHeader(header) + if err != nil { + return err + } + } + + imageLayoutVersion := specs.ImageLayoutVersion{ + Version: "1.0.0", + } + imageLayoutVersionBytes, err := json.Marshal(imageLayoutVersion) + if err != nil { + return err + } + header := &tar.Header{ + Name: "./oci-layout", + Mode: 0666, + Size: int64(len(imageLayoutVersionBytes)), + ModTime: now, + Typeflag: tar.TypeReg, + } + err = tarWriter.WriteHeader(header) + if err != nil { + return err + } + _, err = tarWriter.Write(imageLayoutVersionBytes) + return err +} From b9becbdd508dc7eaba50c1f620bb9584e48a6595 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Tue, 28 Jun 2016 22:08:44 -0700 Subject: [PATCH 12/13] .tool/lint: Ignore dupl complaints for cmd/oci-image-tool/(cas|refs)*.go Don't worry about: $ make lint checking lint cmd/oci-image-tool/cas_get.go:35::warning: duplicate of cmd/oci-image-tool/refs_get.go:34-45 (dupl) cmd/oci-image-tool/refs_get.go:34::warning: duplicate of cmd/oci-image-tool/refs_list.go:33-44 (dupl) cmd/oci-image-tool/refs_list.go:33::warning: duplicate of cmd/oci-image-tool/cas_get.go:35-46 (dupl) cmd/oci-image-tool/cas_get.go:35::warning: duplicate of cmd/oci-image-tool/refs_get.go:34-66 (dupl) cmd/oci-image-tool/refs_get.go:34::warning: duplicate of cmd/oci-image-tool/cas_get.go:35-67 (dupl) make: *** [lint] Error 1 The commands are all similar (open an engine, perform some method, print the result), but are short enough that extracting out helpers would be more trouble and indirection than it's worth. Oddly, dupl seems happy to print "duplicate of oci-image-tool/refs_get.go:..." and "duplicate of refs_get.go:..." if when I exclude "duplicate of cmd/oci-image-tool/refs_get.go:..." or "duplicate of .*oci-image-tool/refs_get.go:...". I want to get "oci-image-tool" in the exclusion regular expression somewhere to avoid accidentally skipping dupl checks for refs_get.go and similar if they show up somewhere else in the repository, so I'm matching on the initial filename. This commit initially landed before the (cas|refs)_put.go commands. After those commands landed I was getting: cmd/oci-image-tool/cas_put.go:34::warning: duplicate of cmd/oci-image-tool/refs_put.go:36-48 (dupl) cmd/oci-image-tool/refs_put.go:36::warning: duplicate of cmd/oci-image-tool/cas_get.go:36-48 (dupl) So I've rebased this commit to land after the put commands and added ignore lines for those files. I have not updates the line numbers from the earlier 'make lint' quote, so they may have gone stale. I expect they're similar to the quoted values if they've changed at all, since I haven't restructured those files in any major way. Signed-off-by: W. Trevor King --- .tool/lint | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.tool/lint b/.tool/lint index a3b0ecbc5..db235977f 100755 --- a/.tool/lint +++ b/.tool/lint @@ -14,6 +14,11 @@ for d in $(find . -type d -not -iwholename '*.git*' -a -not -iname '.tool' -a -n --exclude='error return value not checked.*(Close|Log|Print).*\(errcheck\)$' \ --exclude='.*_test\.go:.*error return value not checked.*\(errcheck\)$' \ --exclude='duplicate of.*_test.go.*\(dupl\)$' \ + --exclude='^cmd/oci-image-tool/cas_get.go:.* duplicate of .* \(dupl\)$' \ + --exclude='^cmd/oci-image-tool/cas_put.go:.* duplicate of .* \(dupl\)$' \ + --exclude='^cmd/oci-image-tool/refs_get.go:.* duplicate of .* \(dupl\)$' \ + --exclude='^cmd/oci-image-tool/refs_list.go:.* duplicate of .* \(dupl\)$' \ + --exclude='^cmd/oci-image-tool/refs_put.go:.* duplicate of .* \(dupl\)$' \ --exclude='schema/fs.go' \ --disable=aligncheck \ --disable=gotype \ From 7e96d03d2058567c8d98453682720866664c8c56 Mon Sep 17 00:00:00 2001 From: "W. Trevor King" Date: Wed, 31 Aug 2016 00:22:19 -0700 Subject: [PATCH 13/13] cmd/oci-image-tool/man: Document the cas, refs, and init commands Signed-off-by: W. Trevor King --- .../man/oci-image-tool-cas-get.1.md | 21 +++++++++ .../man/oci-image-tool-cas-put.1.md | 21 +++++++++ .../man/oci-image-tool-cas.1.md | 39 +++++++++++++++ .../man/oci-image-tool-init-image-layout.1.md | 21 +++++++++ .../man/oci-image-tool-init.1.md | 26 ++++++++++ .../man/oci-image-tool-refs-get.1.md | 21 +++++++++ .../man/oci-image-tool-refs-list.1.md | 21 +++++++++ .../man/oci-image-tool-refs-put.1.md | 21 +++++++++ .../man/oci-image-tool-refs.1.md | 47 +++++++++++++++++++ cmd/oci-image-tool/man/oci-image-tool.1.md | 12 +++++ 10 files changed, 250 insertions(+) create mode 100644 cmd/oci-image-tool/man/oci-image-tool-cas-get.1.md create mode 100644 cmd/oci-image-tool/man/oci-image-tool-cas-put.1.md create mode 100644 cmd/oci-image-tool/man/oci-image-tool-cas.1.md create mode 100644 cmd/oci-image-tool/man/oci-image-tool-init-image-layout.1.md create mode 100644 cmd/oci-image-tool/man/oci-image-tool-init.1.md create mode 100644 cmd/oci-image-tool/man/oci-image-tool-refs-get.1.md create mode 100644 cmd/oci-image-tool/man/oci-image-tool-refs-list.1.md create mode 100644 cmd/oci-image-tool/man/oci-image-tool-refs-put.1.md create mode 100644 cmd/oci-image-tool/man/oci-image-tool-refs.1.md diff --git a/cmd/oci-image-tool/man/oci-image-tool-cas-get.1.md b/cmd/oci-image-tool/man/oci-image-tool-cas-get.1.md new file mode 100644 index 000000000..33a9e1243 --- /dev/null +++ b/cmd/oci-image-tool/man/oci-image-tool-cas-get.1.md @@ -0,0 +1,21 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME +oci-image-tool-cas-get \- Retrieve a blob from the store + +# SYNOPSIS +**oci-image-tool cas get** [OPTIONS] PATH DIGEST + +# DESCRIPTION +`oci-image-tool cas get` retrieves a blob referenced by `DIGEST` from the store at `PATH` and writes it to standard output. + +# OPTIONS +**--help** + Print usage statement + +# SEE ALSO +**oci-image-tool-cas**(1), **oci-image-tool-cas-put**(1) + +# HISTORY +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-image-tool/man/oci-image-tool-cas-put.1.md b/cmd/oci-image-tool/man/oci-image-tool-cas-put.1.md new file mode 100644 index 000000000..2182cee9b --- /dev/null +++ b/cmd/oci-image-tool/man/oci-image-tool-cas-put.1.md @@ -0,0 +1,21 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME +oci-image-tool-cas-put \- Write a blob to the store + +# SYNOPSIS +**oci-image-tool cas put** [OPTIONS] PATH + +# DESCRIPTION +`oci-image-tool cas put` reads a blob from stdin, writes it to the store at `PATH`, and prints the digest to standard output. + +# OPTIONS +**--help** + Print usage statement + +# SEE ALSO +**oci-image-tool-cas**(1), **oci-image-tool-cas-get**(1) + +# HISTORY +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-image-tool/man/oci-image-tool-cas.1.md b/cmd/oci-image-tool/man/oci-image-tool-cas.1.md new file mode 100644 index 000000000..f6d644736 --- /dev/null +++ b/cmd/oci-image-tool/man/oci-image-tool-cas.1.md @@ -0,0 +1,39 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME +oci-image-tool-cas \- Content-addressable storage manipulation + +# SYNOPSIS +**oci-image-tool cas** [command] + +# DESCRIPTION +`oci-image-tool cas` manipulates content-addressable storage. + +# OPTIONS +**--help** + Print usage statement + +# COMMANDS +**get** + Retrieve a blob from the store. + See **oci-image-tool-cas-get**(1) for full documentation on the **get** command. + +**put** + Write a blob to the store. + See **oci-image-tool-cas-put**(1) for full documentation on the **put** command. + +# EXAMPLES +``` +$ oci-image-tool init image-layout image.tar +$ echo hello | oci-image-tool cas put image.tar +sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 +$ oci-image-tool cas get image.tar sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03 +hello +``` + +# SEE ALSO +**oci-image-tool**(1), **oci-image-tool-cas-get**(1), **oci-image-tool-cas-put**(1), **oci-image-tool-init**(1) + +# HISTORY +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-image-tool/man/oci-image-tool-init-image-layout.1.md b/cmd/oci-image-tool/man/oci-image-tool-init-image-layout.1.md new file mode 100644 index 000000000..8d571c031 --- /dev/null +++ b/cmd/oci-image-tool/man/oci-image-tool-init-image-layout.1.md @@ -0,0 +1,21 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME +oci-image-tool-init-image-layout \- Initialize an OCI image-layout repository + +# SYNOPSIS +**oci-image-tool init image-layout** [OPTIONS] PATH + +# DESCRIPTION +`oci-image-tool init image-layout` initializes an image-layout repository at `PATH`. + +# OPTIONS +**--help** + Print usage statement + +# SEE ALSO +**oci-image-tool-init**(1), **oci-image-tool-cas**(1), **oci-image-tool-refs**(1) + +# HISTORY +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-image-tool/man/oci-image-tool-init.1.md b/cmd/oci-image-tool/man/oci-image-tool-init.1.md new file mode 100644 index 000000000..a7c9a8d8d --- /dev/null +++ b/cmd/oci-image-tool/man/oci-image-tool-init.1.md @@ -0,0 +1,26 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME +oci-image-tool-init \- Initialize an OCI image + +# SYNOPSIS +**oci-image-tool init** [command] + +# DESCRIPTION +`oci-image-tool init` Initializes an OCI image. + +# OPTIONS +**--help** + Print usage statement + +# COMMANDS +**image-layout** + Initialize an OCI image-layout repository. + See **oci-image-tool-init-image-layout**(1) for full documentation on the **image-layout** command. + +# SEE ALSO +**oci-image-tool**(1), **oci-image-tool-init-image-layout**(1) + +# HISTORY +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-image-tool/man/oci-image-tool-refs-get.1.md b/cmd/oci-image-tool/man/oci-image-tool-refs-get.1.md new file mode 100644 index 000000000..e664f8b4d --- /dev/null +++ b/cmd/oci-image-tool/man/oci-image-tool-refs-get.1.md @@ -0,0 +1,21 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME +oci-image-tool-refs-get \- Retrieve a reference from the store + +# SYNOPSIS +**oci-image-tool refs get** [OPTIONS] PATH NAME + +# DESCRIPTION +`oci-image-tool refs get` retrieves reference `NAME` from the store at `PATH` and writes the JSON descriptor to standard output. + +# OPTIONS +**--help** + Print usage statement + +# SEE ALSO +**oci-image-tool-refs**(1), **oci-image-tool-refs-list**(1), **oci-image-tool-refs-put**(1) + +# HISTORY +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-image-tool/man/oci-image-tool-refs-list.1.md b/cmd/oci-image-tool/man/oci-image-tool-refs-list.1.md new file mode 100644 index 000000000..d40b4e124 --- /dev/null +++ b/cmd/oci-image-tool/man/oci-image-tool-refs-list.1.md @@ -0,0 +1,21 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME +oci-image-tool-refs-list \- Return available names from the store + +# SYNOPSIS +**oci-image-tool refs list** [OPTIONS] PATH + +# DESCRIPTION +`oci-image-tool refs list` retrieves all names from the store at `PATH` and writes them to standard output. + +# OPTIONS +**--help** + Print usage statement + +# SEE ALSO +**oci-image-tool-refs**(1), **oci-image-tool-refs-get**(1), **oci-image-tool-refs-put**(1) + +# HISTORY +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-image-tool/man/oci-image-tool-refs-put.1.md b/cmd/oci-image-tool/man/oci-image-tool-refs-put.1.md new file mode 100644 index 000000000..aaca0139b --- /dev/null +++ b/cmd/oci-image-tool/man/oci-image-tool-refs-put.1.md @@ -0,0 +1,21 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME +oci-image-tool-refs-put \- Write a reference to the store + +# SYNOPSIS +**oci-image-tool refs put** [OPTIONS] PATH NAME + +# DESCRIPTION +`oci-image-tool refs put` reads descriptor JSON from standard input and writes it to the store at `PATH` as `NAME`. + +# OPTIONS +**--help** + Print usage statement + +# SEE ALSO +**oci-image-tool-refs**(1), **oci-image-tool-refs-get**(1), **oci-image-tool-refs-list**(1) + +# HISTORY +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-image-tool/man/oci-image-tool-refs.1.md b/cmd/oci-image-tool/man/oci-image-tool-refs.1.md new file mode 100644 index 000000000..aa7883e21 --- /dev/null +++ b/cmd/oci-image-tool/man/oci-image-tool-refs.1.md @@ -0,0 +1,47 @@ +% OCI(1) OCI-IMAGE-TOOL User Manuals +% OCI Community +% AUGUST 2016 +# NAME +oci-image-tool-refs \- Name-based reference manipulation + +# SYNOPSIS +**oci-image-tool refs** [command] + +# DESCRIPTION +`oci-image-tool refs` manipulates name-based references. + +# OPTIONS +**--help** + Print usage statement + +# COMMANDS +**get** + Retrieve a reference from the store. + See **oci-image-tool-refs-get**(1) for full documentation on the **get** command. + +**list** + Return available names from the store. + See **oci-image-tool-refs-list**(1) for full documentation on the **list** command. + +**put** + Write a reference to the store. + See **oci-image-tool-refs-put**(1) for full documentation on the **put** command. + +# EXAMPLES +``` +$ oci-image-tool init image-layout image.tar +$ DIGEST=$(echo hello | oci-image-tool cas put image.tar) +$ SIZE=$(echo hello | wc -c) +$ printf '{"mediaType": "text/plain", "digest": "%s", "size": %d}' "${DIGEST}" "${SIZE}" | +> oci-image-tool refs put image.tar greeting +$ oci-image-tool refs list image.tar +greeting +$ oci-image-tool refs get image.tar greeting +{"mediaType":"text/plain","digest":"sha256:5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03","size":6} +``` + +# SEE ALSO +**oci-image-tool**(1), **oci-image-tool-cas-put**(1), **oci-image-tool-refs-get**(1), **oci-image-tool-refs-list**(1), **oci-image-tool-refs-put**(1) + +# HISTORY +August 2016, Originally compiled by W. Trevor King (wking at tremily dot us) diff --git a/cmd/oci-image-tool/man/oci-image-tool.1.md b/cmd/oci-image-tool/man/oci-image-tool.1.md index 2e6b46d64..3d58f2e4a 100644 --- a/cmd/oci-image-tool/man/oci-image-tool.1.md +++ b/cmd/oci-image-tool/man/oci-image-tool.1.md @@ -18,10 +18,22 @@ oci-image-tool \- OCI (Open Container Initiative) image tool Print usage statement # COMMANDS +**cas** + Content-addressable storage manipulation. + See **oci-image-tool-cas**(1) for full documentation on the **cas** command. + **create-runtime-bundle** Create an OCI image runtime bundle See **oci-image-tool-create-runtime-bundle(1)** for full documentation on the **create-runtime-bundle** command. +**init** + Initialize an OCI image. + See **oci-image-tool-init**(1) for full documentation on the **init** command. + +**refs** + Name-based reference manipulation. + See **oci-image-tool-refs**(1) for full documentation on the **refs** command. + **unpack** Unpack an image or image source layout See **oci-image-tool-unpack(1)** for full documentation on the **unpack** command.