diff --git a/examples/advanced/advanced.go b/examples/advanced/advanced.go new file mode 100644 index 00000000..9c4428c0 --- /dev/null +++ b/examples/advanced/advanced.go @@ -0,0 +1,182 @@ +package main + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "oras.land/oras-go/pkg/content" + "oras.land/oras-go/pkg/oras" + "oras.land/oras-go/pkg/target" +) + +func main() { + var verbose int + cmd := &cobra.Command{ + Use: fmt.Sprintf("%s [command]", os.Args[0]), + SilenceUsage: true, + PersistentPreRun: func(cmd *cobra.Command, args []string) { + log.SetLevel(log.InfoLevel) + if verbose > 1 { + log.SetLevel(log.DebugLevel) + } + }, + } + cmd.AddCommand(copyCmd()) + cmd.PersistentFlags().IntVarP(&verbose, "verbose", "v", 1, "set log level") + if err := cmd.Execute(); err != nil { + os.Exit(1) + } +} + +func copyCmd() *cobra.Command { + var ( + fromStr, toStr string + manifestConfig string + opts content.RegistryOptions + ) + cmd := &cobra.Command{ + Use: "copy ", + Short: "Copy artifacts from one location to another", + Long: `Copy artifacts from one location to another +Example - Copy artifacts from local files to local files: + oras copy foo/bar:v1 --from files --to files:path/to/save file1 file2 ... filen +Example - Copy artifacts from registry to local files: + oras copy foo/bar:v1 --from registry --to files:path/to/save +Example - Copy artifacts from registry to oci: + oras copy foo/bar:v1 --from registry --to oci:path/to/oci +Example - Copy artifacts from local files to registry: + oras copy foo/bar:v1 --from files --to registry file1 file2 ... filen + +When the source (--from) is "files", the config by default will be "{}" and of media type +application/vnd.unknown.config.v1+json. You can override it by setting the path, for example: + + oras copy foo/bar:v1 --from files --manifest-config path/to/config:application/vnd.oci.image.config.v1+json --to files:path/to/save file1 file2 ... filen + + +`, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + var ( + ref = args[0] + err error + from, to target.Target + ) + // get the fromStr; it might also have a ':' to add options + fromParts := strings.SplitN(fromStr, ":", 2) + toParts := strings.SplitN(toStr, ":", 2) + switch fromParts[0] { + case "files": + fromFile := content.NewFile("") + descs, err := loadFiles(fromFile, args[1:]...) + if err != nil { + return fmt.Errorf("unable to load files: %v", err) + } + // parse the manifest config + manifestConfigParts := strings.SplitN(manifestConfig, ":", 2) + var manifestConfigPath, manifestConfigMediaType string + switch len(manifestConfigParts) { + case 1: + manifestConfigPath = manifestConfigParts[0] + case 2: + manifestConfigPath = manifestConfigParts[0] + manifestConfigMediaType = manifestConfigParts[1] + } + if err != nil { + return fmt.Errorf("error reading manifest config at %s: %v", manifestConfigPath, err) + } + configDesc, err := fromFile.Add("", manifestConfigMediaType, manifestConfigPath) + if err != nil { + return fmt.Errorf("unable to load manifest config: %v", err) + } + if _, err := fromFile.GenerateManifest(ref, &configDesc, descs...); err != nil { + return fmt.Errorf("unable to generate root manifest: %s", err) + } + rootDesc, rootManifest, err := fromFile.Ref(ref) + if err != nil { + return err + } + log.Debugf("root manifest: %s %v %s", ref, rootDesc, rootManifest) + from = fromFile + case "registry": + from, err = content.NewRegistry(opts) + if err != nil { + return fmt.Errorf("could not create registry target: %v", err) + } + case "oci": + from, err = content.NewOCI(fromParts[1]) + if err != nil { + return fmt.Errorf("could not read OCI layout at %s: %v", fromParts[1], err) + } + default: + return fmt.Errorf("unknown from argyment: %s", from) + } + + switch toParts[0] { + case "files": + to = content.NewFile(toParts[1]) + case "registry": + to, err = content.NewRegistry(opts) + if err != nil { + return fmt.Errorf("could not create registry target: %v", err) + } + case "oci": + to, err = content.NewOCI(toParts[1]) + if err != nil { + return fmt.Errorf("could not read OCI layout at %s: %v", toParts[1], err) + } + default: + return fmt.Errorf("unknown from argyment: %s", from) + } + + if manifestConfig != "" && fromParts[0] != "files" { + return fmt.Errorf("only specify --manifest-config when using --from files") + } + return runCopy(ref, from, to) + }, + } + cmd.Flags().StringVar(&fromStr, "from", "", "source type and possible options") + cmd.MarkFlagRequired("from") + cmd.Flags().StringVar(&toStr, "to", "", "destination type and possible options") + cmd.MarkFlagRequired("to") + cmd.Flags().StringArrayVarP(&opts.Configs, "config", "c", nil, "auth config path") + cmd.Flags().StringVarP(&opts.Username, "username", "u", "", "registry username") + cmd.Flags().StringVarP(&opts.Password, "password", "p", "", "registry password") + cmd.Flags().BoolVarP(&opts.Insecure, "insecure", "", false, "allow connections to SSL registry without certs") + cmd.Flags().BoolVarP(&opts.PlainHTTP, "plain-http", "", false, "use plain http and not https") + cmd.Flags().StringVar(&manifestConfig, "manifest-config", "", "path to manifest config and its media type, e.g. path/to/file.json:application/vnd.oci.image.config.v1+json") + return cmd +} + +func runCopy(ref string, from, to target.Target) error { + desc, err := oras.Copy(context.Background(), from, ref, to, "") + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v", err) + os.Exit(1) + } + fmt.Printf("%#v\n", desc) + return nil +} + +func loadFiles(store *content.File, files ...string) ([]ocispec.Descriptor, error) { + var descs []ocispec.Descriptor + for _, fileRef := range files { + filename, mediaType := parseFileRef(fileRef, "") + name := filepath.Clean(filename) + if !filepath.IsAbs(name) { + // convert to slash-separated path unless it is absolute path + name = filepath.ToSlash(name) + } + desc, err := store.Add(name, mediaType, filename) + if err != nil { + return nil, err + } + descs = append(descs, desc) + } + return descs, nil +} diff --git a/examples/advanced/file_unix.go b/examples/advanced/file_unix.go new file mode 100644 index 00000000..0cbb9401 --- /dev/null +++ b/examples/advanced/file_unix.go @@ -0,0 +1,13 @@ +// +build !windows + +package main + +import "strings" + +func parseFileRef(ref string, mediaType string) (string, string) { + i := strings.LastIndex(ref, ":") + if i < 0 { + return ref, mediaType + } + return ref[:i], ref[i+1:] +} diff --git a/examples/advanced/file_windows.go b/examples/advanced/file_windows.go new file mode 100644 index 00000000..88c4fd08 --- /dev/null +++ b/examples/advanced/file_windows.go @@ -0,0 +1,26 @@ +package main + +import ( + "strings" + "unicode" +) + +// parseFileRef parse file reference on windows. +// Windows systems does not allow ':' in the file path except for drive letter. +func parseFileRef(ref string, mediaType string) (string, string) { + i := strings.Index(ref, ":") + if i < 0 { + return ref, mediaType + } + + // In case it is C:\ + if i == 1 && len(ref) > 2 && ref[2] == '\\' && unicode.IsLetter(rune(ref[0])) { + i = strings.Index(ref[3:], ":") + if i < 0 { + return ref, mediaType + } + i += 3 + } + + return ref[:i], ref[i+1:] +} diff --git a/examples/simple_push_pull.go b/examples/simple/simple_push_pull.go similarity index 74% rename from examples/simple_push_pull.go rename to examples/simple/simple_push_pull.go index a2008000..4f54ecd0 100644 --- a/examples/simple_push_pull.go +++ b/examples/simple/simple_push_pull.go @@ -20,9 +20,6 @@ import ( "fmt" "os" - "github.com/containerd/containerd/remotes/docker" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "oras.land/oras-go/pkg/content" "oras.land/oras-go/pkg/oras" ) @@ -48,23 +45,25 @@ func main() { customMediaType := "my.custom.media.type" ctx := context.Background() - resolver := docker.NewResolver(docker.ResolverOptions{PlainHTTP: true}) // Push file(s) w custom mediatype to registry - memoryStore := content.NewMemoryStore() - desc := memoryStore.Add(fileName, customMediaType, fileContent) - pushContents := []ocispec.Descriptor{desc} + memoryStore := content.NewMemory() + desc, err := memoryStore.Add(fileName, customMediaType, fileContent) + check(err) + _, err = memoryStore.GenerateManifest(ref, nil, desc) + check(err) + registry, err := content.NewRegistry(content.RegistryOptions{PlainHTTP: true}) fmt.Printf("Pushing %s to %s...\n", fileName, ref) - desc, err := oras.Push(ctx, resolver, ref, memoryStore, pushContents) + desc, err = oras.Copy(ctx, memoryStore, ref, registry, "") check(err) fmt.Printf("Pushed to %s with digest %s\n", ref, desc.Digest) // Pull file(s) from registry and save to disk fmt.Printf("Pulling from %s and saving to %s...\n", ref, fileName) - fileStore := content.NewFileStore("") + fileStore := content.NewFile("") defer fileStore.Close() allowedMediaTypes := []string{customMediaType} - desc, _, err = oras.Pull(ctx, resolver, ref, fileStore, oras.WithAllowedMediaTypes(allowedMediaTypes)) + desc, err = oras.Copy(ctx, registry, ref, fileStore, "", oras.WithAllowedMediaTypes(allowedMediaTypes)) check(err) fmt.Printf("Pulled from %s with digest %s\n", ref, desc.Digest) fmt.Printf("Try running 'cat %s'\n", fileName) diff --git a/go.mod b/go.mod index b8c1a30e..9d1d8e61 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.8.1 + github.com/spf13/cobra v1.0.0 // indirect github.com/stretchr/testify v1.7.0 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a golang.org/x/sync v0.0.0-20210220032951-036812b2e83c diff --git a/pkg/content/content_test.go b/pkg/content/content_test.go index 417cd079..129be4fc 100644 --- a/pkg/content/content_test.go +++ b/pkg/content/content_test.go @@ -23,7 +23,7 @@ import ( "path/filepath" "testing" - "github.com/containerd/containerd/content" + "github.com/containerd/containerd/remotes" digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/stretchr/testify/suite" @@ -31,8 +31,8 @@ import ( type ContentTestSuite struct { suite.Suite - TestMemoryStore *Memorystore - TestFileStore *FileStore + TestMemoryStore *Memory + TestFileStore *File } var ( @@ -57,40 +57,46 @@ var ( ) func (suite *ContentTestSuite) SetupSuite() { - testMemoryStore := NewMemoryStore() - testMemoryStore.Add(testRef, "", testContent) + testMemoryStore := NewMemory() + desc, err := testMemoryStore.Add(testRef, "", testContent) + suite.Nil(err, "no error adding testContent to memory store") + _, err = testMemoryStore.GenerateManifest(testRef, nil, desc) + suite.Nil(err, "no error adding ref to memory store") suite.TestMemoryStore = testMemoryStore os.Remove(testFileName) - err := ioutil.WriteFile(testFileName, testContent, 0644) + err = ioutil.WriteFile(testFileName, testContent, 0644) suite.Nil(err, "no error creating test file on disk") - testFileStore := NewFileStore(testDirRoot, WithErrorOnNoName()) - _, err = testFileStore.Add(testRef, "", testFileName) + testFileStore := NewFile(testDirRoot, WithErrorOnNoName()) + desc, err = testFileStore.Add(testRef, "", testFileName) suite.Nil(err, "no error adding item to file store") + _, err = testFileStore.GenerateManifest(testRef, nil, desc) + suite.Nil(err, "no error adding ref to file store") suite.TestFileStore = testFileStore } // Tests all Writers (Ingesters) func (suite *ContentTestSuite) Test_0_Ingesters() { - ingesters := map[string]content.Ingester{ - "memory": suite.TestMemoryStore, - "file": suite.TestFileStore, + + ctx := context.Background() + memPusher, _ := suite.TestMemoryStore.Pusher(ctx, "") + filePusher, _ := suite.TestFileStore.Pusher(ctx, "") + ingesters := map[string]remotes.Pusher{ + "memory": memPusher, + "file": filePusher, } for key, ingester := range ingesters { // Bad ref - ctx := context.Background() - refOpt := content.WithDescriptor(testBadDescriptor) - writer, err := ingester.Writer(ctx, refOpt) + writer, err := ingester.Push(ctx, testBadDescriptor) if key == "file" { suite.NotNil(err, fmt.Sprintf("no error getting writer w bad ref for %s store", key)) } // Good ref ctx = context.Background() - refOpt = content.WithDescriptor(testDescriptor) - writer, err = ingester.Writer(ctx, refOpt) + writer, err = ingester.Push(ctx, testDescriptor) suite.Nil(err, fmt.Sprintf("no error getting writer w good ref for %s store", key)) _, err = writer.Write(testContent) suite.Nil(err, fmt.Sprintf("no error using writer.Write w good ref for %s store", key)) @@ -110,7 +116,7 @@ func (suite *ContentTestSuite) Test_0_Ingesters() { suite.NotNil(err, fmt.Sprintf("error using writer.Commit when closed w good ref for %s store", key)) // re-init writer after closing - writer, _ = ingester.Writer(ctx, refOpt) + writer, _ = ingester.Push(ctx, testDescriptor) writer.Write(testContent) // invalid truncate size @@ -129,7 +135,6 @@ func (suite *ContentTestSuite) Test_0_Ingesters() { suite.NotNil(err, fmt.Sprintf("error using writer.Commit w bad size, good ref for %s store", key)) // bad digest - writer, _ = ingester.Writer(ctx, refOpt) err = writer.Commit(ctx, 0, testBadDescriptor.Digest) suite.NotNil(err, fmt.Sprintf("error using writer.Commit w bad digest, good ref for %s store", key)) } @@ -137,9 +142,12 @@ func (suite *ContentTestSuite) Test_0_Ingesters() { // Tests all Readers (Providers) func (suite *ContentTestSuite) Test_1_Providers() { - providers := map[string]content.Provider{ - "memory": suite.TestMemoryStore, - "file": suite.TestFileStore, + ctx := context.Background() + memFetcher, _ := suite.TestMemoryStore.Fetcher(ctx, testRef) + fileFetcher, _ := suite.TestFileStore.Fetcher(ctx, testRef) + providers := map[string]remotes.Fetcher{ + "memory": memFetcher, + "file": fileFetcher, } // Readers (Providers) @@ -147,26 +155,23 @@ func (suite *ContentTestSuite) Test_1_Providers() { // Bad ref ctx := context.Background() - _, err := provider.ReaderAt(ctx, testBadDescriptor) + _, err := provider.Fetch(ctx, testBadDescriptor) suite.NotNil(err, fmt.Sprintf("error with bad ref for %s store", key)) // Good ref ctx = context.Background() - readerAt, err := provider.ReaderAt(ctx, testDescriptor) + reader, err := provider.Fetch(ctx, testDescriptor) suite.Nil(err, fmt.Sprintf("no error with good ref for %s store", key)) - // readerat Size() - suite.Equal(testDescriptor.Size, readerAt.Size(), fmt.Sprintf("readerat size matches for %s store", key)) - // readerat Close() - err = readerAt.Close() - suite.Nil(err, fmt.Sprintf("no error closing readerat for %s store", key)) + err = reader.Close() + suite.Nil(err, fmt.Sprintf("no error closing reader for %s store", key)) // file missing if key == "file" { os.Remove(testFileName) ctx := context.Background() - _, err := provider.ReaderAt(ctx, testDescriptor) + _, err := provider.Fetch(ctx, testDescriptor) suite.NotNil(err, fmt.Sprintf("error with good ref for %s store (file missing)", key)) } } diff --git a/pkg/content/decompressstore.go b/pkg/content/decompress.go similarity index 75% rename from pkg/content/decompressstore.go rename to pkg/content/decompress.go index 3912fc2f..ecde9c9a 100644 --- a/pkg/content/decompressstore.go +++ b/pkg/content/decompress.go @@ -21,19 +21,21 @@ import ( "strings" ctrcontent "github.com/containerd/containerd/content" + "github.com/containerd/containerd/remotes" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// DecompressStore store to decompress content and extract from tar, if needed, wrapping +// Decompress store to decompress content and extract from tar, if needed, wrapping // another store. By default, a FileStore will simply take each artifact and write it to // a file, as a MemoryStore will do into memory. If the artifact is gzipped or tarred, // you might want to store the actual object inside tar or gzip. Wrap your Store -// with DecompressStore, and it will check the media-type and, if relevant, +// with Decompress, and it will check the media-type and, if relevant, // gunzip and/or untar. // // For example: // // fileStore := NewFileStore(rootPath) -// decompressStore := store.NewDecompressStore(fileStore, WithBlocksize(blocksize)) +// Decompress := store.NewDecompress(fileStore, WithBlocksize(blocksize)) // // The above example works if there is no tar, i.e. each artifact is just a single file, perhaps gzipped, // or if there is only one file in each tar archive. In other words, when each content.Writer has only one target output stream. @@ -42,15 +44,15 @@ import ( // target output stream. In that case, use the following example: // // multiStore := NewMultiStore(rootPath) // some store that can handle different filenames -// decompressStore := store.NewDecompressStore(multiStore, WithBlocksize(blocksize), WithMultiWriterIngester()) +// Decompress := store.NewDecompress(multiStore, WithBlocksize(blocksize), WithMultiWriterIngester()) // -type DecompressStore struct { - ingester ctrcontent.Ingester +type Decompress struct { + pusher remotes.Pusher blocksize int multiWriterIngester bool } -func NewDecompressStore(ingester ctrcontent.Ingester, opts ...WriterOpt) DecompressStore { +func NewDecompress(pusher remotes.Pusher, opts ...WriterOpt) Decompress { // we have to reprocess the opts to find the blocksize var wOpts WriterOpts for _, opt := range opts { @@ -60,50 +62,41 @@ func NewDecompressStore(ingester ctrcontent.Ingester, opts ...WriterOpt) Decompr } } - return DecompressStore{ingester, wOpts.Blocksize, wOpts.MultiWriterIngester} + return Decompress{pusher, wOpts.Blocksize, wOpts.MultiWriterIngester} } -// Writer get a writer -func (d DecompressStore) Writer(ctx context.Context, opts ...ctrcontent.WriterOpt) (ctrcontent.Writer, error) { +// Push get a content.Writer +func (d Decompress) Push(ctx context.Context, desc ocispec.Descriptor) (ctrcontent.Writer, error) { // the logic is straightforward: // - if there is a desc in the opts, and the mediatype is tar or tar+gzip, then pass the correct decompress writer // - else, pass the regular writer var ( writer ctrcontent.Writer err error - multiIngester MultiWriterIngester + multiIngester MultiWriterPusher ok bool ) // check to see if we are supposed to use a MultiWriterIngester if d.multiWriterIngester { - multiIngester, ok = d.ingester.(MultiWriterIngester) + multiIngester, ok = d.pusher.(MultiWriterPusher) if !ok { return nil, errors.New("configured to use multiwriter ingester, but ingester does not implement multiwriter") } } - // we have to reprocess the opts to find the desc - var wOpts ctrcontent.WriterOpts - for _, opt := range opts { - if err := opt(&wOpts); err != nil { - return nil, err - } - } // figure out if compression and/or archive exists - desc := wOpts.Desc // before we pass it down, we need to strip anything we are removing here // and possibly update the digest, since the store indexes things by digest hasGzip, hasTar, modifiedMediaType := checkCompression(desc.MediaType) - wOpts.Desc.MediaType = modifiedMediaType - opts = append(opts, ctrcontent.WithDescriptor(wOpts.Desc)) + desc.MediaType = modifiedMediaType // determine if we pass it blocksize, only if positive writerOpts := []WriterOpt{} if d.blocksize > 0 { writerOpts = append(writerOpts, WithBlocksize(d.blocksize)) } - writer, err = d.ingester.Writer(ctx, opts...) + writer, err = d.pusher.Push(ctx, desc) if err != nil { return nil, err } @@ -114,7 +107,7 @@ func (d DecompressStore) Writer(ctx context.Context, opts ...ctrcontent.WriterOp if multiIngester == nil { writer = NewUntarWriter(writer, writerOpts...) } else { - writers, err := multiIngester.Writers(ctx, opts...) + writers, err := multiIngester.Pushers(ctx, desc) if err != nil { return nil, err } @@ -123,7 +116,7 @@ func (d DecompressStore) Writer(ctx context.Context, opts ...ctrcontent.WriterOp } if hasGzip { if writer == nil { - writer, err = d.ingester.Writer(ctx, opts...) + writer, err = d.pusher.Push(ctx, desc) if err != nil { return nil, err } diff --git a/pkg/content/decompressstore_test.go b/pkg/content/decompress_test.go similarity index 89% rename from pkg/content/decompressstore_test.go rename to pkg/content/decompress_test.go index b69c2b8f..67dbffee 100644 --- a/pkg/content/decompressstore_test.go +++ b/pkg/content/decompress_test.go @@ -22,7 +22,6 @@ import ( "fmt" "testing" - ctrcontent "github.com/containerd/containerd/content" digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -50,10 +49,11 @@ func TestDecompressStore(t *testing.T) { Size: int64(len(gzipContent)), } - memStore := content.NewMemoryStore() - decompressStore := content.NewDecompressStore(memStore, content.WithBlocksize(0)) + memStore := content.NewMemory() ctx := context.Background() - decompressWriter, err := decompressStore.Writer(ctx, ctrcontent.WithDescriptor(gzipDescriptor)) + memPusher, _ := memStore.Pusher(ctx, "") + decompressStore := content.NewDecompress(memPusher, content.WithBlocksize(0)) + decompressWriter, err := decompressStore.Push(ctx, gzipDescriptor) if err != nil { t.Fatalf("unable to get a decompress writer: %v", err) } diff --git a/pkg/content/file.go b/pkg/content/file.go index 19161d59..b51ec1a4 100644 --- a/pkg/content/file.go +++ b/pkg/content/file.go @@ -1,45 +1,33 @@ -/* -Copyright The ORAS Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package content import ( + "bytes" "compress/gzip" "context" + _ "crypto/sha256" + "encoding/json" + "fmt" "io" "io/ioutil" "os" "path/filepath" + "sort" "strings" "sync" "time" "github.com/containerd/containerd/content" "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/remotes" digest "github.com/opencontainers/go-digest" + specs "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + artifact "oras.land/oras-go/pkg/artifact" ) -// ensure interface -var ( - _ ProvideIngester = &FileStore{} -) - -// FileStore provides content from the file system -type FileStore struct { +// File provides content via files from the file system +type File struct { DisableOverwrite bool AllowPathTraversalOnWrite bool @@ -48,13 +36,15 @@ type FileStore struct { root string descriptor *sync.Map // map[digest.Digest]ocispec.Descriptor - pathMap *sync.Map + pathMap *sync.Map // map[name string](file string) + memoryMap *sync.Map // map[digest.Digest]([]byte) + refMap *sync.Map // map[string]ocispec.Descriptor tmpFiles *sync.Map ignoreNoName bool } -// NewFileStore creates a new file store -func NewFileStore(rootPath string, opts ...WriterOpt) *FileStore { +// NewFile creats a new file target. It represents a single root reference and all of its components. +func NewFile(rootPath string, opts ...WriterOpt) *File { // we have to process the opts to find if they told us to change defaults wOpts := DefaultWriterOpts() for _, opt := range opts { @@ -62,17 +52,116 @@ func NewFileStore(rootPath string, opts ...WriterOpt) *FileStore { continue } } - return &FileStore{ + return &File{ root: rootPath, descriptor: &sync.Map{}, pathMap: &sync.Map{}, + memoryMap: &sync.Map{}, + refMap: &sync.Map{}, tmpFiles: &sync.Map{}, ignoreNoName: wOpts.IgnoreNoName, } } -// Add adds a file reference -func (s *FileStore) Add(name, mediaType, path string) (ocispec.Descriptor, error) { +func (s *File) Resolver() remotes.Resolver { + return s +} + +func (s *File) Resolve(ctx context.Context, ref string) (name string, desc ocispec.Descriptor, err error) { + desc, ok := s.getRef(ref) + if !ok { + return "", ocispec.Descriptor{}, fmt.Errorf("unknown reference: %s", ref) + } + return ref, desc, nil +} + +func (s *File) Fetcher(ctx context.Context, ref string) (remotes.Fetcher, error) { + if _, ok := s.refMap.Load(ref); !ok { + return nil, fmt.Errorf("unknown reference: %s", ref) + } + return s, nil +} + +// Fetch get an io.ReadCloser for the specific content +func (s *File) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) { + // first see if it is in the in-memory manifest map + manifest, ok := s.getMemory(desc) + if ok { + return ioutil.NopCloser(bytes.NewReader(manifest)), nil + } + desc, ok = s.get(desc) + if !ok { + return nil, ErrNotFound + } + name, ok := ResolveName(desc) + if !ok { + return nil, ErrNoName + } + path := s.ResolvePath(name) + return os.Open(path) +} + +func (s *File) Pusher(ctx context.Context, ref string) (remotes.Pusher, error) { + var tag, hash string + parts := strings.SplitN(ref, "@", 2) + if len(parts) > 0 { + tag = parts[0] + } + if len(parts) > 1 { + hash = parts[1] + } + return &filePusher{ + store: s, + ref: tag, + hash: hash, + }, nil +} + +type filePusher struct { + store *File + ref string + hash string +} + +func (s *filePusher) Push(ctx context.Context, desc ocispec.Descriptor) (content.Writer, error) { + name, ok := ResolveName(desc) + now := time.Now() + if !ok { + // if we were not told to ignore NoName, then return an error + if !s.store.ignoreNoName { + return nil, ErrNoName + } + + // just return a nil writer - we do not want to calculate the hash, so just use + // whatever was passed in the descriptor + return NewIoContentWriter(ioutil.Discard, WithOutputHash(desc.Digest)), nil + } + path, err := s.store.resolveWritePath(name) + if err != nil { + return nil, err + } + file, afterCommit, err := s.store.createWritePath(path, desc, name) + if err != nil { + return nil, err + } + + return &fileWriter{ + store: s.store, + file: file, + desc: desc, + digester: digest.Canonical.Digester(), + status: content.Status{ + Ref: name, + Total: desc.Size, + StartedAt: now, + UpdatedAt: now, + }, + afterCommit: afterCommit, + }, nil +} + +// Add adds a file reference, updating the root +func (s *File) Add(name, mediaType, path string) (ocispec.Descriptor, error) { if path == "" { path = name } @@ -101,7 +190,21 @@ func (s *FileStore) Add(name, mediaType, path string) (ocispec.Descriptor, error return desc, nil } -func (s *FileStore) descFromFile(info os.FileInfo, mediaType, path string) (ocispec.Descriptor, error) { +// Ref gets a reference's descriptor and content +func (s *File) Ref(ref string) (ocispec.Descriptor, []byte, error) { + desc, ok := s.getRef(ref) + if !ok { + return ocispec.Descriptor{}, nil, ErrNotFound + } + // first see if it is in the in-memory manifest map + manifest, ok := s.getMemory(desc) + if !ok { + return ocispec.Descriptor{}, nil, ErrNotFound + } + return desc, manifest, nil +} + +func (s *File) descFromFile(info os.FileInfo, mediaType, path string) (ocispec.Descriptor, error) { file, err := os.Open(path) if err != nil { return ocispec.Descriptor{}, err @@ -122,7 +225,7 @@ func (s *FileStore) descFromFile(info os.FileInfo, mediaType, path string) (ocis }, nil } -func (s *FileStore) descFromDir(name, mediaType, root string) (ocispec.Descriptor, error) { +func (s *File) descFromDir(name, mediaType, root string) (ocispec.Descriptor, error) { // generate temp file file, err := s.tempFile() if err != nil { @@ -167,7 +270,7 @@ func (s *FileStore) descFromDir(name, mediaType, root string) (ocispec.Descripto }, nil } -func (s *FileStore) tempFile() (*os.File, error) { +func (s *File) tempFile() (*os.File, error) { file, err := ioutil.TempFile("", TempFilePattern) if err != nil { return nil, err @@ -177,7 +280,7 @@ func (s *FileStore) tempFile() (*os.File, error) { } // Close frees up resources used by the file store -func (s *FileStore) Close() error { +func (s *File) Close() error { var errs []string s.tmpFiles.Range(func(name, _ interface{}) bool { if err := os.Remove(name.(string)); err != nil { @@ -188,75 +291,7 @@ func (s *FileStore) Close() error { return errors.New(strings.Join(errs, "; ")) } -// ReaderAt provides contents -func (s *FileStore) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content.ReaderAt, error) { - desc, ok := s.get(desc) - if !ok { - return nil, ErrNotFound - } - name, ok := ResolveName(desc) - if !ok { - return nil, ErrNoName - } - path := s.ResolvePath(name) - file, err := os.Open(path) - if err != nil { - return nil, err - } - - return sizeReaderAt{ - readAtCloser: file, - size: desc.Size, - }, nil -} - -// Writer begins or resumes the active writer identified by desc -func (s *FileStore) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) { - var wOpts content.WriterOpts - for _, opt := range opts { - if err := opt(&wOpts); err != nil { - return nil, err - } - } - desc := wOpts.Desc - - name, ok := ResolveName(desc) - if !ok { - // if we were not told to ignore NoName, then return an error - if !s.ignoreNoName { - return nil, ErrNoName - } - - // just return a nil writer - we do not want to calculate the hash, so just use - // whatever was passed in the descriptor - return NewIoContentWriter(ioutil.Discard, WithOutputHash(desc.Digest)), nil - } - path, err := s.resolveWritePath(name) - if err != nil { - return nil, err - } - file, afterCommit, err := s.createWritePath(path, desc, name) - if err != nil { - return nil, err - } - - now := time.Now() - return &fileWriter{ - store: s, - file: file, - desc: desc, - digester: digest.Canonical.Digester(), - status: content.Status{ - Ref: name, - Total: desc.Size, - StartedAt: now, - UpdatedAt: now, - }, - afterCommit: afterCommit, - }, nil -} - -func (s *FileStore) resolveWritePath(name string) (string, error) { +func (s *File) resolveWritePath(name string) (string, error) { path := s.ResolvePath(name) if !s.AllowPathTraversalOnWrite { base, err := filepath.Abs(s.root) @@ -286,7 +321,7 @@ func (s *FileStore) resolveWritePath(name string) (string, error) { return path, nil } -func (s *FileStore) createWritePath(path string, desc ocispec.Descriptor, prefix string) (*os.File, func() error, error) { +func (s *File) createWritePath(path string, desc ocispec.Descriptor, prefix string) (*os.File, func() error, error) { if value, ok := desc.Annotations[AnnotationUnpack]; !ok || value != "true" { if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { return nil, nil, err @@ -307,14 +342,14 @@ func (s *FileStore) createWritePath(path string, desc ocispec.Descriptor, prefix } // MapPath maps name to path -func (s *FileStore) MapPath(name, path string) string { +func (s *File) MapPath(name, path string) string { path = s.resolvePath(path) s.pathMap.Store(name, path) return path } // ResolvePath returns the path by name -func (s *FileStore) ResolvePath(name string) string { +func (s *File) ResolvePath(name string) string { if value, ok := s.pathMap.Load(name); ok { if path, ok := value.(string); ok { return path @@ -325,18 +360,18 @@ func (s *FileStore) ResolvePath(name string) string { return s.resolvePath(name) } -func (s *FileStore) resolvePath(path string) string { +func (s *File) resolvePath(path string) string { if filepath.IsAbs(path) { return path } return filepath.Join(s.root, path) } -func (s *FileStore) set(desc ocispec.Descriptor) { +func (s *File) set(desc ocispec.Descriptor) { s.descriptor.Store(desc.Digest, desc) } -func (s *FileStore) get(desc ocispec.Descriptor) (ocispec.Descriptor, bool) { +func (s *File) get(desc ocispec.Descriptor) (ocispec.Descriptor, bool) { value, ok := s.descriptor.Load(desc.Digest) if !ok { return ocispec.Descriptor{}, false @@ -345,8 +380,52 @@ func (s *FileStore) get(desc ocispec.Descriptor) (ocispec.Descriptor, bool) { return desc, ok } +func (s *File) getMemory(desc ocispec.Descriptor) ([]byte, bool) { + value, ok := s.memoryMap.Load(desc.Digest) + if !ok { + return nil, false + } + content, ok := value.([]byte) + return content, ok +} + +func (s *File) getRef(ref string) (ocispec.Descriptor, bool) { + value, ok := s.refMap.Load(ref) + if !ok { + return ocispec.Descriptor{}, false + } + desc, ok := value.(ocispec.Descriptor) + return desc, ok +} + +func (s *File) GenerateManifest(ref string, config *ocispec.Descriptor, descs ...ocispec.Descriptor) ([]byte, error) { + var ( + desc ocispec.Descriptor + manifest []byte + err error + ) + // Config + // Config - either it was set, or we have to set it + if config == nil { + configBytes := []byte("{}") + dig := digest.FromBytes(configBytes) + config = &ocispec.Descriptor{ + MediaType: artifact.UnknownConfigMediaType, + Digest: dig, + Size: int64(len(configBytes)), + } + s.memoryMap.Store(dig, configBytes) + } + if manifest, desc, err = pack(*config, descs); err != nil { + return nil, err + } + s.refMap.Store(ref, desc) + s.memoryMap.Store(desc.Digest, manifest) + return manifest, nil +} + type fileWriter struct { - store *FileStore + store *File file *os.File desc ocispec.Descriptor digester digest.Digester @@ -440,3 +519,32 @@ func (w *fileWriter) Truncate(size int64) error { } return w.file.Truncate(0) } + +// pack given a bunch of descriptors, create a manifest that references all of them +func pack(config ocispec.Descriptor, descriptors []ocispec.Descriptor) ([]byte, ocispec.Descriptor, error) { + if descriptors == nil { + descriptors = []ocispec.Descriptor{} // make it an empty array to prevent potential server-side bugs + } + // sort descriptors alphanumerically by sha hash so it always is consistent + sort.Slice(descriptors, func(i, j int) bool { + return descriptors[i].Digest < descriptors[j].Digest + }) + manifest := ocispec.Manifest{ + Versioned: specs.Versioned{ + SchemaVersion: 2, // historical value. does not pertain to OCI or docker version + }, + Config: config, + Layers: descriptors, + } + manifestBytes, err := json.Marshal(manifest) + if err != nil { + return nil, ocispec.Descriptor{}, err + } + manifestDescriptor := ocispec.Descriptor{ + MediaType: ocispec.MediaTypeImageManifest, + Digest: digest.FromBytes(manifestBytes), + Size: int64(len(manifestBytes)), + } + + return manifestBytes, manifestDescriptor, nil +} diff --git a/pkg/content/file_test.go b/pkg/content/file_test.go index 781dc55f..765f38fa 100644 --- a/pkg/content/file_test.go +++ b/pkg/content/file_test.go @@ -21,7 +21,6 @@ import ( "os" "testing" - ctrcontent "github.com/containerd/containerd/content" digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -50,10 +49,10 @@ func TestFileStoreNoName(t *testing.T) { t.Fatalf("error creating tempdir: %v", err) } defer os.RemoveAll(rootPath) - fileStore := content.NewFileStore(rootPath, tt.opts...) + fileStore := content.NewFile(rootPath, tt.opts...) ctx := context.Background() - refOpt := ctrcontent.WithDescriptor(descriptor) - if _, err := fileStore.Writer(ctx, refOpt); err != tt.err { + pusher, _ := fileStore.Pusher(ctx, "") + if _, err := pusher.Push(ctx, descriptor); err != tt.err { t.Errorf("mismatched error, actual '%v', expected '%v'", err, tt.err) } diff --git a/pkg/content/interface.go b/pkg/content/interface.go index b9daee43..f5e312cc 100644 --- a/pkg/content/interface.go +++ b/pkg/content/interface.go @@ -15,10 +15,12 @@ limitations under the License. package content -import "github.com/containerd/containerd/content" +import ( + "github.com/containerd/containerd/remotes" +) // ProvideIngester is the interface that groups the basic Read and Write methods. -type ProvideIngester interface { - content.Provider - content.Ingester +type Store interface { + remotes.Pusher + remotes.Fetcher } diff --git a/pkg/content/memory.go b/pkg/content/memory.go index 0792324d..39981cf8 100644 --- a/pkg/content/memory.go +++ b/pkg/content/memory.go @@ -1,110 +1,103 @@ -/* -Copyright The ORAS Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package content import ( "bytes" "context" + "fmt" + "io" + "io/ioutil" + "strings" "sync" "time" "github.com/containerd/containerd/content" "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/remotes" digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + artifact "oras.land/oras-go/pkg/artifact" ) -// ensure interface -var ( - _ content.Provider = &Memorystore{} - _ content.Ingester = &Memorystore{} -) - -// Memorystore provides content from the memory -type Memorystore struct { +// Memory provides content from the memory +type Memory struct { descriptor map[digest.Digest]ocispec.Descriptor content map[digest.Digest][]byte nameMap map[string]ocispec.Descriptor + refMap map[string]ocispec.Descriptor lock *sync.Mutex } -// NewMemoryStore creates a new memory store -func NewMemoryStore() *Memorystore { - return &Memorystore{ +// NewMemory creats a new memory store +func NewMemory() *Memory { + return &Memory{ descriptor: make(map[digest.Digest]ocispec.Descriptor), content: make(map[digest.Digest][]byte), nameMap: make(map[string]ocispec.Descriptor), + refMap: make(map[string]ocispec.Descriptor), lock: &sync.Mutex{}, } } -// Add adds content -func (s *Memorystore) Add(name, mediaType string, content []byte) ocispec.Descriptor { - var annotations map[string]string - if name != "" { - annotations = map[string]string{ - ocispec.AnnotationTitle: name, - } - } +func (s *Memory) Resolver() remotes.Resolver { + return s +} - if mediaType == "" { - mediaType = DefaultBlobMediaType +func (s *Memory) Resolve(ctx context.Context, ref string) (name string, desc ocispec.Descriptor, err error) { + desc, ok := s.refMap[ref] + if !ok { + return "", ocispec.Descriptor{}, fmt.Errorf("unknown reference: %s", ref) } + return ref, desc, nil +} - desc := ocispec.Descriptor{ - MediaType: mediaType, - Digest: digest.FromBytes(content), - Size: int64(len(content)), - Annotations: annotations, +func (s *Memory) Fetcher(ctx context.Context, ref string) (remotes.Fetcher, error) { + if _, ok := s.refMap[ref]; !ok { + return nil, fmt.Errorf("unknown reference: %s", ref) } - - s.Set(desc, content) - return desc + return s, nil } -// ReaderAt provides contents -func (s *Memorystore) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content.ReaderAt, error) { - desc, content, ok := s.Get(desc) +// Fetch get an io.ReadCloser for the specific content +func (s *Memory) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) { + _, content, ok := s.Get(desc) if !ok { return nil, ErrNotFound } + return ioutil.NopCloser(bytes.NewReader(content)), nil +} - return sizeReaderAt{ - readAtCloser: nopCloser{ - ReaderAt: bytes.NewReader(content), - }, - size: desc.Size, +func (s *Memory) Pusher(ctx context.Context, ref string) (remotes.Pusher, error) { + var tag, hash string + parts := strings.SplitN(ref, "@", 2) + if len(parts) > 0 { + tag = parts[0] + } + if len(parts) > 1 { + hash = parts[1] + } + return &memoryPusher{ + store: s, + ref: tag, + hash: hash, }, nil } -// Writer begins or resumes the active writer identified by desc -func (s *Memorystore) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) { - var wOpts content.WriterOpts - for _, opt := range opts { - if err := opt(&wOpts); err != nil { - return nil, err - } - } - desc := wOpts.Desc +type memoryPusher struct { + store *Memory + ref string + hash string +} +func (s *memoryPusher) Push(ctx context.Context, desc ocispec.Descriptor) (content.Writer, error) { name, _ := ResolveName(desc) now := time.Now() + // is this the root? + if desc.Digest.String() == s.hash { + s.store.refMap[s.ref] = desc + } return &memoryWriter{ - store: s, + store: s.store, buffer: bytes.NewBuffer(nil), desc: desc, digester: digest.Canonical.Digester(), @@ -117,8 +110,32 @@ func (s *Memorystore) Writer(ctx context.Context, opts ...content.WriterOpt) (co }, nil } +// Add adds content +func (s *Memory) Add(name, mediaType string, content []byte) (ocispec.Descriptor, error) { + var annotations map[string]string + if name != "" { + annotations = map[string]string{ + ocispec.AnnotationTitle: name, + } + } + + if mediaType == "" { + mediaType = DefaultBlobMediaType + } + + desc := ocispec.Descriptor{ + MediaType: mediaType, + Digest: digest.FromBytes(content), + Size: int64(len(content)), + Annotations: annotations, + } + + s.Set(desc, content) + return desc, nil +} + // Set adds the content to the store -func (s *Memorystore) Set(desc ocispec.Descriptor, content []byte) { +func (s *Memory) Set(desc ocispec.Descriptor, content []byte) { s.lock.Lock() defer s.lock.Unlock() @@ -131,7 +148,7 @@ func (s *Memorystore) Set(desc ocispec.Descriptor, content []byte) { } // Get finds the content from the store -func (s *Memorystore) Get(desc ocispec.Descriptor) (ocispec.Descriptor, []byte, bool) { +func (s *Memory) Get(desc ocispec.Descriptor) (ocispec.Descriptor, []byte, bool) { s.lock.Lock() defer s.lock.Unlock() @@ -144,7 +161,7 @@ func (s *Memorystore) Get(desc ocispec.Descriptor) (ocispec.Descriptor, []byte, } // GetByName finds the content from the store by name (i.e. AnnotationTitle) -func (s *Memorystore) GetByName(name string) (ocispec.Descriptor, []byte, bool) { +func (s *Memory) GetByName(name string) (ocispec.Descriptor, []byte, bool) { s.lock.Lock() defer s.lock.Unlock() @@ -156,8 +173,49 @@ func (s *Memorystore) GetByName(name string) (ocispec.Descriptor, []byte, bool) return desc, content, ok } +func (s *Memory) GenerateManifest(ref string, config *ocispec.Descriptor, descs ...ocispec.Descriptor) ([]byte, error) { + var ( + desc ocispec.Descriptor + manifest []byte + err error + ) + // Config + // Config - either it was set, or we have to set it + if config == nil { + configBytes := []byte("{}") + s.Add("", artifact.UnknownConfigMediaType, configBytes) + config = &ocispec.Descriptor{ + MediaType: artifact.UnknownConfigMediaType, + Digest: digest.FromBytes(configBytes), + Size: int64(len(configBytes)), + } + } + if manifest, desc, err = pack(*config, descs); err != nil { + return nil, err + } + s.refMap[ref] = desc + s.Add("", desc.MediaType, manifest) + return manifest, nil +} + +func descFromBytes(b []byte, mediaType string) (ocispec.Descriptor, error) { + digest, err := digest.FromReader(bytes.NewReader(b)) + if err != nil { + return ocispec.Descriptor{}, err + } + + if mediaType == "" { + mediaType = DefaultBlobMediaType + } + return ocispec.Descriptor{ + MediaType: mediaType, + Digest: digest, + Size: int64(len(b)), + }, nil +} + type memoryWriter struct { - store *Memorystore + store *Memory buffer *bytes.Buffer desc ocispec.Descriptor digester digest.Digester diff --git a/pkg/content/multireader.go b/pkg/content/multireader.go index d2cf924a..8dba773d 100644 --- a/pkg/content/multireader.go +++ b/pkg/content/multireader.go @@ -18,8 +18,9 @@ package content import ( "context" "fmt" + "io" - "github.com/containerd/containerd/content" + "github.com/containerd/containerd/remotes" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) @@ -34,18 +35,18 @@ import ( // // You now can use multiStore anywhere that content.Provider is accepted type MultiReader struct { - stores []content.Provider + stores []remotes.Fetcher } // AddStore add a store to read from -func (m *MultiReader) AddStore(store ...content.Provider) { +func (m *MultiReader) AddStore(store ...remotes.Fetcher) { m.stores = append(m.stores, store...) } // ReaderAt get a reader -func (m MultiReader) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content.ReaderAt, error) { +func (m MultiReader) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) { for _, store := range m.stores { - r, err := store.ReaderAt(ctx, desc) + r, err := store.Fetch(ctx, desc) if r != nil && err == nil { return r, nil } diff --git a/pkg/content/multireader_test.go b/pkg/content/multireader_test.go index fc96440f..5737f97b 100644 --- a/pkg/content/multireader_test.go +++ b/pkg/content/multireader_test.go @@ -20,7 +20,6 @@ import ( "io/ioutil" "testing" - ctrcontent "github.com/containerd/containerd/content" digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -45,18 +44,18 @@ var ( ) func TestMultiReader(t *testing.T) { - mem1, mem2 := content.NewMemoryStore(), content.NewMemoryStore() + mem1, mem2 := content.NewMemory(), content.NewMemory() mem1.Add("a", ocispec.MediaTypeImageConfig, testContentA) mem2.Add("b", ocispec.MediaTypeImageConfig, testContentB) multiReader := content.MultiReader{} multiReader.AddStore(mem1, mem2) ctx := context.Background() - contentA, err := multiReader.ReaderAt(ctx, testDescriptorA) + contentA, err := multiReader.Fetch(ctx, testDescriptorA) if err != nil { t.Fatalf("failed to get a reader for descriptor A: %v", err) } - outputA, err := ioutil.ReadAll(ctrcontent.NewReader(contentA)) + outputA, err := ioutil.ReadAll(contentA) if err != nil { t.Fatalf("failed to read content for descriptor A: %v", err) } @@ -64,11 +63,11 @@ func TestMultiReader(t *testing.T) { t.Errorf("mismatched content for A, actual '%s', expected '%s'", outputA, testContentA) } - contentB, err := multiReader.ReaderAt(ctx, testDescriptorB) + contentB, err := multiReader.Fetch(ctx, testDescriptorB) if err != nil { t.Fatalf("failed to get a reader for descriptor B: %v", err) } - outputB, err := ioutil.ReadAll(ctrcontent.NewReader(contentB)) + outputB, err := ioutil.ReadAll(contentB) if err != nil { t.Fatalf("failed to read content for descriptor B: %v", err) } diff --git a/pkg/content/multiwriter.go b/pkg/content/multiwriter.go index 2813f54e..1e69bcdf 100644 --- a/pkg/content/multiwriter.go +++ b/pkg/content/multiwriter.go @@ -19,6 +19,8 @@ import ( "context" ctrcontent "github.com/containerd/containerd/content" + "github.com/containerd/containerd/remotes" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) // MultiWriterIngester an ingester that can provide a single writer or multiple writers for a single @@ -29,3 +31,12 @@ type MultiWriterIngester interface { ctrcontent.Ingester Writers(ctx context.Context, opts ...ctrcontent.WriterOpt) (func(string) (ctrcontent.Writer, error), error) } + +// MultiWriterPusher a pusher that can provide a single writer or multiple writers for a single +// descriptor. Useful when the target of a descriptor can have multiple items within it, e.g. a layer +// that is a tar file with multiple files, each of which should go to a different stream, some of which +// should not be handled at all. +type MultiWriterPusher interface { + remotes.Pusher + Pushers(ctx context.Context, desc ocispec.Descriptor) (func(string) (ctrcontent.Writer, error), error) +} diff --git a/pkg/content/oci.go b/pkg/content/oci.go index 4f1b1b32..4a67436c 100644 --- a/pkg/content/oci.go +++ b/pkg/content/oci.go @@ -1,35 +1,27 @@ -/* -Copyright The ORAS Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package content import ( + "context" "encoding/json" + "errors" + "fmt" + "io" "io/ioutil" "os" "path/filepath" + "strings" "github.com/containerd/containerd/content" "github.com/containerd/containerd/content/local" + "github.com/containerd/containerd/remotes" + "github.com/opencontainers/go-digest" specs "github.com/opencontainers/image-spec/specs-go" ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// OCIStore provides content from the file system with the OCI-Image layout. +// OCI provides content from the file system with the OCI-Image layout. // Reference: https://github.com/opencontainers/image-spec/blob/master/image-layout.md -type OCIStore struct { +type OCI struct { content.Store root string @@ -37,14 +29,14 @@ type OCIStore struct { nameMap map[string]ocispec.Descriptor } -// NewOCIStore creates a new OCI store -func NewOCIStore(rootPath string) (*OCIStore, error) { +// NewOCI creates a new OCI store +func NewOCI(rootPath string) (*OCI, error) { fileStore, err := local.NewStore(rootPath) if err != nil { return nil, err } - store := &OCIStore{ + store := &OCI{ Store: fileStore, root: rootPath, } @@ -59,7 +51,7 @@ func NewOCIStore(rootPath string) (*OCIStore, error) { } // LoadIndex reads the index.json from the file system -func (s *OCIStore) LoadIndex() error { +func (s *OCI) LoadIndex() error { path := filepath.Join(s.root, OCIImageIndexFile) indexFile, err := os.Open(path) if err != nil { @@ -92,7 +84,17 @@ func (s *OCIStore) LoadIndex() error { } // SaveIndex writes the index.json to the file system -func (s *OCIStore) SaveIndex() error { +func (s *OCI) SaveIndex() error { + // first need to update the index + var descs []ocispec.Descriptor + for name, desc := range s.nameMap { + if desc.Annotations == nil { + desc.Annotations = map[string]string{} + } + desc.Annotations[ocispec.AnnotationRefName] = name + descs = append(descs, desc) + } + s.index.Manifests = descs indexJSON, err := json.Marshal(s.index) if err != nil { return err @@ -102,8 +104,57 @@ func (s *OCIStore) SaveIndex() error { return ioutil.WriteFile(path, indexJSON, 0644) } +func (s *OCI) Resolver() remotes.Resolver { + return s +} + +func (s *OCI) Resolve(ctx context.Context, ref string) (name string, desc ocispec.Descriptor, err error) { + if err := s.LoadIndex(); err != nil { + return "", ocispec.Descriptor{}, err + } + desc, ok := s.nameMap[ref] + if !ok { + return "", ocispec.Descriptor{}, fmt.Errorf("reference %s not in store", ref) + } + return ref, desc, nil +} + +func (s *OCI) Fetcher(ctx context.Context, ref string) (remotes.Fetcher, error) { + if err := s.LoadIndex(); err != nil { + return nil, err + } + if _, ok := s.nameMap[ref]; !ok { + return nil, fmt.Errorf("reference %s not in store", ref) + } + return s, nil +} + +// Fetch get an io.ReadCloser for the specific content +func (s *OCI) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) { + readerAt, err := s.Store.ReaderAt(ctx, desc) + if err != nil { + return nil, err + } + // just wrap the ReaderAt with a Reader + return ioutil.NopCloser(&ReaderAtWrapper{readerAt: readerAt}), nil +} + +// Pusher get a remotes.Pusher for the given ref +func (s *OCI) Pusher(ctx context.Context, ref string) (remotes.Pusher, error) { + // separate the tag based ref from the hash + var ( + baseRef, hash string + ) + parts := strings.SplitN(ref, "@", 2) + baseRef = parts[0] + if len(parts) > 1 { + hash = parts[1] + } + return &ociPusher{oci: s, ref: baseRef, digest: hash}, nil +} + // AddReference adds or updates an reference to index. -func (s *OCIStore) AddReference(name string, desc ocispec.Descriptor) { +func (s *OCI) AddReference(name string, desc ocispec.Descriptor) { if desc.Annotations == nil { desc.Annotations = map[string]string{ ocispec.AnnotationRefName: name, @@ -133,7 +184,7 @@ func (s *OCIStore) AddReference(name string, desc ocispec.Descriptor) { } // DeleteReference deletes an reference from index. -func (s *OCIStore) DeleteReference(name string) { +func (s *OCI) DeleteReference(name string) { if _, ok := s.nameMap[name]; !ok { return } @@ -149,12 +200,12 @@ func (s *OCIStore) DeleteReference(name string) { } // ListReferences lists all references in index. -func (s *OCIStore) ListReferences() map[string]ocispec.Descriptor { +func (s *OCI) ListReferences() map[string]ocispec.Descriptor { return s.nameMap } // validateOCILayoutFile ensures the `oci-layout` file -func (s *OCIStore) validateOCILayoutFile() error { +func (s *OCI) validateOCILayoutFile() error { layoutFilePath := filepath.Join(s.root, ocispec.ImageLayoutFile) layoutFile, err := os.Open(layoutFilePath) if err != nil { @@ -185,3 +236,88 @@ func (s *OCIStore) validateOCILayoutFile() error { return nil } + +// TODO: implement (needed to create a content.Store) +// TODO: do not return empty content.Info +// Abort completely cancels the ingest operation targeted by ref. +func (s *OCI) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) { + return content.Info{}, nil +} + +// TODO: implement (needed to create a content.Store) +// Update updates mutable information related to content. +// If one or more fieldpaths are provided, only those +// fields will be updated. +// Mutable fields: +// labels.* +func (s *OCI) Update(ctx context.Context, info content.Info, fieldpaths ...string) (content.Info, error) { + return content.Info{}, errors.New("not yet implemented: Update (content.Store interface)") +} + +// TODO: implement (needed to create a content.Store) +// Walk will call fn for each item in the content store which +// match the provided filters. If no filters are given all +// items will be walked. +func (s *OCI) Walk(ctx context.Context, fn content.WalkFunc, filters ...string) error { + return errors.New("not yet implemented: Walk (content.Store interface)") +} + +// TODO: implement (needed to create a content.Store) +// Delete removes the content from the store. +func (s *OCI) Delete(ctx context.Context, dgst digest.Digest) error { + return errors.New("not yet implemented: Delete (content.Store interface)") +} + +// TODO: implement (needed to create a content.Store) +func (s *OCI) Status(ctx context.Context, ref string) (content.Status, error) { + // Status returns the status of the provided ref. + return content.Status{}, errors.New("not yet implemented: Status (content.Store interface)") +} + +// TODO: implement (needed to create a content.Store) +// ListStatuses returns the status of any active ingestions whose ref match the +// provided regular expression. If empty, all active ingestions will be +// returned. +func (s *OCI) ListStatuses(ctx context.Context, filters ...string) ([]content.Status, error) { + return []content.Status{}, errors.New("not yet implemented: ListStatuses (content.Store interface)") +} + +// TODO: implement (needed to create a content.Store) +// Abort completely cancels the ingest operation targeted by ref. +func (s *OCI) Abort(ctx context.Context, ref string) error { + return errors.New("not yet implemented: Abort (content.Store interface)") +} + +// ReaderAt provides contents +func (s *OCI) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content.ReaderAt, error) { + return s.Store.ReaderAt(ctx, desc) +} + +// ociPusher to push content for a single referencem can handle multiple descriptors. +// Needs to be able to recognize when a root manifest is being pushed and to create the tag +// for it. +type ociPusher struct { + oci *OCI + ref string + digest string +} + +// Push get a writer for a single Descriptor +func (p *ociPusher) Push(ctx context.Context, desc ocispec.Descriptor) (content.Writer, error) { + // do we need to create a tag? + switch desc.MediaType { + case ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex: + // if the hash of the content matches that which was provided as the hash for the root, mark it + if p.digest != "" && p.digest == desc.Digest.String() { + if err := p.oci.LoadIndex(); err != nil { + return nil, err + } + p.oci.nameMap[p.ref] = desc + if err := p.oci.SaveIndex(); err != nil { + return nil, err + } + } + } + + return p.oci.Store.Writer(ctx, content.WithDescriptor(desc), content.WithRef(p.ref)) +} diff --git a/pkg/content/passthrough_test.go b/pkg/content/passthrough_test.go index 4ca949c4..73d66d73 100644 --- a/pkg/content/passthrough_test.go +++ b/pkg/content/passthrough_test.go @@ -100,8 +100,9 @@ func TestPassthroughWriter(t *testing.T) { for _, tt := range tests { ctx := context.Background() - mem := content.NewMemoryStore() - memw, err := mem.Writer(ctx, ctrcontent.WithDescriptor(modifiedDescriptor)) + mem := content.NewMemory() + pusher, _ := mem.Pusher(ctx, "") + memw, err := pusher.Push(ctx, modifiedDescriptor) if err != nil { t.Fatalf("unexpected error getting the memory store writer: %v", err) } diff --git a/pkg/content/readerat.go b/pkg/content/readerat.go index 41e892ee..6e423d27 100644 --- a/pkg/content/readerat.go +++ b/pkg/content/readerat.go @@ -1,18 +1,3 @@ -/* -Copyright The ORAS Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package content import ( @@ -40,10 +25,30 @@ func (ra sizeReaderAt) Size() int64 { return ra.size } -type nopCloser struct { +func NopCloserAt(r io.ReaderAt) nopCloserAt { + return nopCloserAt{r} +} + +type nopCloserAt struct { io.ReaderAt } -func (nopCloser) Close() error { +func (n nopCloserAt) Close() error { return nil } + +// readerAtWrapper wraps a ReaderAt to give a Reader +type ReaderAtWrapper struct { + offset int64 + readerAt io.ReaderAt +} + +func (r *ReaderAtWrapper) Read(p []byte) (n int, err error) { + n, err = r.readerAt.ReadAt(p, r.offset) + r.offset += int64(n) + return +} + +func NewReaderAtWrapper(readerAt io.ReaderAt) *ReaderAtWrapper { + return &ReaderAtWrapper{readerAt: readerAt} +} diff --git a/pkg/content/registry.go b/pkg/content/registry.go new file mode 100644 index 00000000..ea611cc1 --- /dev/null +++ b/pkg/content/registry.go @@ -0,0 +1,70 @@ +package content + +import ( + "context" + "crypto/tls" + "fmt" + "net/http" + "os" + + auth "oras.land/oras-go/pkg/auth/docker" + + "github.com/containerd/containerd/remotes" + "github.com/containerd/containerd/remotes/docker" +) + +// RegistryOptions provide configuration options to a Registry +type RegistryOptions struct { + Configs []string + Username string + Password string + Insecure bool + PlainHTTP bool +} + +// Registry provides content from a spec-compliant registry. Create an use a new one for each +// registry with unique configuration of RegistryOptions. +type Registry struct { + remotes.Resolver +} + +// NewRegistry creates a new Registry store +func NewRegistry(opts RegistryOptions) (*Registry, error) { + return &Registry{ + Resolver: newResolver(opts.Username, opts.Password, opts.Insecure, opts.PlainHTTP, opts.Configs...), + }, nil +} + +func newResolver(username, password string, insecure bool, plainHTTP bool, configs ...string) remotes.Resolver { + + opts := docker.ResolverOptions{ + PlainHTTP: plainHTTP, + } + + client := http.DefaultClient + if insecure { + client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + } + opts.Client = client + + if username != "" || password != "" { + opts.Credentials = func(hostName string) (string, string, error) { + return username, password, nil + } + return docker.NewResolver(opts) + } + cli, err := auth.NewClient(configs...) + if err != nil { + fmt.Fprintf(os.Stderr, "WARNING: Error loading auth file: %v\n", err) + } + resolver, err := cli.Resolver(context.Background(), client, plainHTTP) + if err != nil { + fmt.Fprintf(os.Stderr, "WARNING: Error loading resolver: %v\n", err) + resolver = docker.NewResolver(opts) + } + return resolver +} diff --git a/pkg/oras/copy.go b/pkg/oras/copy.go new file mode 100644 index 00000000..e07ad22e --- /dev/null +++ b/pkg/oras/copy.go @@ -0,0 +1,178 @@ +package oras + +import ( + "context" + "fmt" + "sync" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/images" + "github.com/containerd/containerd/log" + "github.com/containerd/containerd/remotes" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "oras.land/oras-go/pkg/target" +) + +// Copy copy a ref from one target.Target to a ref in another target.Target. If toRef is blank, reuses fromRef +// Returns the root +// Descriptor of the copied item. Can use the root to retrieve child elements from target.Target. +func Copy(ctx context.Context, from target.Target, fromRef string, to target.Target, toRef string, opts ...CopyOpt) (ocispec.Descriptor, error) { + if from == nil { + return ocispec.Descriptor{}, ErrFromTargetUndefined + } + if to == nil { + return ocispec.Descriptor{}, ErrToTargetUndefined + } + // blank toRef + if toRef == "" { + toRef = fromRef + } + opt := copyOptsDefaults() + for _, o := range opts { + if err := o(opt); err != nil { + return ocispec.Descriptor{}, err + } + } + + if from == nil { + return ocispec.Descriptor{}, ErrFromResolverUndefined + } + if to == nil { + return ocispec.Descriptor{}, ErrToResolverUndefined + } + + // for the "from", we resolve the ref, then use resolver.Fetcher to fetch the various content blobs + // for the "to", we simply use resolver.Pusher to push the various content blobs + + _, desc, err := from.Resolve(ctx, fromRef) + if err != nil { + return ocispec.Descriptor{}, err + } + + fetcher, err := from.Fetcher(ctx, fromRef) + if err != nil { + return ocispec.Descriptor{}, err + } + // construct the reference we send to the pusher using the digest, so it knows what the root is + pushRef := fmt.Sprintf("%s@%s", toRef, desc.Digest.String()) + pusher, err := to.Pusher(ctx, pushRef) + if err != nil { + return ocispec.Descriptor{}, err + } + + if err := transferContent(ctx, desc, fetcher, pusher, opt); err != nil { + return ocispec.Descriptor{}, err + } + return desc, nil +} + +func transferContent(ctx context.Context, desc ocispec.Descriptor, fetcher remotes.Fetcher, pusher remotes.Pusher, opts *copyOpts) error { + var descriptors, manifests []ocispec.Descriptor + lock := &sync.Mutex{} + picker := images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + if isAllowedMediaType(desc.MediaType, opts.allowedMediaTypes...) { + if opts.filterName(desc) { + lock.Lock() + defer lock.Unlock() + descriptors = append(descriptors, desc) + } + return nil, nil + } + return nil, nil + }) + + // we use a hybrid store - a cache wrapping the underlying pusher - for two reasons: + // 1. so that we can cache the manifests as pushing them, then retrieve them later to push in reverse order after the blobs + // 2. so that we can retrieve them to analyze and find children in the Dispatch routine + store := opts.contentProvideIngesterPusherFetcher + if store == nil { + store = newHybridStoreFromPusher(pusher, opts.cachedMediaTypes, true) + } + + // fetchHandler pushes to the *store*, which may or may not cache it + baseFetchHandler := func(p remotes.Pusher) images.HandlerFunc { + return images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + cw, err := p.Push(ctx, desc) + if err != nil { + if !errdefs.IsAlreadyExists(err) { + return nil, err + } + + return nil, nil + } + defer cw.Close() + + rc, err := fetcher.Fetch(ctx, desc) + if err != nil { + return nil, err + } + defer rc.Close() + return nil, content.Copy(ctx, cw, rc, desc.Size, desc.Digest) + }) + } + + // track all of our manifests that will be cached + fetchHandler := images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + if isAllowedMediaType(desc.MediaType, opts.cachedMediaTypes...) { + lock.Lock() + defer lock.Unlock() + manifests = append(manifests, desc) + } + return baseFetchHandler(store)(ctx, desc) + }) + + handlers := []images.Handler{ + filterHandler(opts, opts.allowedMediaTypes...), + } + handlers = append(handlers, opts.baseHandlers...) + handlers = append(handlers, + fetchHandler, + picker, + images.ChildrenHandler(&ProviderWrapper{Fetcher: store}), + ) + handlers = append(handlers, opts.callbackHandlers...) + + if err := opts.dispatch(ctx, images.Handlers(handlers...), nil, desc); err != nil { + return err + } + + // finally, we cached all of the manifests, so push those out + // Iterate in reverse order as seen, parent always uploaded after child + for i := len(manifests) - 1; i >= 0; i-- { + _, err := baseFetchHandler(pusher)(ctx, manifests[i]) + if err != nil { + return err + } + } + return nil +} + +func filterHandler(opts *copyOpts, allowedMediaTypes ...string) images.HandlerFunc { + return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + switch { + case isAllowedMediaType(desc.MediaType, ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex): + return nil, nil + case isAllowedMediaType(desc.MediaType, allowedMediaTypes...): + if opts.filterName(desc) { + return nil, nil + } + log.G(ctx).Warnf("blob no name: %v", desc.Digest) + default: + log.G(ctx).Warnf("unknown type: %v", desc.MediaType) + } + return nil, images.ErrStopHandler + } +} + +func isAllowedMediaType(mediaType string, allowedMediaTypes ...string) bool { + if len(allowedMediaTypes) == 0 { + return true + } + for _, allowedMediaType := range allowedMediaTypes { + if mediaType == allowedMediaType { + return true + } + } + return false +} diff --git a/pkg/oras/errors.go b/pkg/oras/errors.go index 054af764..6c5f9f24 100644 --- a/pkg/oras/errors.go +++ b/pkg/oras/errors.go @@ -22,7 +22,11 @@ import ( // Common errors var ( - ErrResolverUndefined = errors.New("resolver undefined") + ErrResolverUndefined = errors.New("resolver undefined") + ErrFromResolverUndefined = errors.New("from target resolver undefined") + ErrToResolverUndefined = errors.New("to target resolver undefined") + ErrFromTargetUndefined = errors.New("from target undefined") + ErrToTargetUndefined = errors.New("from target undefined") ) // Path validation related errors diff --git a/pkg/oras/opts.go b/pkg/oras/opts.go new file mode 100644 index 00000000..f2f2e86a --- /dev/null +++ b/pkg/oras/opts.go @@ -0,0 +1,259 @@ +package oras + +import ( + "context" + "fmt" + "io" + "path/filepath" + "strings" + "sync" + + "github.com/containerd/containerd/images" + "github.com/opencontainers/go-digest" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "golang.org/x/sync/semaphore" + orascontent "oras.land/oras-go/pkg/content" +) + +func copyOptsDefaults() *copyOpts { + return ©Opts{ + dispatch: images.Dispatch, + filterName: filterName, + cachedMediaTypes: []string{ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex}, + validateName: ValidateNameAsPath, + } +} + +type CopyOpt func(o *copyOpts) error + +type copyOpts struct { + allowedMediaTypes []string + dispatch func(context.Context, images.Handler, *semaphore.Weighted, ...ocispec.Descriptor) error + baseHandlers []images.Handler + callbackHandlers []images.Handler + contentProvideIngesterPusherFetcher orascontent.Store + filterName func(ocispec.Descriptor) bool + cachedMediaTypes []string + + config *ocispec.Descriptor + configMediaType string + configAnnotations map[string]string + manifest *ocispec.Descriptor + manifestAnnotations map[string]string + validateName func(desc ocispec.Descriptor) error + + userAgent string +} + +// ValidateNameAsPath validates name in the descriptor as file path in order +// to generate good packages intended to be pulled using the FileStore or +// the oras cli. +// For cross-platform considerations, only unix paths are accepted. +func ValidateNameAsPath(desc ocispec.Descriptor) error { + // no empty name + path, ok := orascontent.ResolveName(desc) + if !ok || path == "" { + return orascontent.ErrNoName + } + + // path should be clean + if target := filepath.ToSlash(filepath.Clean(path)); target != path { + return errors.Wrap(ErrDirtyPath, path) + } + + // path should be slash-separated + if strings.Contains(path, "\\") { + return errors.Wrap(ErrPathNotSlashSeparated, path) + } + + // disallow absolute path: covers unix and windows format + if strings.HasPrefix(path, "/") { + return errors.Wrap(ErrAbsolutePathDisallowed, path) + } + if len(path) > 2 { + c := path[0] + if path[1] == ':' && path[2] == '/' && ('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') { + return errors.Wrap(ErrAbsolutePathDisallowed, path) + } + } + + // disallow path traversal + if strings.HasPrefix(path, "../") || path == ".." { + return errors.Wrap(ErrPathTraversalDisallowed, path) + } + + return nil +} + +// dispatchBFS behaves the same as images.Dispatch() but in sequence with breath-first search. +func dispatchBFS(ctx context.Context, handler images.Handler, weighted *semaphore.Weighted, descs ...ocispec.Descriptor) error { + for i := 0; i < len(descs); i++ { + desc := descs[i] + children, err := handler.Handle(ctx, desc) + if err != nil { + switch err := errors.Cause(err); err { + case images.ErrSkipDesc: + continue // don't traverse the children. + case ErrStopProcessing: + return nil + } + return err + } + descs = append(descs, children...) + } + return nil +} + +func filterName(desc ocispec.Descriptor) bool { + // needs to be filled in + return true +} + +// WithAdditionalCachedMediaTypes adds media types normally cached in memory when pulling. +// This does not replace the default media types, but appends to them +func WithAdditionalCachedMediaTypes(cachedMediaTypes ...string) CopyOpt { + return func(o *copyOpts) error { + o.cachedMediaTypes = append(o.cachedMediaTypes, cachedMediaTypes...) + return nil + } +} + +// WithAllowedMediaType sets the allowed media types +func WithAllowedMediaType(allowedMediaTypes ...string) CopyOpt { + return func(o *copyOpts) error { + o.allowedMediaTypes = append(o.allowedMediaTypes, allowedMediaTypes...) + return nil + } +} + +// WithAllowedMediaTypes sets the allowed media types +func WithAllowedMediaTypes(allowedMediaTypes []string) CopyOpt { + return func(o *copyOpts) error { + o.allowedMediaTypes = append(o.allowedMediaTypes, allowedMediaTypes...) + return nil + } +} + +// WithPullByBFS opt to pull in sequence with breath-first search +func WithPullByBFS(o *copyOpts) error { + o.dispatch = dispatchBFS + return nil +} + +// WithPullBaseHandler provides base handlers, which will be called before +// any pull specific handlers. +func WithPullBaseHandler(handlers ...images.Handler) CopyOpt { + return func(o *copyOpts) error { + o.baseHandlers = append(o.baseHandlers, handlers...) + return nil + } +} + +// WithPullCallbackHandler provides callback handlers, which will be called after +// any pull specific handlers. +func WithPullCallbackHandler(handlers ...images.Handler) CopyOpt { + return func(o *copyOpts) error { + o.callbackHandlers = append(o.callbackHandlers, handlers...) + return nil + } +} + +// WithContentProvideIngester opt to the provided Provider and Ingester +// for file system I/O, including caches. +func WithContentStore(store orascontent.Store) CopyOpt { + return func(o *copyOpts) error { + o.contentProvideIngesterPusherFetcher = store + return nil + } +} + +// WithPullEmptyNameAllowed allows pulling blobs with empty name. +func WithPullEmptyNameAllowed() CopyOpt { + return func(o *copyOpts) error { + o.filterName = func(ocispec.Descriptor) bool { + return true + } + return nil + } +} + +// WithPullStatusTrack report results to stdout +func WithPullStatusTrack(writer io.Writer) CopyOpt { + return WithPullCallbackHandler(pullStatusTrack(writer)) +} + +func pullStatusTrack(writer io.Writer) images.Handler { + var printLock sync.Mutex + return images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { + if name, ok := orascontent.ResolveName(desc); ok { + digestString := desc.Digest.String() + if err := desc.Digest.Validate(); err == nil { + if algo := desc.Digest.Algorithm(); algo == digest.SHA256 { + digestString = desc.Digest.Encoded()[:12] + } + } + printLock.Lock() + defer printLock.Unlock() + fmt.Fprintln(writer, "Downloaded", digestString, name) + } + return nil, nil + }) +} + +// WithConfig overrides the config - setting this will ignore WithConfigMediaType and WithConfigAnnotations +func WithConfig(config ocispec.Descriptor) CopyOpt { + return func(o *copyOpts) error { + o.config = &config + return nil + } +} + +// WithConfigMediaType overrides the config media type +func WithConfigMediaType(mediaType string) CopyOpt { + return func(o *copyOpts) error { + o.configMediaType = mediaType + return nil + } +} + +// WithConfigAnnotations overrides the config annotations +func WithConfigAnnotations(annotations map[string]string) CopyOpt { + return func(o *copyOpts) error { + o.configAnnotations = annotations + return nil + } +} + +// WithManifest overrides the manifest - setting this will ignore WithManifestConfigAnnotations +func WithManifest(manifest ocispec.Descriptor) CopyOpt { + return func(o *copyOpts) error { + o.manifest = &manifest + return nil + } +} + +// WithManifestAnnotations overrides the manifest annotations +func WithManifestAnnotations(annotations map[string]string) CopyOpt { + return func(o *copyOpts) error { + o.manifestAnnotations = annotations + return nil + } +} + +// WithNameValidation validates the image title in the descriptor. +// Pass nil to disable name validation. +func WithNameValidation(validate func(desc ocispec.Descriptor) error) CopyOpt { + return func(o *copyOpts) error { + o.validateName = validate + return nil + } +} + +// WithUserAgent set the user agent string in http communications +func WithUserAgent(agent string) CopyOpt { + return func(o *copyOpts) error { + o.userAgent = agent + return nil + } +} diff --git a/pkg/oras/push_opts_test.go b/pkg/oras/opts_test.go similarity index 100% rename from pkg/oras/push_opts_test.go rename to pkg/oras/opts_test.go diff --git a/pkg/oras/oras_test.go b/pkg/oras/oras_test.go index 181f9090..1501de20 100644 --- a/pkg/oras/oras_test.go +++ b/pkg/oras/oras_test.go @@ -30,16 +30,16 @@ import ( "time" orascontent "oras.land/oras-go/pkg/content" + "oras.land/oras-go/pkg/target" "github.com/containerd/containerd/images" - "github.com/containerd/containerd/remotes" - "github.com/containerd/containerd/remotes/docker" "github.com/distribution/distribution/v3/configuration" "github.com/distribution/distribution/v3/registry" _ "github.com/distribution/distribution/v3/registry/storage/driver/inmemory" digest "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "github.com/phayes/freeport" + "github.com/stretchr/testify/suite" ) @@ -66,8 +66,9 @@ func newContext() context.Context { return context.Background() } -func newResolver() remotes.Resolver { - return docker.NewResolver(docker.ResolverOptions{}) +func newResolver() target.Target { + reg, _ := orascontent.NewRegistry(orascontent.RegistryOptions{}) + return reg } // Start Docker registry @@ -88,40 +89,46 @@ func (suite *ORASTestSuite) SetupSuite() { } // Push files to docker registry -func (suite *ORASTestSuite) Test_0_Push() { +func (suite *ORASTestSuite) Test_0_Copy() { var ( err error ref string desc ocispec.Descriptor descriptors []ocispec.Descriptor - store *orascontent.FileStore + store *orascontent.File + memStore *orascontent.Memory ) - _, err = Push(newContext(), nil, ref, nil, descriptors) + _, err = Copy(newContext(), nil, ref, nil, ref) suite.NotNil(err, "error pushing with empty resolver") - _, err = Push(newContext(), newResolver(), ref, nil, descriptors) - suite.NotNil(err, "error pushing when context missing hostname") + _, err = Copy(newContext(), orascontent.NewMemory(), ref, newResolver(), "") + suite.NotNil(err, "error pushing when ref missing hostname") ref = fmt.Sprintf("%s/empty:test", suite.DockerRegistryHost) - _, err = Push(newContext(), newResolver(), ref, nil, descriptors) + + memStore = orascontent.NewMemory() + _, _ = memStore.GenerateManifest(ref, nil) + _, err = Copy(newContext(), memStore, ref, newResolver(), "") suite.Nil(err, "no error pushing with empty descriptors") // Load descriptors with test chart tgz (as single layer) - store = orascontent.NewFileStore("") + ref = fmt.Sprintf("%s/chart-tgz:test", suite.DockerRegistryHost) + + store = orascontent.NewFile("") basename := filepath.Base(testTarball) desc, err = store.Add(basename, "", testTarball) suite.Nil(err, "no error loading test chart") - descriptors = []ocispec.Descriptor{desc} + manifest, _ := store.GenerateManifest(ref, nil, desc) + fmt.Printf("%s\n", manifest) - ref = fmt.Sprintf("%s/chart-tgz:test", suite.DockerRegistryHost) - _, err = Push(newContext(), newResolver(), ref, store, descriptors) + _, err = Copy(newContext(), store, ref, newResolver(), "") suite.Nil(err, "no error pushing test chart tgz (as single layer)") // Load descriptors with test chart dir (each file as layer) testDirAbs, err := filepath.Abs(testDir) suite.Nil(err, "no error parsing test directory") - store = orascontent.NewFileStore(testDirAbs) + store = orascontent.NewFile(testDirAbs) descriptors = []ocispec.Descriptor{} var ff = func(pathX string, infoX os.FileInfo, errX error) error { if !infoX.IsDir() { @@ -142,34 +149,36 @@ func (suite *ORASTestSuite) Test_0_Push() { os.Chdir(cwd) ref = fmt.Sprintf("%s/chart-dir:test", suite.DockerRegistryHost) - _, err = Push(newContext(), newResolver(), ref, store, descriptors) + store.GenerateManifest(ref, nil, descriptors...) + _, err = Copy(newContext(), store, ref, newResolver(), "") suite.Nil(err, "no error pushing test chart dir (each file as layer)") } // Pull files and verify descriptors func (suite *ORASTestSuite) Test_1_Pull() { var ( - err error - ref string - descriptors []ocispec.Descriptor - store *orascontent.Memorystore + err error + ref string + desc ocispec.Descriptor + store *orascontent.Memory + emptyDesc ocispec.Descriptor ) - _, descriptors, err = Pull(newContext(), nil, ref, nil) + desc, err = Copy(newContext(), nil, ref, nil, ref) suite.NotNil(err, "error pulling with empty resolver") - suite.Nil(descriptors, "descriptors nil pulling with empty resolver") + suite.Equal(desc, emptyDesc, "descriptor empty pulling with empty resolver") - // Pull non-existant - store = orascontent.NewMemoryStore() - ref = fmt.Sprintf("%s/nonexistant:test", suite.DockerRegistryHost) - _, descriptors, err = Pull(newContext(), newResolver(), ref, store) - suite.NotNil(err, "error pulling non-existant ref") - suite.Nil(descriptors, "descriptors empty with error") + // Pull non-existent + store = orascontent.NewMemory() + ref = fmt.Sprintf("%s/nonexistent:test", suite.DockerRegistryHost) + desc, err = Copy(newContext(), newResolver(), ref, store, ref) + suite.NotNil(err, "error pulling non-existent ref") + suite.Equal(desc, emptyDesc, "descriptor empty with error") // Pull chart-tgz - store = orascontent.NewMemoryStore() + store = orascontent.NewMemory() ref = fmt.Sprintf("%s/chart-tgz:test", suite.DockerRegistryHost) - _, descriptors, err = Pull(newContext(), newResolver(), ref, store) + _, err = Copy(newContext(), newResolver(), ref, store, ref) suite.Nil(err, "no error pulling chart-tgz ref") // Verify the descriptors, single layer/file @@ -181,9 +190,9 @@ func (suite *ORASTestSuite) Test_1_Pull() { suite.Equal(content, actualContent, ".tgz content matches on pull") // Pull chart-dir - store = orascontent.NewMemoryStore() + store = orascontent.NewMemory() ref = fmt.Sprintf("%s/chart-dir:test", suite.DockerRegistryHost) - _, descriptors, err = Pull(newContext(), newResolver(), ref, store) + desc, err = Copy(newContext(), newResolver(), ref, store, ref) suite.Nil(err, "no error pulling chart-dir ref") // Verify the descriptors, multiple layers/files @@ -209,26 +218,26 @@ func (suite *ORASTestSuite) Test_2_MediaType() { err error ref string descriptors []ocispec.Descriptor - store *orascontent.Memorystore + store *orascontent.Memory ) // Push content with customized media types - store = orascontent.NewMemoryStore() + store = orascontent.NewMemory() descriptors = nil for _, data := range testData { - desc := store.Add(data[0], data[1], []byte(data[2])) + desc, _ := store.Add(data[0], data[1], []byte(data[2])) descriptors = append(descriptors, desc) } ref = fmt.Sprintf("%s/media-type:test", suite.DockerRegistryHost) - _, err = Push(newContext(), newResolver(), ref, store, descriptors) + _, _ = store.GenerateManifest(ref, nil, descriptors...) + _, err = Copy(newContext(), store, ref, newResolver(), ref) suite.Nil(err, "no error pushing test data with customized media type") // Pull with all media types - store = orascontent.NewMemoryStore() + store = orascontent.NewMemory() ref = fmt.Sprintf("%s/media-type:test", suite.DockerRegistryHost) - _, descriptors, err = Pull(newContext(), newResolver(), ref, store) + _, err = Copy(newContext(), newResolver(), ref, store, ref) suite.Nil(err, "no error pulling media-type ref") - suite.Equal(2, len(descriptors), "number of contents matches on pull") for _, data := range testData { _, actualContent, ok := store.GetByName(data[0]) suite.True(ok, "find in memory") @@ -237,11 +246,10 @@ func (suite *ORASTestSuite) Test_2_MediaType() { } // Pull with specified media type - store = orascontent.NewMemoryStore() + store = orascontent.NewMemory() ref = fmt.Sprintf("%s/media-type:test", suite.DockerRegistryHost) - _, descriptors, err = Pull(newContext(), newResolver(), ref, store, WithAllowedMediaType(testData[0][1])) + _, err = Copy(newContext(), newResolver(), ref, store, ref, WithAllowedMediaType(testData[0][1])) suite.Nil(err, "no error pulling media-type ref") - suite.Equal(1, len(descriptors), "number of contents matches on pull") for _, data := range testData[:1] { _, actualContent, ok := store.GetByName(data[0]) suite.True(ok, "find in memory") @@ -249,12 +257,11 @@ func (suite *ORASTestSuite) Test_2_MediaType() { suite.Equal(content, actualContent, "test content matches on pull") } - // Pull with non-existing media type - store = orascontent.NewMemoryStore() + // Pull with non-existing media type, so only should do root manifest + store = orascontent.NewMemory() ref = fmt.Sprintf("%s/media-type:test", suite.DockerRegistryHost) - _, descriptors, err = Pull(newContext(), newResolver(), ref, store, WithAllowedMediaType("non.existing.media.type")) + _, err = Copy(newContext(), newResolver(), ref, store, ref, WithAllowedMediaType("non.existing.media.type")) suite.Nil(err, "no error pulling media-type ref") - suite.Equal(0, len(descriptors), "number of contents matches on pull") } // Pull with condition @@ -267,27 +274,27 @@ func (suite *ORASTestSuite) Test_3_Conditional_Pull() { err error ref string descriptors []ocispec.Descriptor - store *orascontent.Memorystore + store *orascontent.Memory stop bool ) // Push test content - store = orascontent.NewMemoryStore() + store = orascontent.NewMemory() descriptors = nil for _, data := range testData { - desc := store.Add(data[0], "", []byte(data[1])) + desc, _ := store.Add(data[0], "", []byte(data[1])) descriptors = append(descriptors, desc) } ref = fmt.Sprintf("%s/conditional-pull:test", suite.DockerRegistryHost) - _, err = Push(newContext(), newResolver(), ref, store, descriptors) + store.GenerateManifest(ref, nil, descriptors...) + _, err = Copy(newContext(), store, ref, newResolver(), ref) suite.Nil(err, "no error pushing test data") // Pull all contents in sequence - store = orascontent.NewMemoryStore() + store = orascontent.NewMemory() ref = fmt.Sprintf("%s/conditional-pull:test", suite.DockerRegistryHost) - _, descriptors, err = Pull(newContext(), newResolver(), ref, store, WithPullByBFS) + _, err = Copy(newContext(), newResolver(), ref, store, ref, WithPullByBFS) suite.Nil(err, "no error pulling ref") - suite.Equal(2, len(descriptors), "number of contents matches on pull") for i, data := range testData { _, actualContent, ok := store.GetByName(data[0]) suite.True(ok, "find in memory") @@ -298,9 +305,9 @@ func (suite *ORASTestSuite) Test_3_Conditional_Pull() { } // Selective pull contents: stop at the very beginning - store = orascontent.NewMemoryStore() + store = orascontent.NewMemory() ref = fmt.Sprintf("%s/conditional-pull:test", suite.DockerRegistryHost) - _, descriptors, err = Pull(newContext(), newResolver(), ref, store, WithPullByBFS, + _, err = Copy(newContext(), newResolver(), ref, store, ref, WithPullByBFS, WithPullBaseHandler(images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { if name, ok := orascontent.ResolveName(desc); ok && name == testData[0][0] { return nil, ErrStopProcessing @@ -308,13 +315,12 @@ func (suite *ORASTestSuite) Test_3_Conditional_Pull() { return nil, nil }))) suite.Nil(err, "no error pulling ref") - suite.Equal(0, len(descriptors), "number of contents matches on pull") // Selective pull contents: stop in the middle - store = orascontent.NewMemoryStore() + store = orascontent.NewMemory() ref = fmt.Sprintf("%s/conditional-pull:test", suite.DockerRegistryHost) stop = false - _, descriptors, err = Pull(newContext(), newResolver(), ref, store, WithPullByBFS, + _, err = Copy(newContext(), newResolver(), ref, store, ref, WithPullByBFS, WithPullBaseHandler(images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { if stop { return nil, ErrStopProcessing @@ -325,7 +331,6 @@ func (suite *ORASTestSuite) Test_3_Conditional_Pull() { return nil, nil }))) suite.Nil(err, "no error pulling ref") - suite.Equal(1, len(descriptors), "number of contents matches on pull") for _, data := range testData[:1] { _, actualContent, ok := store.GetByName(data[0]) suite.True(ok, "find in memory") @@ -365,10 +370,11 @@ func (suite *ORASTestSuite) Test_4_GHSA_g5v4_5x39_vwhx() { } // Step 3: upload malicious artifact to registry - memoryStore := orascontent.NewMemoryStore() + memoryStore := orascontent.NewMemory() memoryStore.Set(evilDesc, buf.Bytes()) ref := fmt.Sprintf("%s/evil:%s", suite.DockerRegistryHost, tag) - _, err = Push(newContext(), newResolver(), ref, memoryStore, []ocispec.Descriptor{evilDesc}) + memoryStore.GenerateManifest(ref, nil, evilDesc) + _, err = Copy(newContext(), memoryStore, ref, newResolver(), ref) suite.Nil(err, "no error pushing test data") // Step 4: pull malicious tar with oras filestore and ensure error @@ -377,10 +383,10 @@ func (suite *ORASTestSuite) Test_4_GHSA_g5v4_5x39_vwhx() { suite.FailNow("error creating temp directory", err) } defer os.RemoveAll(tempDir) - store := orascontent.NewFileStore(tempDir) + store := orascontent.NewFile(tempDir) defer store.Close() ref = fmt.Sprintf("%s/evil:%s", suite.DockerRegistryHost, tag) - _, _, err = Pull(newContext(), newResolver(), ref, store) + _, err = Copy(newContext(), newResolver(), ref, store, ref) suite.NotNil(err, "error expected pulling malicious tar") suite.Contains(err.Error(), expectedError, diff --git a/pkg/oras/provider.go b/pkg/oras/provider.go new file mode 100644 index 00000000..2ce528aa --- /dev/null +++ b/pkg/oras/provider.go @@ -0,0 +1,65 @@ +package oras + +import ( + "context" + "errors" + "io" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/remotes" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// ProviderWrapper wraps a remote.Fetcher to make a content.Provider, which is useful for things +type ProviderWrapper struct { + Fetcher remotes.Fetcher +} + +func (p *ProviderWrapper) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content.ReaderAt, error) { + if p.Fetcher == nil { + return nil, errors.New("no Fetcher provided") + } + return &fetcherReaderAt{ + ctx: ctx, + fetcher: p.Fetcher, + desc: desc, + offset: 0, + }, nil +} + +type fetcherReaderAt struct { + ctx context.Context + fetcher remotes.Fetcher + desc ocispec.Descriptor + rc io.ReadCloser + offset int64 +} + +func (f *fetcherReaderAt) Close() error { + if f.rc == nil { + return nil + } + return f.rc.Close() +} + +func (f *fetcherReaderAt) Size() int64 { + return f.desc.Size +} + +func (f *fetcherReaderAt) ReadAt(p []byte, off int64) (n int, err error) { + // if we do not have a readcloser, get it + if f.rc == nil || f.offset != off { + rc, err := f.fetcher.Fetch(f.ctx, f.desc) + if err != nil { + return 0, err + } + f.rc = rc + } + + n, err = f.rc.Read(p) + if err != nil { + return n, err + } + f.offset += int64(n) + return n, err +} diff --git a/pkg/oras/pull.go b/pkg/oras/pull.go deleted file mode 100644 index 8fe55091..00000000 --- a/pkg/oras/pull.go +++ /dev/null @@ -1,150 +0,0 @@ -/* -Copyright The ORAS Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package oras - -import ( - "context" - "sync" - - "github.com/containerd/containerd/content" - "github.com/containerd/containerd/images" - "github.com/containerd/containerd/log" - "github.com/containerd/containerd/remotes" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" - "golang.org/x/sync/semaphore" - - orascontent "oras.land/oras-go/pkg/content" -) - -// Pull pull files from the remote -func Pull(ctx context.Context, resolver remotes.Resolver, ref string, ingester content.Ingester, opts ...PullOpt) (ocispec.Descriptor, []ocispec.Descriptor, error) { - if resolver == nil { - return ocispec.Descriptor{}, nil, ErrResolverUndefined - } - opt := pullOptsDefaults() - for _, o := range opts { - if err := o(opt); err != nil { - return ocispec.Descriptor{}, nil, err - } - } - - _, desc, err := resolver.Resolve(ctx, ref) - if err != nil { - return ocispec.Descriptor{}, nil, err - } - - fetcher, err := resolver.Fetcher(ctx, ref) - if err != nil { - return ocispec.Descriptor{}, nil, err - } - - layers, err := fetchContent(ctx, fetcher, desc, ingester, opt) - if err != nil { - return ocispec.Descriptor{}, nil, err - } - return desc, layers, nil -} - -func fetchContent(ctx context.Context, fetcher remotes.Fetcher, desc ocispec.Descriptor, ingester content.Ingester, opts *pullOpts) ([]ocispec.Descriptor, error) { - var descriptors []ocispec.Descriptor - lock := &sync.Mutex{} - picker := images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - if isAllowedMediaType(desc.MediaType, opts.allowedMediaTypes...) { - if opts.filterName(desc) { - lock.Lock() - defer lock.Unlock() - descriptors = append(descriptors, desc) - } - return nil, nil - } - return nil, nil - }) - - store := opts.contentProvideIngester - if store == nil { - store = newHybridStoreFromIngester(ingester, opts.cachedMediaTypes) - } - handlers := []images.Handler{ - filterHandler(opts, opts.allowedMediaTypes...), - } - handlers = append(handlers, opts.baseHandlers...) - handlers = append(handlers, - remotes.FetchHandler(store, fetcher), - picker, - images.ChildrenHandler(store), - ) - handlers = append(handlers, opts.callbackHandlers...) - - if err := opts.dispatch(ctx, images.Handlers(handlers...), nil, desc); err != nil { - return nil, err - } - - return descriptors, nil -} - -func filterHandler(opts *pullOpts, allowedMediaTypes ...string) images.HandlerFunc { - return func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - switch { - case isAllowedMediaType(desc.MediaType, ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex): - return nil, nil - case isAllowedMediaType(desc.MediaType, allowedMediaTypes...): - if opts.filterName(desc) { - return nil, nil - } - log.G(ctx).Warnf("blob no name: %v", desc.Digest) - default: - log.G(ctx).Warnf("unknown type: %v", desc.MediaType) - } - return nil, images.ErrStopHandler - } -} - -func isAllowedMediaType(mediaType string, allowedMediaTypes ...string) bool { - if len(allowedMediaTypes) == 0 { - return true - } - for _, allowedMediaType := range allowedMediaTypes { - if mediaType == allowedMediaType { - return true - } - } - return false -} - -// dispatchBFS behaves the same as images.Dispatch() but in sequence with breath-first search. -func dispatchBFS(ctx context.Context, handler images.Handler, weighted *semaphore.Weighted, descs ...ocispec.Descriptor) error { - for i := 0; i < len(descs); i++ { - desc := descs[i] - children, err := handler.Handle(ctx, desc) - if err != nil { - switch err := errors.Cause(err); err { - case images.ErrSkipDesc: - continue // don't traverse the children. - case ErrStopProcessing: - return nil - } - return err - } - descs = append(descs, children...) - } - return nil -} - -func filterName(desc ocispec.Descriptor) bool { - name, ok := orascontent.ResolveName(desc) - return ok && len(name) > 0 -} diff --git a/pkg/oras/pull_opts.go b/pkg/oras/pull_opts.go deleted file mode 100644 index 907c27cf..00000000 --- a/pkg/oras/pull_opts.go +++ /dev/null @@ -1,150 +0,0 @@ -/* -Copyright The ORAS Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package oras - -import ( - "context" - "fmt" - "io" - "sync" - - "github.com/containerd/containerd/images" - "github.com/opencontainers/go-digest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "golang.org/x/sync/semaphore" - - orascontent "oras.land/oras-go/pkg/content" -) - -type pullOpts struct { - allowedMediaTypes []string - dispatch func(context.Context, images.Handler, *semaphore.Weighted, ...ocispec.Descriptor) error - baseHandlers []images.Handler - callbackHandlers []images.Handler - contentProvideIngester orascontent.ProvideIngester - filterName func(ocispec.Descriptor) bool - cachedMediaTypes []string -} - -// PullOpt allows callers to set options on the oras pull -type PullOpt func(o *pullOpts) error - -func pullOptsDefaults() *pullOpts { - return &pullOpts{ - dispatch: images.Dispatch, - filterName: filterName, - cachedMediaTypes: []string{ocispec.MediaTypeImageManifest, ocispec.MediaTypeImageIndex}, - } -} - -// WithCachedMediaTypes sets the media types normally cached in memory when pulling. -func WithCachedMediaTypes(cachedMediaTypes ...string) PullOpt { - return func(o *pullOpts) error { - o.cachedMediaTypes = cachedMediaTypes - return nil - } -} - -// WithAdditionalCachedMediaTypes adds media types normally cached in memory when pulling. -// This does not replace the default media types, but appends to them -func WithAdditionalCachedMediaTypes(cachedMediaTypes ...string) PullOpt { - return func(o *pullOpts) error { - o.cachedMediaTypes = append(o.cachedMediaTypes, cachedMediaTypes...) - return nil - } -} - -// WithAllowedMediaType sets the allowed media types -func WithAllowedMediaType(allowedMediaTypes ...string) PullOpt { - return func(o *pullOpts) error { - o.allowedMediaTypes = append(o.allowedMediaTypes, allowedMediaTypes...) - return nil - } -} - -// WithAllowedMediaTypes sets the allowed media types -func WithAllowedMediaTypes(allowedMediaTypes []string) PullOpt { - return func(o *pullOpts) error { - o.allowedMediaTypes = append(o.allowedMediaTypes, allowedMediaTypes...) - return nil - } -} - -// WithPullByBFS opt to pull in sequence with breath-first search -func WithPullByBFS(o *pullOpts) error { - o.dispatch = dispatchBFS - return nil -} - -// WithPullBaseHandler provides base handlers, which will be called before -// any pull specific handlers. -func WithPullBaseHandler(handlers ...images.Handler) PullOpt { - return func(o *pullOpts) error { - o.baseHandlers = append(o.baseHandlers, handlers...) - return nil - } -} - -// WithPullCallbackHandler provides callback handlers, which will be called after -// any pull specific handlers. -func WithPullCallbackHandler(handlers ...images.Handler) PullOpt { - return func(o *pullOpts) error { - o.callbackHandlers = append(o.callbackHandlers, handlers...) - return nil - } -} - -// WithContentProvideIngester opt to the provided Provider and Ingester -// for file system I/O, including caches. -func WithContentProvideIngester(store orascontent.ProvideIngester) PullOpt { - return func(o *pullOpts) error { - o.contentProvideIngester = store - return nil - } -} - -// WithPullEmptyNameAllowed allows pulling blobs with empty name. -func WithPullEmptyNameAllowed() PullOpt { - return func(o *pullOpts) error { - o.filterName = func(ocispec.Descriptor) bool { - return true - } - return nil - } -} - -// WithPullStatusTrack report results to stdout -func WithPullStatusTrack(writer io.Writer) PullOpt { - return WithPullCallbackHandler(pullStatusTrack(writer)) -} - -func pullStatusTrack(writer io.Writer) images.Handler { - var printLock sync.Mutex - return images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - if name, ok := orascontent.ResolveName(desc); ok { - digestString := desc.Digest.String() - if err := desc.Digest.Validate(); err == nil { - if algo := desc.Digest.Algorithm(); algo == digest.SHA256 { - digestString = desc.Digest.Encoded()[:12] - } - } - printLock.Lock() - defer printLock.Unlock() - fmt.Fprintln(writer, "Downloaded", digestString, name) - } - return nil, nil - }) -} diff --git a/pkg/oras/push.go b/pkg/oras/push.go deleted file mode 100644 index 93542f5c..00000000 --- a/pkg/oras/push.go +++ /dev/null @@ -1,126 +0,0 @@ -/* -Copyright The ORAS Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package oras - -import ( - "context" - "encoding/json" - - "github.com/containerd/containerd/content" - "github.com/containerd/containerd/images" - "github.com/containerd/containerd/remotes" - digest "github.com/opencontainers/go-digest" - specs "github.com/opencontainers/image-spec/specs-go" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - - artifact "oras.land/oras-go/pkg/artifact" -) - -// Push pushes files to the remote -func Push(ctx context.Context, resolver remotes.Resolver, ref string, provider content.Provider, descriptors []ocispec.Descriptor, opts ...PushOpt) (ocispec.Descriptor, error) { - if resolver == nil { - return ocispec.Descriptor{}, ErrResolverUndefined - } - opt := pushOptsDefaults() - for _, o := range opts { - if err := o(opt); err != nil { - return ocispec.Descriptor{}, err - } - } - if opt.validateName != nil { - for _, desc := range descriptors { - if err := opt.validateName(desc); err != nil { - return ocispec.Descriptor{}, err - } - } - } - - pusher, err := resolver.Pusher(ctx, ref) - if err != nil { - return ocispec.Descriptor{}, err - } - - desc, store, err := pack(provider, descriptors, opt) - if err != nil { - return ocispec.Descriptor{}, err - } - - var wrapper func(images.Handler) images.Handler - if len(opt.baseHandlers) > 0 { - wrapper = func(h images.Handler) images.Handler { - return images.Handlers(append(opt.baseHandlers, h)...) - } - } - - if err := remotes.PushContent(ctx, pusher, desc, store, nil, nil, wrapper); err != nil { - return ocispec.Descriptor{}, err - } - return desc, nil -} - -//func pack(store *hybridStore, descriptors []ocispec.Descriptor, opts *pushOpts) (ocispec.Descriptor, error) { -func pack(provider content.Provider, descriptors []ocispec.Descriptor, opts *pushOpts) (ocispec.Descriptor, content.Store, error) { - store := newHybridStoreFromProvider(provider, nil) - - // Config - var config ocispec.Descriptor - if opts.config == nil { - configBytes := []byte("{}") - config = ocispec.Descriptor{ - MediaType: artifact.UnknownConfigMediaType, - Digest: digest.FromBytes(configBytes), - Size: int64(len(configBytes)), - } - store.Set(config, configBytes) - } else { - config = *opts.config - } - if opts.configAnnotations != nil { - config.Annotations = opts.configAnnotations - } - if opts.configMediaType != "" { - config.MediaType = opts.configMediaType - } - - // Manifest - if opts.manifest != nil { - return *opts.manifest, store, nil - } - - if descriptors == nil { - descriptors = []ocispec.Descriptor{} // make it an empty array to prevent potential server-side bugs - } - manifest := ocispec.Manifest{ - Versioned: specs.Versioned{ - SchemaVersion: 2, // historical value. does not pertain to OCI or docker version - }, - Config: config, - Layers: descriptors, - Annotations: opts.manifestAnnotations, - } - manifestBytes, err := json.Marshal(manifest) - if err != nil { - return ocispec.Descriptor{}, nil, err - } - manifestDescriptor := ocispec.Descriptor{ - MediaType: ocispec.MediaTypeImageManifest, - Digest: digest.FromBytes(manifestBytes), - Size: int64(len(manifestBytes)), - } - store.Set(manifestDescriptor, manifestBytes) - - return manifestDescriptor, store, nil -} diff --git a/pkg/oras/push_opts.go b/pkg/oras/push_opts.go deleted file mode 100644 index 1ae7b638..00000000 --- a/pkg/oras/push_opts.go +++ /dev/null @@ -1,165 +0,0 @@ -/* -Copyright The ORAS Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package oras - -import ( - "context" - "fmt" - "io" - "path/filepath" - "strings" - "sync" - - "github.com/containerd/containerd/images" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" - - orascontent "oras.land/oras-go/pkg/content" -) - -type pushOpts struct { - config *ocispec.Descriptor - configMediaType string - configAnnotations map[string]string - manifest *ocispec.Descriptor - manifestAnnotations map[string]string - validateName func(desc ocispec.Descriptor) error - baseHandlers []images.Handler -} - -func pushOptsDefaults() *pushOpts { - return &pushOpts{ - validateName: ValidateNameAsPath, - } -} - -// PushOpt allows callers to set options on the oras push -type PushOpt func(o *pushOpts) error - -// WithConfig overrides the config - setting this will ignore WithConfigMediaType and WithConfigAnnotations -func WithConfig(config ocispec.Descriptor) PushOpt { - return func(o *pushOpts) error { - o.config = &config - return nil - } -} - -// WithConfigMediaType overrides the config media type -func WithConfigMediaType(mediaType string) PushOpt { - return func(o *pushOpts) error { - o.configMediaType = mediaType - return nil - } -} - -// WithConfigAnnotations overrides the config annotations -func WithConfigAnnotations(annotations map[string]string) PushOpt { - return func(o *pushOpts) error { - o.configAnnotations = annotations - return nil - } -} - -// WithManifest overrides the manifest - setting this will ignore WithManifestConfigAnnotations -func WithManifest(manifest ocispec.Descriptor) PushOpt { - return func(o *pushOpts) error { - o.manifest = &manifest - return nil - } -} - -// WithManifestAnnotations overrides the manifest annotations -func WithManifestAnnotations(annotations map[string]string) PushOpt { - return func(o *pushOpts) error { - o.manifestAnnotations = annotations - return nil - } -} - -// WithNameValidation validates the image title in the descriptor. -// Pass nil to disable name validation. -func WithNameValidation(validate func(desc ocispec.Descriptor) error) PushOpt { - return func(o *pushOpts) error { - o.validateName = validate - return nil - } -} - -// ValidateNameAsPath validates name in the descriptor as file path in order -// to generate good packages intended to be pulled using the FileStore or -// the oras cli. -// For cross-platform considerations, only unix paths are accepted. -func ValidateNameAsPath(desc ocispec.Descriptor) error { - // no empty name - path, ok := orascontent.ResolveName(desc) - if !ok || path == "" { - return orascontent.ErrNoName - } - - // path should be clean - if target := filepath.ToSlash(filepath.Clean(path)); target != path { - return errors.Wrap(ErrDirtyPath, path) - } - - // path should be slash-separated - if strings.Contains(path, "\\") { - return errors.Wrap(ErrPathNotSlashSeparated, path) - } - - // disallow absolute path: covers unix and windows format - if strings.HasPrefix(path, "/") { - return errors.Wrap(ErrAbsolutePathDisallowed, path) - } - if len(path) > 2 { - c := path[0] - if path[1] == ':' && path[2] == '/' && ('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') { - return errors.Wrap(ErrAbsolutePathDisallowed, path) - } - } - - // disallow path traversal - if strings.HasPrefix(path, "../") || path == ".." { - return errors.Wrap(ErrPathTraversalDisallowed, path) - } - - return nil -} - -// WithPushBaseHandler provides base handlers, which will be called before -// any push specific handlers. -func WithPushBaseHandler(handlers ...images.Handler) PushOpt { - return func(o *pushOpts) error { - o.baseHandlers = append(o.baseHandlers, handlers...) - return nil - } -} - -// WithPushStatusTrack report results to a provided writer -func WithPushStatusTrack(writer io.Writer) PushOpt { - return WithPushBaseHandler(pushStatusTrack(writer)) -} - -func pushStatusTrack(writer io.Writer) images.Handler { - var printLock sync.Mutex - return images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - if name, ok := orascontent.ResolveName(desc); ok { - printLock.Lock() - defer printLock.Unlock() - fmt.Fprintln(writer, "Uploading", desc.Digest.Encoded()[:12], name) - } - return nil, nil - }) -} diff --git a/pkg/oras/store.go b/pkg/oras/store.go index 33d87345..edb83562 100644 --- a/pkg/oras/store.go +++ b/pkg/oras/store.go @@ -17,11 +17,13 @@ package oras import ( "context" - "errors" "io" + "io/ioutil" "time" "github.com/containerd/containerd/content" + "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/remotes" "github.com/opencontainers/go-digest" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "golang.org/x/sync/errgroup" @@ -29,31 +31,24 @@ import ( orascontent "oras.land/oras-go/pkg/content" ) -// ensure interface -var ( - _ content.Store = &hybridStore{} -) - type hybridStore struct { - cache *orascontent.Memorystore + cache *orascontent.Memory cachedMediaTypes []string + cacheOnly bool provider content.Provider ingester content.Ingester } -func newHybridStoreFromProvider(provider content.Provider, cachedMediaTypes []string) *hybridStore { - return &hybridStore{ - cache: orascontent.NewMemoryStore(), - cachedMediaTypes: cachedMediaTypes, - provider: provider, +func newHybridStoreFromPusher(pusher remotes.Pusher, cachedMediaTypes []string, cacheOnly bool) *hybridStore { + // construct an ingester from a pusher + ingester := pusherIngester{ + pusher: pusher, } -} - -func newHybridStoreFromIngester(ingester content.Ingester, cachedMediaTypes []string) *hybridStore { return &hybridStore{ - cache: orascontent.NewMemoryStore(), + cache: orascontent.NewMemory(), cachedMediaTypes: cachedMediaTypes, ingester: ingester, + cacheOnly: cacheOnly, } } @@ -61,18 +56,22 @@ func (s *hybridStore) Set(desc ocispec.Descriptor, content []byte) { s.cache.Set(desc, content) } -// ReaderAt provides contents -func (s *hybridStore) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content.ReaderAt, error) { - readerAt, err := s.cache.ReaderAt(ctx, desc) +func (s *hybridStore) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) { + reader, err := s.cache.Fetch(ctx, desc) if err == nil { - return readerAt, nil + return reader, err } if s.provider != nil { - return s.provider.ReaderAt(ctx, desc) + rat, err := s.provider.ReaderAt(ctx, desc) + return ioutil.NopCloser(orascontent.NewReaderAtWrapper(rat)), err } return nil, err } +func (s *hybridStore) Push(ctx context.Context, desc ocispec.Descriptor) (content.Writer, error) { + return s.Writer(ctx, content.WithDescriptor(desc)) +} + // Writer begins or resumes the active writer identified by desc func (s *hybridStore) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) { var wOpts content.WriterOpts @@ -83,70 +82,30 @@ func (s *hybridStore) Writer(ctx context.Context, opts ...content.WriterOpt) (co } if isAllowedMediaType(wOpts.Desc.MediaType, s.cachedMediaTypes...) || s.ingester == nil { - cacheWriter, err := s.cache.Writer(ctx, opts...) + pusher, err := s.cache.Pusher(ctx, "") if err != nil { return nil, err } - ingesterWriter, err := s.ingester.Writer(ctx, opts...) + cacheWriter, err := pusher.Push(ctx, wOpts.Desc) if err != nil { return nil, err } - return newTeeWriter(wOpts.Desc, cacheWriter, ingesterWriter), nil + // if we cache it only, do not pass it through + if s.cacheOnly { + return cacheWriter, nil + } + ingesterWriter, err := s.ingester.Writer(ctx, opts...) + switch { + case err == nil: + return newTeeWriter(wOpts.Desc, cacheWriter, ingesterWriter), nil + case errdefs.IsAlreadyExists(err): + return cacheWriter, nil + } + return nil, err } return s.ingester.Writer(ctx, opts...) } -// TODO: implement (needed to create a content.Store) -// TODO: do not return empty content.Info -// Abort completely cancels the ingest operation targeted by ref. -func (s *hybridStore) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) { - return content.Info{}, nil -} - -// TODO: implement (needed to create a content.Store) -// Update updates mutable information related to content. -// If one or more fieldpaths are provided, only those -// fields will be updated. -// Mutable fields: -// labels.* -func (s *hybridStore) Update(ctx context.Context, info content.Info, fieldpaths ...string) (content.Info, error) { - return content.Info{}, errors.New("not yet implemented: Update (content.Store interface)") -} - -// TODO: implement (needed to create a content.Store) -// Walk will call fn for each item in the content store which -// match the provided filters. If no filters are given all -// items will be walked. -func (s *hybridStore) Walk(ctx context.Context, fn content.WalkFunc, filters ...string) error { - return errors.New("not yet implemented: Walk (content.Store interface)") -} - -// TODO: implement (needed to create a content.Store) -// Delete removes the content from the store. -func (s *hybridStore) Delete(ctx context.Context, dgst digest.Digest) error { - return errors.New("not yet implemented: Delete (content.Store interface)") -} - -// TODO: implement (needed to create a content.Store) -func (s *hybridStore) Status(ctx context.Context, ref string) (content.Status, error) { - // Status returns the status of the provided ref. - return content.Status{}, errors.New("not yet implemented: Status (content.Store interface)") -} - -// TODO: implement (needed to create a content.Store) -// ListStatuses returns the status of any active ingestions whose ref match the -// provided regular expression. If empty, all active ingestions will be -// returned. -func (s *hybridStore) ListStatuses(ctx context.Context, filters ...string) ([]content.Status, error) { - return []content.Status{}, errors.New("not yet implemented: ListStatuses (content.Store interface)") -} - -// TODO: implement (needed to create a content.Store) -// Abort completely cancels the ingest operation targeted by ref. -func (s *hybridStore) Abort(ctx context.Context, ref string) error { - return errors.New("not yet implemented: Abort (content.Store interface)") -} - // teeWriter tees the content to one or more content.Writer type teeWriter struct { writers []content.Writer @@ -237,3 +196,18 @@ func (t *teeWriter) Truncate(size int64) error { } return g.Wait() } + +// pusherIngester simple wrapper to get an ingester from a pusher +type pusherIngester struct { + pusher remotes.Pusher +} + +func (p pusherIngester) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) { + var wOpts content.WriterOpts + for _, opt := range opts { + if err := opt(&wOpts); err != nil { + return nil, err + } + } + return p.pusher.Push(ctx, wOpts.Desc) +} diff --git a/pkg/target/target.go b/pkg/target/target.go new file mode 100644 index 00000000..16734f2a --- /dev/null +++ b/pkg/target/target.go @@ -0,0 +1,12 @@ +package target + +import ( + "github.com/containerd/containerd/remotes" +) + +// Target represents a place to which one can send/push or retrieve/pull artifacts. +// Anything that implements the Target interface can be used as a place to send or +// retrieve artifacts. +type Target interface { + remotes.Resolver +} diff --git a/scripts/acceptance.sh b/scripts/acceptance.sh index 3706572d..b04abe29 100755 --- a/scripts/acceptance.sh +++ b/scripts/acceptance.sh @@ -21,11 +21,12 @@ LOCAL_REGISTRY_HOSTNAME="${LOCAL_REGISTRY_HOSTNAME:-localhost}" # Cleanup from previous runs rm -f hello.txt -rm -f bin/oras-acceptance || true +rm -f bin/oras-acceptance-* || true docker rm -f oras-acceptance-registry || true -# Build the example into a binary -CGO_ENABLED=0 go build -v -o bin/oras-acceptance ./examples/ +# Build the examples into binaries +CGO_ENABLED=0 go build -v -o bin/oras-acceptance-simple ./examples/simple +CGO_ENABLED=0 go build -v -o bin/oras-acceptance-advanced ./examples/advanced # Run a test registry and expose at localhost:5000 trap "docker rm -f oras-acceptance-registry" EXIT @@ -54,7 +55,7 @@ done sleep 5 # Run the example binary -bin/oras-acceptance +bin/oras-acceptance-simple # Ensure hello.txt exists and contains expected content grep '^Hello World!$' hello.txt