diff --git a/cmd/client/commands/copy.go b/cmd/client/commands/copy.go new file mode 100644 index 00000000..244d3071 --- /dev/null +++ b/cmd/client/commands/copy.go @@ -0,0 +1,147 @@ +package commands + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + + "github.com/spf13/cobra" + "github.com/uor-framework/uor-client-go/attributes/matchers" + "github.com/uor-framework/uor-client-go/cmd/client/commands/options" + "github.com/uor-framework/uor-client-go/config" + "github.com/uor-framework/uor-client-go/content/layout" + "github.com/uor-framework/uor-client-go/manager/defaultmanager" + "github.com/uor-framework/uor-client-go/registryclient/orasclient" + "github.com/uor-framework/uor-client-go/util/examples" + "github.com/uor-framework/uor-client-go/util/workspace" +) + +// PushOptions describe configuration options that can +// be set using the push subcommand. + +type CopyOptions struct { + PullOptions + Add string + Delete string +} + +var clientCopyExamples = examples.Example{ + RootCommand: filepath.Base(os.Args[0]), + Descriptions: []string{"Copy Collection."}, + CommandString: "copy localhost:5000/myartifacts:v0.1.0 localhost:5000/myartifacts:v0.1.1", +} + +// NewPushCmd creates a new cobra.Command for the push subcommand. +func NewCopyCmd(common *options.Common) *cobra.Command { + o := CopyOptions{} + + o.PullOptions.Common = common + + cmd := &cobra.Command{ + Use: "copy SRC DST", + Short: "Copy a UOR Collection", + Example: examples.FormatExamples(clientCopyExamples), + SilenceErrors: false, + SilenceUsage: false, + Args: cobra.ExactArgs(2), + Run: func(cmd *cobra.Command, args []string) { + cobra.CheckErr(o.Complete(args)) + cobra.CheckErr(o.Validate()) + cobra.CheckErr(o.Run(cmd.Context())) + }, + } + + o.Remote.BindFlags(cmd.Flags()) + o.RemoteAuth.BindFlags(cmd.Flags()) + cmd.Flags().StringVarP(&o.Add, "add-content", "", o.Add, "add content to a UOR Collection from a directory") + cmd.Flags().StringVarP(&o.AttributeQuery, "remove-content", "", o.AttributeQuery, "remove content from a UOR Collection from an attribute query") + cmd.Flags().BoolVarP(&o.NoVerify, "no-verify", "", o.NoVerify, "skip collection signature verification") + + return cmd +} + +func (o *CopyOptions) Complete(args []string) error { + if len(args) < 2 { + return errors.New("bug: expecting two arguments") + } + o.Source = args[0] + o.Output = args[1] + + return nil +} + +func (o *CopyOptions) Validate() error { + return nil +} + +func (o *CopyOptions) Run(ctx context.Context) error { + + if !o.NoVerify { + o.Logger.Infof("Checking signature of %s", o.Source) + if err := verifyCollection(ctx, o.Source, o.RemoteAuth.Configs, o.Remote); err != nil { + return err + } + + } + + matcher := matchers.PartialAttributeMatcher{} + if o.AttributeQuery != "" { + query, err := config.ReadAttributeQuery(o.AttributeQuery) + if err != nil { + return err + } + + attributeSet, err := config.ConvertToModel(query.Attributes) + if err != nil { + return err + } + matcher = attributeSet.List() + } + + var space workspace.Workspace + if o.Add != "" { + var err error + if space, err = workspace.NewLocalWorkspace(o.Add); err != nil { + return err + } + } + + absCache, err := filepath.Abs(o.CacheDir) + if err != nil { + return err + } + cache, err := layout.NewWithContext(ctx, absCache) + if err != nil { + return err + } + + client, err := orasclient.NewClient( + orasclient.SkipTLSVerify(o.Insecure), + orasclient.WithAuthConfigs(o.Configs), + orasclient.WithPlainHTTP(o.PlainHTTP), + orasclient.WithPullableAttributes(matcher), + orasclient.WithCache(cache), + ) + if err != nil { + return fmt.Errorf("error configuring client: %v", err) + } + defer func() { + if err := client.Destroy(); err != nil { + o.Logger.Errorf(err.Error()) + } + }() + + manager := defaultmanager.New(cache, o.Logger) + var remove bool + if o.AttributeQuery != "" { + remove = true + } + var add bool + if o.Add != "" { + add = true + } + _, err = manager.Update(ctx, space, o.Source, o.Output, add, remove, client) + return err +} diff --git a/cmd/client/commands/root.go b/cmd/client/commands/root.go index 173b2865..50375ab4 100644 --- a/cmd/client/commands/root.go +++ b/cmd/client/commands/root.go @@ -64,6 +64,7 @@ func NewRootCmd() *cobra.Command { cmd.AddCommand(NewPushCmd(&o)) cmd.AddCommand(NewPullCmd(&o)) cmd.AddCommand(NewServeCmd(&o)) + cmd.AddCommand(NewCopyCmd(&o)) cmd.AddCommand(NewVersionCmd(&o)) return cmd diff --git a/manager/defaultmanager/build.go b/manager/defaultmanager/build.go index e2792cd5..fd278b5d 100644 --- a/manager/defaultmanager/build.go +++ b/manager/defaultmanager/build.go @@ -21,37 +21,12 @@ import ( ) func (d DefaultManager) Build(ctx context.Context, space workspace.Workspace, config clientapi.DataSetConfiguration, reference string, client registryclient.Client) (string, error) { - var files []string - err := space.Walk(func(path string, info os.FileInfo, err error) error { - if err != nil { - return fmt.Errorf("traversing %s: %v", path, err) - } - if info == nil { - return fmt.Errorf("no file info") - } - if info.Mode().IsRegular() { - files = append(files, path) - } - return nil - }) + files, attributesByFile, err := addFiles(space, d, config, reference) if err != nil { return "", err } - attributesByFile := map[string]model.AttributeSet{} - for _, file := range config.Collection.Files { - set, err := load.ConvertToModel(file.Attributes) - if err != nil { - return "", err - } - attributesByFile[file.File] = set - } - - if len(files) == 0 { - return "", fmt.Errorf("path %q empty workspace", space.Path(".")) - } - // If a schema is present, pull it and do the validation before // processing the files to get quick feedback to the user. collectionManifestAnnotations := map[string]string{} @@ -231,3 +206,41 @@ func deduplicate(in []string) []string { } return out } + +func addFiles(space workspace.Workspace, d DefaultManager, config clientapi.DataSetConfiguration, dest string) ([]string, map[string]model.AttributeSet, error) { + attributesByFile := map[string]model.AttributeSet{} + var files []string + + err := space.Walk(func(path string, info os.FileInfo, err error) error { + if err != nil { + return fmt.Errorf("traversing %s: %v", path, err) + } + if info == nil { + return fmt.Errorf("no file info") + } + + if info.Mode().IsRegular() { + files = append(files, path) + d.logger.Infof("adding %s to %s", path, dest) + + } + return nil + }) + if err != nil { + return nil, nil, err + } + + for _, file := range config.Collection.Files { + set, err := load.ConvertToModel(file.Attributes) + if err != nil { + return nil, nil, err + } + attributesByFile[file.File] = set + } + + if len(files) == 0 { + return nil, nil, fmt.Errorf("path %q empty workspace", space.Path(".")) + } + + return files, attributesByFile, err +} diff --git a/manager/defaultmanager/copy.go b/manager/defaultmanager/copy.go new file mode 100644 index 00000000..de6f0650 --- /dev/null +++ b/manager/defaultmanager/copy.go @@ -0,0 +1,165 @@ +package defaultmanager + +import ( + "context" + "encoding/json" + "fmt" + "os" + + ocispec "github.com/opencontainers/image-spec/specs-go/v1" + load "github.com/uor-framework/uor-client-go/config" + "github.com/uor-framework/uor-client-go/model" + "github.com/uor-framework/uor-client-go/ocimanifest" + "github.com/uor-framework/uor-client-go/registryclient" + "github.com/uor-framework/uor-client-go/util/workspace" + "oras.land/oras-go/v2/content/memory" +) + +func (d DefaultManager) Update(ctx context.Context, space workspace.Workspace, src string, dest string, add bool, remove bool, client registryclient.Client) (string, error) { + + // Pull the dataset config from the target collection + _, manBytes, err := client.GetManifest(ctx, src) + if err != nil { + return "", err + } + + var manifest ocispec.Manifest + if err := json.NewDecoder(manBytes).Decode(&manifest); err != nil { + return "", err + } + manconfig, err := client.GetContent(ctx, src, manifest.Config) + if err != nil { + return "", err + } + + config, err := load.LoadDataSetConfig(manconfig) + if err != nil { + return "", err + } + + if remove { + pruneDescs, err := d.pullCollection(ctx, src, memory.New(), client) + if err != nil { + return "", err + } + pDesc := map[string]struct{}{} + for _, s := range pruneDescs { + pDesc[s.Digest.String()] = struct{}{} + } + + var newLayers []ocispec.Descriptor + for _, v := range manifest.Layers { + if _, found := pDesc[v.Digest.String()]; !found { + newLayers = append(newLayers, v) + } + manifest.Layers = newLayers + } + } + var attributesByFile map[string]model.AttributeSet + var files []string + + if add { + files, attributesByFile, err = addFiles(space, d, config, dest) + if err != nil { + return "", err + } + } + + // If a schema is present, pull it and do the validation before + // processing the files to get quick feedback to the user. + collectionManifestAnnotations := map[string]string{} + var descs []ocispec.Descriptor + + if config.Collection.SchemaAddress != "" { + d.logger.Infof("Validating dataset configuration against schema %s", config.Collection.SchemaAddress) + collectionManifestAnnotations[ocimanifest.AnnotationSchema] = config.Collection.SchemaAddress + if err != nil { + return "", fmt.Errorf("error configuring client: %v", err) + } + + _, _, err = client.Pull(ctx, config.Collection.SchemaAddress, d.store) + if err != nil { + return "", err + } + + schemaDoc, err := fetchJSONSchema(ctx, config.Collection.SchemaAddress, d.store) + if err != nil { + return "", err + } + if add { + for file, attr := range attributesByFile { + valid, err := schemaDoc.Validate(attr) + if err != nil { + return "", fmt.Errorf("schema validation error: %w", err) + } + if !valid { + return "", fmt.Errorf("attributes for file %s are not valid for schema %s", file, config.Collection.SchemaAddress) + } + } + + // To allow the files to be loaded relative to the render + // workspace, change to the render directory. This is required + // to get path correct in the description annotations. + cwd, err := os.Getwd() + if err != nil { + return "", err + } + + if err := os.Chdir(space.Path()); err != nil { + return "", err + } + defer func() { + if err := os.Chdir(cwd); err != nil { + d.logger.Errorf("%v", err) + } + }() + + descs, err = client.AddFiles(ctx, "", files...) + if err != nil { + return "", err + } + } + + descs = append(descs, manifest.Layers...) + descs, err = ocimanifest.UpdateLayerDescriptors(descs, attributesByFile) + if err != nil { + return "", err + } + } + + // Store the DataSetConfiguration file in the manifest config of the OCI artifact for + // later use. + configJSON, err := json.Marshal(config) + if err != nil { + return "", err + } + configDesc, err := client.AddContent(ctx, ocimanifest.UORConfigMediaType, configJSON, nil) + if err != nil { + return "", err + } + + linkedDescs, linkedSchemas, err := gatherLinkedCollections(ctx, config, client) + if err != nil { + return "", err + } + + descs = append(descs, linkedDescs...) + // Write the root collection attributes + if len(linkedDescs) > 0 { + collectionManifestAnnotations[ocimanifest.AnnotationSchemaLinks] = formatLinks(linkedSchemas) + collectionManifestAnnotations[ocimanifest.AnnotationCollectionLinks] = formatLinks(config.Collection.LinkedCollections) + } + + _, err = client.AddManifest(ctx, dest, configDesc, collectionManifestAnnotations, descs...) + if err != nil { + return "", err + } + + desc, err := client.Save(ctx, dest, d.store) + if err != nil { + return "", fmt.Errorf("client save error for reference %s: %v", dest, err) + } + d.logger.Infof("Artifact %s built with reference name %s\n", desc.Digest, dest) + + return desc.Digest.String(), nil +} diff --git a/manager/manager.go b/manager/manager.go index db0b9409..5d5fbc38 100644 --- a/manager/manager.go +++ b/manager/manager.go @@ -25,4 +25,7 @@ type Manager interface { // PullAll is similar to Pull with the exception that it walks a graph of linked collections // starting with the source collection reference. PullAll(ctx context.Context, source string, remote registryclient.Remote, destination content.Store) ([]string, error) + // Update adds and removes content from a collection and stores the collection in the + // underlying content store. If successful, the root descriptor is returned. + Update(ctx context.Context, space workspace.Workspace, src string, dest string, add bool, remove bool, client registryclient.Client) (string, error) }