-
Notifications
You must be signed in to change notification settings - Fork 205
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
nydusify chunkdict generate --sources
Add the 'nydus-image chunkdict save' command with the "--sources" followed by the nydus image of registry (e.g.,'registry.com/busybox:nydus-v1,registry.com/busybox:nydus-v2') Signed-off-by: Zhao Yuan <[email protected]>
- Loading branch information
1 parent
8a93024
commit 8b59e19
Showing
4 changed files
with
253 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,106 @@ | ||
| package generator | ||
|
|
||
| import ( | ||
| "context" | ||
| "io/fs" | ||
| "os" | ||
| "path/filepath" | ||
| "strings" | ||
|
|
||
| "github.com/pkg/errors" | ||
| "github.com/sirupsen/logrus" | ||
|
|
||
| "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/build" | ||
| "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/parser" | ||
| "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/provider" | ||
| "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils" | ||
| ) | ||
|
|
||
| // Opt defines Chunkdict generate options. | ||
| // Note: sources is one or more Nydus image references. | ||
| type Opt struct { | ||
| WorkDir string | ||
| Sources []string | ||
| SourceInsecure bool | ||
| NydusImagePath string | ||
| ExpectedArch string | ||
| } | ||
|
|
||
| // Generator generates chunkdict by deduplicating multiple nydus images | ||
| // invoking "nydus-image chunkdict save" to save image information into database. | ||
| type Generator struct { | ||
| Opt | ||
| sourcesParser []*parser.Parser | ||
| } | ||
|
|
||
| // New creates Generator instance. | ||
| func New(opt Opt) (*Generator, error) { | ||
| // TODO: support sources image resolver | ||
| var sourcesParser []*parser.Parser | ||
| for _, source := range opt.Sources { | ||
| sourcesRemote, err := provider.DefaultRemote(source, opt.SourceInsecure) | ||
| if err != nil { | ||
| return nil, errors.Wrap(err, "Init source image parser") | ||
| } | ||
| sourceParser, err := parser.New(sourcesRemote, opt.ExpectedArch) | ||
| sourcesParser = append(sourcesParser, sourceParser) | ||
| if err != nil { | ||
| return nil, errors.Wrap(err, "Failed to create parser") | ||
| } | ||
| } | ||
|
|
||
| generator := &Generator{ | ||
| Opt: opt, | ||
| sourcesParser: sourcesParser, | ||
| } | ||
|
|
||
| return generator, nil | ||
| } | ||
|
|
||
| // Generate saves multiple Nydus bootstraps into the database one by one. | ||
| func (generator *Generator) Generate(ctx context.Context) error { | ||
| for index := range generator.Sources { | ||
| if err := generator.save(ctx, index); err != nil { | ||
| if utils.RetryWithHTTP(err) { | ||
| generator.sourcesParser[index].Remote.MaybeWithHTTP(err) | ||
| } | ||
| if err := generator.save(ctx, index); err != nil { | ||
| return err | ||
| } | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // "save" stores information of chunk and blob of a Nydus Image in the database | ||
| func (generator *Generator) save(ctx context.Context, index int) error { | ||
| sourceParsed, err := generator.sourcesParser[index].Parse(ctx) | ||
| if err != nil { | ||
| return errors.Wrap(err, "parse Nydus image") | ||
| } | ||
|
|
||
| // Create a directory to store the image bootstrap | ||
| nydusImageName := strings.Replace(generator.Sources[index], "/", ":", -1) | ||
| folderPath := filepath.Join(generator.WorkDir, nydusImageName) | ||
| if err := os.MkdirAll(folderPath, fs.ModePerm); err != nil { | ||
| return errors.Wrap(err, "creat work directory") | ||
| } | ||
| if err := generator.Output(ctx, sourceParsed, folderPath, index); err != nil { | ||
| return errors.Wrap(err, "output image information") | ||
| } | ||
|
|
||
| // Invoke "nydus-image save" command | ||
| builder := build.NewBuilder(generator.NydusImagePath) | ||
| if err := builder.Save(build.SaveOption{ | ||
| BootstrapPath: filepath.Join(folderPath, "nydus_bootstrap"), | ||
| }); err != nil { | ||
| return errors.Wrap(err, "invalid nydus bootstrap format") | ||
| } | ||
|
|
||
| logrus.Infof("Save chunk information from image %s", generator.sourcesParser[index].Remote.Ref) | ||
|
|
||
| if err := os.RemoveAll(folderPath); err != nil { | ||
| return errors.Wrap(err, "remove work directory") | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| package generator | ||
|
|
||
| import ( | ||
| "context" | ||
| "encoding/json" | ||
| "fmt" | ||
| "os" | ||
| "path/filepath" | ||
|
|
||
| "github.com/pkg/errors" | ||
| "github.com/sirupsen/logrus" | ||
|
|
||
| "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/parser" | ||
| "github.com/dragonflyoss/image-service/contrib/nydusify/pkg/utils" | ||
| ) | ||
|
|
||
| func prettyDump(obj interface{}, name string) error { | ||
| bytes, err := json.MarshalIndent(obj, "", " ") | ||
| if err != nil { | ||
| return err | ||
| } | ||
| return os.WriteFile(name, bytes, 0644) | ||
| } | ||
|
|
||
| // Output outputs Nydus image nydus_bootstrap file and manifest, config to JSON file. | ||
| func (generator *Generator) Output( | ||
| ctx context.Context, sourceParsed *parser.Parsed, outputPath string, index int, | ||
| ) error { | ||
| if sourceParsed.Index != nil { | ||
| if err := prettyDump( | ||
| sourceParsed.Index, | ||
| filepath.Join(outputPath, "nydus_index.json"), | ||
| ); err != nil { | ||
| return errors.Wrap(err, "output nydus index file") | ||
| } | ||
| } | ||
| if sourceParsed.NydusImage != nil { | ||
| if err := prettyDump( | ||
| sourceParsed.NydusImage.Manifest, | ||
| filepath.Join(outputPath, "nydus_manifest.json"), | ||
| ); err != nil { | ||
| return errors.Wrap(err, "output Nydus manifest file") | ||
| } | ||
| if err := prettyDump( | ||
| sourceParsed.NydusImage.Config, | ||
| filepath.Join(outputPath, "nydus_config.json"), | ||
| ); err != nil { | ||
| return errors.Wrap(err, "output Nydus config file") | ||
| } | ||
| source := filepath.Join(outputPath, "nydus_bootstrap") | ||
| logrus.Infof("Pulling Nydus bootstrap to %s", source) | ||
| bootstrapReader, err := generator.sourcesParser[index].PullNydusBootstrap(ctx, sourceParsed.NydusImage) | ||
| if err != nil { | ||
| return errors.Wrap(err, "pull Nydus bootstrap layer") | ||
| } | ||
| defer bootstrapReader.Close() | ||
|
|
||
| if err := utils.UnpackFile(bootstrapReader, utils.BootstrapFileNameInLayer, source); err != nil { | ||
| return errors.Wrap(err, "unpack Nydus bootstrap layer") | ||
| } | ||
| } else { | ||
| err := fmt.Errorf("the %s is not a Nydus image", generator.sourcesParser[index].Remote.Ref) | ||
| return err | ||
| } | ||
| return nil | ||
| } |