diff --git a/contrib/nydusify/cmd/nydusify.go b/contrib/nydusify/cmd/nydusify.go index dd0b1859151..553a86095bb 100644 --- a/contrib/nydusify/cmd/nydusify.go +++ b/contrib/nydusify/cmd/nydusify.go @@ -391,6 +391,12 @@ func main() { EnvVars: []string{"FS_CHUNK_SIZE"}, Aliases: []string{"chunk-size"}, }, + &cli.StringFlag{ + Name: "batch-size", + Value: "0", + Usage: "size of batch data chunks, must be power of two, between 0x1000-0x1000000 or zero, [default: 0]", + EnvVars: []string{"BATCH_SIZE"}, + }, &cli.StringFlag{ Name: "work-dir", Value: "./tmp", @@ -492,6 +498,7 @@ func main() { FsAlignChunk: c.Bool("backend-aligned-chunk") || c.Bool("fs-align-chunk"), Compressor: c.String("compressor"), ChunkSize: c.String("chunk-size"), + BatchSize: c.String("batch-size"), OCIRef: c.Bool("oci-ref"), AllPlatforms: c.Bool("all-platforms"), diff --git a/contrib/nydusify/pkg/converter/config.go b/contrib/nydusify/pkg/converter/config.go index 0ee12f4d0df..388fcf23ab8 100644 --- a/contrib/nydusify/pkg/converter/config.go +++ b/contrib/nydusify/pkg/converter/config.go @@ -28,6 +28,7 @@ func getConfig(opt Opt) map[string]string { cfg["fs_version"] = opt.FsVersion cfg["fs_align_chunk"] = strconv.FormatBool(opt.FsAlignChunk) cfg["fs_chunk_size"] = opt.ChunkSize + cfg["batch_size"] = opt.BatchSize // FIXME: still needs to be supported by acceld converter package. cfg["cache_ref"] = opt.CacheRef diff --git a/contrib/nydusify/pkg/converter/converter.go b/contrib/nydusify/pkg/converter/converter.go index da04cc16e09..1bb55674af8 100644 --- a/contrib/nydusify/pkg/converter/converter.go +++ b/contrib/nydusify/pkg/converter/converter.go @@ -42,6 +42,7 @@ type Opt struct { FsAlignChunk bool Compressor string ChunkSize string + BatchSize string PrefetchPatterns string OCIRef bool diff --git a/contrib/nydusify/tests/e2e_test.go b/contrib/nydusify/tests/e2e_test.go index 5ba94f1d215..2282b9b2945 100644 --- a/contrib/nydusify/tests/e2e_test.go +++ b/contrib/nydusify/tests/e2e_test.go @@ -25,7 +25,7 @@ func testBasicConvert(t *testing.T, fsVersion string) { registry := NewRegistry(t) defer registry.Destroy(t) registry.Build(t, "image-basic") - nydusify := NewNydusify(registry, "image-basic", "image-basic-nydus", "", "", fsVersion) + nydusify := NewNydusify(registry, "image-basic", "image-basic-nydus", "", "", fsVersion, "") nydusify.Convert(t) nydusify.Check(t) } @@ -34,7 +34,7 @@ func testBasicAuth(t *testing.T, fsVersion string) { registry := NewAuthRegistry(t) defer registry.Destroy(t) registry.AuthBuild(t, "image-basic") - nydusify := NewNydusify(registry, "image-basic", "image-basic-nydus", "", "", fsVersion) + nydusify := NewNydusify(registry, "image-basic", "image-basic-nydus", "", "", fsVersion, "") nydusify.Convert(t) nydusify.Check(t) } @@ -48,7 +48,7 @@ func testReproducibleBuild(t *testing.T, fsVersion string) { for i := 0; i < 5; i++ { workDir := fmt.Sprintf("./tmp-%d", i) os.Setenv("WORKDIR", workDir) - nydusify := NewNydusify(registry, "image-basic", "image-basic-nydus", "", "", fsVersion) + nydusify := NewNydusify(registry, "image-basic", "image-basic-nydus", "", "", fsVersion, "") nydusify.Convert(t) nydusify.Check(t) hash, err := utils.HashFile(nydusify.GetBootstarpFilePath()) @@ -68,20 +68,20 @@ func testConvertWithCache(t *testing.T, fsVersion string) { defer registry.Destroy(t) registry.Build(t, "image-basic") - nydusify1 := NewNydusify(registry, "image-basic", "image-basic-nydus-1", "cache:v1", "", fsVersion) + nydusify1 := NewNydusify(registry, "image-basic", "image-basic-nydus-1", "cache:v1", "", fsVersion, "") nydusify1.Convert(t) nydusify1.Check(t) - nydusify2 := NewNydusify(registry, "image-basic", "image-basic-nydus-2", "cache:v1", "", fsVersion) + nydusify2 := NewNydusify(registry, "image-basic", "image-basic-nydus-2", "cache:v1", "", fsVersion, "") nydusify2.Convert(t) nydusify2.Check(t) registry.Build(t, "image-from-1") - nydusify3 := NewNydusify(registry, "image-from-1", "image-from-nydus-1", "cache:v1", "", fsVersion) + nydusify3 := NewNydusify(registry, "image-from-1", "image-from-nydus-1", "cache:v1", "", fsVersion, "") nydusify3.Convert(t) nydusify3.Check(t) registry.Build(t, "image-from-2") - nydusify4 := NewNydusify(registry, "image-from-2", "image-from-nydus-2", "cache:v1", "", fsVersion) + nydusify4 := NewNydusify(registry, "image-from-2", "image-from-nydus-2", "cache:v1", "", fsVersion, "") nydusify4.Convert(t) nydusify4.Check(t) } @@ -92,28 +92,28 @@ func testConvertWithChunkDict(t *testing.T, fsVersion string) { registry.Build(t, "chunk-dict-1") // build chunk-dict-1 bootstrap - nydusify1 := NewNydusify(registry, "chunk-dict-1", "nydus:chunk-dict-1", "", "", fsVersion) + nydusify1 := NewNydusify(registry, "chunk-dict-1", "nydus:chunk-dict-1", "", "", fsVersion, "") nydusify1.Convert(t) nydusify1.Check(t) chunkDictOpt := fmt.Sprintf("bootstrap:registry:%s/%s", registry.Host(), "nydus:chunk-dict-1") // build without build-cache registry.Build(t, "image-basic") - nydusify2 := NewNydusify(registry, "image-basic", "nydus:image-basic", "", chunkDictOpt, fsVersion) + nydusify2 := NewNydusify(registry, "image-basic", "nydus:image-basic", "", chunkDictOpt, fsVersion, "") nydusify2.Convert(t) nydusify2.Check(t) // build with build-cache registry.Build(t, "image-from-1") - nydusify3 := NewNydusify(registry, "image-from-1", "nydus:image-from-1", "nydus:cache_v1", chunkDictOpt, fsVersion) + nydusify3 := NewNydusify(registry, "image-from-1", "nydus:image-from-1", "nydus:cache_v1", chunkDictOpt, fsVersion, "") nydusify3.Convert(t) nydusify3.Check(t) // change chunk dict registry.Build(t, "chunk-dict-2") - nydusify4 := NewNydusify(registry, "chunk-dict-2", "nydus:chunk-dict-2", "", "", fsVersion) + nydusify4 := NewNydusify(registry, "chunk-dict-2", "nydus:chunk-dict-2", "", "", fsVersion, "") nydusify4.Convert(t) nydusify4.Check(t) chunkDictOpt = fmt.Sprintf("bootstrap:registry:%s/%s", registry.Host(), "nydus:chunk-dict-2") registry.Build(t, "image-from-2") - nydusify5 := NewNydusify(registry, "image-from-2", "nydus:image-from-2", "nydus:cache_v1", chunkDictOpt, fsVersion) + nydusify5 := NewNydusify(registry, "image-from-2", "nydus:image-from-2", "nydus:cache_v1", chunkDictOpt, fsVersion, "") nydusify5.Convert(t) nydusify5.Check(t) } @@ -194,11 +194,30 @@ func testConvertWithS3Backend(t *testing.T, fsVersion string) { logrus.Infof("use s3 backend config: %s", backendConfig) - nydusify := NewNydusify(registry, "image-basic", "image-basic-nydus", "", "", fsVersion) + nydusify := NewNydusify(registry, "image-basic", "image-basic-nydus", "", "", fsVersion, "") nydusify.Convert(t) nydusify.Check(t) } +func testConvertWithBatchSize(t *testing.T, fsVersion string) { + registry := NewRegistry(t) + defer registry.Destroy(t) + + registry.Build(t, "image-basic") + nydusify1 := NewNydusify(registry, "image-basic", "image-basic-nydus-1", "", "", fsVersion, "0x1000") + nydusify1.Convert(t) + nydusify1.Check(t) + nydusify2 := NewNydusify(registry, "image-basic", "image-basic-nydus-2", "", "", fsVersion, "0x10000") + nydusify2.Convert(t) + nydusify2.Check(t) + nydusify3 := NewNydusify(registry, "image-basic", "image-basic-nydus-3", "", "", fsVersion, "0x100000") + nydusify3.Convert(t) + nydusify3.Check(t) + nydusify4 := NewNydusify(registry, "image-basic", "image-basic-nydus-4", "", "", fsVersion, "0x1000000") + nydusify4.Convert(t) + nydusify4.Check(t) +} + func TestSmoke(t *testing.T) { fsVersions := [2]string{"5", "6"} for _, v := range fsVersions { @@ -209,4 +228,6 @@ func TestSmoke(t *testing.T) { testConvertWithChunkDict(t, v) testConvertWithS3Backend(t, v) } + // batch-size only support v6 + testConvertWithBatchSize(t, fsVersions[1]) } diff --git a/contrib/nydusify/tests/image_test.go b/contrib/nydusify/tests/image_test.go index 100551be35e..2228279fcf3 100644 --- a/contrib/nydusify/tests/image_test.go +++ b/contrib/nydusify/tests/image_test.go @@ -41,7 +41,7 @@ func convert(t *testing.T, ref string, fsVersion string) { registry := NewRegistry(t) defer registry.Destroy(t) transfer(t, ref) - nydusify := NewNydusify(registry, ref, fmt.Sprintf("%s-nydus", ref), "", "", fsVersion) + nydusify := NewNydusify(registry, ref, fmt.Sprintf("%s-nydus", ref), "", "", fsVersion, "") nydusify.Convert(t) nydusify.Check(t) } diff --git a/contrib/nydusify/tests/nydusify.go b/contrib/nydusify/tests/nydusify.go index cb11294b187..4467d35e64d 100644 --- a/contrib/nydusify/tests/nydusify.go +++ b/contrib/nydusify/tests/nydusify.go @@ -41,9 +41,10 @@ type Nydusify struct { chunkDictArgs string fsVersion string workDir string + batchSize string } -func NewNydusify(registry *Registry, source, target, cache string, chunkDictArgs string, fsVersion string) *Nydusify { +func NewNydusify(registry *Registry, source, target, cache string, chunkDictArgs string, fsVersion string, batchSize string) *Nydusify { backendType := "" if os.Getenv("BACKEND_TYPE") != "" { backendType = os.Getenv("BACKEND_TYPE") @@ -61,6 +62,10 @@ func NewNydusify(registry *Registry, source, target, cache string, chunkDictArgs workDir = os.Getenv("WORKDIR") } + if len(batchSize) == 0 { + batchSize = "0" + } + return &Nydusify{ Registry: registry, Source: source, @@ -71,6 +76,7 @@ func NewNydusify(registry *Registry, source, target, cache string, chunkDictArgs chunkDictArgs: chunkDictArgs, fsVersion: fsVersion, workDir: workDir, + batchSize: batchSize, } } @@ -104,6 +110,8 @@ func (nydusify *Nydusify) Convert(t *testing.T) { BackendConfig: nydusify.backendConfig, FsVersion: nydusify.fsVersion, + + BatchSize: nydusify.batchSize, } err := converter.Convert(context.Background(), opt)