feat: support the arg batch-size for small chunks mergence for nydusify

contrib/nydusify/cmd/nydusify.go

@@ -391,6 +391,12 @@ func main() {
 					EnvVars: []string{"FS_CHUNK_SIZE"},
 					Aliases: []string{"chunk-size"},
 				},
+				&cli.StringFlag{
+					Name:    "batch-size",
+					Value:   "0",
+					Usage:   "size of batch data chunks, must be power of two, between 0x1000-0x1000000 or zero, [default: 0]",
+					EnvVars: []string{"BATCH_SIZE"},
+				},
 				&cli.StringFlag{
 					Name:    "work-dir",
 					Value:   "./tmp",
@@ -492,6 +498,7 @@ func main() {
 					FsAlignChunk:     c.Bool("backend-aligned-chunk") || c.Bool("fs-align-chunk"),
 					Compressor:       c.String("compressor"),
 					ChunkSize:        c.String("chunk-size"),
+					BatchSize:        c.String("batch-size"),
 
 					OCIRef:       c.Bool("oci-ref"),
 					AllPlatforms: c.Bool("all-platforms"),

contrib/nydusify/pkg/converter/config.go

@@ -28,6 +28,7 @@ func getConfig(opt Opt) map[string]string {
 	cfg["fs_version"] = opt.FsVersion
 	cfg["fs_align_chunk"] = strconv.FormatBool(opt.FsAlignChunk)
 	cfg["fs_chunk_size"] = opt.ChunkSize
+	cfg["batch_size"] = opt.BatchSize
 
 	// FIXME: still needs to be supported by acceld converter package.
 	cfg["cache_ref"] = opt.CacheRef

contrib/nydusify/pkg/converter/converter.go

@@ -42,6 +42,7 @@ type Opt struct {
 	FsAlignChunk     bool
 	Compressor       string
 	ChunkSize        string
+	BatchSize        string
 	PrefetchPatterns string
 	OCIRef           bool
 

contrib/nydusify/tests/e2e_test.go

@@ -25,7 +25,7 @@ func testBasicConvert(t *testing.T, fsVersion string) {
 	registry := NewRegistry(t)
 	defer registry.Destroy(t)
 	registry.Build(t, "image-basic")
-	nydusify := NewNydusify(registry, "image-basic", "image-basic-nydus", "", "", fsVersion)
+	nydusify := NewNydusify(registry, "image-basic", "image-basic-nydus", "", "", fsVersion, "")
 	nydusify.Convert(t)
 	nydusify.Check(t)
 }
@@ -34,7 +34,7 @@ func testBasicAuth(t *testing.T, fsVersion string) {
 	registry := NewAuthRegistry(t)
 	defer registry.Destroy(t)
 	registry.AuthBuild(t, "image-basic")
-	nydusify := NewNydusify(registry, "image-basic", "image-basic-nydus", "", "", fsVersion)
+	nydusify := NewNydusify(registry, "image-basic", "image-basic-nydus", "", "", fsVersion, "")
 	nydusify.Convert(t)
 	nydusify.Check(t)
 }
@@ -48,7 +48,7 @@ func testReproducibleBuild(t *testing.T, fsVersion string) {
 	for i := 0; i < 5; i++ {
 		workDir := fmt.Sprintf("./tmp-%d", i)
 		os.Setenv("WORKDIR", workDir)
-		nydusify := NewNydusify(registry, "image-basic", "image-basic-nydus", "", "", fsVersion)
+		nydusify := NewNydusify(registry, "image-basic", "image-basic-nydus", "", "", fsVersion, "")
 		nydusify.Convert(t)
 		nydusify.Check(t)
 		hash, err := utils.HashFile(nydusify.GetBootstarpFilePath())
@@ -68,20 +68,20 @@ func testConvertWithCache(t *testing.T, fsVersion string) {
 	defer registry.Destroy(t)
 
 	registry.Build(t, "image-basic")
-	nydusify1 := NewNydusify(registry, "image-basic", "image-basic-nydus-1", "cache:v1", "", fsVersion)
+	nydusify1 := NewNydusify(registry, "image-basic", "image-basic-nydus-1", "cache:v1", "", fsVersion, "")
 	nydusify1.Convert(t)
 	nydusify1.Check(t)
-	nydusify2 := NewNydusify(registry, "image-basic", "image-basic-nydus-2", "cache:v1", "", fsVersion)
+	nydusify2 := NewNydusify(registry, "image-basic", "image-basic-nydus-2", "cache:v1", "", fsVersion, "")
 	nydusify2.Convert(t)
 	nydusify2.Check(t)
 
 	registry.Build(t, "image-from-1")
-	nydusify3 := NewNydusify(registry, "image-from-1", "image-from-nydus-1", "cache:v1", "", fsVersion)
+	nydusify3 := NewNydusify(registry, "image-from-1", "image-from-nydus-1", "cache:v1", "", fsVersion, "")
 	nydusify3.Convert(t)
 	nydusify3.Check(t)
 
 	registry.Build(t, "image-from-2")
-	nydusify4 := NewNydusify(registry, "image-from-2", "image-from-nydus-2", "cache:v1", "", fsVersion)
+	nydusify4 := NewNydusify(registry, "image-from-2", "image-from-nydus-2", "cache:v1", "", fsVersion, "")
 	nydusify4.Convert(t)
 	nydusify4.Check(t)
 }
@@ -92,28 +92,28 @@ func testConvertWithChunkDict(t *testing.T, fsVersion string) {
 
 	registry.Build(t, "chunk-dict-1")
 	// build chunk-dict-1 bootstrap
-	nydusify1 := NewNydusify(registry, "chunk-dict-1", "nydus:chunk-dict-1", "", "", fsVersion)
+	nydusify1 := NewNydusify(registry, "chunk-dict-1", "nydus:chunk-dict-1", "", "", fsVersion, "")
 	nydusify1.Convert(t)
 	nydusify1.Check(t)
 	chunkDictOpt := fmt.Sprintf("bootstrap:registry:%s/%s", registry.Host(), "nydus:chunk-dict-1")
 	// build without build-cache
 	registry.Build(t, "image-basic")
-	nydusify2 := NewNydusify(registry, "image-basic", "nydus:image-basic", "", chunkDictOpt, fsVersion)
+	nydusify2 := NewNydusify(registry, "image-basic", "nydus:image-basic", "", chunkDictOpt, fsVersion, "")
 	nydusify2.Convert(t)
 	nydusify2.Check(t)
 	// build with build-cache
 	registry.Build(t, "image-from-1")
-	nydusify3 := NewNydusify(registry, "image-from-1", "nydus:image-from-1", "nydus:cache_v1", chunkDictOpt, fsVersion)
+	nydusify3 := NewNydusify(registry, "image-from-1", "nydus:image-from-1", "nydus:cache_v1", chunkDictOpt, fsVersion, "")
 	nydusify3.Convert(t)
 	nydusify3.Check(t)
 	// change chunk dict
 	registry.Build(t, "chunk-dict-2")
-	nydusify4 := NewNydusify(registry, "chunk-dict-2", "nydus:chunk-dict-2", "", "", fsVersion)
+	nydusify4 := NewNydusify(registry, "chunk-dict-2", "nydus:chunk-dict-2", "", "", fsVersion, "")
 	nydusify4.Convert(t)
 	nydusify4.Check(t)
 	chunkDictOpt = fmt.Sprintf("bootstrap:registry:%s/%s", registry.Host(), "nydus:chunk-dict-2")
 	registry.Build(t, "image-from-2")
-	nydusify5 := NewNydusify(registry, "image-from-2", "nydus:image-from-2", "nydus:cache_v1", chunkDictOpt, fsVersion)
+	nydusify5 := NewNydusify(registry, "image-from-2", "nydus:image-from-2", "nydus:cache_v1", chunkDictOpt, fsVersion, "")
 	nydusify5.Convert(t)
 	nydusify5.Check(t)
 }
@@ -194,11 +194,30 @@ func testConvertWithS3Backend(t *testing.T, fsVersion string) {
 
 	logrus.Infof("use s3 backend config: %s", backendConfig)
 
-	nydusify := NewNydusify(registry, "image-basic", "image-basic-nydus", "", "", fsVersion)
+	nydusify := NewNydusify(registry, "image-basic", "image-basic-nydus", "", "", fsVersion, "")
 	nydusify.Convert(t)
 	nydusify.Check(t)
 }
 
+func testConvertWithBatchSize(t *testing.T, fsVersion string) {
+	registry := NewRegistry(t)
+	defer registry.Destroy(t)
+
+	registry.Build(t, "image-basic")
+	nydusify1 := NewNydusify(registry, "image-basic", "image-basic-nydus-1", "", "", fsVersion, "0x1000")
+	nydusify1.Convert(t)
+	nydusify1.Check(t)
+	nydusify2 := NewNydusify(registry, "image-basic", "image-basic-nydus-2", "", "", fsVersion, "0x10000")
+	nydusify2.Convert(t)
+	nydusify2.Check(t)
+	nydusify3 := NewNydusify(registry, "image-basic", "image-basic-nydus-3", "", "", fsVersion, "0x100000")
+	nydusify3.Convert(t)
+	nydusify3.Check(t)
+	nydusify4 := NewNydusify(registry, "image-basic", "image-basic-nydus-4", "", "", fsVersion, "0x1000000")
+	nydusify4.Convert(t)
+	nydusify4.Check(t)
+}
+
 func TestSmoke(t *testing.T) {
 	fsVersions := [2]string{"5", "6"}
 	for _, v := range fsVersions {
@@ -209,4 +228,6 @@ func TestSmoke(t *testing.T) {
 		testConvertWithChunkDict(t, v)
 		testConvertWithS3Backend(t, v)
 	}
+	// batch-size only support v6
+	testConvertWithBatchSize(t, fsVersions[1])
 }

contrib/nydusify/tests/image_test.go

@@ -41,7 +41,7 @@ func convert(t *testing.T, ref string, fsVersion string) {
 	registry := NewRegistry(t)
 	defer registry.Destroy(t)
 	transfer(t, ref)
-	nydusify := NewNydusify(registry, ref, fmt.Sprintf("%s-nydus", ref), "", "", fsVersion)
+	nydusify := NewNydusify(registry, ref, fmt.Sprintf("%s-nydus", ref), "", "", fsVersion, "")
 	nydusify.Convert(t)
 	nydusify.Check(t)
 }

contrib/nydusify/tests/nydusify.go

@@ -41,9 +41,10 @@ type Nydusify struct {
 	chunkDictArgs string
 	fsVersion     string
 	workDir       string
+	batchSize     string
 }
 
-func NewNydusify(registry *Registry, source, target, cache string, chunkDictArgs string, fsVersion string) *Nydusify {
+func NewNydusify(registry *Registry, source, target, cache string, chunkDictArgs string, fsVersion string, batchSize string) *Nydusify {
 	backendType := ""
 	if os.Getenv("BACKEND_TYPE") != "" {
 		backendType = os.Getenv("BACKEND_TYPE")
@@ -61,6 +62,10 @@ func NewNydusify(registry *Registry, source, target, cache string, chunkDictArgs
 		workDir = os.Getenv("WORKDIR")
 	}
 
+	if len(batchSize) == 0 {
+		batchSize = "0"
+	}
+
 	return &Nydusify{
 		Registry:      registry,
 		Source:        source,
@@ -71,6 +76,7 @@ func NewNydusify(registry *Registry, source, target, cache string, chunkDictArgs
 		chunkDictArgs: chunkDictArgs,
 		fsVersion:     fsVersion,
 		workDir:       workDir,
+		batchSize:     batchSize,
 	}
 }
 
@@ -104,6 +110,8 @@ func (nydusify *Nydusify) Convert(t *testing.T) {
 		BackendConfig: nydusify.backendConfig,
 
 		FsVersion: nydusify.fsVersion,
+
+		BatchSize: nydusify.batchSize,
 	}
 
 	err := converter.Convert(context.Background(), opt)