Support providing chunk size in multi part upload (#2517)

.github/workflows/gradleTests.yml

@@ -30,6 +30,11 @@ jobs:
         with:
           go-version: 1.20.x
           cache: false
+      - name: Setup Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '11'
       - name: Setup Gradle
         uses: gradle/gradle-build-action@v2
         with:

artifactory/cli_test.go

@@ -130,12 +130,14 @@ var createUploadConfigurationCases = []struct {
 	expectedSplitCount int
 	expectedThreads    int
 	expectedDeb        string
+	expectedChunkSize  int64
 }{
-	{"empty", []string{}, cliutils.UploadMinSplitMb, cliutils.UploadSplitCount, commonCliUtils.Threads, ""},
-	{"min-split", []string{"min-split=101"}, 101, cliutils.UploadSplitCount, commonCliUtils.Threads, ""},
-	{"split-count", []string{"split-count=6"}, cliutils.UploadMinSplitMb, 6, commonCliUtils.Threads, ""},
-	{"threads", []string{"threads=6"}, cliutils.UploadMinSplitMb, cliutils.UploadSplitCount, 6, ""},
-	{"deb", []string{"deb=jammy/main/i386"}, cliutils.UploadMinSplitMb, cliutils.UploadSplitCount, commonCliUtils.Threads, "jammy/main/i386"},
+	{"empty", []string{}, cliutils.UploadMinSplitMb, cliutils.UploadSplitCount, commonCliUtils.Threads, "", cliutils.UploadChunkSizeMb},
+	{"min-split", []string{"min-split=101"}, 101, cliutils.UploadSplitCount, commonCliUtils.Threads, "", cliutils.UploadChunkSizeMb},
+	{"split-count", []string{"split-count=6"}, cliutils.UploadMinSplitMb, 6, commonCliUtils.Threads, "", cliutils.UploadChunkSizeMb},
+	{"threads", []string{"threads=6"}, cliutils.UploadMinSplitMb, cliutils.UploadSplitCount, 6, "", cliutils.UploadChunkSizeMb},
+	{"deb", []string{"deb=jammy/main/i386"}, cliutils.UploadMinSplitMb, cliutils.UploadSplitCount, commonCliUtils.Threads, "jammy/main/i386", cliutils.UploadChunkSizeMb},
+	{"chunk-size", []string{"chunk-size=123"}, cliutils.UploadMinSplitMb, cliutils.UploadSplitCount, commonCliUtils.Threads, "", 123},
 }
 
 func TestCreateUploadConfiguration(t *testing.T) {

go.mod

@@ -137,9 +137,9 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
-// replace github.com/jfrog/jfrog-cli-core/v2 => github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240408074156-13680c04f22e
+replace github.com/jfrog/jfrog-cli-core/v2 => github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240424135031-dac0f92a2aae
 
-// replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20240408071430-62ee0279ac58
+replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20240424133643-5bf715f66eac
 
 // replace github.com/jfrog/jfrog-cli-security => github.com/jfrog/jfrog-cli-security v1.0.6-0.20240408061620-c9b84da33d5e
 

go.sum

@@ -135,12 +135,12 @@ github.com/jfrog/gofrog v1.7.1 h1:ME1Meg4hukAT/7X6HUQCVSe4DNjMZACCP8aCY37EW/w=
 github.com/jfrog/gofrog v1.7.1/go.mod h1:X7bjfWoQDN0Z4FQGbE91j3gbPP7Urwzm4Z8tkvrlbRI=
 github.com/jfrog/jfrog-apps-config v1.0.1 h1:mtv6k7g8A8BVhlHGlSveapqf4mJfonwvXYLipdsOFMY=
 github.com/jfrog/jfrog-apps-config v1.0.1/go.mod h1:8AIIr1oY9JuH5dylz2S6f8Ym2MaadPLR6noCBO4C22w=
-github.com/jfrog/jfrog-cli-core/v2 v2.51.0 h1:nESbCpSTPZx1av0W9tdmWLxKaPSL1SaZinbZGtYNeFI=
-github.com/jfrog/jfrog-cli-core/v2 v2.51.0/go.mod h1:064wSSHVI3ZIVi/a94yJqzs+ACM+9JK/u9tQ1sfTK6A=
+github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240424135031-dac0f92a2aae h1:Lgn3WxoSc4YwhitPF2sPlqVviDZ4WRMI8eLOQqV9GZs=
+github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20240424135031-dac0f92a2aae/go.mod h1:/rNY4nIB8hIsL+MLGrZOgOYpY4QnJ+CHeTHPQDdGmOk=
 github.com/jfrog/jfrog-cli-security v1.1.0 h1:ifCjFJSa1D1pWyW/ADYPqnMkOddzkAT/WY4vHAufn1g=
 github.com/jfrog/jfrog-cli-security v1.1.0/go.mod h1:086t7e/einVAGfBXxRdEGDKovWt67I6SqUb1rcpdiZc=
-github.com/jfrog/jfrog-client-go v1.40.1 h1:ISSSV7/IUS8R+QCPfH2lVKLburbv2Xn07fvNyDc17rI=
-github.com/jfrog/jfrog-client-go v1.40.1/go.mod h1:FprEW0Sqhj6ZSFTFk9NCni+ovFAYMA3zCBmNX4hGXgQ=
+github.com/jfrog/jfrog-client-go v1.28.1-0.20240424133643-5bf715f66eac h1:e/QfkFN/Qtkn1rBg/p7JCOMFLmE2EgfPBx57m2yIF1k=
+github.com/jfrog/jfrog-client-go v1.28.1-0.20240424133643-5bf715f66eac/go.mod h1:FprEW0Sqhj6ZSFTFk9NCni+ovFAYMA3zCBmNX4hGXgQ=
 github.com/jszwec/csvutil v1.10.0 h1:upMDUxhQKqZ5ZDCs/wy+8Kib8rZR8I8lOR34yJkdqhI=
 github.com/jszwec/csvutil v1.10.0/go.mod h1:/E4ONrmGkwmWsk9ae9jpXnv9QT8pLHEPcCirMFhxG9I=
 github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4=

plugins_test.go

@@ -25,7 +25,7 @@ import (
 )
 
 const officialPluginForTest = "rt-fs"
-const officialPluginVersion = "v1.0.0"
+const officialPluginVersion = "v1.1.5"
 const customPluginName = "custom-plugin"
 
 func TestPluginInstallUninstallOfficialRegistry(t *testing.T) {

utils/cliutils/cli_consts.go

@@ -28,6 +28,7 @@ const (
 	UploadMinSplitMb    = 200
 	UploadSplitCount    = 5
 	UploadMaxSplitCount = 100
+	UploadChunkSizeMb   = 20
 
 	// Common
 	Retries                       = 3

utils/cliutils/commandsflags.go

@@ -198,6 +198,7 @@ const (
 	Status                  = "status"
 	MinSplit                = "min-split"
 	SplitCount              = "split-count"
+	ChunkSize               = "chunk-size"
 
 	// Config flags
 	interactive   = "interactive"
@@ -799,11 +800,15 @@ var flagsMap = map[string]cli.Flag{
 	},
 	uploadMinSplit: cli.StringFlag{
 		Name:  MinSplit,
-		Usage: "[Default: " + strconv.Itoa(UploadMinSplitMb) + "] The minimum file size in MiB required to attempt a multi-part upload. This option, as well as the functionality of multi-part upload, requires Artifactory with S3 storage.` `",
+		Usage: "[Default: " + strconv.Itoa(UploadMinSplitMb) + "] The minimum file size in MiB required to attempt a multi-part upload. This option, as well as the functionality of multi-part upload, requires Artifactory with S3 or GCP storage.` `",
 	},
 	uploadSplitCount: cli.StringFlag{
 		Name:  SplitCount,
-		Usage: "[Default: " + strconv.Itoa(UploadSplitCount) + "] The maximum number of parts that can be concurrently uploaded per file during a multi-part upload. Set to 0 to disable multi-part upload. This option, as well as the functionality of multi-part upload, requires Artifactory with S3 storage.` `",
+		Usage: "[Default: " + strconv.Itoa(UploadSplitCount) + "] The maximum number of parts that can be concurrently uploaded per file during a multi-part upload. Set to 0 to disable multi-part upload. This option, as well as the functionality of multi-part upload, requires Artifactory with S3 or GCP storage.` `",
+	},
+	ChunkSize: cli.StringFlag{
+		Name:  ChunkSize,
+		Usage: "[Default: " + strconv.Itoa(UploadChunkSizeMb) + "] The upload chunk size in MiB that can be concurrently uploaded during a multi-part upload. This option, as well as the functionality of multi-part upload, requires Artifactory with S3 or GCP storage.` `",
 	},
 	syncDeletesQuiet: cli.BoolFlag{
 		Name:  quiet,
@@ -1736,7 +1741,7 @@ var commandFlags = map[string][]string{
 		ClientCertKeyPath, specFlag, specVars, buildName, buildNumber, module, uploadExclusions, deb,
 		uploadRecursive, uploadFlat, uploadRegexp, retries, retryWaitTime, dryRun, uploadExplode, symlinks, includeDirs,
 		failNoOp, threads, uploadSyncDeletes, syncDeletesQuiet, InsecureTls, detailedSummary, Project,
-		uploadAnt, uploadArchive, uploadMinSplit, uploadSplitCount,
+		uploadAnt, uploadArchive, uploadMinSplit, uploadSplitCount, ChunkSize,
 	},
 	Download: {
 		url, user, password, accessToken, sshPassphrase, sshKeyPath, serverId, ClientCertPath,

utils/cliutils/utils.go

@@ -279,6 +279,10 @@ func CreateUploadConfiguration(c *cli.Context) (uploadConfiguration *artifactory
 	if err != nil {
 		return nil, err
 	}
+	uploadConfiguration.ChunkSizeMB, err = getUploadChunkSize(c, UploadChunkSizeMb)
+	if err != nil {
+		return nil, err
+	}
 	uploadConfiguration.SplitCount, err = getSplitCount(c, UploadSplitCount, UploadMaxSplitCount)
 	if err != nil {
 		return nil, err
@@ -767,6 +771,19 @@ func getMinSplit(c *cli.Context, defaultMinSplit int64) (minSplitSize int64, err
 	return minSplitSize, nil
 }
 
+func getUploadChunkSize(c *cli.Context, defaultChunkSize int64) (chunkSize int64, err error) {
+	chunkSize = defaultChunkSize
+	if c.String(ChunkSize) != "" {
+		chunkSize, err = strconv.ParseInt(c.String(ChunkSize), 10, 64)
+		if err != nil {
+			err = fmt.Errorf("the '--%s' option should have a numeric value. %s", ChunkSize, GetDocumentationMessage())
+			return 0, err
+		}
+	}
+
+	return chunkSize, nil
+}
+
 func getDebFlag(c *cli.Context) (deb string, err error) {
 	deb = c.String("deb")
 	slashesCount := strings.Count(deb, "/") - strings.Count(deb, "\\/")