New command - Transfer config merge (#1802)

.github/workflows/accessTests.yml

@@ -3,7 +3,7 @@ on:
   push:
   # Triggers the workflow on labeled PRs only.
   pull_request_target:
-    types: [labeled]
+    types: [ labeled ]
 # Ensures that only the latest commit is running for each PR at a time.
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.ref }}
@@ -15,7 +15,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Install Go
@@ -32,5 +32,12 @@ jobs:
         uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
+      - name: Setup Artifactory
+        run: |
+          go install github.com/jfrog/jfrog-testing-infra/local-rt-setup@latest
+          ~/go/bin/local-rt-setup
+        env:
+          RTLIC: ${{secrets.RTLIC}}
+          GOPROXY: direct
       - name: Run Access tests
-        run: go test -v github.com/jfrog/jfrog-cli --timeout 0 --test.access --jfrog.url=${{ secrets.PLATFORM_URL }} --jfrog.adminToken=${{ secrets.PLATFORM_ADMIN_TOKEN }} --jfrog.user=${{ secrets.PLATFORM_USER }} --ci.runId=${{ runner.os }}-access
+        run: go test -v github.com/jfrog/jfrog-cli --timeout 0 --test.access  --ci.runId=${{ runner.os }}-access

.github/workflows/artifactoryTests.yml

@@ -3,7 +3,7 @@ on:
   push:
   # Triggers the workflow on labeled PRs only.
   pull_request_target:
-    types: [labeled]
+    types: [ labeled ]
 # Ensures that only the latest commit is running for each PR at a time.
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.ref }}
@@ -15,8 +15,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        suite: [artifactory, artifactoryProject]
-        os: [ubuntu-latest, macos-latest, windows-latest]
+        suite: [ artifactory, artifactoryProject ]
+        os: [ ubuntu-latest, macos-latest, windows-latest ]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Install Go
@@ -40,10 +40,11 @@ jobs:
         env:
           RTLIC: ${{secrets.RTLIC}}
           GOPROXY: direct
-        if: ${{ matrix.suite == 'artifactory' }}
+
       - name: Run Artifactory tests
         run: go test -v github.com/jfrog/jfrog-cli --timeout 0 --test.artifactory
         if: ${{ matrix.suite == 'artifactory' }}
+
       - name: Run Artifactory projects tests
-        run: go test -v github.com/jfrog/jfrog-cli --timeout 0 --test.artifactoryProject --jfrog.url=${{ secrets.PLATFORM_URL }} --jfrog.adminToken=${{ secrets.PLATFORM_ADMIN_TOKEN }} --jfrog.user=${{ secrets.PLATFORM_USER }} --ci.runId=${{ runner.os }}-${{ matrix.suite }}
+        run: go test -v github.com/jfrog/jfrog-cli --timeout 0 --test.artifactoryProject --ci.runId=${{ runner.os }}-${{ matrix.suite }}
         if: ${{ matrix.suite == 'artifactoryProject' }}

artifactory/cli.go

@@ -21,7 +21,7 @@ import (
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/replication"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/repository"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transfer"
-	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferconfig"
+	transferconfigcore "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferconfig"
 	transferfilescore "github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/transferfiles"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/commands/usersmanagement"
 	"github.com/jfrog/jfrog-cli-core/v2/artifactory/utils"
@@ -44,7 +44,6 @@ import (
 	"github.com/jfrog/jfrog-cli/docs/artifactory/buildpromote"
 	"github.com/jfrog/jfrog-cli/docs/artifactory/buildpublish"
 	"github.com/jfrog/jfrog-cli/docs/artifactory/buildscan"
-	"github.com/jfrog/jfrog-cli/docs/artifactory/configtransfer"
 	copydocs "github.com/jfrog/jfrog-cli/docs/artifactory/copy"
 	curldocs "github.com/jfrog/jfrog-cli/docs/artifactory/curl"
 	"github.com/jfrog/jfrog-cli/docs/artifactory/delete"
@@ -93,6 +92,8 @@ import (
 	"github.com/jfrog/jfrog-cli/docs/artifactory/repoupdate"
 	"github.com/jfrog/jfrog-cli/docs/artifactory/search"
 	"github.com/jfrog/jfrog-cli/docs/artifactory/setprops"
+	"github.com/jfrog/jfrog-cli/docs/artifactory/transferconfig"
+	"github.com/jfrog/jfrog-cli/docs/artifactory/transferconfigmerge"
 	"github.com/jfrog/jfrog-cli/docs/artifactory/transferfiles"
 	"github.com/jfrog/jfrog-cli/docs/artifactory/transfersettings"
 	"github.com/jfrog/jfrog-cli/docs/artifactory/upload"
@@ -959,15 +960,27 @@ func GetCommands() []cli.Command {
 		{
 			Name:         "transfer-config",
 			Flags:        cliutils.GetCommandFlags(cliutils.TransferConfig),
-			Usage:        configtransfer.GetDescription(),
-			HelpName:     corecommon.CreateUsage("rt transfer-config", configtransfer.GetDescription(), configtransfer.Usage),
-			UsageText:    configtransfer.GetArguments(),
+			Usage:        transferconfig.GetDescription(),
+			HelpName:     corecommon.CreateUsage("rt transfer-config", transferconfig.GetDescription(), transferconfig.Usage),
+			UsageText:    transferconfig.GetArguments(),
 			ArgsUsage:    common.CreateEnvVars(),
 			BashComplete: corecommon.CreateBashCompletionFunc(),
 			Action: func(c *cli.Context) error {
 				return transferConfigCmd(c)
 			},
 		},
+		{
+			Name:         "transfer-config-merge",
+			Flags:        cliutils.GetCommandFlags(cliutils.TransferConfigMerge),
+			Usage:        transferconfigmerge.GetDescription(),
+			HelpName:     corecommon.CreateUsage("rt transfer-config-merge", transferconfigmerge.GetDescription(), transferconfigmerge.Usage),
+			UsageText:    transferconfigmerge.GetArguments(),
+			ArgsUsage:    common.CreateEnvVars(),
+			BashComplete: corecommon.CreateBashCompletionFunc(),
+			Action: func(c *cli.Context) error {
+				return transferConfigMergeCmd(c)
+			},
+		},
 		{
 			Name:         "transfer-files",
 			Flags:        cliutils.GetCommandFlags(cliutils.TransferFiles),
@@ -2316,7 +2329,7 @@ func transferConfigCmd(c *cli.Context) error {
 		return cliutils.WrongNumberOfArgumentsHandler(c)
 	}
 
-	// Get source artifactory server
+	// Get source Artifactory server
 	sourceServerDetails, err := coreConfig.GetSpecificConfig(c.Args()[0], false, true)
 	if err != nil {
 		return err
@@ -2329,7 +2342,7 @@ func transferConfigCmd(c *cli.Context) error {
 	}
 
 	// Run transfer config command
-	transferConfigCmd := transferconfig.NewTransferConfigCommand(sourceServerDetails, targetServerDetails).SetForce(c.Bool(cliutils.Force)).
+	transferConfigCmd := transferconfigcore.NewTransferConfigCommand(sourceServerDetails, targetServerDetails).SetForce(c.Bool(cliutils.Force)).
 		SetVerbose(c.Bool(cliutils.Verbose)).SetPreChecks(c.Bool(cliutils.PreChecks)).SetWorkingDir(c.String(cliutils.WorkingDir))
 	includeReposPatterns, excludeReposPatterns := getTransferIncludeExcludeRepos(c)
 	transferConfigCmd.SetIncludeReposPatterns(includeReposPatterns)
@@ -2338,6 +2351,33 @@ func transferConfigCmd(c *cli.Context) error {
 	return transferConfigCmd.Run()
 }
 
+func transferConfigMergeCmd(c *cli.Context) error {
+	if c.NArg() != 2 {
+		return cliutils.WrongNumberOfArgumentsHandler(c)
+	}
+
+	// Get source Artifactory server
+	sourceServerDetails, err := coreConfig.GetSpecificConfig(c.Args()[0], false, true)
+	if err != nil {
+		return err
+	}
+
+	// Get target artifactory server
+	targetServerDetails, err := coreConfig.GetSpecificConfig(c.Args()[1], false, true)
+	if err != nil {
+		return err
+	}
+
+	// Run transfer config command
+	includeReposPatterns, excludeReposPatterns := getTransferIncludeExcludeRepos(c)
+	includeProjectsPatterns, excludeProjectsPatterns := getTransferIncludeExcludeProjects(c)
+	transferConfigMergeCmd := transferconfigcore.NewTransferConfigMergeCommand(sourceServerDetails, targetServerDetails).
+		SetIncludeReposPatterns(includeReposPatterns).SetExcludeReposPatterns(excludeReposPatterns).
+		SetIncludeProjectsPatterns(includeProjectsPatterns).SetExcludeProjectsPatterns(excludeProjectsPatterns)
+	_, err = transferConfigMergeCmd.Run()
+	return err
+}
+
 func dataTransferPluginInstallCmd(c *cli.Context) error {
 	// Get the Artifactory serverID from the argument or use default if not exists
 	serverID := ""
@@ -2386,7 +2426,7 @@ func transferFilesCmd(c *cli.Context) error {
 		return cliutils.WrongNumberOfArgumentsHandler(c)
 	}
 
-	// Get source artifactory server
+	// Get source Artifactory server
 	sourceServerDetails, err := coreConfig.GetSpecificConfig(c.Args()[0], false, true)
 	if err != nil {
 		return err
@@ -2424,6 +2464,17 @@ func getTransferIncludeExcludeRepos(c *cli.Context) (includeReposPatterns, exclu
 	return
 }
 
+func getTransferIncludeExcludeProjects(c *cli.Context) (includeProjectsPatterns, excludeProjectsPatterns []string) {
+	const patternSeparator = ";"
+	if c.IsSet(cliutils.IncludeProjects) {
+		includeProjectsPatterns = strings.Split(c.String(cliutils.IncludeProjects), patternSeparator)
+	}
+	if c.IsSet(cliutils.ExcludeProjects) {
+		excludeProjectsPatterns = strings.Split(c.String(cliutils.ExcludeProjects), patternSeparator)
+	}
+	return
+}
+
 func transferSettingsCmd() error {
 	transferSettingsCmd := transfer.NewTransferSettingsCommand()
 	return commands.Exec(transferSettingsCmd)

artifactory_test.go

@@ -59,8 +59,6 @@ import (
 
 // Access does not support creating an admin token without UI. Skipping projects tests till this functionality will be implemented.
 // https://jira.jfrog.org/browse/JA-2620
-const projectsTokenMinArtifactoryVersion = "7.41.0"
-
 // Minimum Artifactory version with Terraform support
 const terraformMinArtifactoryVersion = "7.38.4"
 
@@ -3252,25 +3250,23 @@ func TestArtifactoryDownloadByBuildUsingSimpleDownload(t *testing.T) {
 }
 
 func TestArtifactoryDownloadByBuildUsingSimpleDownloadWithProject(t *testing.T) {
-	initArtifactoryProjectTest(t, projectsTokenMinArtifactoryVersion)
+	initArtifactoryTest(t, "")
 	accessManager, err := utils.CreateAccessServiceManager(serverDetails, false)
 	assert.NoError(t, err)
-	timestamp := strconv.FormatInt(time.Now().Unix(), 10)
-	projectKey := "prj" + timestamp[len(timestamp)-3:]
 	// Delete the project if already exists
-	deleteProjectIfExists(t, accessManager, projectKey)
+	deleteProjectIfExists(t, accessManager, tests.ProjectKey)
 
 	// Create new project
 	projectParams := accessServices.ProjectParams{
 		ProjectDetails: accessServices.Project{
-			DisplayName: "testProject " + projectKey,
-			ProjectKey:  projectKey,
+			DisplayName: "testProject " + tests.ProjectKey,
+			ProjectKey:  tests.ProjectKey,
 		},
 	}
 	err = accessManager.CreateProject(projectParams)
 	assert.NoError(t, err)
 	// Assign the repository to the project
-	err = accessManager.AssignRepoToProject(tests.RtRepo1, projectKey, true)
+	err = accessManager.AssignRepoToProject(tests.RtRepo1, tests.ProjectKey, true)
 	assert.NoError(t, err)
 
 	// Delete the build if exists
@@ -3281,14 +3277,14 @@ func TestArtifactoryDownloadByBuildUsingSimpleDownloadWithProject(t *testing.T)
 	buildNumberA := "123"
 
 	// Upload files with buildName, buildNumber and project flags
-	runRt(t, "upload", "--spec="+specFileB, "--build-name="+tests.RtBuildName1, "--build-number="+buildNumberA, "--project="+projectKey)
+	runRt(t, "upload", "--spec="+specFileB, "--build-name="+tests.RtBuildName1, "--build-number="+buildNumberA, "--project="+tests.ProjectKey)
 
 	// Publish buildInfo with project flag
-	runRt(t, "build-publish", tests.RtBuildName1, buildNumberA, "--project="+projectKey)
+	runRt(t, "build-publish", tests.RtBuildName1, buildNumberA, "--project="+tests.ProjectKey)
 
 	// Download by project, b1 should be downloaded
 	runRt(t, "download", tests.RtRepo1+"/data/b1.in", filepath.Join(tests.Out, "download", "simple_by_build")+fileutils.GetFileSeparator(),
-		"--build="+tests.RtBuildName1, "--project="+projectKey)
+		"--build="+tests.RtBuildName1, "--project="+tests.ProjectKey)
 
 	// Validate files are downloaded by build number
 	paths, err := fileutils.ListFilesRecursiveWalkIntoDirSymlink(tests.Out, false)
@@ -3300,31 +3296,29 @@ func TestArtifactoryDownloadByBuildUsingSimpleDownloadWithProject(t *testing.T)
 	inttestutils.DeleteBuild(serverDetails.ArtifactoryUrl, tests.RtBuildName1, artHttpDetails)
 	err = accessManager.UnassignRepoFromProject(tests.RtRepo1)
 	assert.NoError(t, err)
-	err = accessManager.DeleteProject(projectKey)
+	err = accessManager.DeleteProject(tests.ProjectKey)
 	assert.NoError(t, err)
 	cleanArtifactoryTest()
 }
 
 func TestArtifactoryDownloadWithEnvProject(t *testing.T) {
-	initArtifactoryProjectTest(t, projectsTokenMinArtifactoryVersion)
+	initArtifactoryTest(t, "")
 	accessManager, err := utils.CreateAccessServiceManager(serverDetails, false)
 	assert.NoError(t, err)
-	timestamp := strconv.FormatInt(time.Now().Unix(), 10)
-	projectKey := "prj" + timestamp[len(timestamp)-3:]
 	// Delete the project if already exists
-	deleteProjectIfExists(t, accessManager, projectKey)
+	deleteProjectIfExists(t, accessManager, tests.ProjectKey)
 
 	// Create new project
 	projectParams := accessServices.ProjectParams{
 		ProjectDetails: accessServices.Project{
-			DisplayName: "testProject " + projectKey,
-			ProjectKey:  projectKey,
+			DisplayName: "testProject " + tests.ProjectKey,
+			ProjectKey:  tests.ProjectKey,
 		},
 	}
 	err = accessManager.CreateProject(projectParams)
 	assert.NoError(t, err)
 	// Assign the repository to the project
-	err = accessManager.AssignRepoToProject(tests.RtRepo1, projectKey, true)
+	err = accessManager.AssignRepoToProject(tests.RtRepo1, tests.ProjectKey, true)
 	assert.NoError(t, err)
 
 	// Delete the build if exists
@@ -3337,7 +3331,7 @@ func TestArtifactoryDownloadWithEnvProject(t *testing.T) {
 	defer setEnvCallBack()
 	setEnvCallBack = clientTestUtils.SetEnvWithCallbackAndAssert(t, coreutils.BuildNumber, buildNumberA)
 	defer setEnvCallBack()
-	setEnvCallBack = clientTestUtils.SetEnvWithCallbackAndAssert(t, coreutils.Project, projectKey)
+	setEnvCallBack = clientTestUtils.SetEnvWithCallbackAndAssert(t, coreutils.Project, tests.ProjectKey)
 	defer setEnvCallBack()
 	// Upload files with buildName, buildNumber and project flags
 	runRt(t, "upload", "--spec="+specFileB)
@@ -3347,7 +3341,7 @@ func TestArtifactoryDownloadWithEnvProject(t *testing.T) {
 
 	// Download by project, b1 should be downloaded
 	runRt(t, "download", tests.RtRepo1+"/data/b1.in", filepath.Join(tests.Out, "download", "simple_by_build")+fileutils.GetFileSeparator(),
-		"--build="+tests.RtBuildName1, "--project="+projectKey)
+		"--build="+tests.RtBuildName1, "--project="+tests.ProjectKey)
 
 	// Validate files are downloaded by build number
 	paths, err := fileutils.ListFilesRecursiveWalkIntoDirSymlink(tests.Out, false)
@@ -3359,7 +3353,7 @@ func TestArtifactoryDownloadWithEnvProject(t *testing.T) {
 	inttestutils.DeleteBuild(serverDetails.ArtifactoryUrl, tests.RtBuildName1, artHttpDetails)
 	err = accessManager.UnassignRepoFromProject(tests.RtRepo1)
 	assert.NoError(t, err)
-	err = accessManager.DeleteProject(projectKey)
+	err = accessManager.DeleteProject(tests.ProjectKey)
 	assert.NoError(t, err)
 	cleanArtifactoryTest()
 }
@@ -4518,15 +4512,6 @@ func initArtifactoryTest(t *testing.T, minVersion string) {
 	}
 }
 
-func initArtifactoryProjectTest(t *testing.T, minVersion string) {
-	if !*tests.TestArtifactoryProject {
-		t.Skip("Skipping artifactory project test. To run artifactory test add the '-test.artifactoryProject=true' option.")
-	}
-	if minVersion != "" {
-		validateArtifactoryVersion(t, minVersion)
-	}
-}
-
 func validateArtifactoryVersion(t *testing.T, minVersion string) {
 	rtVersion, err := getArtifactoryVersion()
 	if err != nil {
@@ -5046,7 +5031,7 @@ func TestAccessTokenCreate(t *testing.T) {
 	if *tests.JfrogAccessToken != "" {
 		// Use Artifactory CLI with basic auth to allow running `jfrog rt atc` without arguments
 		origAccessToken := *tests.JfrogAccessToken
-		origUsername, origPassword := tests.SetBasicAuthFromAccessToken(t)
+		origUsername, origPassword := tests.SetBasicAuthFromAccessToken()
 		defer func() {
 			*tests.JfrogUser = origUsername
 			*tests.JfrogPassword = origPassword

docker_test.go

@@ -478,7 +478,7 @@ func runKaniko(t *testing.T, imageToPush string) string {
 	dockerFile := "TestKanikoBuildCollect"
 	KanikoOutputFile := "image-file"
 	if *tests.JfrogAccessToken != "" {
-		origUsername, origPassword := tests.SetBasicAuthFromAccessToken(t)
+		origUsername, origPassword := tests.SetBasicAuthFromAccessToken()
 		defer func() {
 			*tests.JfrogUser = origUsername
 			*tests.JfrogPassword = origPassword

docs/artifactory/configtransfer/help.go → docs/artifactory/transferconfig/help.go

@@ -1,4 +1,4 @@
-package configtransfer
+package transferconfig
 
 var Usage = []string{"rt transfer-config [command options] <source-server-id> <target-server-id>"}
 

docs/artifactory/transferconfigmerge/help.go

@@ -0,0 +1,15 @@
+package transferconfigmerge
+
+var Usage = []string{"rt transfer-config-merge [command options] <source-server-id> <target-server-id>"}
+
+func GetDescription() string {
+	return "Merge projects and repositories from a source Artifactory instance to a target Artifactory instance, if no conflicts are found"
+}
+
+func GetArguments() string {
+	return `	source-server-id
+		The source server ID. The configuration will be exported from this server.
+
+	target-server-id
+		The target server ID. The configuration will be imported to this server.`
+}