New exclusions options for go-publish (#2044)

* gp --exclusion new flag

buildtools/cli.go

@@ -636,7 +636,7 @@ func GoPublishCmd(c *cli.Context) (err error) {
 	version := c.Args().Get(0)
 	printDeploymentView, detailedSummary := log.IsStdErrTerminal(), c.Bool("detailed-summary")
 	goPublishCmd := golang.NewGoPublishCommand()
-	goPublishCmd.SetConfigFilePath(configFilePath).SetBuildConfiguration(buildConfiguration).SetVersion(version).SetDetailedSummary(detailedSummary || printDeploymentView)
+	goPublishCmd.SetConfigFilePath(configFilePath).SetBuildConfiguration(buildConfiguration).SetVersion(version).SetDetailedSummary(detailedSummary || printDeploymentView).SetExcludedPatterns(cliutils.GetStringsArrFlagValue(c, "exclusions"))
 	err = commands.Exec(goPublishCmd)
 	result := goPublishCmd.Result()
 	defer cliutils.CleanupResult(result, &err)

go.mod

@@ -123,8 +123,8 @@ require (
 
 // replace github.com/jfrog/build-info-go => github.com/jfrog/build-info-go
 
-// replace github.com/jfrog/jfrog-cli-core/v2 => github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20230705084348-c7d33487e393
+replace github.com/jfrog/jfrog-cli-core/v2 => github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20230717092920-cd976eb1e5c3
 
 // replace github.com/jfrog/gofrog => github.com/jfrog/gofrog v1.2.6-0.20230418122323-2bf299dd6d27
 
-// replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20230705083849-6fd087a5e228
+replace github.com/jfrog/jfrog-client-go => github.com/jfrog/jfrog-client-go v1.28.1-0.20230717090738-b2e0c7bcc026

go.sum

@@ -238,10 +238,10 @@ github.com/jfrog/build-info-go v1.9.6 h1:lCJ2j5uXAlJsSwDe5J8WD7Co1f/hUlZvMfwfb5A
 github.com/jfrog/build-info-go v1.9.6/go.mod h1:GbuFS+viHCKZYx9nWHYu7ab1DgQkFdtVN3BJPUNb2D4=
 github.com/jfrog/gofrog v1.3.0 h1:o4zgsBZE4QyDbz2M7D4K6fXPTBJht+8lE87mS9bw7Gk=
 github.com/jfrog/gofrog v1.3.0/go.mod h1:IFMc+V/yf7rA5WZ74CSbXe+Lgf0iApEQLxRZVzKRUR0=
-github.com/jfrog/jfrog-cli-core/v2 v2.38.0 h1:lHylMjp0+IbZAUKVWi++keVktpyvI/0UwewIdbCoI/A=
-github.com/jfrog/jfrog-cli-core/v2 v2.38.0/go.mod h1:Ws5UvSUITSZGuVVNNb/lDFPG0UAyiwpKv5o86M8By9I=
-github.com/jfrog/jfrog-client-go v1.31.1 h1:lmunA5ZpRsrWTXgEGvnvVPIfwEqB3gn6+eVNpV2VBzU=
-github.com/jfrog/jfrog-client-go v1.31.1/go.mod h1:qEJxoe68sUtqHJ1YhXv/7pKYP/9p1D5tJrruzJKYeoI=
+github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20230717092920-cd976eb1e5c3 h1:ED7QzMz/KA/7ps2bNvRiWbqBP/BbExWlUrJWcAoGiHM=
+github.com/jfrog/jfrog-cli-core/v2 v2.31.1-0.20230717092920-cd976eb1e5c3/go.mod h1:QjTOL5xFgplonx47OnXXXurRtpQGDquGmrTbpe011EY=
+github.com/jfrog/jfrog-client-go v1.28.1-0.20230717090738-b2e0c7bcc026 h1:Xam/SD9ZqanqexbX2iW2H1fH5MLB9qx6vN8SK8wBMhA=
+github.com/jfrog/jfrog-client-go v1.28.1-0.20230717090738-b2e0c7bcc026/go.mod h1:qEJxoe68sUtqHJ1YhXv/7pKYP/9p1D5tJrruzJKYeoI=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/jszwec/csvutil v1.8.0 h1:G7vS2LGdpZZDH1HmHeNbxOaJ/ZnJlpwGFvOkTkJzzNk=
@@ -674,6 +674,7 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
 golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

go_test.go

@@ -203,6 +203,56 @@ func TestGoPublishWithDeploymentView(t *testing.T) {
 	clientTestUtils.ChangeDirAndAssert(t, wd)
 }
 
+func TestGoPublishWithExclusions(t *testing.T) {
+	_, goCleanupFunc := initGoTest(t)
+	defer goCleanupFunc()
+	wd, err := os.Getwd()
+	assert.NoError(t, err, "Failed to get current dir")
+	searchFilePath, err := tests.CreateSpec(tests.GoPublishRepoExcludes)
+	assert.NoError(t, err)
+
+	var testData = []struct {
+		exclusions                 string
+		expectedExistFilesPaths    []string
+		expectedNotExistFilesPaths []string
+	}{
+		{"./dir1/*", tests.GetGoPublishWithExclusionsExpectedFiles1(), tests.GetGoPublishWithExclusionsExcludedFiles1()},
+		{"./dir1/dir2/*", tests.GetGoPublishWithExclusionsExpectedFiles2(), tests.GetGoPublishWithExclusionsExcludedFiles2()},
+		{"*.txt", nil, tests.GetGoPublishWithExclusionsExcludedFiles3()},
+	}
+	for _, test := range testData {
+		prepareGoProject("project4", t, true)
+		jfrogCli := tests.NewJfrogCli(execMain, "jf", "")
+		err = execGo(jfrogCli, "gp", "v1.1.1", "--exclusions", test.exclusions)
+		assert.NoError(t, err)
+
+		// Verify that go-publish successfully published expected files and directories to Artifactory.
+		inttestutils.VerifyExistInArtifactory(tests.GetGoPublishWithExclusionsExpectedRepoGo(), searchFilePath, serverDetails, t)
+		// Creating a temporary directory to download for it the content of the zip file from artifactory.
+		tmpDir, createTempDirCallback := coretests.CreateTempDirWithCallbackAndAssert(t)
+		err = os.RemoveAll(tmpDir)
+		assert.NoError(t, err)
+		assert.NoError(t, os.Mkdir(tmpDir, 0777))
+		runRt(t, "download", tests.GoRepo, tmpDir+"/", "--explode")
+		// Checking whether the expected files exist in the zip file after downloading from artifactory with unzipping it.
+		for _, path := range test.expectedExistFilesPaths {
+			result, err := fileutils.IsFileExists(filepath.Join(tmpDir, path), true)
+			assert.NoError(t, err)
+			assert.True(t, result, "This file"+path+"does not exist")
+		}
+		// Checking whether the excluded files do not exist in the zip file after downloading from Artifactory with unzipping it.
+		for _, path := range test.expectedNotExistFilesPaths {
+			result, err := fileutils.IsFileExists(filepath.Join(tmpDir, path), true)
+			assert.NoError(t, err)
+			assert.False(t, result)
+		}
+		// Delete the temporary dir.
+		createTempDirCallback()
+		// Restore workspace.
+		clientTestUtils.ChangeDirAndAssert(t, wd)
+	}
+}
+
 func TestGoVcsFallback(t *testing.T) {
 	_, cleanUpFunc := initGoTest(t)
 	defer cleanUpFunc()

testdata/filespecs/go_publish_repo_excludes.json

@@ -0,0 +1,7 @@
+{
+  "files": [
+    {
+      "pattern": "${GO_REPO}/*"
+    }
+  ]
+}

testdata/go/project3/.jfrog/projects/go.yaml

@@ -0,0 +1,9 @@
+version: 1
+type: go
+
+resolver:
+  repo: ${GO_VIRTUAL_REPO}
+  serverID: default
+deployer:
+  repo: ${GO_REPO}
+  serverID: default

testdata/go/project3/createGoProject_go.mod_suffix

@@ -0,0 +1,8 @@
+module github.com/jfrog/dependency
+
+require (
+	github.com/pkg/errors v0.8.0
+	rsc.io/quote v1.5.2
+)
+
+go 1.13

testdata/go/project3/dependency.go

@@ -0,0 +1,11 @@
+package dependency
+
+import (
+	"fmt"
+	"github.com/pkg/errors"
+)
+
+func PrintHello() error {
+	fmt.Println("Hello")
+	return errors.New("abc")
+}

testdata/go/project3/go.sum

@@ -0,0 +1,7 @@
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c h1:qgOY6WgZOaTkIIMiVjBQcw93ERBE4m30iBm00nkL0i8=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+rsc.io/quote v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y=
+rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0=
+rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

testdata/go/project4/.jfrog/projects/go.yaml

@@ -0,0 +1,9 @@
+version: 1
+type: go
+
+resolver:
+  repo: ${GO_VIRTUAL_REPO}
+  serverID: default
+deployer:
+  repo: ${GO_REPO}
+  serverID: default

testdata/go/project4/createGoProject_go.mod_suffix

@@ -0,0 +1,8 @@
+module github.com/jfrog/dependency
+
+require (
+	github.com/pkg/errors v0.8.0
+	rsc.io/quote v1.5.2
+)
+
+go 1.13

testdata/go/project4/dependency.go

@@ -0,0 +1,11 @@
+package dependency
+
+import (
+	"fmt"
+	"github.com/pkg/errors"
+)
+
+func PrintHello() error {
+	fmt.Println("Hello World")
+	return errors.New("abc")
+}

testdata/go/project4/dir1/a.txt

@@ -0,0 +1 @@
+a.txt

testdata/go/project4/dir1/dir2/b.txt

@@ -0,0 +1 @@
+b.txt

testdata/go/project4/dir1/dir2/dir3/c.txt

@@ -0,0 +1 @@
+c.txt

testdata/go/project4/dir4/d.txt

@@ -0,0 +1 @@
+d.txt

testdata/go/project4/go.sum

@@ -0,0 +1,7 @@
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c h1:qgOY6WgZOaTkIIMiVjBQcw93ERBE4m30iBm00nkL0i8=
+golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+rsc.io/quote v1.5.2 h1:w5fcysjrx7yqtD/aO+QwRjYZOKnaM9Uh2b40tElTs3Y=
+rsc.io/quote v1.5.2/go.mod h1:LzX7hefJvL54yjefDEDHNONDjII0t9xZLPXsUe+TKr0=
+rsc.io/sampler v1.3.0 h1:7uVkIFmeBqHfdjD+gZwtXXI+RODJ2Wc4O7MPEh/QiW4=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

utils/cliutils/commandsflags.go

@@ -270,6 +270,9 @@ const (
 	propsProps        = propertiesPrefix + props
 	propsExcludeProps = propertiesPrefix + excludeProps
 
+	// Unique go publish flags
+	goPublishExclusions = GoPublish + exclusions
+
 	// Unique build-publish flags
 	buildPublishPrefix = "bp-"
 	bpDryRun           = buildPublishPrefix + dryRun
@@ -1407,6 +1410,10 @@ var flagsMap = map[string]cli.Flag{
 		Name:  Go,
 		Usage: "[Default: false] Set to true to request audit for a Go project.` `",
 	},
+	goPublishExclusions: cli.StringFlag{
+		Name:  exclusions,
+		Usage: "[Optional] Semicolon-separated list of exclusions. Exclusions can include the * and the ? wildcards.` `",
+	},
 	rescan: cli.BoolFlag{
 		Name:  rescan,
 		Usage: "[Default: false] Set to true when scanning an already successfully scanned build, for example after adding an ignore rule.` `",
@@ -1768,7 +1775,7 @@ var commandFlags = map[string][]string{
 		global, serverIdResolve, serverIdDeploy, repoResolve, repoDeploy,
 	},
 	GoPublish: {
-		url, user, password, accessToken, buildName, buildNumber, module, project, detailedSummary,
+		url, user, password, accessToken, buildName, buildNumber, module, project, detailedSummary, goPublishExclusions,
 	},
 	Go: {
 		buildName, buildNumber, module, project, noFallback,

utils/tests/consts.go

@@ -94,6 +94,7 @@ const (
 	SearchAllProdRepo                                     = "search_all_prod_repo.json"
 	SearchDistRepoByInSuffix                              = "search_dist_repo_by_in_suffix.json"
 	SearchRepo1ByInSuffix                                 = "search_repo1_by_in_suffix.json"
+	GoPublishRepoExcludes                                 = "go_publish_repo_excludes.json"
 	SearchRepo1IncludeDirs                                = "search_repo1_include_dirs.json"
 	SearchRepo1NonExistFile                               = "search_repo1_ant_test_file.json"
 	SearchRepo1NonExistFileAntExclusions                  = "search_repo1_ant_and_exclusions_test_file.json"
@@ -215,6 +216,7 @@ var (
 	LcRbName1                   = "cli-lc-rb1"
 	LcRbName2                   = "cli-lc-rb2"
 	LcRbName3                   = "cli-lc-rb3"
+	GoPublishWithExclusionPath  = "github.com/jfrog/dependency/@v/github.com/jfrog/[email protected]/"
 
 	// Users
 	UserName1  = "alice"
@@ -262,6 +264,16 @@ func GetExpectedExcludeUploadPart2() []string {
 		RtRepo1 + "/",
 	}
 }
+func GetExpectedExcludeUpload2() []string {
+	return []string{
+		RtRepo1 + "/b3.in",
+		RtRepo1 + "/a2.in",
+		RtRepo1 + "/a3.in",
+		RtRepo1 + "/a1.in",
+		RtRepo1 + "/c",
+		RtRepo1 + "/",
+	}
+}
 func GetExpectedExcludeUploadIncludeDir() []string {
 	return []string{
 		RtRepo1 + "/a2.in",
@@ -272,6 +284,14 @@ func GetExpectedExcludeUploadIncludeDir() []string {
 	}
 }
 
+func GetUploadLegacyPropsExpected() []string {
+	return []string{
+		RtRepo1 + "/data/a1.in",
+		RtRepo1 + "/data/a2.in",
+		RtRepo1 + "/data/a3.in",
+	}
+}
+
 func GetSearchAppendedBuildNoPatternExpected() []string {
 	return []string{
 		RtRepo1 + "/data/a1.in",
@@ -2078,3 +2098,54 @@ func GetExpectedLifecycleArtifacts() []string {
 		RtProdRepo + "/c3.in",
 	}
 }
+
+func GetGoPublishWithExclusionsExpectedRepoGo() []string {
+	var expected = []string{
+		GoRepo + "/github.com/jfrog/dependency/@v/v1.1.1.info",
+		GoRepo + "/github.com/jfrog/dependency/@v/v1.1.1.mod",
+		GoRepo + "/github.com/jfrog/dependency/@v/v1.1.1.zip",
+	}
+	return expected
+}
+
+func GetGoPublishWithExclusionsExpectedFiles1() []string {
+	var expected = []string{
+		GoPublishWithExclusionPath + "dir4/d.txt",
+	}
+	return expected
+}
+
+func GetGoPublishWithExclusionsExcludedFiles1() []string {
+	var excluded = []string{
+		GoPublishWithExclusionPath + "dir1/a.txt",
+		GoPublishWithExclusionPath + "dir1/dir2/b.txt",
+		GoPublishWithExclusionPath + "dir1/dir2/dir3/c.txt",
+	}
+	return excluded
+}
+
+func GetGoPublishWithExclusionsExpectedFiles2() []string {
+	var expected = []string{
+		GoPublishWithExclusionPath + "dir4/d.txt",
+		GoPublishWithExclusionPath + "dir1/a.txt",
+	}
+	return expected
+}
+
+func GetGoPublishWithExclusionsExcludedFiles2() []string {
+	var excluded = []string{
+		GoPublishWithExclusionPath + "dir1/dir2/b.txt",
+		GoPublishWithExclusionPath + "dir1/dir2/dir3/c.txt",
+	}
+	return excluded
+}
+
+func GetGoPublishWithExclusionsExcludedFiles3() []string {
+	var excluded = []string{
+		GoPublishWithExclusionPath + "dir1/a.txt",
+		GoPublishWithExclusionPath + "dir1/dir2/b.txt",
+		GoPublishWithExclusionPath + "dir1/dir2/dir3/c.txt",
+		GoPublishWithExclusionPath + "dir4/d.txt",
+	}
+	return excluded
+}

xray_test.go

@@ -293,7 +293,7 @@ func TestXrayAuditNoTech(t *testing.T) {
 	defer clientTestUtils.ChangeDirAndAssert(t, prevWd)
 	// Run audit on empty folder, expect an error
 	err := xrayCli.Exec("audit")
-	assert.EqualError(t, err, "could not determine the package manager / build tool used by this project.")
+	assert.NoError(t, err)
 }
 
 func TestXrayAuditDetectTech(t *testing.T) {