Skip to content

Commit

Permalink
Merge branch 'dev'
Browse files Browse the repository at this point in the history
  • Loading branch information
eyalbe4 committed Sep 19, 2024
2 parents f029a88 + e1da0a4 commit fe73fb2
Show file tree
Hide file tree
Showing 28 changed files with 456 additions and 133 deletions.
5 changes: 3 additions & 2 deletions artifactory/commands/generic/upload.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package generic
import (
"errors"
"github.com/jfrog/jfrog-cli-core/v2/artifactory/utils/commandsummary"
"github.com/jfrog/jfrog-client-go/artifactory"
"os"

buildInfo "github.com/jfrog/build-info-go/entities"
Expand Down Expand Up @@ -135,7 +136,7 @@ func (uc *UploadCommand) upload() (err error) {
var artifactsDetailsReader *content.ContentReader = nil
if uc.DetailedSummary() || toCollect {
var summary *rtServicesUtils.OperationSummary
summary, err = servicesManager.UploadFilesWithSummary(uploadParamsArray...)
summary, err = servicesManager.UploadFilesWithSummary(artifactory.UploadServiceOptions{}, uploadParamsArray...)
if err != nil {
errorOccurred = true
log.Error(err)
Expand All @@ -162,7 +163,7 @@ func (uc *UploadCommand) upload() (err error) {
}
}
} else {
successCount, failCount, err = servicesManager.UploadFiles(uploadParamsArray...)
successCount, failCount, err = servicesManager.UploadFiles(artifactory.UploadServiceOptions{}, uploadParamsArray...)
if err != nil {
errorOccurred = true
log.Error(err)
Expand Down
5 changes: 3 additions & 2 deletions artifactory/commands/npm/publish.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"github.com/jfrog/jfrog-cli-core/v2/common/spec"
"github.com/jfrog/jfrog-cli-core/v2/utils/config"
"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
"github.com/jfrog/jfrog-client-go/artifactory"
"github.com/jfrog/jfrog-client-go/artifactory/services"
specutils "github.com/jfrog/jfrog-client-go/artifactory/services/utils"
clientutils "github.com/jfrog/jfrog-client-go/utils"
Expand Down Expand Up @@ -344,7 +345,7 @@ func (npc *NpmPublishCommand) doDeploy(target string, artDetails *config.ServerD
return err
}
}
summary, err := servicesManager.UploadFilesWithSummary(up)
summary, err := servicesManager.UploadFilesWithSummary(artifactory.UploadServiceOptions{}, up)
if err != nil {
return err
}
Expand All @@ -367,7 +368,7 @@ func (npc *NpmPublishCommand) doDeploy(target string, artDetails *config.ServerD
}
}
} else {
_, totalFailed, err = servicesManager.UploadFiles(up)
_, totalFailed, err = servicesManager.UploadFiles(artifactory.UploadServiceOptions{}, up)
if err != nil {
return err
}
Expand Down
195 changes: 195 additions & 0 deletions artifactory/commands/python/twine.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,195 @@
package python

import (
"errors"
"fmt"
"github.com/jfrog/build-info-go/build"
"github.com/jfrog/build-info-go/utils/pythonutils"
buildUtils "github.com/jfrog/jfrog-cli-core/v2/common/build"
"github.com/jfrog/jfrog-cli-core/v2/utils/config"
"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
"github.com/jfrog/jfrog-client-go/auth"
"github.com/jfrog/jfrog-client-go/utils"
"github.com/jfrog/jfrog-client-go/utils/errorutils"
"github.com/jfrog/jfrog-client-go/utils/log"
"os"
"os/exec"
"strings"
)

const (
_configFileOptionKey = "--config-file"
_repositoryUrlOptionKey = "--repository-url"
_usernameOptionKey = "--username"
_passwordOptionKey = "--password"
_usernamePrefixOptionKey = "-u"
_passwordPrefixOptionKey = "-p"
_repositoryUrlEnvKey = "TWINE_REPOSITORY_URL"
_usernameEnvKey = "TWINE_USERNAME"
_passwordEnvKey = "TWINE_PASSWORD"
// Artifactory endpoint for pypi deployment.
_apiPypi = "api/pypi/"
_twineExecName = "twine"
_uploadCmdName = "upload"
)

var twineRepoConfigFlags = []string{_configFileOptionKey, _repositoryUrlOptionKey, _usernameOptionKey, _passwordOptionKey, _usernamePrefixOptionKey, _passwordPrefixOptionKey}

type TwineCommand struct {
serverDetails *config.ServerDetails
commandName string
args []string
targetRepo string
buildConfiguration *buildUtils.BuildConfiguration
}

func NewTwineCommand(commandName string) *TwineCommand {
return &TwineCommand{
commandName: commandName,
}
}

func (tc *TwineCommand) CommandName() string {
return "twine_" + tc.commandName
}

func (tc *TwineCommand) ServerDetails() (*config.ServerDetails, error) {
return tc.serverDetails, nil
}

func (tc *TwineCommand) SetServerDetails(serverDetails *config.ServerDetails) *TwineCommand {
tc.serverDetails = serverDetails
return tc
}

func (tc *TwineCommand) SetTargetRepo(targetRepo string) *TwineCommand {
tc.targetRepo = targetRepo
return tc
}

func (tc *TwineCommand) SetArgs(args []string) *TwineCommand {
tc.args = args
return tc
}

func (tc *TwineCommand) Run() (err error) {
// Assert no forbidden flags were provided.
if tc.isRepoConfigFlagProvided() {
return errorutils.CheckErrorf(tc.getRepoConfigFlagProvidedErr())
}
if err = tc.extractAndFilterArgs(tc.args); err != nil {
return err
}
callbackFunc, err := tc.setAuthEnvVars()
defer func() {
err = errors.Join(err, callbackFunc())
}()

collectBuild, err := tc.buildConfiguration.IsCollectBuildInfo()
if err != nil {
return err
}
// If build info is not collected, or this is not an upload command, run the twine command directly.
if !collectBuild || tc.commandName != _uploadCmdName {
return tc.runPlainTwineCommand()
}
return tc.uploadAndCollectBuildInfo()
}

func (tc *TwineCommand) extractAndFilterArgs(args []string) (err error) {
cleanArgs := append([]string(nil), args...)
cleanArgs, tc.buildConfiguration, err = buildUtils.ExtractBuildDetailsFromArgs(cleanArgs)
if err != nil {
return
}
tc.args = cleanArgs
return
}

func (tc *TwineCommand) setAuthEnvVars() (callbackFunc func() error, err error) {
oldRepoUrl := os.Getenv(_repositoryUrlEnvKey)
oldUsername := os.Getenv(_usernameEnvKey)
oldPassword := os.Getenv(_passwordEnvKey)
callbackFunc = func() error {
return errors.Join(os.Setenv(_repositoryUrlOptionKey, oldRepoUrl), os.Setenv(_usernameEnvKey, oldUsername), os.Setenv(_passwordEnvKey, oldPassword))
}

if err = os.Setenv(_repositoryUrlEnvKey, utils.AddTrailingSlashIfNeeded(tc.serverDetails.ArtifactoryUrl)+_apiPypi+tc.targetRepo); err != nil {
return
}

username := tc.serverDetails.User
password := tc.serverDetails.Password
// Get credentials from access-token if exists.
if tc.serverDetails.GetAccessToken() != "" {
if username == "" {
username = auth.ExtractUsernameFromAccessToken(tc.serverDetails.GetAccessToken())
}
password = tc.serverDetails.GetAccessToken()
}

if err = os.Setenv(_usernameEnvKey, username); err != nil {
return
}
err = os.Setenv(_passwordEnvKey, password)
return
}

func (tc *TwineCommand) runPlainTwineCommand() error {
log.Debug("Running twine command:", tc.commandName, strings.Join(tc.args, " "))
args := append([]string{tc.commandName}, tc.args...)
cmd := exec.Command(_twineExecName, args...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
return cmd.Run()
}

func (tc *TwineCommand) uploadAndCollectBuildInfo() error {
buildInfo, err := buildUtils.PrepareBuildPrerequisites(tc.buildConfiguration)
if err != nil {
return err
}

defer func() {
if buildInfo != nil && err != nil {
err = errors.Join(err, buildInfo.Clean())
}
}()

var pythonModule *build.PythonModule
pythonModule, err = buildInfo.AddPythonModule("", pythonutils.Twine)
if err != nil {
return err
}
if tc.buildConfiguration.GetModule() != "" {
pythonModule.SetName(tc.buildConfiguration.GetModule())
}

artifacts, err := pythonModule.TwineUploadWithLogParsing(tc.args)
if err != nil {
return err
}
for i := range artifacts {
artifacts[i].OriginalDeploymentRepo = tc.targetRepo
}
if err = pythonModule.AddArtifacts(artifacts); err != nil {
return err
}
log.Debug(fmt.Sprintf("Command finished successfully. %d artifacs were added to build info.", len(artifacts)))
return nil
}

func (tc *TwineCommand) isRepoConfigFlagProvided() bool {
for _, arg := range tc.args {
for _, flag := range twineRepoConfigFlags {
if strings.HasPrefix(arg, flag) {
return true
}
}
}
return false
}

func (tc *TwineCommand) getRepoConfigFlagProvidedErr() string {
return "twine command must not be executed with the following flags: " + coreutils.ListToText(twineRepoConfigFlags)
}
3 changes: 2 additions & 1 deletion artifactory/commands/terraform/terraformpublish.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/jfrog/jfrog-cli-core/v2/common/project"
"github.com/jfrog/jfrog-cli-core/v2/utils/config"
"github.com/jfrog/jfrog-cli-core/v2/utils/coreutils"
"github.com/jfrog/jfrog-client-go/artifactory"
"github.com/jfrog/jfrog-client-go/artifactory/services"
servicesUtils "github.com/jfrog/jfrog-client-go/artifactory/services/utils"
clientUtils "github.com/jfrog/jfrog-client-go/utils"
Expand Down Expand Up @@ -232,7 +233,7 @@ func createServiceManagerAndUpload(serverDetails *config.ServerDetails, uploadPa
if err != nil {
return nil, err
}
return serviceManager.UploadFilesWithSummary(*uploadParams)
return serviceManager.UploadFilesWithSummary(artifactory.UploadServiceOptions{}, *uploadParams)
}

func (tpc *TerraformPublishCommand) walkDirAndUploadTerraformModules(pwd string, producer parallel.Runner, errorsQueue *clientUtils.ErrorsQueue, uploadSummary *[][]*servicesUtils.OperationSummary, produceTaskFunc ProduceTaskFunc) error {
Expand Down
7 changes: 6 additions & 1 deletion artifactory/commands/transferfiles/fulltransfer.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package transferfiles

import (
"fmt"
"github.com/jfrog/gofrog/safeconvert"
"path"
"time"

Expand Down Expand Up @@ -250,7 +251,11 @@ func (m *fullTransferPhase) handleFoundFile(pcWrapper *producerConsumerWrapper,
return
}
// Increment the files count in the directory's node in the snapshot manager, to track its progress.
err = node.IncrementFilesCount(uint64(file.Size))
unsignedFileSize, err := safeconvert.Int64ToUint64(file.Size)
if err != nil {
return fmt.Errorf("failed to convert file size to uint64: %w", err)
}
err = node.IncrementFilesCount(unsignedFileSize)
if err != nil {
return
}
Expand Down
5 changes: 4 additions & 1 deletion artifactory/commands/transferfiles/longpropertycheck_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package transferfiles

import (
"encoding/json"
"github.com/jfrog/gofrog/safeconvert"
"io"
"net/http"
"net/http/httptest"
Expand Down Expand Up @@ -56,7 +57,9 @@ func TestProperty(t *testing.T) {
}

func testProperty(t *testing.T, property Property, isLong bool) {
assert.Len(t, property.Value, int(property.valueLength()))
signedValueLen, err := safeconvert.UintToInt(property.valueLength())
assert.NoError(t, err)
assert.Len(t, property.Value, signedValueLen)
long := isLongProperty(property)
assert.Equal(t, isLong, long)
}
Expand Down
2 changes: 1 addition & 1 deletion artifactory/commands/transferfiles/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -416,7 +416,7 @@ func updateProgress(phase *phaseBase, timeEstMng *state.TimeEstimationManager,
}

if timeEstMng != nil {
timeEstMng.AddChunkStatus(chunk, time.Since(chunkSentTime).Milliseconds())
return timeEstMng.AddChunkStatus(chunk, time.Since(chunkSentTime).Milliseconds())
}
return nil
}
Expand Down
14 changes: 12 additions & 2 deletions artifactory/commands/transferfiles/state/statemanager.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package state

import (
"fmt"
"github.com/jfrog/gofrog/safeconvert"
"path/filepath"
"time"

Expand Down Expand Up @@ -94,7 +96,11 @@ func (ts *TransferStateManager) SetRepoState(repoKey string, totalSizeBytes, tot
log.Info("Calculated transferred files from previous run:", transferredFiles)
log.Info("Calculated transferred bytes from previous run:", transferredSizeBytes)
transferState.CurrentRepo.Phase1Info.TransferredUnits = int64(transferredFiles)
transferState.CurrentRepo.Phase1Info.TransferredSizeBytes = int64(transferredSizeBytes)
signedTransferredSizeBytes, err := safeconvert.Uint64ToInt64(transferredSizeBytes)
if err != nil {
return fmt.Errorf("failed to set transferred size bytes: %w", err)
}
transferState.CurrentRepo.Phase1Info.TransferredSizeBytes = signedTransferredSizeBytes
}

ts.TransferState = transferState
Expand All @@ -110,7 +116,11 @@ func (ts *TransferStateManager) SetRepoState(repoKey string, totalSizeBytes, tot
transferRunStatus.VisitedFolders = 0

transferRunStatus.OverallTransfer.TransferredUnits += int64(transferredFiles)
transferRunStatus.OverallTransfer.TransferredSizeBytes += int64(transferredSizeBytes)
signedTransferredSizeBytes, err := safeconvert.Uint64ToInt64(transferredSizeBytes)
if err != nil {
return fmt.Errorf("failed to set transferred size bytes: %w", err)
}
transferRunStatus.OverallTransfer.TransferredSizeBytes += signedTransferredSizeBytes
return nil
})
}
Expand Down
21 changes: 16 additions & 5 deletions artifactory/commands/transferfiles/state/statemanager_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package state

import (
"github.com/jfrog/gofrog/safeconvert"
"sync"
"testing"
"time"
Expand Down Expand Up @@ -340,9 +341,15 @@ func TestStateConcurrency(t *testing.T) {
assert.Equal(t, 1000, int(stateManager.CurrentRepo.Phase3Info.TransferredSizeBytes))
assert.Equal(t, 1000, int(stateManager.CurrentRepo.Phase3Info.TransferredUnits))
assert.Equal(t, 1000, int(stateManager.OverallTransfer.TransferredSizeBytes))
assert.Equal(t, 1000, int(stateManager.VisitedFolders))
assert.Equal(t, 1000, int(stateManager.DelayedFiles))
assert.Equal(t, 1000, int(stateManager.TransferFailures))
signedVisitedFolders, err := safeconvert.Uint64ToInt(stateManager.VisitedFolders)
assert.NoError(t, err)
assert.Equal(t, 1000, signedVisitedFolders)
signedDelayedFiles, err := safeconvert.Uint64ToInt(stateManager.DelayedFiles)
assert.NoError(t, err)
assert.Equal(t, 1000, signedDelayedFiles)
signedTransferFailures, err := safeconvert.Uint64ToInt(stateManager.TransferFailures)
assert.NoError(t, err)
assert.Equal(t, 1000, signedTransferFailures)

// Concurrently decrement delayed artifacts and transfer failures
for i := 0; i < 500; i++ {
Expand All @@ -356,6 +363,10 @@ func TestStateConcurrency(t *testing.T) {
wg.Wait()

// Assert 500 in delayed artifacts and transfer failures
assert.Equal(t, 500, int(stateManager.DelayedFiles))
assert.Equal(t, 500, int(stateManager.TransferFailures))
signedDelayedFiles, err = safeconvert.Uint64ToInt(stateManager.DelayedFiles)
assert.NoError(t, err)
assert.Equal(t, 500, signedDelayedFiles)
signedTransferFailures, err = safeconvert.Uint64ToInt(stateManager.TransferFailures)
assert.NoError(t, err)
assert.Equal(t, 500, signedTransferFailures)
}
Loading

0 comments on commit fe73fb2

Please sign in to comment.