From 40bba0fb00a7a1601b294b01ef8a9dac0d6e98e4 Mon Sep 17 00:00:00 2001 From: Gabor Nyerges Date: Mon, 2 May 2022 17:58:33 +0200 Subject: [PATCH 01/14] check minio for cached elements --- backend/src/apiserver/client_manager.go | 3 +- .../apiserver/resource/client_manager_fake.go | 6 ++ backend/src/apiserver/storage/object_store.go | 21 ++++-- .../apiserver/storage/object_store_fake.go | 2 +- backend/src/cache/client_manager.go | 24 +++++-- backend/src/cache/main.go | 33 ++++++++- .../src/cache/server/client_manager_fake.go | 6 ++ backend/src/cache/server/model.go | 72 +++++++++++++++++++ backend/src/cache/server/mutation.go | 67 +++++++++++++++++ backend/src/cache/server/mutation_test.go | 10 +-- 10 files changed, 225 insertions(+), 19 deletions(-) create mode 100644 backend/src/cache/server/model.go diff --git a/backend/src/apiserver/client_manager.go b/backend/src/apiserver/client_manager.go index e667a90a664..813b546daa7 100644 --- a/backend/src/apiserver/client_manager.go +++ b/backend/src/apiserver/client_manager.go @@ -402,7 +402,8 @@ func initMinioClient(initConnectionTimeout time.Duration) storage.ObjectStoreInt secretKey, minioServiceSecure, minioServiceRegion, initConnectionTimeout) createMinioBucket(minioClient, bucketName, minioServiceRegion) - return storage.NewMinioObjectStore(&storage.MinioClient{Client: minioClient}, bucketName, pipelinePath, disableMultipart) + return storage.NewMinioObjectStore(&storage.MinioClient{Client: minioClient}, minioServiceRegion, + bucketName, pipelinePath, disableMultipart) } func createMinioBucket(minioClient *minio.Client, bucketName, region string) { diff --git a/backend/src/apiserver/resource/client_manager_fake.go b/backend/src/apiserver/resource/client_manager_fake.go index 5b2f0ef4ebf..223b691bdea 100644 --- a/backend/src/apiserver/resource/client_manager_fake.go +++ b/backend/src/apiserver/resource/client_manager_fake.go @@ -20,6 +20,7 @@ import ( "github.com/kubeflow/pipelines/backend/src/apiserver/auth" "github.com/kubeflow/pipelines/backend/src/apiserver/client" "github.com/kubeflow/pipelines/backend/src/apiserver/storage" + api_storage "github.com/kubeflow/pipelines/backend/src/apiserver/storage" "github.com/kubeflow/pipelines/backend/src/common/util" ) @@ -43,6 +44,7 @@ type FakeClientManager struct { ArgoClientFake *client.FakeArgoClient swfClientFake *client.FakeSwfClient k8sCoreClientFake *client.FakeKuberneteCoreClient + apiClient api_storage.ObjectStoreInterface SubjectAccessReviewClientFake client.SubjectAccessReviewInterface tokenReviewClientFake client.TokenReviewInterface logArchive archive.LogArchiveInterface @@ -179,6 +181,10 @@ func (f *FakeClientManager) KubernetesCoreClient() client.KubernetesCoreInterfac return f.k8sCoreClientFake } +func (f *FakeClientManager) MinioClient() api_storage.ObjectStoreInterface { + return f.apiClient +} + func (f *FakeClientManager) SubjectAccessReviewClient() client.SubjectAccessReviewInterface { return f.SubjectAccessReviewClientFake } diff --git a/backend/src/apiserver/storage/object_store.go b/backend/src/apiserver/storage/object_store.go index 88ce2556c52..70012a326b7 100644 --- a/backend/src/apiserver/storage/object_store.go +++ b/backend/src/apiserver/storage/object_store.go @@ -21,7 +21,7 @@ import ( "github.com/ghodss/yaml" "github.com/kubeflow/pipelines/backend/src/common/util" - minio "github.com/minio/minio-go" + "github.com/minio/minio-go" ) const ( @@ -40,10 +40,13 @@ type ObjectStoreInterface interface { // Managing pipeline using Minio type MinioObjectStore struct { - minioClient MinioClientInterface - bucketName string - baseFolder string - disableMultipart bool + minioClient MinioClientInterface + bucketName string + baseFolder string + disableMultipart bool + defaultBucketName string + region string + hasPipelineNamespacedBucketEnabled bool } // GetPipelineKey adds the configured base folder to pipeline id. @@ -126,6 +129,10 @@ func buildPath(folder, file string) string { return folder + "/" + file } -func NewMinioObjectStore(minioClient MinioClientInterface, bucketName string, baseFolder string, disableMultipart bool) *MinioObjectStore { - return &MinioObjectStore{minioClient: minioClient, bucketName: bucketName, baseFolder: baseFolder, disableMultipart: disableMultipart} +func NewMinioObjectStore(minioClient MinioClientInterface, region string, defaultBucketName string, + baseFolder string, disableMultipart bool) *MinioObjectStore { + return &MinioObjectStore{minioClient: minioClient, region: region, + defaultBucketName: defaultBucketName, baseFolder: baseFolder, + disableMultipart: disableMultipart, + } } diff --git a/backend/src/apiserver/storage/object_store_fake.go b/backend/src/apiserver/storage/object_store_fake.go index ca38d66c203..c84c5c332d0 100644 --- a/backend/src/apiserver/storage/object_store_fake.go +++ b/backend/src/apiserver/storage/object_store_fake.go @@ -16,5 +16,5 @@ package storage // Return the object store with faked minio client. func NewFakeObjectStore() ObjectStoreInterface { - return NewMinioObjectStore(NewFakeMinioClient(), "", "pipelines", false) + return NewMinioObjectStore(NewFakeMinioClient(), "", "", "pipelines", false) } diff --git a/backend/src/cache/client_manager.go b/backend/src/cache/client_manager.go index 115f919d422..932a43a4bf6 100644 --- a/backend/src/cache/client_manager.go +++ b/backend/src/cache/client_manager.go @@ -16,14 +16,16 @@ package main import ( "database/sql" + "encoding/json" "fmt" "log" "time" - "encoding/json" "github.com/cenkalti/backoff" "github.com/golang/glog" "github.com/jinzhu/gorm" + api_client "github.com/kubeflow/pipelines/backend/src/apiserver/client" + api_storage "github.com/kubeflow/pipelines/backend/src/apiserver/storage" "github.com/kubeflow/pipelines/backend/src/cache/client" "github.com/kubeflow/pipelines/backend/src/cache/model" "github.com/kubeflow/pipelines/backend/src/cache/storage" @@ -38,6 +40,7 @@ type ClientManager struct { db *storage.DB cacheStore storage.ExecutionCacheStoreInterface k8sCoreClient client.KubernetesCoreInterface + minioClient api_storage.ObjectStoreInterface time util.TimeInterface } @@ -49,11 +52,15 @@ func (c *ClientManager) KubernetesCoreClient() client.KubernetesCoreInterface { return c.k8sCoreClient } +func (c *ClientManager) MinioClient() api_storage.ObjectStoreInterface { + return c.minioClient +} + func (c *ClientManager) Close() { c.db.Close() } -func (c *ClientManager) init(params WhSvrDBParameters, clientParams util.ClientParameters) { +func (c *ClientManager) init(params WhSvrDBParameters, clientParams util.ClientParameters, s3params S3Params) { timeoutDuration, _ := time.ParseDuration(DefaultConnectionTimeout) db := initDBClient(params, timeoutDuration) @@ -61,6 +68,15 @@ func (c *ClientManager) init(params WhSvrDBParameters, clientParams util.ClientP c.db = db c.cacheStore = storage.NewExecutionCacheStore(db, c.time) c.k8sCoreClient = client.CreateKubernetesCoreOrFatal(timeoutDuration, clientParams) + c.minioClient = NewMinioClient(timeoutDuration, s3params) +} + +func NewMinioClient(initConnectionTimeout time.Duration, params S3Params) api_storage.ObjectStoreInterface { + glog.Infof("NewMinioClient %v", params) + minioClient := api_client.CreateMinioClientOrFatal(params.serviceHost, params.servicePort, + params.accessKey, params.secretKey, params.isServiceSecure, params.region, initConnectionTimeout) + return api_storage.NewMinioObjectStore(&api_storage.MinioClient{Client: minioClient}, + params.region, params.bucketName, params.pipelinePath, params.disableMultipart) } func initDBClient(params WhSvrDBParameters, initConnectionTimeout time.Duration) *storage.DB { @@ -169,9 +185,9 @@ func initMysql(params WhSvrDBParameters, initConnectionTimeout time.Duration) st return mysqlConfig.FormatDSN() } -func NewClientManager(params WhSvrDBParameters, clientParams util.ClientParameters) ClientManager { +func NewClientManager(params WhSvrDBParameters, clientParams util.ClientParameters, s3params S3Params) ClientManager { clientManager := ClientManager{} - clientManager.init(params, clientParams) + clientManager.init(params, clientParams, s3params) return clientManager } diff --git a/backend/src/cache/main.go b/backend/src/cache/main.go index adcc42c6c3f..03d9ffa5acd 100644 --- a/backend/src/cache/main.go +++ b/backend/src/cache/main.go @@ -19,6 +19,7 @@ import ( "flag" "log" "net/http" + "os" "path/filepath" "github.com/kubeflow/pipelines/backend/src/cache/server" @@ -57,9 +58,22 @@ type WhSvrDBParameters struct { namespaceToWatch string } +type S3Params struct { + serviceHost string + servicePort string + region string + accessKey string + secretKey string + isServiceSecure bool + bucketName string + pipelinePath string + disableMultipart bool +} + func main() { var params WhSvrDBParameters var clientParams util.ClientParameters + var s3params S3Params flag.StringVar(¶ms.dbDriver, "db_driver", mysqlDBDriverDefault, "Database driver name, mysql is the default value") flag.StringVar(¶ms.dbHost, "db_host", mysqlDBHostDefault, "Database host name.") flag.StringVar(¶ms.dbPort, "db_port", mysqlDBPortDefault, "Database port number.") @@ -74,10 +88,12 @@ func main() { flag.Float64Var(&clientParams.QPS, "kube_client_qps", 5, "The maximum QPS to the master from this client.") flag.IntVar(&clientParams.Burst, "kube_client_burst", 10, "Maximum burst for throttle from this client.") + s3params = parseS3Flags() + //s3 endpoint params flag.Parse() log.Println("Initing client manager....") - clientManager := NewClientManager(params, clientParams) + clientManager := NewClientManager(params, clientParams, s3params) ctx := context.Background() go server.WatchPods(ctx, params.namespaceToWatch, &clientManager) @@ -94,3 +110,18 @@ func main() { } log.Fatal(server.ListenAndServeTLS(certPath, keyPath)) } + +func parseS3Flags() S3Params { + var params S3Params + flag.StringVar(¶ms.serviceHost, "s3_service_host", os.Getenv("MINIO_SERVICE_SERVICE_HOST"), "hostname of S3.") + flag.StringVar(¶ms.servicePort, "s3_service_port", os.Getenv("MINIO_SERVICE_SERVICE_PORT"), "port of S3.") + flag.StringVar(¶ms.region, "s3_region", os.Getenv("MINIO_SERVICE_REGION"), "Region of S3 service.") + flag.StringVar(¶ms.accessKey, "s3_access_key", os.Getenv("OBJECTSTORECONFIG_ACCESSKEY"), "accessKey of S3.") + flag.StringVar(¶ms.secretKey, "s3_secret_key", os.Getenv("OBJECTSTORECONFIG_SECRETACCESSKEY"), "secretKey of S3.") + flag.BoolVar(¶ms.isServiceSecure, "s3_is_service_secure", false, "is ssl enabled on S3.") + flag.StringVar(¶ms.bucketName, "s3_bucketname", os.Getenv("S3_BUCKETNAME"), "default bucketname on S3.") + flag.StringVar(¶ms.pipelinePath, "s3_pipelinepath", "pipelines", "pipelinepath of S3.") + flag.BoolVar(¶ms.disableMultipart, "s3_disable_multipart", true, "if multipart request are disabled on S3.") + log.Printf("S3Params %v\n", params) + return params +} diff --git a/backend/src/cache/server/client_manager_fake.go b/backend/src/cache/server/client_manager_fake.go index d459eae18eb..f22c25f4e87 100644 --- a/backend/src/cache/server/client_manager_fake.go +++ b/backend/src/cache/server/client_manager_fake.go @@ -16,6 +16,7 @@ package server import ( "github.com/golang/glog" + api_storage "github.com/kubeflow/pipelines/backend/src/apiserver/storage" "github.com/kubeflow/pipelines/backend/src/cache/client" "github.com/kubeflow/pipelines/backend/src/cache/storage" "github.com/kubeflow/pipelines/backend/src/common/util" @@ -25,6 +26,7 @@ type FakeClientManager struct { db *storage.DB cacheStore storage.ExecutionCacheStoreInterface k8sCoreClientFake *client.FakeKuberneteCoreClient + apiClient api_storage.ObjectStoreInterface time util.TimeInterface } @@ -73,3 +75,7 @@ func (f *FakeClientManager) Close() error { func (f *FakeClientManager) KubernetesCoreClient() client.KubernetesCoreInterface { return f.k8sCoreClientFake } + +func (f *FakeClientManager) MinioClient() api_storage.ObjectStoreInterface { + return f.apiClient +} diff --git a/backend/src/cache/server/model.go b/backend/src/cache/server/model.go new file mode 100644 index 00000000000..231ff8c4dec --- /dev/null +++ b/backend/src/cache/server/model.go @@ -0,0 +1,72 @@ +package server + +type ExecutionTemplate struct { + Name string `json:"name"` + Inputs struct { + Artifacts []struct { + Name string `json:"name"` + Path string `json:"path"` + Raw struct { + Data string `json:"data"` + } `json:"raw"` + } `json:"artifacts"` + } `json:"inputs"` + Outputs struct { + Artifacts []struct { + Name string `json:"name"` + Path string `json:"path"` + } `json:"artifacts"` + } `json:"outputs"` + Metadata struct { + Annotations struct { + PipelinesKubeflowOrgComponentRef string `json:"pipelines.kubeflow.org/component_ref"` + PipelinesKubeflowOrgComponentSpec string `json:"pipelines.kubeflow.org/component_spec"` + SidecarIstioIoInject string `json:"sidecar.istio.io/inject"` + } `json:"annotations"` + Labels struct { + PipelinesKubeflowOrgCacheEnabled string `json:"pipelines.kubeflow.org/cache_enabled"` + PipelinesKubeflowOrgEnableCaching string `json:"pipelines.kubeflow.org/enable_caching"` + PipelinesKubeflowOrgKfpSdkVersion string `json:"pipelines.kubeflow.org/kfp_sdk_version"` + PipelinesKubeflowOrgPipelineSdkType string `json:"pipelines.kubeflow.org/pipeline-sdk-type"` + } `json:"labels"` + } `json:"metadata"` + Container struct { + Name string `json:"name"` + Image string `json:"image"` + Command []string `json:"command"` + Args []string `json:"args"` + Resources struct { + } `json:"resources"` + } `json:"container"` + ArchiveLocation struct { + ArchiveLogs bool `json:"archiveLogs"` + S3 struct { + Endpoint string `json:"endpoint"` + Bucket string `json:"bucket"` + Insecure bool `json:"insecure"` + AccessKeySecret struct { + Name string `json:"name"` + Key string `json:"key"` + } `json:"accessKeySecret"` + SecretKeySecret struct { + Name string `json:"name"` + Key string `json:"key"` + } `json:"secretKeySecret"` + Key string `json:"key"` + } `json:"s3"` + } `json:"archiveLocation"` +} + +type ExecutionOutput struct { + PipelinesKubeflowOrgMetadataExecutionID string `json:"pipelines.kubeflow.org/metadata_execution_id"` + WorkflowsArgoprojIoOutputs string `json:"workflows.argoproj.io/outputs"` +} + +type WorkflowsArgoprojIoOutputs struct { + Artifacts []struct { + Name string `json:"name"` + S3 struct { + Key string `json:"key"` + } `json:"s3"` + } `json:"artifacts"` +} diff --git a/backend/src/cache/server/mutation.go b/backend/src/cache/server/mutation.go index 01cdaf23b6b..f3c29810da2 100644 --- a/backend/src/cache/server/mutation.go +++ b/backend/src/cache/server/mutation.go @@ -24,9 +24,11 @@ import ( "strconv" "strings" + api_storage "github.com/kubeflow/pipelines/backend/src/apiserver/storage" "github.com/kubeflow/pipelines/backend/src/cache/client" "github.com/kubeflow/pipelines/backend/src/cache/model" "github.com/kubeflow/pipelines/backend/src/cache/storage" + "github.com/kubeflow/pipelines/backend/src/common/util" "k8s.io/api/admission/v1beta1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -61,6 +63,7 @@ var ( type ClientManagerInterface interface { CacheStore() storage.ExecutionCacheStoreInterface KubernetesCoreClient() client.KubernetesCoreInterface + MinioClient() api_storage.ObjectStoreInterface } // MutatePodIfCached will check whether the execution has already been run before from MLMD and apply the output into pod.metadata.output @@ -127,9 +130,21 @@ func MutatePodIfCached(req *v1beta1.AdmissionRequest, clientMgr ClientManagerInt var cachedExecution *model.ExecutionCache cachedExecution, err = clientMgr.CacheStore().GetExecutionCache(executionHashKey, maxCacheStalenessInSeconds) + cacheKeyWithFileName := getCacheItemKey(cachedExecution) + log.Println("cache key extracted from output: " + cacheKeyWithFileName) + if err != nil { log.Println(err.Error()) } + + cachedExecution, err = retrieveCacheItemIfReallyExists(clientMgr.MinioClient(), cachedExecution, cacheKeyWithFileName) + + if err != nil { + log.Printf("deleteItemFromCacheStore %s ", executionHashKey) + err = clientMgr.CacheStore().DeleteExecutionCache(executionHashKey) + log.Println(err) + } + // Found cached execution, add cached output and cache_id and replace container images. if cachedExecution != nil { log.Println("Cached output: " + cachedExecution.ExecutionOutput) @@ -210,6 +225,58 @@ func MutatePodIfCached(req *v1beta1.AdmissionRequest, clientMgr ClientManagerInt return patches, nil } +//retrieveCacheItemIfReallyExists checks the objects +func retrieveCacheItemIfReallyExists(minioClient api_storage.ObjectStoreInterface, execution *model.ExecutionCache, fileName string) (*model.ExecutionCache, error) { + if execution != nil && minioClient != nil { + log.Printf("retrieveCacheItemIfReallyExists %s ", execution.ExecutionTemplate) + bucket, key := getCacheItemS3Data(execution) + log.Printf("bucket %s , key: %s, filename: %s", bucket, key, fileName) + if minioClient != nil { + result, err := minioClient.GetFile(fileName) + log.Printf("cache retrieve result length %d %v", len(result), err) + if len(result) == 0 || err != nil { + return nil, util.NewInternalServerError(err, "Failed to get %v", key) + } + } + } + return execution, nil +} + +func getCacheItemS3Data(execution *model.ExecutionCache) (string, string) { + var outputMap = ExecutionTemplate{} + b := []byte(execution.ExecutionTemplate) + err := json.Unmarshal(b, &outputMap) + if err != nil { + panic(err) + return "", "" + } + return outputMap.ArchiveLocation.S3.Bucket, outputMap.ArchiveLocation.S3.Key +} + +func getCacheItemKey(execution *model.ExecutionCache) string { + var output = ExecutionOutput{} + if execution != nil { + b := []byte(execution.ExecutionOutput) + err := json.Unmarshal(b, &output) + if err != nil { + panic(err) + } + var workflowsArgoprojIoOutputs = WorkflowsArgoprojIoOutputs{} + b = []byte(output.WorkflowsArgoprojIoOutputs) + err = json.Unmarshal(b, &workflowsArgoprojIoOutputs) + if err != nil { + panic(err) + } + + for _, artifact := range workflowsArgoprojIoOutputs.Artifacts { + return artifact.S3.Key + } + } + + return "" + +} + // intersectStructureWithSkeleton recursively intersects two maps // nil values in the skeleton map mean that the whole value (which can also be a map) should be kept. func intersectStructureWithSkeleton(src map[string]interface{}, skeleton map[string]interface{}) map[string]interface{} { diff --git a/backend/src/cache/server/mutation_test.go b/backend/src/cache/server/mutation_test.go index 28ab114e1e5..ce1fac38cc5 100644 --- a/backend/src/cache/server/mutation_test.go +++ b/backend/src/cache/server/mutation_test.go @@ -170,7 +170,7 @@ func TestMutatePodIfCached(t *testing.T) { func TestMutatePodIfCachedWithCacheEntryExist(t *testing.T) { executionCache := &model.ExecutionCache{ ExecutionCacheKey: "f5fe913be7a4516ebfe1b5de29bcb35edd12ecc776b2f33f10ca19709ea3b2f0", - ExecutionOutput: "testOutput", + ExecutionOutput: `{"pipelines.kubeflow.org/metadata_execution_id":"1","workflows.argoproj.io/outputs":"{}"}`, ExecutionTemplate: `{"container":{"command":["echo", "Hello"],"image":"python:3.7"}}`, MaxCacheStaleness: -1, } @@ -189,7 +189,7 @@ func TestMutatePodIfCachedWithCacheEntryExist(t *testing.T) { func TestDefaultImage(t *testing.T) { executionCache := &model.ExecutionCache{ ExecutionCacheKey: "f5fe913be7a4516ebfe1b5de29bcb35edd12ecc776b2f33f10ca19709ea3b2f0", - ExecutionOutput: "testOutput", + ExecutionOutput: `{"pipelines.kubeflow.org/metadata_execution_id":"1","workflows.argoproj.io/outputs":"{}"}`, ExecutionTemplate: `{"container":{"command":["echo", "Hello"],"image":"python:3.7"}}`, MaxCacheStaleness: -1, } @@ -208,7 +208,7 @@ func TestSetImage(t *testing.T) { executionCache := &model.ExecutionCache{ ExecutionCacheKey: "f5fe913be7a4516ebfe1b5de29bcb35edd12ecc776b2f33f10ca19709ea3b2f0", - ExecutionOutput: "testOutput", + ExecutionOutput: `{"pipelines.kubeflow.org/metadata_execution_id":"1","workflows.argoproj.io/outputs":"{}"}`, ExecutionTemplate: `{"container":{"command":["echo", "Hello"],"image":"python:3.7"}}`, MaxCacheStaleness: -1, } @@ -225,7 +225,7 @@ func TestCacheNodeRestriction(t *testing.T) { executionCache := &model.ExecutionCache{ ExecutionCacheKey: "f5fe913be7a4516ebfe1b5de29bcb35edd12ecc776b2f33f10ca19709ea3b2f0", - ExecutionOutput: "testOutput", + ExecutionOutput: `{"pipelines.kubeflow.org/metadata_execution_id":"1","workflows.argoproj.io/outputs":"{}"}`, ExecutionTemplate: `{"container":{"command":["echo", "Hello"],"image":"python:3.7"},"nodeSelector":{"disktype":"ssd"}}`, MaxCacheStaleness: -1, } @@ -240,7 +240,7 @@ func TestCacheNodeRestriction(t *testing.T) { func TestMutatePodIfCachedWithTeamplateCleanup(t *testing.T) { executionCache := &model.ExecutionCache{ ExecutionCacheKey: "5a20e3f2e74863b363291953082d9812a58e25f7117bface1c76d40ef0ee88fc", - ExecutionOutput: "testOutput", + ExecutionOutput: `{"pipelines.kubeflow.org/metadata_execution_id":"1","workflows.argoproj.io/outputs":"{}"}`, ExecutionTemplate: `Cache key was calculated from this: {"container":{"command":["echo", "Hello"],"image":"python:3.7"},"outputs":"anything"}`, MaxCacheStaleness: -1, } From fb78be36159d5a04b3c1b0a4a1b442809aad2a7f Mon Sep 17 00:00:00 2001 From: Gabor Nyerges Date: Tue, 3 May 2022 13:22:10 +0200 Subject: [PATCH 02/14] remove structed executiontemplate parse json dynamically with tests --- backend/src/cache/server/model.go | 72 ----------------------- backend/src/cache/server/mutation.go | 47 ++++++++++----- backend/src/cache/server/mutation_test.go | 37 ++++++++++++ 3 files changed, 68 insertions(+), 88 deletions(-) delete mode 100644 backend/src/cache/server/model.go diff --git a/backend/src/cache/server/model.go b/backend/src/cache/server/model.go deleted file mode 100644 index 231ff8c4dec..00000000000 --- a/backend/src/cache/server/model.go +++ /dev/null @@ -1,72 +0,0 @@ -package server - -type ExecutionTemplate struct { - Name string `json:"name"` - Inputs struct { - Artifacts []struct { - Name string `json:"name"` - Path string `json:"path"` - Raw struct { - Data string `json:"data"` - } `json:"raw"` - } `json:"artifacts"` - } `json:"inputs"` - Outputs struct { - Artifacts []struct { - Name string `json:"name"` - Path string `json:"path"` - } `json:"artifacts"` - } `json:"outputs"` - Metadata struct { - Annotations struct { - PipelinesKubeflowOrgComponentRef string `json:"pipelines.kubeflow.org/component_ref"` - PipelinesKubeflowOrgComponentSpec string `json:"pipelines.kubeflow.org/component_spec"` - SidecarIstioIoInject string `json:"sidecar.istio.io/inject"` - } `json:"annotations"` - Labels struct { - PipelinesKubeflowOrgCacheEnabled string `json:"pipelines.kubeflow.org/cache_enabled"` - PipelinesKubeflowOrgEnableCaching string `json:"pipelines.kubeflow.org/enable_caching"` - PipelinesKubeflowOrgKfpSdkVersion string `json:"pipelines.kubeflow.org/kfp_sdk_version"` - PipelinesKubeflowOrgPipelineSdkType string `json:"pipelines.kubeflow.org/pipeline-sdk-type"` - } `json:"labels"` - } `json:"metadata"` - Container struct { - Name string `json:"name"` - Image string `json:"image"` - Command []string `json:"command"` - Args []string `json:"args"` - Resources struct { - } `json:"resources"` - } `json:"container"` - ArchiveLocation struct { - ArchiveLogs bool `json:"archiveLogs"` - S3 struct { - Endpoint string `json:"endpoint"` - Bucket string `json:"bucket"` - Insecure bool `json:"insecure"` - AccessKeySecret struct { - Name string `json:"name"` - Key string `json:"key"` - } `json:"accessKeySecret"` - SecretKeySecret struct { - Name string `json:"name"` - Key string `json:"key"` - } `json:"secretKeySecret"` - Key string `json:"key"` - } `json:"s3"` - } `json:"archiveLocation"` -} - -type ExecutionOutput struct { - PipelinesKubeflowOrgMetadataExecutionID string `json:"pipelines.kubeflow.org/metadata_execution_id"` - WorkflowsArgoprojIoOutputs string `json:"workflows.argoproj.io/outputs"` -} - -type WorkflowsArgoprojIoOutputs struct { - Artifacts []struct { - Name string `json:"name"` - S3 struct { - Key string `json:"key"` - } `json:"s3"` - } `json:"artifacts"` -} diff --git a/backend/src/cache/server/mutation.go b/backend/src/cache/server/mutation.go index f3c29810da2..bee29288d64 100644 --- a/backend/src/cache/server/mutation.go +++ b/backend/src/cache/server/mutation.go @@ -243,38 +243,53 @@ func retrieveCacheItemIfReallyExists(minioClient api_storage.ObjectStoreInterfac } func getCacheItemS3Data(execution *model.ExecutionCache) (string, string) { - var outputMap = ExecutionTemplate{} + key := "" + bucket := "" + var executionTemplate map[string]interface{} b := []byte(execution.ExecutionTemplate) - err := json.Unmarshal(b, &outputMap) + err := json.Unmarshal(b, &executionTemplate) if err != nil { panic(err) - return "", "" + return bucket, key } - return outputMap.ArchiveLocation.S3.Bucket, outputMap.ArchiveLocation.S3.Key + if archiveLocation, ok := executionTemplate["archiveLocation"]; ok { + if s3, ok := archiveLocation.(map[string]interface{})["s3"]; ok { + if s3key, ok := s3.(map[string]interface{})["key"]; ok { + key = s3key.(string) + } + if s3bucket, ok := s3.(map[string]interface{})["bucket"]; ok { + bucket = s3bucket.(string) + } + } + } + return bucket, key } func getCacheItemKey(execution *model.ExecutionCache) string { - var output = ExecutionOutput{} + var output map[string]interface{} if execution != nil { b := []byte(execution.ExecutionOutput) err := json.Unmarshal(b, &output) if err != nil { panic(err) } - var workflowsArgoprojIoOutputs = WorkflowsArgoprojIoOutputs{} - b = []byte(output.WorkflowsArgoprojIoOutputs) - err = json.Unmarshal(b, &workflowsArgoprojIoOutputs) - if err != nil { - panic(err) - } - - for _, artifact := range workflowsArgoprojIoOutputs.Artifacts { - return artifact.S3.Key + if output, ok := output["workflows.argoproj.io/outputs"]; ok { + var artifacts map[string]interface{} + if err := json.Unmarshal([]byte(output.(string)), &artifacts); err != nil { + panic(err) + } + if artifacts, ok := artifacts["artifacts"]; ok { + for _, artifact := range artifacts.([]interface{}) { + if s3, ok := artifact.(map[string]interface{})["s3"]; ok { + if s3key, ok := s3.(map[string]interface{})["key"]; ok { + return s3key.(string) + } + } + } + } } } - return "" - } // intersectStructureWithSkeleton recursively intersects two maps diff --git a/backend/src/cache/server/mutation_test.go b/backend/src/cache/server/mutation_test.go index ce1fac38cc5..0a8d8b64bc2 100644 --- a/backend/src/cache/server/mutation_test.go +++ b/backend/src/cache/server/mutation_test.go @@ -270,3 +270,40 @@ func TestMutatePodIfCachedWithTeamplateCleanup(t *testing.T) { require.Equal(t, patchOperation[1].Op, OperationTypeAdd) require.Equal(t, patchOperation[2].Op, OperationTypeAdd) } + +func TestGetCacheItemKey(t *testing.T) { + t.Run("test cache retrieval", func(t *testing.T) { + cacheKey := getCacheItemKey(&model.ExecutionCache{ + ID: 0, + ExecutionCacheKey: "", + ExecutionTemplate: "", + ExecutionOutput: ` + { + "pipelines.kubeflow.org/metadata_execution_id":"1", + "workflows.argoproj.io/outputs": "{\"artifacts\": [{\"name\": \"example-name\",\"s3\": {\"key\":\"expected-cachekey\"}}]}" + }`, + MaxCacheStaleness: 0, + StartedAtInSec: 0, + EndedAtInSec: 0, + }) + require.Equal(t, "expected-cachekey", cacheKey) + + }) +} + +func TestGetCacheItemS3Data(t *testing.T) { + t.Run("test s3 data retrieval", func(t *testing.T) { + bucket, key := getCacheItemS3Data(&model.ExecutionCache{ExecutionTemplate: ` + { + "archiveLocation": { + "archiveLogs": true, + "s3": { + "key": "key-name", + "bucket": "bucket-name" + } + } + }`}) + require.Equal(t, "bucket-name", bucket) + require.Equal(t, "key-name", key) + }) +} From d6376fe1fc1aa4690e241734b159f86b72b37bec Mon Sep 17 00:00:00 2001 From: Gabor Nyerges Date: Tue, 3 May 2022 13:32:29 +0200 Subject: [PATCH 03/14] remove not used fields in upstream --- backend/src/apiserver/storage/object_store.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/backend/src/apiserver/storage/object_store.go b/backend/src/apiserver/storage/object_store.go index 70012a326b7..4d7e95bbb23 100644 --- a/backend/src/apiserver/storage/object_store.go +++ b/backend/src/apiserver/storage/object_store.go @@ -40,13 +40,11 @@ type ObjectStoreInterface interface { // Managing pipeline using Minio type MinioObjectStore struct { - minioClient MinioClientInterface - bucketName string - baseFolder string - disableMultipart bool - defaultBucketName string - region string - hasPipelineNamespacedBucketEnabled bool + minioClient MinioClientInterface + bucketName string + baseFolder string + disableMultipart bool + region string } // GetPipelineKey adds the configured base folder to pipeline id. @@ -129,10 +127,10 @@ func buildPath(folder, file string) string { return folder + "/" + file } -func NewMinioObjectStore(minioClient MinioClientInterface, region string, defaultBucketName string, +func NewMinioObjectStore(minioClient MinioClientInterface, region string, bucketName string, baseFolder string, disableMultipart bool) *MinioObjectStore { return &MinioObjectStore{minioClient: minioClient, region: region, - defaultBucketName: defaultBucketName, baseFolder: baseFolder, + bucketName: bucketName, baseFolder: baseFolder, disableMultipart: disableMultipart, } } From d82290f5748e7a244e0cb83f529f3855b80841e8 Mon Sep 17 00:00:00 2001 From: Gabor Nyerges Date: Tue, 3 May 2022 14:39:31 +0200 Subject: [PATCH 04/14] add manifests envs --- backend/src/cache/main.go | 9 ++++++--- .../kustomize/base/cache/cache-deployment.yaml | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/backend/src/cache/main.go b/backend/src/cache/main.go index 03d9ffa5acd..5b84da0a39e 100644 --- a/backend/src/cache/main.go +++ b/backend/src/cache/main.go @@ -22,6 +22,7 @@ import ( "os" "path/filepath" + "github.com/kubeflow/pipelines/backend/src/apiserver/common" "github.com/kubeflow/pipelines/backend/src/cache/server" "github.com/kubeflow/pipelines/backend/src/common/util" ) @@ -118,10 +119,12 @@ func parseS3Flags() S3Params { flag.StringVar(¶ms.region, "s3_region", os.Getenv("MINIO_SERVICE_REGION"), "Region of S3 service.") flag.StringVar(¶ms.accessKey, "s3_access_key", os.Getenv("OBJECTSTORECONFIG_ACCESSKEY"), "accessKey of S3.") flag.StringVar(¶ms.secretKey, "s3_secret_key", os.Getenv("OBJECTSTORECONFIG_SECRETACCESSKEY"), "secretKey of S3.") - flag.BoolVar(¶ms.isServiceSecure, "s3_is_service_secure", false, "is ssl enabled on S3.") - flag.StringVar(¶ms.bucketName, "s3_bucketname", os.Getenv("S3_BUCKETNAME"), "default bucketname on S3.") + flag.BoolVar(¶ms.isServiceSecure, "s3_is_service_secure", + common.GetBoolFromStringWithDefault(os.Getenv("MINIO_SERVICE_SECURE"), false), "is ssl enabled on S3.") + flag.StringVar(¶ms.bucketName, "s3_bucketname", os.Getenv("MINIO_PIPELINE_BUCKET_NAME"), "default bucketname on S3.") flag.StringVar(¶ms.pipelinePath, "s3_pipelinepath", "pipelines", "pipelinepath of S3.") - flag.BoolVar(¶ms.disableMultipart, "s3_disable_multipart", true, "if multipart request are disabled on S3.") + flag.BoolVar(¶ms.disableMultipart, "s3_disable_multipart", + common.GetBoolConfigWithDefault(os.Getenv("MINIO_PIPELINE_BUCKET_DISABLE_MULTIPART"), true), "if multipart request are disabled on S3.") log.Printf("S3Params %v\n", params) return params } diff --git a/manifests/kustomize/base/cache/cache-deployment.yaml b/manifests/kustomize/base/cache/cache-deployment.yaml index 58512e1e6ea..6b9370c7a59 100644 --- a/manifests/kustomize/base/cache/cache-deployment.yaml +++ b/manifests/kustomize/base/cache/cache-deployment.yaml @@ -59,6 +59,23 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: MINIO_SERVICE_SERVICE_HOST + - name: MINIO_SERVICE_SERVICE_PORT + - name: MINIO_SERVICE_REGION + - name: MINIO_SERVICE_SECURE + - name: MINIO_PIPELINE_BUCKET_NAME + - name: MINIO_PIPELINE_PATH + - name: MINIO_PIPELINE_BUCKET_DISABLE_MULTIPART + - name: OBJECTSTORECONFIG_ACCESSKEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: accesskey + - name: OBJECTSTORECONFIG_SECRETACCESSKEY + valueFrom: + secretKeyRef: + name: mlpipeline-minio-artifact + key: secretkey args: ["--db_driver=$(DBCONFIG_DRIVER)", "--db_host=$(DBCONFIG_HOST_NAME)", "--db_port=$(DBCONFIG_PORT)", From 5d0ae16b943590eaee7b31608463cb4cfd231f1e Mon Sep 17 00:00:00 2001 From: Gabor Nyerges Date: Fri, 24 Jun 2022 11:40:51 +0200 Subject: [PATCH 05/14] minioServiceRegion -> region --- backend/src/apiserver/client_manager.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/src/apiserver/client_manager.go b/backend/src/apiserver/client_manager.go index 813b546daa7..a1aa2acccff 100644 --- a/backend/src/apiserver/client_manager.go +++ b/backend/src/apiserver/client_manager.go @@ -388,7 +388,7 @@ func initMinioClient(initConnectionTimeout time.Duration) storage.ObjectStoreInt "ObjectStoreConfig.Host", os.Getenv(minioServiceHost)) minioServicePort := common.GetStringConfigWithDefault( "ObjectStoreConfig.Port", os.Getenv(minioServicePort)) - minioServiceRegion := common.GetStringConfigWithDefault( + region := common.GetStringConfigWithDefault( "ObjectStoreConfig.Region", os.Getenv(minioServiceRegion)) minioServiceSecure := common.GetBoolConfigWithDefault( "ObjectStoreConfig.Secure", common.GetBoolFromStringWithDefault(os.Getenv(minioServiceSecure), false)) @@ -399,10 +399,10 @@ func initMinioClient(initConnectionTimeout time.Duration) storage.ObjectStoreInt disableMultipart := common.GetBoolConfigWithDefault("ObjectStoreConfig.Multipart.Disable", true) minioClient := client.CreateMinioClientOrFatal(minioServiceHost, minioServicePort, accessKey, - secretKey, minioServiceSecure, minioServiceRegion, initConnectionTimeout) - createMinioBucket(minioClient, bucketName, minioServiceRegion) + secretKey, minioServiceSecure, region, initConnectionTimeout) + createMinioBucket(minioClient, bucketName, region) - return storage.NewMinioObjectStore(&storage.MinioClient{Client: minioClient}, minioServiceRegion, + return storage.NewMinioObjectStore(&storage.MinioClient{Client: minioClient}, region, bucketName, pipelinePath, disableMultipart) } From 22d08a062d3c027a3e5d748b6145942e1294ef78 Mon Sep 17 00:00:00 2001 From: Gabor Nyerges Date: Fri, 24 Jun 2022 11:44:19 +0200 Subject: [PATCH 06/14] api_storage -> apiserver_storage --- backend/src/cache/server/client_manager_fake.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/src/cache/server/client_manager_fake.go b/backend/src/cache/server/client_manager_fake.go index f22c25f4e87..1ffabc0d829 100644 --- a/backend/src/cache/server/client_manager_fake.go +++ b/backend/src/cache/server/client_manager_fake.go @@ -16,7 +16,7 @@ package server import ( "github.com/golang/glog" - api_storage "github.com/kubeflow/pipelines/backend/src/apiserver/storage" + apiserver_storage "github.com/kubeflow/pipelines/backend/src/apiserver/storage" "github.com/kubeflow/pipelines/backend/src/cache/client" "github.com/kubeflow/pipelines/backend/src/cache/storage" "github.com/kubeflow/pipelines/backend/src/common/util" @@ -26,7 +26,7 @@ type FakeClientManager struct { db *storage.DB cacheStore storage.ExecutionCacheStoreInterface k8sCoreClientFake *client.FakeKuberneteCoreClient - apiClient api_storage.ObjectStoreInterface + apiClient apiserver_storage.ObjectStoreInterface time util.TimeInterface } @@ -76,6 +76,6 @@ func (f *FakeClientManager) KubernetesCoreClient() client.KubernetesCoreInterfac return f.k8sCoreClientFake } -func (f *FakeClientManager) MinioClient() api_storage.ObjectStoreInterface { +func (f *FakeClientManager) MinioClient() apiserver_storage.ObjectStoreInterface { return f.apiClient } From 4ae15a745fd2611b54c67379335d7d1133a2d5a6 Mon Sep 17 00:00:00 2001 From: Gabor Nyerges Date: Fri, 24 Jun 2022 11:47:08 +0200 Subject: [PATCH 07/14] api_* -> apiserver_* --- backend/src/cache/client_manager.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/src/cache/client_manager.go b/backend/src/cache/client_manager.go index 932a43a4bf6..b0ff87b8f86 100644 --- a/backend/src/cache/client_manager.go +++ b/backend/src/cache/client_manager.go @@ -24,8 +24,8 @@ import ( "github.com/cenkalti/backoff" "github.com/golang/glog" "github.com/jinzhu/gorm" - api_client "github.com/kubeflow/pipelines/backend/src/apiserver/client" - api_storage "github.com/kubeflow/pipelines/backend/src/apiserver/storage" + apiserver_client "github.com/kubeflow/pipelines/backend/src/apiserver/client" + apiserver_storage "github.com/kubeflow/pipelines/backend/src/apiserver/storage" "github.com/kubeflow/pipelines/backend/src/cache/client" "github.com/kubeflow/pipelines/backend/src/cache/model" "github.com/kubeflow/pipelines/backend/src/cache/storage" @@ -40,7 +40,7 @@ type ClientManager struct { db *storage.DB cacheStore storage.ExecutionCacheStoreInterface k8sCoreClient client.KubernetesCoreInterface - minioClient api_storage.ObjectStoreInterface + minioClient apiserver_storage.ObjectStoreInterface time util.TimeInterface } @@ -52,7 +52,7 @@ func (c *ClientManager) KubernetesCoreClient() client.KubernetesCoreInterface { return c.k8sCoreClient } -func (c *ClientManager) MinioClient() api_storage.ObjectStoreInterface { +func (c *ClientManager) MinioClient() apiserver_storage.ObjectStoreInterface { return c.minioClient } @@ -71,11 +71,11 @@ func (c *ClientManager) init(params WhSvrDBParameters, clientParams util.ClientP c.minioClient = NewMinioClient(timeoutDuration, s3params) } -func NewMinioClient(initConnectionTimeout time.Duration, params S3Params) api_storage.ObjectStoreInterface { +func NewMinioClient(initConnectionTimeout time.Duration, params S3Params) apiserver_storage.ObjectStoreInterface { glog.Infof("NewMinioClient %v", params) - minioClient := api_client.CreateMinioClientOrFatal(params.serviceHost, params.servicePort, + minioClient := apiserver_client.CreateMinioClientOrFatal(params.serviceHost, params.servicePort, params.accessKey, params.secretKey, params.isServiceSecure, params.region, initConnectionTimeout) - return api_storage.NewMinioObjectStore(&api_storage.MinioClient{Client: minioClient}, + return apiserver_storage.NewMinioObjectStore(&apiserver_storage.MinioClient{Client: minioClient}, params.region, params.bucketName, params.pipelinePath, params.disableMultipart) } From 6df6f450183a1770eb46ba63b748838a4ffde30d Mon Sep 17 00:00:00 2001 From: Gabor Nyerges Date: Fri, 24 Jun 2022 11:50:34 +0200 Subject: [PATCH 08/14] api_* -> apiserver_* --- backend/src/apiserver/resource/client_manager_fake.go | 6 +++--- backend/src/cache/server/mutation.go | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/src/apiserver/resource/client_manager_fake.go b/backend/src/apiserver/resource/client_manager_fake.go index 223b691bdea..61b1a04cda8 100644 --- a/backend/src/apiserver/resource/client_manager_fake.go +++ b/backend/src/apiserver/resource/client_manager_fake.go @@ -20,7 +20,7 @@ import ( "github.com/kubeflow/pipelines/backend/src/apiserver/auth" "github.com/kubeflow/pipelines/backend/src/apiserver/client" "github.com/kubeflow/pipelines/backend/src/apiserver/storage" - api_storage "github.com/kubeflow/pipelines/backend/src/apiserver/storage" + apiserver_storage "github.com/kubeflow/pipelines/backend/src/apiserver/storage" "github.com/kubeflow/pipelines/backend/src/common/util" ) @@ -44,7 +44,7 @@ type FakeClientManager struct { ArgoClientFake *client.FakeArgoClient swfClientFake *client.FakeSwfClient k8sCoreClientFake *client.FakeKuberneteCoreClient - apiClient api_storage.ObjectStoreInterface + apiClient apiserver_storage.ObjectStoreInterface SubjectAccessReviewClientFake client.SubjectAccessReviewInterface tokenReviewClientFake client.TokenReviewInterface logArchive archive.LogArchiveInterface @@ -181,7 +181,7 @@ func (f *FakeClientManager) KubernetesCoreClient() client.KubernetesCoreInterfac return f.k8sCoreClientFake } -func (f *FakeClientManager) MinioClient() api_storage.ObjectStoreInterface { +func (f *FakeClientManager) MinioClient() apiserver_storage.ObjectStoreInterface { return f.apiClient } diff --git a/backend/src/cache/server/mutation.go b/backend/src/cache/server/mutation.go index bee29288d64..42df238681e 100644 --- a/backend/src/cache/server/mutation.go +++ b/backend/src/cache/server/mutation.go @@ -24,7 +24,7 @@ import ( "strconv" "strings" - api_storage "github.com/kubeflow/pipelines/backend/src/apiserver/storage" + apiserver_storage "github.com/kubeflow/pipelines/backend/src/apiserver/storage" "github.com/kubeflow/pipelines/backend/src/cache/client" "github.com/kubeflow/pipelines/backend/src/cache/model" "github.com/kubeflow/pipelines/backend/src/cache/storage" @@ -63,7 +63,7 @@ var ( type ClientManagerInterface interface { CacheStore() storage.ExecutionCacheStoreInterface KubernetesCoreClient() client.KubernetesCoreInterface - MinioClient() api_storage.ObjectStoreInterface + MinioClient() apiserver_storage.ObjectStoreInterface } // MutatePodIfCached will check whether the execution has already been run before from MLMD and apply the output into pod.metadata.output @@ -226,7 +226,7 @@ func MutatePodIfCached(req *v1beta1.AdmissionRequest, clientMgr ClientManagerInt } //retrieveCacheItemIfReallyExists checks the objects -func retrieveCacheItemIfReallyExists(minioClient api_storage.ObjectStoreInterface, execution *model.ExecutionCache, fileName string) (*model.ExecutionCache, error) { +func retrieveCacheItemIfReallyExists(minioClient apiserver_storage.ObjectStoreInterface, execution *model.ExecutionCache, fileName string) (*model.ExecutionCache, error) { if execution != nil && minioClient != nil { log.Printf("retrieveCacheItemIfReallyExists %s ", execution.ExecutionTemplate) bucket, key := getCacheItemS3Data(execution) From a294e75b3a1ac52028f7c9fb3308add389194619 Mon Sep 17 00:00:00 2001 From: Gabor Nyerges Date: Fri, 24 Jun 2022 12:00:24 +0200 Subject: [PATCH 09/14] env var names were simplified --- backend/src/cache/main.go | 12 ++++++------ .../kustomize/base/cache/cache-deployment.yaml | 14 +++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/backend/src/cache/main.go b/backend/src/cache/main.go index 63a65a9bec5..c301618216f 100644 --- a/backend/src/cache/main.go +++ b/backend/src/cache/main.go @@ -121,17 +121,17 @@ func main() { func parseS3Flags() S3Params { var params S3Params - flag.StringVar(¶ms.serviceHost, "s3_service_host", os.Getenv("MINIO_SERVICE_SERVICE_HOST"), "hostname of S3.") - flag.StringVar(¶ms.servicePort, "s3_service_port", os.Getenv("MINIO_SERVICE_SERVICE_PORT"), "port of S3.") - flag.StringVar(¶ms.region, "s3_region", os.Getenv("MINIO_SERVICE_REGION"), "Region of S3 service.") + flag.StringVar(¶ms.serviceHost, "s3_service_host", os.Getenv("SERVICE_HOST"), "hostname of S3.") + flag.StringVar(¶ms.servicePort, "s3_service_port", os.Getenv("SERVICE_PORT"), "port of S3.") + flag.StringVar(¶ms.region, "s3_region", os.Getenv("SERVICE_REGION"), "Region of S3 service.") flag.StringVar(¶ms.accessKey, "s3_access_key", os.Getenv("OBJECTSTORECONFIG_ACCESSKEY"), "accessKey of S3.") flag.StringVar(¶ms.secretKey, "s3_secret_key", os.Getenv("OBJECTSTORECONFIG_SECRETACCESSKEY"), "secretKey of S3.") flag.BoolVar(¶ms.isServiceSecure, "s3_is_service_secure", - common.GetBoolFromStringWithDefault(os.Getenv("MINIO_SERVICE_SECURE"), false), "is ssl enabled on S3.") - flag.StringVar(¶ms.bucketName, "s3_bucketname", os.Getenv("MINIO_PIPELINE_BUCKET_NAME"), "default bucketname on S3.") + common.GetBoolFromStringWithDefault(os.Getenv("SERVICE_SECURE"), false), "is ssl enabled on S3.") + flag.StringVar(¶ms.bucketName, "s3_bucketname", os.Getenv("PIPELINE_BUCKET_NAME"), "default bucketname on S3.") flag.StringVar(¶ms.pipelinePath, "s3_pipelinepath", "pipelines", "pipelinepath of S3.") flag.BoolVar(¶ms.disableMultipart, "s3_disable_multipart", - common.GetBoolConfigWithDefault(os.Getenv("MINIO_PIPELINE_BUCKET_DISABLE_MULTIPART"), true), "if multipart request are disabled on S3.") + common.GetBoolConfigWithDefault(os.Getenv("PIPELINE_BUCKET_DISABLE_MULTIPART"), true), "if multipart request are disabled on S3.") log.Printf("S3Params %v\n", params) return params } diff --git a/manifests/kustomize/base/cache/cache-deployment.yaml b/manifests/kustomize/base/cache/cache-deployment.yaml index 6b9370c7a59..065bd7c0b4a 100644 --- a/manifests/kustomize/base/cache/cache-deployment.yaml +++ b/manifests/kustomize/base/cache/cache-deployment.yaml @@ -59,13 +59,13 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - - name: MINIO_SERVICE_SERVICE_HOST - - name: MINIO_SERVICE_SERVICE_PORT - - name: MINIO_SERVICE_REGION - - name: MINIO_SERVICE_SECURE - - name: MINIO_PIPELINE_BUCKET_NAME - - name: MINIO_PIPELINE_PATH - - name: MINIO_PIPELINE_BUCKET_DISABLE_MULTIPART + - name: SERVICE_HOST + - name: SERVICE_PORT + - name: SERVICE_REGION + - name: SERVICE_SECURE + - name: PIPELINE_BUCKET_NAME + - name: PIPELINE_PATH + - name: PIPELINE_BUCKET_DISABLE_MULTIPART - name: OBJECTSTORECONFIG_ACCESSKEY valueFrom: secretKeyRef: From f05f02c27e8be52a81525cf459132146f56d37a7 Mon Sep 17 00:00:00 2001 From: julius Date: Tue, 28 Jun 2022 22:32:52 +0200 Subject: [PATCH 10/14] fix-licences --- backend/third_party_licenses/cache_server.csv | 18 ++++++++++++++++++ .../kustomize/base/cache/cache-deployment.yaml | 7 ------- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/backend/third_party_licenses/cache_server.csv b/backend/third_party_licenses/cache_server.csv index f33f7bf4259..532af703bbb 100644 --- a/backend/third_party_licenses/cache_server.csv +++ b/backend/third_party_licenses/cache_server.csv @@ -1,11 +1,15 @@ +github.com/Masterminds/squirrel,https://github.com/Masterminds/squirrel/blob/fa735ea14f09/LICENSE.txt,MIT github.com/PuerkitoBio/purell,https://github.com/PuerkitoBio/purell/blob/v1.1.1/LICENSE,BSD-3-Clause github.com/PuerkitoBio/urlesc,https://github.com/PuerkitoBio/urlesc/blob/de5bf2ad4578/LICENSE,BSD-3-Clause +github.com/VividCortex/mysqlerr,https://github.com/VividCortex/mysqlerr/blob/6c6b55f8796f/LICENSE,MIT github.com/argoproj/argo-workflows/v3,https://github.com/argoproj/argo-workflows/blob/v3.2.3/LICENSE,Apache-2.0 github.com/asaskevich/govalidator,https://github.com/asaskevich/govalidator/blob/7a23bdc65eef/LICENSE,MIT github.com/cenkalti/backoff,https://github.com/cenkalti/backoff/blob/v2.2.1/LICENSE,MIT github.com/davecgh/go-spew/spew,https://github.com/davecgh/go-spew/blob/v1.1.1/LICENSE,ISC github.com/emicklei/go-restful,https://github.com/emicklei/go-restful/blob/v2.15.0/LICENSE,MIT +github.com/fsnotify/fsnotify,https://github.com/fsnotify/fsnotify/blob/v1.4.9/LICENSE,BSD-3-Clause github.com/ghodss/yaml,https://github.com/ghodss/yaml/blob/25d852aebe32/LICENSE,MIT +github.com/go-ini/ini,https://github.com/go-ini/ini/blob/v1.51.1/LICENSE,Apache-2.0 github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v0.4.0/LICENSE,Apache-2.0 github.com/go-openapi/errors,https://github.com/go-openapi/errors/blob/v0.19.9/LICENSE,Apache-2.0 github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.19.5/LICENSE,Apache-2.0 @@ -24,22 +28,35 @@ github.com/google/gofuzz,https://github.com/google/gofuzz/blob/v1.1.0/LICENSE,Ap github.com/google/uuid,https://github.com/google/uuid/blob/v1.1.2/LICENSE,BSD-3-Clause github.com/googleapis/gnostic,https://github.com/googleapis/gnostic/blob/v0.5.5/LICENSE,Apache-2.0 github.com/grpc-ecosystem/grpc-gateway,https://github.com/grpc-ecosystem/grpc-gateway/blob/v1.16.0/LICENSE.txt,BSD-3-Clause +github.com/hashicorp/hcl,https://github.com/hashicorp/hcl/blob/v1.0.0/LICENSE,MPL-2.0 github.com/imdario/mergo,https://github.com/imdario/mergo/blob/v0.3.12/LICENSE,BSD-3-Clause github.com/jinzhu/gorm,https://github.com/jinzhu/gorm/blob/v1.9.1/License,MIT github.com/jinzhu/inflection,https://github.com/jinzhu/inflection/blob/04140366298a/LICENSE,MIT github.com/josharian/intern,https://github.com/josharian/intern/blob/v1.0.0/license.md,MIT github.com/json-iterator/go,https://github.com/json-iterator/go/blob/v1.1.11/LICENSE,MIT github.com/kubeflow/pipelines/backend,https://github.com/kubeflow/pipelines/blob/HEAD/LICENSE,Apache-2.0 +github.com/lann/builder,https://github.com/lann/builder/blob/47ae307949d0/LICENSE,MIT +github.com/lann/ps,https://github.com/lann/ps/blob/62de8c46ede0/LICENSE,MIT github.com/lestrrat-go/strftime,https://github.com/lestrrat-go/strftime/blob/v1.0.4/LICENSE,MIT +github.com/magiconair/properties,https://github.com/magiconair/properties/blob/v1.8.5/LICENSE.md,BSD-2-Clause github.com/mailru/easyjson,https://github.com/mailru/easyjson/blob/v0.7.6/LICENSE,MIT github.com/mattn/go-sqlite3,https://github.com/mattn/go-sqlite3/blob/v1.9.0/LICENSE,MIT +github.com/minio/minio-go,https://github.com/minio/minio-go/blob/v6.0.14/LICENSE,Apache-2.0 +github.com/mitchellh/go-homedir,https://github.com/mitchellh/go-homedir/blob/v1.1.0/LICENSE,MIT github.com/mitchellh/mapstructure,https://github.com/mitchellh/mapstructure/blob/v1.4.1/LICENSE,MIT github.com/modern-go/concurrent,https://github.com/modern-go/concurrent/blob/bacd9c7ef1dd/LICENSE,Apache-2.0 github.com/modern-go/reflect2,https://github.com/modern-go/reflect2/blob/v1.0.1/LICENSE,Apache-2.0 +github.com/pelletier/go-toml,https://github.com/pelletier/go-toml/blob/v1.9.3/LICENSE,Apache-2.0 github.com/peterhellberg/duration,https://github.com/peterhellberg/duration/blob/ec6baeebcd10/LICENSE,MIT github.com/pkg/errors,https://github.com/pkg/errors/blob/v0.9.1/LICENSE,BSD-2-Clause +github.com/spf13/afero,https://github.com/spf13/afero/blob/v1.6.0/LICENSE.txt,Apache-2.0 +github.com/spf13/cast,https://github.com/spf13/cast/blob/v1.3.1/LICENSE,MIT +github.com/spf13/jwalterweatherman,https://github.com/spf13/jwalterweatherman/blob/v1.1.0/LICENSE,MIT github.com/spf13/pflag,https://github.com/spf13/pflag/blob/v1.0.5/LICENSE,BSD-3-Clause +github.com/spf13/viper,https://github.com/spf13/viper/blob/v1.8.1/LICENSE,MIT +github.com/subosito/gotenv,https://github.com/subosito/gotenv/blob/v1.2.0/LICENSE,MIT go.mongodb.org/mongo-driver,https://github.com/mongodb/mongo-go-driver/blob/v1.4.4/LICENSE,Apache-2.0 +golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/a769d52b:LICENSE,BSD-3-Clause golang.org/x/net,https://cs.opensource.google/go/x/net/+/fe4d6282:LICENSE,BSD-3-Clause golang.org/x/oauth2,https://cs.opensource.google/go/x/oauth2/+/2e8d9340:LICENSE,BSD-3-Clause golang.org/x/sys,https://cs.opensource.google/go/x/sys/+/1d35b9e2:LICENSE,BSD-3-Clause @@ -50,6 +67,7 @@ google.golang.org/genproto,https://github.com/googleapis/go-genproto/blob/197313 google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.44.0/LICENSE,Apache-2.0 google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.27.1/LICENSE,BSD-3-Clause gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause +gopkg.in/ini.v1,https://github.com/go-ini/ini/blob/v1.62.0/LICENSE,Apache-2.0 gopkg.in/yaml.v2,https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE,Apache-2.0 gopkg.in/yaml.v3,https://github.com/go-yaml/yaml/blob/496545a6307b/LICENSE,MIT k8s.io/api,https://github.com/kubernetes/api/blob/v0.21.5/LICENSE,Apache-2.0 diff --git a/manifests/kustomize/base/cache/cache-deployment.yaml b/manifests/kustomize/base/cache/cache-deployment.yaml index 065bd7c0b4a..e0037407ba6 100644 --- a/manifests/kustomize/base/cache/cache-deployment.yaml +++ b/manifests/kustomize/base/cache/cache-deployment.yaml @@ -59,13 +59,6 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace - - name: SERVICE_HOST - - name: SERVICE_PORT - - name: SERVICE_REGION - - name: SERVICE_SECURE - - name: PIPELINE_BUCKET_NAME - - name: PIPELINE_PATH - - name: PIPELINE_BUCKET_DISABLE_MULTIPART - name: OBJECTSTORECONFIG_ACCESSKEY valueFrom: secretKeyRef: From 1c5c91ae3b208654170f0cdf12e3285f723ce81f Mon Sep 17 00:00:00 2001 From: julius Date: Tue, 28 Jun 2022 22:40:56 +0200 Subject: [PATCH 11/14] revert licence to master for force push --- backend/third_party_licenses/cache_server.csv | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/backend/third_party_licenses/cache_server.csv b/backend/third_party_licenses/cache_server.csv index 532af703bbb..bd163e7ab07 100644 --- a/backend/third_party_licenses/cache_server.csv +++ b/backend/third_party_licenses/cache_server.csv @@ -1,15 +1,17 @@ -github.com/Masterminds/squirrel,https://github.com/Masterminds/squirrel/blob/fa735ea14f09/LICENSE.txt,MIT +github.com/Masterminds/goutils,https://github.com/Masterminds/goutils/blob/v1.1.1/LICENSE.txt,Apache-2.0 +github.com/Masterminds/semver/v3,https://github.com/Masterminds/semver/blob/v3.1.1/LICENSE.txt,MIT +github.com/Masterminds/sprig/v3,https://github.com/Masterminds/sprig/blob/v3.2.2/LICENSE.txt,MIT github.com/PuerkitoBio/purell,https://github.com/PuerkitoBio/purell/blob/v1.1.1/LICENSE,BSD-3-Clause github.com/PuerkitoBio/urlesc,https://github.com/PuerkitoBio/urlesc/blob/de5bf2ad4578/LICENSE,BSD-3-Clause -github.com/VividCortex/mysqlerr,https://github.com/VividCortex/mysqlerr/blob/6c6b55f8796f/LICENSE,MIT +github.com/antonmedv/expr,https://github.com/antonmedv/expr/blob/v1.8.9/LICENSE,MIT github.com/argoproj/argo-workflows/v3,https://github.com/argoproj/argo-workflows/blob/v3.2.3/LICENSE,Apache-2.0 +github.com/argoproj/pkg,https://github.com/argoproj/pkg/blob/v0.11.0/LICENSE,Apache-2.0 github.com/asaskevich/govalidator,https://github.com/asaskevich/govalidator/blob/7a23bdc65eef/LICENSE,MIT github.com/cenkalti/backoff,https://github.com/cenkalti/backoff/blob/v2.2.1/LICENSE,MIT github.com/davecgh/go-spew/spew,https://github.com/davecgh/go-spew/blob/v1.1.1/LICENSE,ISC +github.com/doublerebel/bellows,https://github.com/doublerebel/bellows/blob/f177d92a03d3/LICENSE,MIT github.com/emicklei/go-restful,https://github.com/emicklei/go-restful/blob/v2.15.0/LICENSE,MIT -github.com/fsnotify/fsnotify,https://github.com/fsnotify/fsnotify/blob/v1.4.9/LICENSE,BSD-3-Clause github.com/ghodss/yaml,https://github.com/ghodss/yaml/blob/25d852aebe32/LICENSE,MIT -github.com/go-ini/ini,https://github.com/go-ini/ini/blob/v1.51.1/LICENSE,Apache-2.0 github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v0.4.0/LICENSE,Apache-2.0 github.com/go-openapi/errors,https://github.com/go-openapi/errors/blob/v0.19.9/LICENSE,Apache-2.0 github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.19.5/LICENSE,Apache-2.0 @@ -27,34 +29,34 @@ github.com/google/go-cmp/cmp,https://github.com/google/go-cmp/blob/v0.5.6/LICENS github.com/google/gofuzz,https://github.com/google/gofuzz/blob/v1.1.0/LICENSE,Apache-2.0 github.com/google/uuid,https://github.com/google/uuid/blob/v1.1.2/LICENSE,BSD-3-Clause github.com/googleapis/gnostic,https://github.com/googleapis/gnostic/blob/v0.5.5/LICENSE,Apache-2.0 +github.com/gorilla/websocket,https://github.com/gorilla/websocket/blob/v1.4.2/LICENSE,BSD-2-Clause github.com/grpc-ecosystem/grpc-gateway,https://github.com/grpc-ecosystem/grpc-gateway/blob/v1.16.0/LICENSE.txt,BSD-3-Clause -github.com/hashicorp/hcl,https://github.com/hashicorp/hcl/blob/v1.0.0/LICENSE,MPL-2.0 +github.com/hashicorp/golang-lru,https://github.com/hashicorp/golang-lru/blob/v0.5.4/LICENSE,MPL-2.0 +github.com/huandu/xstrings,https://github.com/huandu/xstrings/blob/v1.3.2/LICENSE,MIT github.com/imdario/mergo,https://github.com/imdario/mergo/blob/v0.3.12/LICENSE,BSD-3-Clause github.com/jinzhu/gorm,https://github.com/jinzhu/gorm/blob/v1.9.1/License,MIT github.com/jinzhu/inflection,https://github.com/jinzhu/inflection/blob/04140366298a/LICENSE,MIT github.com/josharian/intern,https://github.com/josharian/intern/blob/v1.0.0/license.md,MIT github.com/json-iterator/go,https://github.com/json-iterator/go/blob/v1.1.11/LICENSE,MIT github.com/kubeflow/pipelines/backend,https://github.com/kubeflow/pipelines/blob/HEAD/LICENSE,Apache-2.0 -github.com/lann/builder,https://github.com/lann/builder/blob/47ae307949d0/LICENSE,MIT -github.com/lann/ps,https://github.com/lann/ps/blob/62de8c46ede0/LICENSE,MIT github.com/lestrrat-go/strftime,https://github.com/lestrrat-go/strftime/blob/v1.0.4/LICENSE,MIT -github.com/magiconair/properties,https://github.com/magiconair/properties/blob/v1.8.5/LICENSE.md,BSD-2-Clause github.com/mailru/easyjson,https://github.com/mailru/easyjson/blob/v0.7.6/LICENSE,MIT github.com/mattn/go-sqlite3,https://github.com/mattn/go-sqlite3/blob/v1.9.0/LICENSE,MIT -github.com/minio/minio-go,https://github.com/minio/minio-go/blob/v6.0.14/LICENSE,Apache-2.0 -github.com/mitchellh/go-homedir,https://github.com/mitchellh/go-homedir/blob/v1.1.0/LICENSE,MIT +github.com/mitchellh/copystructure,https://github.com/mitchellh/copystructure/blob/v1.2.0/LICENSE,MIT github.com/mitchellh/mapstructure,https://github.com/mitchellh/mapstructure/blob/v1.4.1/LICENSE,MIT +github.com/mitchellh/reflectwalk,https://github.com/mitchellh/reflectwalk/blob/v1.0.2/LICENSE,MIT +github.com/moby/spdystream,https://github.com/moby/spdystream/blob/v0.2.0/LICENSE,Apache-2.0 github.com/modern-go/concurrent,https://github.com/modern-go/concurrent/blob/bacd9c7ef1dd/LICENSE,Apache-2.0 github.com/modern-go/reflect2,https://github.com/modern-go/reflect2/blob/v1.0.1/LICENSE,Apache-2.0 -github.com/pelletier/go-toml,https://github.com/pelletier/go-toml/blob/v1.9.3/LICENSE,Apache-2.0 +github.com/oliveagle/jsonpath,https://github.com/oliveagle/jsonpath/blob/2e52cf6e6852/LICENSE,MIT github.com/peterhellberg/duration,https://github.com/peterhellberg/duration/blob/ec6baeebcd10/LICENSE,MIT github.com/pkg/errors,https://github.com/pkg/errors/blob/v0.9.1/LICENSE,BSD-2-Clause -github.com/spf13/afero,https://github.com/spf13/afero/blob/v1.6.0/LICENSE.txt,Apache-2.0 +github.com/shopspring/decimal,https://github.com/shopspring/decimal/blob/v1.2.0/LICENSE,MIT +github.com/sirupsen/logrus,https://github.com/sirupsen/logrus/blob/v1.7.0/LICENSE,MIT github.com/spf13/cast,https://github.com/spf13/cast/blob/v1.3.1/LICENSE,MIT -github.com/spf13/jwalterweatherman,https://github.com/spf13/jwalterweatherman/blob/v1.1.0/LICENSE,MIT github.com/spf13/pflag,https://github.com/spf13/pflag/blob/v1.0.5/LICENSE,BSD-3-Clause -github.com/spf13/viper,https://github.com/spf13/viper/blob/v1.8.1/LICENSE,MIT -github.com/subosito/gotenv,https://github.com/subosito/gotenv/blob/v1.2.0/LICENSE,MIT +github.com/valyala/bytebufferpool,https://github.com/valyala/bytebufferpool/blob/v1.0.0/LICENSE,MIT +github.com/valyala/fasttemplate,https://github.com/valyala/fasttemplate/blob/v1.1.0/LICENSE,MIT go.mongodb.org/mongo-driver,https://github.com/mongodb/mongo-go-driver/blob/v1.4.4/LICENSE,Apache-2.0 golang.org/x/crypto,https://cs.opensource.google/go/x/crypto/+/a769d52b:LICENSE,BSD-3-Clause golang.org/x/net,https://cs.opensource.google/go/x/net/+/fe4d6282:LICENSE,BSD-3-Clause @@ -67,16 +69,15 @@ google.golang.org/genproto,https://github.com/googleapis/go-genproto/blob/197313 google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.44.0/LICENSE,Apache-2.0 google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.27.1/LICENSE,BSD-3-Clause gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause -gopkg.in/ini.v1,https://github.com/go-ini/ini/blob/v1.62.0/LICENSE,Apache-2.0 gopkg.in/yaml.v2,https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE,Apache-2.0 gopkg.in/yaml.v3,https://github.com/go-yaml/yaml/blob/496545a6307b/LICENSE,MIT k8s.io/api,https://github.com/kubernetes/api/blob/v0.21.5/LICENSE,Apache-2.0 k8s.io/apimachinery/pkg,https://github.com/kubernetes/apimachinery/blob/v0.21.5/LICENSE,Apache-2.0 -k8s.io/apimachinery/third_party/forked/golang/reflect,https://github.com/kubernetes/apimachinery/blob/v0.21.5/third_party/forked/golang/LICENSE,BSD-3-Clause +k8s.io/apimachinery/third_party/forked/golang,https://github.com/kubernetes/apimachinery/blob/v0.21.5/third_party/forked/golang/LICENSE,BSD-3-Clause k8s.io/client-go,https://github.com/kubernetes/client-go/blob/v0.21.5/LICENSE,Apache-2.0 k8s.io/klog/v2,https://github.com/kubernetes/klog/blob/v2.8.0/LICENSE,Apache-2.0 k8s.io/kube-openapi/pkg/common,https://github.com/kubernetes/kube-openapi/blob/591a79e4bda7/LICENSE,Apache-2.0 k8s.io/kubernetes/pkg/apis/core,https://github.com/kubernetes/kubernetes/blob/v1.11.1/LICENSE,Apache-2.0 -k8s.io/utils/integer,https://github.com/kubernetes/utils/blob/efc7438f0176/LICENSE,Apache-2.0 +k8s.io/utils,https://github.com/kubernetes/utils/blob/efc7438f0176/LICENSE,Apache-2.0 sigs.k8s.io/structured-merge-diff/v4,https://github.com/kubernetes-sigs/structured-merge-diff/blob/v4.1.2/LICENSE,Apache-2.0 -sigs.k8s.io/yaml,https://github.com/kubernetes-sigs/yaml/blob/v1.2.0/LICENSE,MIT +sigs.k8s.io/yaml,https://github.com/kubernetes-sigs/yaml/blob/v1.2.0/LICENSE,MIT \ No newline at end of file From 4a2b8573949cb23cc0c6366ed1dd9f8f00b8efbb Mon Sep 17 00:00:00 2001 From: julius Date: Tue, 28 Jun 2022 22:43:33 +0200 Subject: [PATCH 12/14] fix-licences --- backend/third_party_licenses/cache_server.csv | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/backend/third_party_licenses/cache_server.csv b/backend/third_party_licenses/cache_server.csv index bd163e7ab07..82fcd10106c 100644 --- a/backend/third_party_licenses/cache_server.csv +++ b/backend/third_party_licenses/cache_server.csv @@ -1,8 +1,10 @@ github.com/Masterminds/goutils,https://github.com/Masterminds/goutils/blob/v1.1.1/LICENSE.txt,Apache-2.0 github.com/Masterminds/semver/v3,https://github.com/Masterminds/semver/blob/v3.1.1/LICENSE.txt,MIT github.com/Masterminds/sprig/v3,https://github.com/Masterminds/sprig/blob/v3.2.2/LICENSE.txt,MIT +github.com/Masterminds/squirrel,https://github.com/Masterminds/squirrel/blob/fa735ea14f09/LICENSE.txt,MIT github.com/PuerkitoBio/purell,https://github.com/PuerkitoBio/purell/blob/v1.1.1/LICENSE,BSD-3-Clause github.com/PuerkitoBio/urlesc,https://github.com/PuerkitoBio/urlesc/blob/de5bf2ad4578/LICENSE,BSD-3-Clause +github.com/VividCortex/mysqlerr,https://github.com/VividCortex/mysqlerr/blob/6c6b55f8796f/LICENSE,MIT github.com/antonmedv/expr,https://github.com/antonmedv/expr/blob/v1.8.9/LICENSE,MIT github.com/argoproj/argo-workflows/v3,https://github.com/argoproj/argo-workflows/blob/v3.2.3/LICENSE,Apache-2.0 github.com/argoproj/pkg,https://github.com/argoproj/pkg/blob/v0.11.0/LICENSE,Apache-2.0 @@ -11,7 +13,9 @@ github.com/cenkalti/backoff,https://github.com/cenkalti/backoff/blob/v2.2.1/LICE github.com/davecgh/go-spew/spew,https://github.com/davecgh/go-spew/blob/v1.1.1/LICENSE,ISC github.com/doublerebel/bellows,https://github.com/doublerebel/bellows/blob/f177d92a03d3/LICENSE,MIT github.com/emicklei/go-restful,https://github.com/emicklei/go-restful/blob/v2.15.0/LICENSE,MIT +github.com/fsnotify/fsnotify,https://github.com/fsnotify/fsnotify/blob/v1.4.9/LICENSE,BSD-3-Clause github.com/ghodss/yaml,https://github.com/ghodss/yaml/blob/25d852aebe32/LICENSE,MIT +github.com/go-ini/ini,https://github.com/go-ini/ini/blob/v1.51.1/LICENSE,Apache-2.0 github.com/go-logr/logr,https://github.com/go-logr/logr/blob/v0.4.0/LICENSE,Apache-2.0 github.com/go-openapi/errors,https://github.com/go-openapi/errors/blob/v0.19.9/LICENSE,Apache-2.0 github.com/go-openapi/jsonpointer,https://github.com/go-openapi/jsonpointer/blob/v0.19.5/LICENSE,Apache-2.0 @@ -32,6 +36,7 @@ github.com/googleapis/gnostic,https://github.com/googleapis/gnostic/blob/v0.5.5/ github.com/gorilla/websocket,https://github.com/gorilla/websocket/blob/v1.4.2/LICENSE,BSD-2-Clause github.com/grpc-ecosystem/grpc-gateway,https://github.com/grpc-ecosystem/grpc-gateway/blob/v1.16.0/LICENSE.txt,BSD-3-Clause github.com/hashicorp/golang-lru,https://github.com/hashicorp/golang-lru/blob/v0.5.4/LICENSE,MPL-2.0 +github.com/hashicorp/hcl,https://github.com/hashicorp/hcl/blob/v1.0.0/LICENSE,MPL-2.0 github.com/huandu/xstrings,https://github.com/huandu/xstrings/blob/v1.3.2/LICENSE,MIT github.com/imdario/mergo,https://github.com/imdario/mergo/blob/v0.3.12/LICENSE,BSD-3-Clause github.com/jinzhu/gorm,https://github.com/jinzhu/gorm/blob/v1.9.1/License,MIT @@ -39,22 +44,32 @@ github.com/jinzhu/inflection,https://github.com/jinzhu/inflection/blob/041403662 github.com/josharian/intern,https://github.com/josharian/intern/blob/v1.0.0/license.md,MIT github.com/json-iterator/go,https://github.com/json-iterator/go/blob/v1.1.11/LICENSE,MIT github.com/kubeflow/pipelines/backend,https://github.com/kubeflow/pipelines/blob/HEAD/LICENSE,Apache-2.0 +github.com/lann/builder,https://github.com/lann/builder/blob/47ae307949d0/LICENSE,MIT +github.com/lann/ps,https://github.com/lann/ps/blob/62de8c46ede0/LICENSE,MIT github.com/lestrrat-go/strftime,https://github.com/lestrrat-go/strftime/blob/v1.0.4/LICENSE,MIT +github.com/magiconair/properties,https://github.com/magiconair/properties/blob/v1.8.5/LICENSE.md,BSD-2-Clause github.com/mailru/easyjson,https://github.com/mailru/easyjson/blob/v0.7.6/LICENSE,MIT github.com/mattn/go-sqlite3,https://github.com/mattn/go-sqlite3/blob/v1.9.0/LICENSE,MIT +github.com/minio/minio-go,https://github.com/minio/minio-go/blob/v6.0.14/LICENSE,Apache-2.0 github.com/mitchellh/copystructure,https://github.com/mitchellh/copystructure/blob/v1.2.0/LICENSE,MIT +github.com/mitchellh/go-homedir,https://github.com/mitchellh/go-homedir/blob/v1.1.0/LICENSE,MIT github.com/mitchellh/mapstructure,https://github.com/mitchellh/mapstructure/blob/v1.4.1/LICENSE,MIT github.com/mitchellh/reflectwalk,https://github.com/mitchellh/reflectwalk/blob/v1.0.2/LICENSE,MIT github.com/moby/spdystream,https://github.com/moby/spdystream/blob/v0.2.0/LICENSE,Apache-2.0 github.com/modern-go/concurrent,https://github.com/modern-go/concurrent/blob/bacd9c7ef1dd/LICENSE,Apache-2.0 github.com/modern-go/reflect2,https://github.com/modern-go/reflect2/blob/v1.0.1/LICENSE,Apache-2.0 github.com/oliveagle/jsonpath,https://github.com/oliveagle/jsonpath/blob/2e52cf6e6852/LICENSE,MIT +github.com/pelletier/go-toml,https://github.com/pelletier/go-toml/blob/v1.9.3/LICENSE,Apache-2.0 github.com/peterhellberg/duration,https://github.com/peterhellberg/duration/blob/ec6baeebcd10/LICENSE,MIT github.com/pkg/errors,https://github.com/pkg/errors/blob/v0.9.1/LICENSE,BSD-2-Clause github.com/shopspring/decimal,https://github.com/shopspring/decimal/blob/v1.2.0/LICENSE,MIT github.com/sirupsen/logrus,https://github.com/sirupsen/logrus/blob/v1.7.0/LICENSE,MIT +github.com/spf13/afero,https://github.com/spf13/afero/blob/v1.6.0/LICENSE.txt,Apache-2.0 github.com/spf13/cast,https://github.com/spf13/cast/blob/v1.3.1/LICENSE,MIT +github.com/spf13/jwalterweatherman,https://github.com/spf13/jwalterweatherman/blob/v1.1.0/LICENSE,MIT github.com/spf13/pflag,https://github.com/spf13/pflag/blob/v1.0.5/LICENSE,BSD-3-Clause +github.com/spf13/viper,https://github.com/spf13/viper/blob/v1.8.1/LICENSE,MIT +github.com/subosito/gotenv,https://github.com/subosito/gotenv/blob/v1.2.0/LICENSE,MIT github.com/valyala/bytebufferpool,https://github.com/valyala/bytebufferpool/blob/v1.0.0/LICENSE,MIT github.com/valyala/fasttemplate,https://github.com/valyala/fasttemplate/blob/v1.1.0/LICENSE,MIT go.mongodb.org/mongo-driver,https://github.com/mongodb/mongo-go-driver/blob/v1.4.4/LICENSE,Apache-2.0 @@ -69,6 +84,7 @@ google.golang.org/genproto,https://github.com/googleapis/go-genproto/blob/197313 google.golang.org/grpc,https://github.com/grpc/grpc-go/blob/v1.44.0/LICENSE,Apache-2.0 google.golang.org/protobuf,https://github.com/protocolbuffers/protobuf-go/blob/v1.27.1/LICENSE,BSD-3-Clause gopkg.in/inf.v0,https://github.com/go-inf/inf/blob/v0.9.1/LICENSE,BSD-3-Clause +gopkg.in/ini.v1,https://github.com/go-ini/ini/blob/v1.62.0/LICENSE,Apache-2.0 gopkg.in/yaml.v2,https://github.com/go-yaml/yaml/blob/v2.4.0/LICENSE,Apache-2.0 gopkg.in/yaml.v3,https://github.com/go-yaml/yaml/blob/496545a6307b/LICENSE,MIT k8s.io/api,https://github.com/kubernetes/api/blob/v0.21.5/LICENSE,Apache-2.0 @@ -80,4 +96,4 @@ k8s.io/kube-openapi/pkg/common,https://github.com/kubernetes/kube-openapi/blob/5 k8s.io/kubernetes/pkg/apis/core,https://github.com/kubernetes/kubernetes/blob/v1.11.1/LICENSE,Apache-2.0 k8s.io/utils,https://github.com/kubernetes/utils/blob/efc7438f0176/LICENSE,Apache-2.0 sigs.k8s.io/structured-merge-diff/v4,https://github.com/kubernetes-sigs/structured-merge-diff/blob/v4.1.2/LICENSE,Apache-2.0 -sigs.k8s.io/yaml,https://github.com/kubernetes-sigs/yaml/blob/v1.2.0/LICENSE,MIT \ No newline at end of file +sigs.k8s.io/yaml,https://github.com/kubernetes-sigs/yaml/blob/v1.2.0/LICENSE,MIT From e8b5b26eba40784dc5fabe6367db5966fbcd9cbb Mon Sep 17 00:00:00 2001 From: Gabor Nyerges Date: Wed, 29 Jun 2022 11:03:49 +0200 Subject: [PATCH 13/14] setup general defaults --- backend/src/apiserver/common/config.go | 7 +++++++ backend/src/cache/main.go | 6 +++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/backend/src/apiserver/common/config.go b/backend/src/apiserver/common/config.go index def9c655db0..168a6fb24ba 100644 --- a/backend/src/apiserver/common/config.go +++ b/backend/src/apiserver/common/config.go @@ -104,6 +104,13 @@ func GetPodNamespace() string { return GetStringConfig(PodNamespace) } +func GetFromStringWithDefault(value string, defaultValue string) string { + if len(value) == 0 { + return defaultValue + } + return value +} + func GetBoolFromStringWithDefault(value string, defaultValue bool) bool { boolVal, err := strconv.ParseBool(value) if err != nil { diff --git a/backend/src/cache/main.go b/backend/src/cache/main.go index c301618216f..bb2202e3779 100644 --- a/backend/src/cache/main.go +++ b/backend/src/cache/main.go @@ -121,9 +121,9 @@ func main() { func parseS3Flags() S3Params { var params S3Params - flag.StringVar(¶ms.serviceHost, "s3_service_host", os.Getenv("SERVICE_HOST"), "hostname of S3.") - flag.StringVar(¶ms.servicePort, "s3_service_port", os.Getenv("SERVICE_PORT"), "port of S3.") - flag.StringVar(¶ms.region, "s3_region", os.Getenv("SERVICE_REGION"), "Region of S3 service.") + flag.StringVar(¶ms.serviceHost, "s3_service_host", common.GetFromStringWithDefault(os.Getenv("SERVICE_HOST"), "minio-service.kubeflow.svc"), "hostname of S3.") + flag.StringVar(¶ms.servicePort, "s3_service_port", common.GetFromStringWithDefault(os.Getenv("SERVICE_PORT"), "9000"), "port of S3.") + flag.StringVar(¶ms.region, "s3_region", common.GetFromStringWithDefault(os.Getenv("SERVICE_REGION"), ""), "Region of S3 service.") flag.StringVar(¶ms.accessKey, "s3_access_key", os.Getenv("OBJECTSTORECONFIG_ACCESSKEY"), "accessKey of S3.") flag.StringVar(¶ms.secretKey, "s3_secret_key", os.Getenv("OBJECTSTORECONFIG_SECRETACCESSKEY"), "secretKey of S3.") flag.BoolVar(¶ms.isServiceSecure, "s3_is_service_secure", From 8c8b04c5854955c12519b4e4d4705cc96ffaf3fa Mon Sep 17 00:00:00 2001 From: Gabor Nyerges Date: Wed, 29 Jun 2022 13:02:47 +0200 Subject: [PATCH 14/14] default bucketname from configmap --- manifests/kustomize/base/cache/cache-deployment.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/manifests/kustomize/base/cache/cache-deployment.yaml b/manifests/kustomize/base/cache/cache-deployment.yaml index e0037407ba6..a12b2464e10 100644 --- a/manifests/kustomize/base/cache/cache-deployment.yaml +++ b/manifests/kustomize/base/cache/cache-deployment.yaml @@ -59,6 +59,11 @@ spec: valueFrom: fieldRef: fieldPath: metadata.namespace + - name: PIPELINE_BUCKET_NAME + valueFrom: + configMapKeyRef: + name: pipeline-install-config + key: bucketName - name: OBJECTSTORECONFIG_ACCESSKEY valueFrom: secretKeyRef: