ACLs: Modify superuser command and add migrate flow (#8087)

* ACLs: Modify superuser command and add migrate flow

* Fix tests

* lint

* more fixes

* more fixes

* last fix

* CR Fixes

.github/workflows/esti.yaml

@@ -1353,7 +1353,7 @@ jobs:
           fi
 
   run-system-aws-s3-basic-auth:
-    name: Run latest lakeFS app on AWS S3
+    name: Run latest lakeFS app on AWS S3 + Basic Auth
     needs: [deploy-image, login-to-amazon-ecr]
     runs-on: ubuntu-22.04
     env:

cmd/lakefs/cmd/superuser.go

@@ -25,6 +25,12 @@ import (
 var superuserCmd = &cobra.Command{
 	Use:   "superuser",
 	Short: "Create additional user with admin credentials",
+	Long: `Create additional user with admin credentials.
+This command can be used to import an admin user when moving from lakeFS version 
+with previously configured users to a lakeFS with basic auth version.
+To do that provide the user name as well as the access key ID to import. 
+If the wrong user or credentials were chosen it is possible to delete the user and perform the action again.
+`,
 	Run: func(cmd *cobra.Command, args []string) {
 		cfg := loadConfig()
 		if cfg.Auth.UIConfig.RBAC == config.AuthRBACExternal {
@@ -60,7 +66,32 @@ var superuserCmd = &cobra.Command{
 			fmt.Printf("Failed to open KV store: %s\n", err)
 			os.Exit(1)
 		}
-		authService := acl.NewAuthService(kvStore, crypt.NewSecretStore([]byte(cfg.Auth.Encrypt.SecretKey)), authparams.ServiceCache(cfg.Auth.Cache))
+
+		var authService auth.Service
+		secretStore := crypt.NewSecretStore([]byte(cfg.Auth.Encrypt.SecretKey))
+		authLogger := logger.WithField("service", "auth_api")
+		addToAdmins := true
+		switch {
+		case cfg.IsAuthBasic():
+			authService = auth.NewBasicAuthService(kvStore, secretStore, authparams.ServiceCache(cfg.Auth.Cache), authLogger)
+			addToAdmins = false
+		case cfg.IsAuthUISimplified() && cfg.IsAuthenticationTypeAPI(): // ACL server
+			authService, err = auth.NewAPIAuthService(
+				cfg.Auth.API.Endpoint,
+				cfg.Auth.API.Token.SecureValue(),
+				cfg.Auth.AuthenticationAPI.ExternalPrincipalsEnabled,
+				secretStore,
+				authparams.ServiceCache(cfg.Auth.Cache),
+				authLogger)
+			if err != nil {
+				fmt.Printf("Failed to initialize auth service: %s\n", err)
+				os.Exit(1)
+			}
+		// TODO (niro): This needs to be removed
+		default:
+			authService = acl.NewAuthService(kvStore, secretStore, authparams.ServiceCache(cfg.Auth.Cache))
+		}
+
 		authMetadataManager := auth.NewKVMetadataManager(version.Version, cfg.Installation.FixedID, cfg.Database.Type, kvStore)
 
 		metadataProvider := stats.BuildMetadataProvider(logger, cfg)
@@ -72,7 +103,7 @@ var superuserCmd = &cobra.Command{
 			},
 			AccessKeyID:     accessKeyID,
 			SecretAccessKey: secretAccessKey,
-		}, true)
+		}, addToAdmins)
 		if err != nil {
 			fmt.Printf("Failed to setup admin user: %s\n", err)
 			os.Exit(1)

contrib/auth/acl/service.go

@@ -243,7 +243,7 @@ func (s *AuthService) ListUserCredentials(ctx context.Context, username string,
 	if err != nil {
 		return nil, nil, err
 	}
-	creds, err := model.ConvertCredDataList(s.secretStore, msgs)
+	creds, err := model.ConvertCredDataList(s.secretStore, msgs, false)
 	if err != nil {
 		return nil, nil, err
 	}

esti/golden/lakefs/help.golden

@@ -0,0 +1,23 @@
+lakeFS is a data lake management platform
+
+Usage:
+  lakefs [command]
+
+Available Commands:
+  completion  Generate completion script
+  flare       collect configuration, environment variables, and logs for debugging and troubleshooting
+  help        Help about any command
+  migrate     Manage migrations
+  run         Run lakeFS
+  setup       Setup a new lakeFS instance with initial credentials
+  superuser   Create additional user with admin credentials
+
+Flags:
+  -c, --config string    config file (default is $HOME/.lakefs.yaml)
+  -h, --help             help for lakefs
+      --local-settings   Use lakeFS local default configuration
+      --quickstart       Use lakeFS quickstart configuration
+  -t, --toggle           Help message for toggle
+  -v, --version          version for lakefs
+
+Use "lakefs [command] --help" for more information about a command.

esti/lakectl_test.go

@@ -412,13 +412,9 @@ func TestLakectlAuthUsers(t *testing.T) {
 		"ID": userName,
 	}
 	isSupported := !isBasicAuth()
-	expected := "Not implemented\n501 Not Implemented\n"
-	if isSupported {
-		expected = "user not found\n404 Not Found\n"
-	}
 
 	// Not Found
-	RunCmdAndVerifyFailure(t, Lakectl()+" auth users delete --id "+userName, false, expected, vars)
+	RunCmdAndVerifyFailure(t, Lakectl()+" auth users delete --id "+userName, false, "user not found\n404 Not Found\n", vars)
 
 	// Check unique
 	if isSupported {
@@ -427,6 +423,7 @@ func TestLakectlAuthUsers(t *testing.T) {
 	RunCmdAndVerifyFailure(t, Lakectl()+" auth users create --id "+userName, false, "Already exists\n409 Conflict\n", vars)
 
 	// Cleanup
+	expected := "user not found\n404 Not Found\n"
 	if isSupported {
 		expected = "User deleted successfully\n"
 	}

esti/lakectl_util.go

@@ -34,10 +34,12 @@ var (
 	rePhysicalAddress = regexp.MustCompile(`/data/[0-9a-v]{20}/(?:[0-9a-v]{20}(?:,.+)?)?`)
 	reVariable        = regexp.MustCompile(`\$\{([^${}]+)}`)
 	rePreSignURL      = regexp.MustCompile(`https://\S+\?\S+`)
+	reSecretAccessKey = regexp.MustCompile(`secret_access_key: \S{16,128}`)
+	reAccessKeyID     = regexp.MustCompile(`access_key_id: AKIA\S{12,124}`)
 )
 
 func lakectlLocation() string {
-	return viper.GetString("lakectl_dir") + "/lakectl"
+	return viper.GetString("binaries_dir") + "/lakectl"
 }
 
 func LakectlWithParams(accessKeyID, secretAccessKey, endPointURL string) string {
@@ -145,6 +147,8 @@ func sanitize(output string, vars map[string]string) string {
 	s = normalizeCommitID(s)
 	s = normalizeChecksum(s)
 	s = normalizeShortCommitID(s)
+	s = normalizeAccessKeyID(s)
+	s = normalizeSecretAccessKey(s)
 	return s
 }
 
@@ -267,3 +271,11 @@ func normalizeEndpoint(output string, endpoint string) string {
 func normalizePreSignURL(output string) string {
 	return rePreSignURL.ReplaceAllString(output, "<PRE_SIGN_URL>")
 }
+
+func normalizeAccessKeyID(output string) string {
+	return reAccessKeyID.ReplaceAllString(output, "access_key_id: <ACCESS_KEY_ID>")
+}
+
+func normalizeSecretAccessKey(output string) string {
+	return reSecretAccessKey.ReplaceAllString(output, "secret_access_key: <SECRET_ACCESS_KEY>")
+}

esti/lakefs_test.go

@@ -0,0 +1,34 @@
+package esti
+
+import "testing"
+
+func TestLakefsHelp(t *testing.T) {
+	RunCmdAndVerifySuccessWithFile(t, Lakefs(), false, "lakefs/help", emptyVars)
+	RunCmdAndVerifySuccessWithFile(t, Lakefs()+" --help", false, "lakefs/help", emptyVars)
+	RunCmdAndVerifySuccessWithFile(t, Lakefs(), true, "lakefs/help", emptyVars)
+	RunCmdAndVerifySuccessWithFile(t, Lakefs()+" --help", true, "lakefs/help", emptyVars)
+}
+
+func TestLakefsSuperuser_basic(t *testing.T) {
+	RequirePostgresDB(t)
+	lakefsCmd := Lakefs()
+	outputString := "credentials:\n  access_key_id: <ACCESS_KEY_ID>\n  secret_access_key: <SECRET_ACCESS_KEY>\n"
+	username := t.Name()
+	expectFailure := false
+	if isBasicAuth() {
+		lakefsCmd = LakefsWithBasicAuth()
+		outputString = "already exists"
+		expectFailure = true
+	}
+	runCmdAndVerifyContainsText(t, lakefsCmd+" superuser --user-name "+username, expectFailure, false, outputString, nil)
+}
+
+func TestLakefsSuperuser_alreadyExists(t *testing.T) {
+	RequirePostgresDB(t)
+	lakefsCmd := Lakefs()
+	if isBasicAuth() {
+		lakefsCmd = LakefsWithBasicAuth()
+	}
+	// On init - the AdminUsername is already created and expected error should be "already exist" (also in basic auth mode)
+	RunCmdAndVerifyFailureContainsText(t, lakefsCmd+" superuser --user-name "+AdminUsername, false, "already exists", nil)
+}

esti/lakefs_util.go

@@ -0,0 +1,41 @@
+package esti
+
+import (
+	"strconv"
+	"testing"
+
+	"github.com/spf13/viper"
+)
+
+func LakefsWithParams(connectionString string) string {
+	return LakefsWithParamsWithBasicAuth(connectionString, false)
+}
+
+func LakefsWithParamsWithBasicAuth(connectionString string, basicAuth bool) string {
+	lakefsCmdline := "LAKEFS_DATABASE_TYPE=postgres" +
+		" LAKEFS_DATABASE_POSTGRES_CONNECTION_STRING=" + connectionString +
+		" LAKEFS_AUTH_INTERNAL_BASIC=" + strconv.FormatBool(basicAuth) +
+		" LAKEFS_BLOCKSTORE_TYPE=" + viper.GetString("blockstore_type") +
+		" LAKEFS_AUTH_ENCRYPT_SECRET_KEY='some random secret string' " + lakefsLocation()
+
+	return lakefsCmdline
+}
+
+func lakefsLocation() string {
+	return viper.GetString("binaries_dir") + "/lakefs"
+}
+
+func LakefsWithBasicAuth() string {
+	return LakefsWithParamsWithBasicAuth(viper.GetString("database_connection_string"), true)
+}
+
+func Lakefs() string {
+	return LakefsWithParams(viper.GetString("database_connection_string"))
+}
+
+func RequirePostgresDB(t *testing.T) {
+	dbString := viper.GetString("database_connection_string")
+	if dbString == "" {
+		t.Skip("skip test - not postgres")
+	}
+}

esti/main_test.go

@@ -23,6 +23,7 @@ import (
 const (
 	DefaultAdminAccessKeyID     = "AKIAIOSFDNN7EXAMPLEQ"
 	DefaultAdminSecretAccessKey = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+	AdminUsername               = "esti"
 )
 
 type (
@@ -234,7 +235,7 @@ func TestMain(m *testing.M) {
 	}
 
 	params := testutil.SetupTestingEnvParams{
-		Name:      "esti",
+		Name:      AdminUsername,
 		StorageNS: "esti-system-testing",
 	}
 

esti/ops/docker-compose-acl.yaml

@@ -69,7 +69,7 @@ services:
       - ESTI_SETUP_LAKEFS
       - ESTI_AWS_SECRET_ACCESS_KEY
       - ESTI_ENDPOINT_URL=http://lakefs:8000
-      - ESTI_LAKECTL_DIR=/app
+      - ESTI_BINARIES_DIR=/app
       - ESTI_DATABASE_CONNECTION_STRING=postgres://lakefs:lakefs@postgres/postgres?sslmode=disable
       - ESTI_GOTEST_FLAGS
       - ESTI_FLAGS

esti/ops/docker-compose-dynamodb.yaml

@@ -52,7 +52,7 @@ services:
       - ESTI_SETUP_LAKEFS
       - ESTI_AWS_SECRET_ACCESS_KEY
       - ESTI_ENDPOINT_URL=http://lakefs:8000
-      - ESTI_LAKECTL_DIR=/app
+      - ESTI_BINARIES_DIR=/app
       - ESTI_DATABASE_KV_ENABLED
       - ESTI_KV_MIGRATION=${ESTI_KV_MIGRATION:-none}
       - ESTI_POST_MIGRATE

esti/ops/docker-compose-external-db.yaml

@@ -47,7 +47,7 @@ services:
       - ESTI_SETUP_LAKEFS
       - ESTI_AWS_SECRET_ACCESS_KEY
       - ESTI_ENDPOINT_URL=http://lakefs:8000
-      - ESTI_LAKECTL_DIR=/app
+      - ESTI_BINARIES_DIR=/app
       - ESTI_GOTEST_FLAGS
       - ESTI_FLAGS
       - ESTI_LARGE_OBJECT_PATH

esti/ops/docker-compose.yaml

@@ -55,7 +55,7 @@ services:
       - ESTI_SETUP_LAKEFS
       - ESTI_AWS_SECRET_ACCESS_KEY
       - ESTI_ENDPOINT_URL=http://lakefs:8000
-      - ESTI_LAKECTL_DIR=/app
+      - ESTI_BINARIES_DIR=/app
       - ESTI_DATABASE_CONNECTION_STRING=postgres://lakefs:lakefs@postgres/postgres?sslmode=disable
       - ESTI_GOTEST_FLAGS
       - ESTI_FLAGS