Merge branch 'main' into AN/fix-ui-entrypoint

charts/kuberpult/templates/cd-service.yaml

@@ -381,9 +381,5 @@ metadata:
   name: kuberpult-rbac
 data:
   policy.csv: {{ .Values.auth.dexAuth.policy_csv | quote}}
-kind: ConfigMap
-metadata:
-  name: kuberpult-rbac
-data:
   team.csv: {{ .Values.auth.dexAuth.team_csv| quote}}
 {{- end }}

pkg/db/db.go

@@ -88,6 +88,11 @@ const (
 	AppStateChangeDelete  AppStateChange = "AppStateChangeDelete"
 )
 
+const (
+	MigrationCommitEventUUID = "00000000-0000-0000-0000-000000000000"
+	MigrationCommitEventHash = "0000000000000000000000000000000000000000"
+)
+
 func (h *DBHandler) ShouldUseEslTable() bool {
 	return h != nil
 }
@@ -1003,6 +1008,29 @@ func (h *DBHandler) DBSelectAnyEvent(ctx context.Context, transaction *sql.Tx) (
 	return h.processSingleEventsRow(ctx, rows, err)
 }
 
+func (h *DBHandler) DBContainsMigrationCommitEvent(ctx context.Context, transaction *sql.Tx) (bool, error) {
+	if h == nil {
+		return false, nil
+	}
+	if transaction == nil {
+		return false, fmt.Errorf("DBContainsMigrationCommitEvent: no transaction provided")
+	}
+	span, ctx := tracer.StartSpanFromContext(ctx, "DBContainsMigrationCommitEvent")
+	defer span.Finish()
+
+	query := h.AdaptQuery("SELECT uuid, timestamp, commitHash, eventType, json, transformereslVersion FROM commit_events WHERE commitHash = (?) ORDER BY timestamp DESC LIMIT 1;")
+	span.SetTag("query", query)
+	rows, err := transaction.QueryContext(ctx, query, MigrationCommitEventHash)
+
+	row, err := h.processSingleEventsRow(ctx, rows, err)
+
+	if err != nil {
+		return false, err
+	}
+
+	return row != nil, nil
+}
+
 func (h *DBHandler) DBSelectAllCommitEventsForTransformer(ctx context.Context, transaction *sql.Tx, transformerID TransformerID, eventType event.EventType, limit uint) ([]event.DBEventGo, error) {
 	if h == nil {
 		return nil, nil
@@ -1332,11 +1360,7 @@ func (h *DBHandler) RunCustomMigrations(
 	if err != nil {
 		return err
 	}
-	err = h.RunCustomMigrationAllAppsTable(ctx, getAllAppsFun)
-	if err != nil {
-		return err
-	}
-	err = h.RunCustomMigrationApps(ctx, getAllAppsFun)
+	err = h.RunAllCustomMigrationsForApps(ctx, getAllAppsFun)
 	if err != nil {
 		return err
 	}
@@ -2124,19 +2148,25 @@ func (h *DBHandler) RunCustomMigrationsCommitEvents(ctx context.Context, getAllE
 				}
 			}
 		}
+		//Migration event
+		err = h.writeEvent(ctx, transaction, 0, MigrationCommitEventUUID, event.EventTypeDBMigrationEventType, MigrationCommitEventHash, []byte("{}"))
+		if err != nil {
+			return fmt.Errorf("error writing migration commit event to the database: %v\n", err)
+		}
 		return nil
 	})
 }
 
 func (h *DBHandler) needsCommitEventsMigrations(ctx context.Context, transaction *sql.Tx) (bool, error) {
 	l := logger.FromContext(ctx).Sugar()
 
-	ev, err := h.DBSelectAnyEvent(ctx, transaction)
+	//Checks for 'migration' commit event with hash 0000(...)0000
+	contains, err := h.DBContainsMigrationCommitEvent(ctx, transaction)
 	if err != nil {
 		return true, err
 	}
-	if ev != nil {
-		l.Infof("There are already commit events in the DB - skipping migrations")
+	if contains {
+		l.Infof("detected migration commit event on the database - skipping migrations")
 		return false, nil
 	}
 	return true, nil
@@ -2194,7 +2224,6 @@ func (h *DBHandler) NeedsMigrations(ctx context.Context) (bool, error) {
 	txError := h.WithTransaction(ctx, true, func(ctx context.Context, transaction *sql.Tx) error {
 		var checkFunctions = []CheckFun{
 			(*DBHandler).NeedsEventSourcingLightMigrations,
-			(*DBHandler).needsAllAppsMigrations,
 			(*DBHandler).needsAppsMigrations,
 			(*DBHandler).needsDeploymentsMigrations,
 			(*DBHandler).needsReleasesMigrations,
@@ -2294,12 +2323,14 @@ func (h *DBHandler) DBWriteMigrationsTransformer(ctx context.Context, transactio
 	return nil
 }
 
-func (h *DBHandler) RunCustomMigrationAllAppsTable(ctx context.Context, getAllAppsFun GetAllAppsFun) error {
-	span, ctx := tracer.StartSpanFromContext(ctx, "RunCustomMigrationAllAppsTable")
+// RunAllCustomMigrationsForApps : Performs necessary migrations for the apps and all_apps table
+func (h *DBHandler) RunAllCustomMigrationsForApps(ctx context.Context, getAllAppsFun GetAllAppsFun) error {
+	span, ctx := tracer.StartSpanFromContext(ctx, "RunAllCustomMigrationsForApps")
 	defer span.Finish()
 
+	//We need to join the all_apps and the apps table together as they need to be committed together on the same transaction
 	return h.WithTransaction(ctx, false, func(ctx context.Context, transaction *sql.Tx) error {
-		needMigrating, err := h.needsAllAppsMigrations(ctx, transaction)
+		needMigrating, err := h.needsAppsMigrations(ctx, transaction)
 		if err != nil {
 			return err
 		}
@@ -2309,18 +2340,31 @@ func (h *DBHandler) RunCustomMigrationAllAppsTable(ctx context.Context, getAllAp
 
 		allAppsRepo, err := getAllAppsFun()
 		if err != nil {
-			return fmt.Errorf("could not get applications to run custom migrations: %v", err)
+			return fmt.Errorf("could not get applications from manifest to run custom migrations: %v", err)
 		}
 
-		sortedApps := sorting.SortKeys(allAppsRepo)
+		err = h.runCustomMigrationAllAppsTable(ctx, transaction, &allAppsRepo)
 
-		// if there is any difference, we assume the manifest wins over the database state,
-		// so we use `allAppsRepo`:
-		return h.DBWriteAllApplications(ctx, transaction, 0, sortedApps)
+		if err != nil {
+			return fmt.Errorf("could not perform all_apps table migration: %v\n", err)
+		}
+
+		err = h.runCustomMigrationApps(ctx, transaction, &allAppsRepo)
+		if err != nil {
+			return fmt.Errorf("could not perform apps table migration: %v\n", err)
+		}
+		return nil
 	})
 }
 
-func (h *DBHandler) needsAllAppsMigrations(ctx context.Context, transaction *sql.Tx) (bool, error) {
+func (h *DBHandler) runCustomMigrationAllAppsTable(ctx context.Context, transaction *sql.Tx, allAppsRepo *map[string]string) error {
+	span, ctx := tracer.StartSpanFromContext(ctx, "runCustomMigrationAllAppsTable")
+	defer span.Finish()
+	sortedApps := sorting.SortKeys(*allAppsRepo)
+	return h.DBWriteAllApplications(ctx, transaction, 0, sortedApps)
+}
+
+func (h *DBHandler) needsAppsMigrations(ctx context.Context, transaction *sql.Tx) (bool, error) {
 	l := logger.FromContext(ctx).Sugar()
 	allAppsDb, err := h.DBSelectAllApplications(ctx, transaction)
 	if err != nil {
@@ -2330,47 +2374,18 @@ func (h *DBHandler) needsAllAppsMigrations(ctx context.Context, transaction *sql
 	return allAppsDb == nil, nil
 }
 
-func (h *DBHandler) RunCustomMigrationApps(ctx context.Context, getAllAppsFun GetAllAppsFun) error {
-	span, ctx := tracer.StartSpanFromContext(ctx, "RunCustomMigrationApps")
+// runCustomMigrationApps : Runs custom migrations for provided apps.
+func (h *DBHandler) runCustomMigrationApps(ctx context.Context, transaction *sql.Tx, appsMap *map[string]string) error {
+	span, ctx := tracer.StartSpanFromContext(ctx, "runCustomMigrationApps")
 	defer span.Finish()
 
-	return h.WithTransaction(ctx, false, func(ctx context.Context, transaction *sql.Tx) error {
-		needMigrating, err := h.needsAppsMigrations(ctx, transaction)
+	for app, team := range *appsMap {
+		err := h.DBInsertApplication(ctx, transaction, app, InitialEslVersion, AppStateChangeMigrate, DBAppMetaData{Team: team})
 		if err != nil {
-			return err
+			return fmt.Errorf("could not write dbApp %s: %v", app, err)
 		}
-		if !needMigrating {
-			logger.FromContext(ctx).Sugar().Warnf("no need to migrate apps")
-			return nil
-		}
-
-		appsMap, err := getAllAppsFun()
-		if err != nil {
-			return fmt.Errorf("could not get dbApp to run custom migrations: %v", err)
-		}
-
-		for app := range appsMap {
-			team := appsMap[app]
-			err = h.DBInsertApplication(ctx, transaction, app, InitialEslVersion, AppStateChangeMigrate, DBAppMetaData{Team: team})
-			if err != nil {
-				return fmt.Errorf("could not write dbApp %s: %v", app, err)
-			}
-		}
-		return nil
-	})
-}
-
-func (h *DBHandler) needsAppsMigrations(ctx context.Context, transaction *sql.Tx) (bool, error) {
-	dbApp, err := h.DBSelectAnyApp(ctx, transaction)
-	if err != nil {
-		return true, err
 	}
-	if dbApp != nil {
-		// the migration was already done
-		logger.FromContext(ctx).Info("migration to apps was done already")
-		return false, nil
-	}
-	return true, nil
+	return nil
 }
 
 // ENV LOCKS

pkg/db/db_test.go

@@ -278,6 +278,133 @@ func TestCustomMigrationReleases(t *testing.T) {
 	}
 }
 
+func TestCustomMigrationsApps(t *testing.T) {
+	const appName = "my-app"
+	const teamName = "my-team"
+	tcs := []struct {
+		Name            string
+		expectedApps    []*DBAppWithMetaData
+		expectedAllApps []string
+		allAppsFunc     GetAllAppsFun
+	}{
+		{
+			Name: "Simple migration",
+			expectedApps: []*DBAppWithMetaData{
+				{
+					EslVersion:  2,
+					App:         appName,
+					StateChange: AppStateChangeMigrate,
+					Metadata: DBAppMetaData{
+						Team: teamName,
+					},
+				},
+			},
+			expectedAllApps: []string{appName},
+			allAppsFunc: func() (map[string]string, error) {
+				result := map[string]string{
+					appName: teamName,
+				}
+				return result, nil
+			},
+		},
+		{
+			Name:            "No apps still populate all_apps table",
+			expectedApps:    []*DBAppWithMetaData{},
+			expectedAllApps: []string{},
+			allAppsFunc: func() (map[string]string, error) {
+				result := map[string]string{}
+				return result, nil
+			},
+		},
+	}
+	for _, tc := range tcs {
+		tc := tc
+		t.Run(tc.Name, func(t *testing.T) {
+			t.Parallel()
+			ctx := context.Background()
+
+			dbHandler := SetupRepositoryTestWithDB(t)
+			err3 := dbHandler.WithTransaction(ctx, false, func(ctx context.Context, transaction *sql.Tx) error {
+				err2 := dbHandler.RunAllCustomMigrationsForApps(ctx, tc.allAppsFunc)
+				if err2 != nil {
+					return fmt.Errorf("error: %v", err2)
+				}
+
+				allApps, err2 := dbHandler.DBSelectAllApplications(ctx, transaction)
+				if err2 != nil {
+					return fmt.Errorf("error: %v", err2)
+				}
+				if diff := cmp.Diff(tc.expectedAllApps, allApps.Apps); diff != "" {
+					t.Errorf("error mismatch (-want, +got):\n%s", diff)
+				}
+				for i := range tc.expectedApps {
+					expectedApp := tc.expectedApps[i]
+
+					app, err := dbHandler.DBSelectApp(ctx, transaction, appName)
+					if err != nil {
+						return err
+					}
+					if diff := cmp.Diff(expectedApp, app); diff != "" {
+						t.Errorf("error mismatch (-want, +got):\n%s", diff)
+					}
+				}
+				return nil
+			})
+			if err3 != nil {
+				t.Fatalf("expected no error, got %v", err3)
+			}
+		})
+	}
+}
+
+func TestMigrationCommitEvent(t *testing.T) {
+	var getAllCommitEvents = /*getAllCommitEvents*/ func(ctx context.Context) (AllCommitEvents, error) {
+		result := AllCommitEvents{}
+		return result, nil
+	}
+	tcs := []struct {
+		Name           string
+		expectedEvents []*event.DBEventGo
+	}{
+		{
+			Name: "Test migration event",
+		},
+	}
+	for _, tc := range tcs {
+		tc := tc
+		t.Run(tc.Name, func(t *testing.T) {
+			t.Parallel()
+			ctx := context.Background()
+
+			dbHandler := SetupRepositoryTestWithDB(t)
+			err3 := dbHandler.WithTransaction(ctx, false, func(ctx context.Context, transaction *sql.Tx) error {
+				err2 := dbHandler.RunCustomMigrationsEventSourcingLight(ctx)
+				if err2 != nil {
+					return fmt.Errorf("error: %v", err2)
+				}
+
+				err2 = dbHandler.RunCustomMigrationsCommitEvents(ctx, getAllCommitEvents)
+				if err2 != nil {
+					return fmt.Errorf("error: %v", err2)
+				}
+				//Check for migration event
+				contains, err := dbHandler.DBContainsMigrationCommitEvent(ctx, transaction)
+				if err != nil {
+					t.Errorf("could not get migration event: %v\n", err)
+
+				}
+				if !contains {
+					t.Errorf("migration event was not created: %v\n", err)
+				}
+				return nil
+			})
+			if err3 != nil {
+				t.Fatalf("expected no error, got %v", err3)
+			}
+		})
+	}
+}
+
 func TestCommitEvents(t *testing.T) {
 
 	tcs := []struct {

pkg/event/event.go

@@ -35,6 +35,7 @@ const (
 	EventTypeLockPreventeDeployment EventType = "lock-prevented-deployment"
 	EventTypeReplaceBy              EventType = "replaced-by"
 	EventTypeNewRelease             EventType = "new-release"
+	EventTypeDBMigrationEventType   EventType = "db-migration"
 )
 
 type eventType struct {