Fix race conditions in runtime

.vscode/settings.json

@@ -20,5 +20,6 @@
     "stretchr",
     "TLSCA",
     "Wrapf"
-  ]
+  ],
+  "go.testFlags": ["-race"]
 }

plugins.go

@@ -72,6 +72,10 @@ func (r *Runtime) pluginsLoaded() bool {
 		return false
 	}
 
+	forceBundleUpdate := false
+
+	statusChannel := make(chan (bool), 1)
+
 	pluginStates := r.pluginsManager.PluginStatus()
 	for pluginName, status := range pluginStates {
 		if status == nil || status.State == plugins.StateOK {
@@ -88,20 +92,36 @@ func (r *Runtime) pluginsLoaded() bool {
 		if pluginName == bundlePluginName || status.State == plugins.StateNotReady {
 			bundles, err := getBundles(context.Background(), r)
 			if err == nil && len(bundles) > 0 {
-				// if bundle plugin state is not ready after a reconfiguration, forcefully update plugin state if bundles are loaded.
-				r.pluginsManager.UpdatePluginStatus(pluginName, &plugins.Status{State: plugins.StateOK})
+				forceBundleUpdate = true
 			}
 		}
 
 		r.Logger.Trace().Str("state", string(status.State)).Str("plugin-name", pluginName).Msg("plugin not ready")
-		return false
+		statusChannel <- false
 	}
 
-	return true
+	if forceBundleUpdate {
+		// if bundle plugin state is not ready after a reconfiguration, forcefully update plugin state if bundles are loaded.
+		r.pluginsManager.UpdatePluginStatus(bundlePluginName, &plugins.Status{State: plugins.StateOK})
+	}
+
+	select {
+	case result := <-statusChannel:
+		{
+			return result
+		}
+	default:
+		{
+			return true
+		}
+	}
 }
 
 // nolint TODO: This change would require upstream changes in OPA
 func (r *Runtime) bundlesStatusCallback(status bundle.Status) {
+	if r.lock.TryLock() {
+		defer r.lock.Unlock()
+	}
 	errs := status.Errors
 	if status.Code == bundleErrorCode {
 		errs = append(errs, errors.Errorf("bundle error: %s", status.Message))
@@ -122,6 +142,9 @@ func (r *Runtime) bundlesStatusCallback(status bundle.Status) {
 //nolint // hugeParam - the status is heavy 200 bytes, upstream changes might be welcomed
 
 func (r *Runtime) pluginStatusCallback(statusDetails map[string]*plugins.Status) {
+	if r.lock.TryLock() {
+		defer r.lock.Unlock()
+	}
 	for n, s := range statusDetails {
 		if n == bundlePluginName && !r.bundlesCallbackRegistered {
 			plugin := r.pluginsManager.Plugin(bundlePluginName)
@@ -158,6 +181,5 @@ func (r *Runtime) pluginStatusCallback(statusDetails map[string]*plugins.Status)
 			r.pluginStates.Store(n, &pluginState{loaded: true})
 		}
 	}
-
 	r.latestState = r.status()
 }

runtime.go

@@ -52,9 +52,9 @@ type Runtime struct {
 	bundleStates                *sync.Map
 	bundlesCallbackRegistered   bool
 	discoveryCallbackRegistered bool
-
-	storage     storage.Store
-	latestState *State
+	storage                     storage.Store
+	latestState                 *State
+	lock                        sync.Mutex
 }
 
 type BundleState struct {
@@ -69,6 +69,7 @@ type State struct {
 	Ready   bool
 	Errors  []error
 	Bundles []BundleState
+	lock    sync.Mutex
 }
 
 var builtinsLock sync.Mutex
@@ -256,6 +257,9 @@ func (r *Runtime) status() *State {
 		Errors:  []error{},
 		Bundles: []BundleState{},
 	}
+	if result.lock.TryLock() {
+		defer result.lock.Unlock()
+	}
 
 	r.pluginStates.Range(func(key, value interface{}) bool {
 		pluginName := key.(string)
@@ -267,7 +271,6 @@ func (r *Runtime) status() *State {
 		if state.err != nil {
 			result.Errors = append(result.Errors, errors.Wrapf(state.err, "plugin '%s' encountered an error", pluginName))
 		}
-
 		return true
 	})