diff --git a/components/ws-manager-api/go/crd/v1/workspace_types.go b/components/ws-manager-api/go/crd/v1/workspace_types.go index 5afacaf9acb67b..86c65b662eba27 100644 --- a/components/ws-manager-api/go/crd/v1/workspace_types.go +++ b/components/ws-manager-api/go/crd/v1/workspace_types.go @@ -171,6 +171,8 @@ type WorkspaceStatus struct { // +kubebuilder:validation:Optional Runtime *WorkspaceRuntimeStatus `json:"runtime,omitempty"` + + LastActivity *metav1.Time `json:"lastActivity,omitempty"` } func (s *WorkspaceStatus) SetCondition(cond metav1.Condition) { diff --git a/components/ws-manager-api/go/crd/v1/zz_generated.deepcopy.go b/components/ws-manager-api/go/crd/v1/zz_generated.deepcopy.go index dbbe95b7297628..d3280f1139c53a 100644 --- a/components/ws-manager-api/go/crd/v1/zz_generated.deepcopy.go +++ b/components/ws-manager-api/go/crd/v1/zz_generated.deepcopy.go @@ -428,6 +428,10 @@ func (in *WorkspaceStatus) DeepCopyInto(out *WorkspaceStatus) { *out = new(WorkspaceRuntimeStatus) **out = **in } + if in.LastActivity != nil { + in, out := &in.LastActivity, &out.LastActivity + *out = (*in).DeepCopy() + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkspaceStatus. diff --git a/components/ws-manager-mk2/cmd/sample-workspace/main.go b/components/ws-manager-mk2/cmd/sample-workspace/main.go index 228869ba8281bf..845d702b754a24 100644 --- a/components/ws-manager-mk2/cmd/sample-workspace/main.go +++ b/components/ws-manager-mk2/cmd/sample-workspace/main.go @@ -10,10 +10,11 @@ import ( "log" "time" - workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/utils/pointer" "sigs.k8s.io/yaml" + + workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1" ) func main() { diff --git a/components/ws-manager-mk2/config/crd/bases/workspace.gitpod.io_workspaces.yaml b/components/ws-manager-mk2/config/crd/bases/workspace.gitpod.io_workspaces.yaml index 8d4ffdddae0119..2138a11b23fb97 100644 --- a/components/ws-manager-mk2/config/crd/bases/workspace.gitpod.io_workspaces.yaml +++ b/components/ws-manager-mk2/config/crd/bases/workspace.gitpod.io_workspaces.yaml @@ -522,6 +522,9 @@ spec: type: string type: array type: object + lastActivity: + format: date-time + type: string ownerToken: type: string phase: diff --git a/components/ws-manager-mk2/config/manager/manager.yaml b/components/ws-manager-mk2/config/manager/manager.yaml index d69189ccb79980..ba50abe568ed21 100644 --- a/components/ws-manager-mk2/config/manager/manager.yaml +++ b/components/ws-manager-mk2/config/manager/manager.yaml @@ -33,8 +33,6 @@ spec: containers: - command: - /manager - args: - - --leader-elect image: controller:latest name: manager securityContext: diff --git a/components/ws-manager-mk2/controllers/maintenance_controller.go b/components/ws-manager-mk2/controllers/maintenance_controller.go index c33f50516bb71a..0754baca12e5e1 100644 --- a/components/ws-manager-mk2/controllers/maintenance_controller.go +++ b/components/ws-manager-mk2/controllers/maintenance_controller.go @@ -8,6 +8,7 @@ import ( "context" "encoding/json" "fmt" + "os" "sync" "time" @@ -17,7 +18,12 @@ import ( "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/source" ) var ( @@ -106,9 +112,46 @@ func (r *MaintenanceReconciler) setEnabledUntil(ctx context.Context, enabledUnti log.FromContext(ctx).Info("maintenance mode state change", "enabledUntil", enabledUntil) } -func (r *MaintenanceReconciler) SetupWithManager(mgr ctrl.Manager) error { - return ctrl.NewControllerManagedBy(mgr). - Named("maintenance"). - For(&corev1.ConfigMap{}). - Complete(r) +func (r *MaintenanceReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { + // We need to use an unmanaged controller to avoid issues when the pod is in standby mode. + // In that scenario, the controllers are not started and don't watch changes and only + // observe the maintenance mode during the initialization. + c, err := controller.NewUnmanaged("maintenance-controller", mgr, controller.Options{Reconciler: r}) + if err != nil { + return err + } + + go func() { + err = c.Start(ctx) + if err != nil { + log.FromContext(ctx).Error(err, "cannot start maintenance reconciler") + os.Exit(1) + } + }() + + return c.Watch(source.Kind(mgr.GetCache(), &corev1.ConfigMap{}), &handler.EnqueueRequestForObject{}, &filterConfigMap{}) +} + +type filterConfigMap struct { + predicate.Funcs +} + +func (f filterConfigMap) Create(e event.CreateEvent) bool { + return f.filter(e.Object) +} + +func (f filterConfigMap) Update(e event.UpdateEvent) bool { + return f.filter(e.ObjectNew) +} + +func (f filterConfigMap) Generic(e event.GenericEvent) bool { + return f.filter(e.Object) +} + +func (f filterConfigMap) filter(obj client.Object) bool { + if obj == nil { + return false + } + + return obj.GetName() == configMapKey.Name && obj.GetNamespace() == configMapKey.Namespace } diff --git a/components/ws-manager-mk2/controllers/subscriber_controller.go b/components/ws-manager-mk2/controllers/subscriber_controller.go new file mode 100644 index 00000000000000..5c98242eb14f03 --- /dev/null +++ b/components/ws-manager-mk2/controllers/subscriber_controller.go @@ -0,0 +1,80 @@ +// Copyright (c) 2022 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License-AGPL.txt in the project root for license information. + +package controllers + +import ( + "context" + "os" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/source" + + config "github.com/gitpod-io/gitpod/ws-manager/api/config" + workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1" +) + +func NewSubscriberReconciler(c client.Client, cfg *config.Configuration) (*SubscriberReconciler, error) { + reconciler := &SubscriberReconciler{ + Client: c, + Config: cfg, + } + + return reconciler, nil +} + +type SubscriberReconciler struct { + client.Client + + Config *config.Configuration + + OnReconcile func(ctx context.Context, ws *workspacev1.Workspace) +} + +func (r *SubscriberReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + log := log.FromContext(ctx) + + var workspace workspacev1.Workspace + if err := r.Get(ctx, req.NamespacedName, &workspace); err != nil { + if !errors.IsNotFound(err) { + log.Error(err, "unable to fetch workspace") + } + + return ctrl.Result{}, client.IgnoreNotFound(err) + } + + if workspace.Status.Conditions == nil { + workspace.Status.Conditions = []metav1.Condition{} + } + + if r.OnReconcile != nil { + r.OnReconcile(ctx, &workspace) + } + + return ctrl.Result{}, nil +} + +// SetupWithManager sets up the controller with the Manager. +func (r *SubscriberReconciler) SetupWithManager(ctx context.Context, mgr ctrl.Manager) error { + c, err := controller.NewUnmanaged("subscribers-controller", mgr, controller.Options{Reconciler: r}) + if err != nil { + return err + } + + go func() { + err = c.Start(ctx) + if err != nil { + log.FromContext(ctx).Error(err, "cannot start Subscriber reconciler") + os.Exit(1) + } + }() + + return c.Watch(source.Kind(mgr.GetCache(), &workspacev1.Workspace{}), &handler.EnqueueRequestForObject{}) +} diff --git a/components/ws-manager-mk2/controllers/suite_test.go b/components/ws-manager-mk2/controllers/suite_test.go index e0bb3ec83ae76c..ea0569f43f0df4 100644 --- a/components/ws-manager-mk2/controllers/suite_test.go +++ b/components/ws-manager-mk2/controllers/suite_test.go @@ -23,7 +23,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/metrics" "github.com/gitpod-io/gitpod/common-go/util" - "github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/activity" "github.com/gitpod-io/gitpod/ws-manager/api/config" workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1" //+kubebuilder:scaffold:imports @@ -50,10 +49,9 @@ func TestAPIs(t *testing.T) { } var ( - ctx context.Context - cancel context.CancelFunc - wsActivity *activity.WorkspaceActivity - wsMetrics *controllerMetrics + ctx context.Context + cancel context.CancelFunc + wsMetrics *controllerMetrics ) var _ = BeforeSuite(func() { @@ -111,8 +109,7 @@ var _ = BeforeSuite(func() { Expect(err).ToNot(HaveOccurred()) Expect(wsReconciler.SetupWithManager(k8sManager)).To(Succeed()) - wsActivity = activity.NewWorkspaceActivity() - timeoutReconciler, err := NewTimeoutReconciler(k8sManager.GetClient(), k8sManager.GetEventRecorderFor("workspace"), conf, wsActivity, maintenance) + timeoutReconciler, err := NewTimeoutReconciler(k8sManager.GetClient(), k8sManager.GetEventRecorderFor("workspace"), conf, maintenance) Expect(err).ToNot(HaveOccurred()) Expect(timeoutReconciler.SetupWithManager(k8sManager)).To(Succeed()) diff --git a/components/ws-manager-mk2/controllers/timeout_controller.go b/components/ws-manager-mk2/controllers/timeout_controller.go index 9ba9a9631fcf36..4c41bdbd2dfa1d 100644 --- a/components/ws-manager-mk2/controllers/timeout_controller.go +++ b/components/ws-manager-mk2/controllers/timeout_controller.go @@ -20,13 +20,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/log" "github.com/gitpod-io/gitpod/common-go/util" - wsactivity "github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/activity" + "github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/activity" "github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/maintenance" config "github.com/gitpod-io/gitpod/ws-manager/api/config" workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1" ) -func NewTimeoutReconciler(c client.Client, recorder record.EventRecorder, cfg config.Configuration, activity *wsactivity.WorkspaceActivity, maintenance maintenance.Maintenance) (*TimeoutReconciler, error) { +func NewTimeoutReconciler(c client.Client, recorder record.EventRecorder, cfg config.Configuration, maintenance maintenance.Maintenance) (*TimeoutReconciler, error) { if cfg.HeartbeatInterval == 0 { return nil, fmt.Errorf("invalid heartbeat interval, must not be 0") } @@ -38,7 +38,6 @@ func NewTimeoutReconciler(c client.Client, recorder record.EventRecorder, cfg co return &TimeoutReconciler{ Client: c, Config: cfg, - activity: activity, reconcileInterval: reconcileInterval, recorder: recorder, maintenance: maintenance, @@ -53,7 +52,6 @@ type TimeoutReconciler struct { client.Client Config config.Configuration - activity *wsactivity.WorkspaceActivity reconcileInterval time.Duration recorder record.EventRecorder maintenance maintenance.Maintenance @@ -157,7 +155,7 @@ func (r *TimeoutReconciler) isWorkspaceTimedOut(ws *workspacev1.Workspace) (reas } start := ws.ObjectMeta.CreationTimestamp.Time - lastActivity := r.activity.GetLastActivity(ws) + lastActivity := activity.Last(ws) isClosed := ws.IsConditionTrue(workspacev1.WorkspaceConditionClosed) switch phase { diff --git a/components/ws-manager-mk2/controllers/timeout_controller_test.go b/components/ws-manager-mk2/controllers/timeout_controller_test.go index a1093b096eb362..0aa6697f124f7e 100644 --- a/components/ws-manager-mk2/controllers/timeout_controller_test.go +++ b/components/ws-manager-mk2/controllers/timeout_controller_test.go @@ -8,7 +8,6 @@ import ( "time" wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes" - "github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/activity" workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1" "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" @@ -36,7 +35,7 @@ var _ = Describe("TimeoutController", func() { // Use a fake client instead of the envtest's k8s client, such that we can add objects // with custom CreationTimestamps and check timeout logic. fakeClient = fake.NewClientBuilder().WithStatusSubresource(&workspacev1.Workspace{}).WithScheme(k8sClient.Scheme()).Build() - r, err = NewTimeoutReconciler(fakeClient, record.NewFakeRecorder(100), conf, activity.NewWorkspaceActivity(), &fakeMaintenance{enabled: false}) + r, err = NewTimeoutReconciler(fakeClient, record.NewFakeRecorder(100), conf, &fakeMaintenance{enabled: false}) Expect(err).ToNot(HaveOccurred()) }) @@ -48,7 +47,6 @@ var _ = Describe("TimeoutController", func() { customMaxLifetime *time.Duration update func(ws *workspacev1.Workspace) updateStatus func(ws *workspacev1.Workspace) - controllerRestart time.Time expectTimeout bool } DescribeTable("workspace timeouts", @@ -56,12 +54,14 @@ var _ = Describe("TimeoutController", func() { By("creating a workspace") ws := newWorkspace(uuid.NewString(), "default") ws.CreationTimestamp = metav1.NewTime(now.Add(-tc.age)) - Expect(fakeClient.Create(ctx, ws)).To(Succeed()) if tc.lastActivityAgo != nil { - r.activity.Store(ws.Name, now.Add(-*tc.lastActivityAgo)) + now := metav1.NewTime(now.Add(-*tc.lastActivityAgo)) + ws.Status.LastActivity = &now } + Expect(fakeClient.Create(ctx, ws)).To(Succeed()) + updateObjWithRetries(fakeClient, ws, false, func(ws *workspacev1.Workspace) { if tc.customTimeout != nil { ws.Spec.Timeout.Time = &metav1.Duration{Duration: *tc.customTimeout} @@ -80,14 +80,6 @@ var _ = Describe("TimeoutController", func() { } }) - // Set controller (re)start time. - if tc.controllerRestart.IsZero() { - // Bit arbitrary, but default to the controller running for ~2 days. - r.activity.ManagerStartedAt = now.Add(-48 * time.Hour) - } else { - r.activity.ManagerStartedAt = tc.controllerRestart - } - // Run the timeout controller for this workspace. By("running the TimeoutController reconcile()") _, err := r.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{Name: ws.Name, Namespace: ws.Namespace}}) @@ -159,32 +151,11 @@ var _ = Describe("TimeoutController", func() { lastActivityAgo: pointer.Duration(1 * time.Minute), expectTimeout: true, }), - Entry("shouldn't timeout after controller restart", testCase{ - phase: workspacev1.WorkspacePhaseRunning, - updateStatus: func(ws *workspacev1.Workspace) { - // Add FirstUserActivity condition from 5 hours ago. - // From this condition the controller should deduce that the workspace - // has had user activity, but since lastActivity is nil, it's been cleared on - // a restart. The controller therefore should not timeout the workspace and - // wait for new user activity. Or timeout once user activity doesn't come - // eventually after the controller restart. - ws.Status.Conditions = wsk8s.AddUniqueCondition(ws.Status.Conditions, metav1.Condition{ - Type: string(workspacev1.WorkspaceConditionFirstUserActivity), - Status: metav1.ConditionTrue, - LastTransitionTime: metav1.NewTime(now.Add(-5 * time.Hour)), - }) - }, - age: 5 * time.Hour, - lastActivityAgo: nil, // No last activity recorded yet after controller restart. - controllerRestart: now, - expectTimeout: false, - }), Entry("should timeout after controller restart if no FirstUserActivity", testCase{ - phase: workspacev1.WorkspacePhaseRunning, - age: 5 * time.Hour, - lastActivityAgo: nil, // No last activity recorded yet after controller restart. - controllerRestart: now, - expectTimeout: true, + phase: workspacev1.WorkspacePhaseRunning, + age: 5 * time.Hour, + lastActivityAgo: nil, // No last activity recorded yet after controller restart. + expectTimeout: true, }), Entry("should timeout eventually with no user activity after controller restart", testCase{ phase: workspacev1.WorkspacePhaseRunning, @@ -195,10 +166,9 @@ var _ = Describe("TimeoutController", func() { LastTransitionTime: metav1.NewTime(now.Add(-5 * time.Hour)), }) }, - age: 5 * time.Hour, - lastActivityAgo: nil, - controllerRestart: now.Add(-2 * time.Hour), - expectTimeout: true, + age: 5 * time.Hour, + lastActivityAgo: nil, + expectTimeout: true, }), ) }) @@ -207,7 +177,7 @@ var _ = Describe("TimeoutController", func() { var r *TimeoutReconciler BeforeEach(func() { var err error - r, err = NewTimeoutReconciler(k8sClient, record.NewFakeRecorder(100), newTestConfig(), activity.NewWorkspaceActivity(), &fakeMaintenance{enabled: false}) + r, err = NewTimeoutReconciler(k8sClient, record.NewFakeRecorder(100), newTestConfig(), &fakeMaintenance{enabled: false}) Expect(err).ToNot(HaveOccurred()) }) diff --git a/components/ws-manager-mk2/controllers/workspace_controller.go b/components/ws-manager-mk2/controllers/workspace_controller.go index c378e0462b16b5..5b148299d92caa 100644 --- a/components/ws-manager-mk2/controllers/workspace_controller.go +++ b/components/ws-manager-mk2/controllers/workspace_controller.go @@ -72,7 +72,6 @@ type WorkspaceReconciler struct { metrics *controllerMetrics maintenance maintenance.Maintenance Recorder record.EventRecorder - OnReconcile func(ctx context.Context, ws *workspacev1.Workspace) } //+kubebuilder:rbac:groups=workspace.gitpod.io,resources=workspaces,verbs=get;list;watch;create;update;patch;delete @@ -108,14 +107,6 @@ func (r *WorkspaceReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( workspace.Status.Conditions = []metav1.Condition{} } - if r.OnReconcile != nil { - // Publish to subscribers in a goroutine, to prevent blocking the main reconcile loop. - ws := workspace.DeepCopy() - go func() { - r.OnReconcile(ctx, ws) - }() - } - log.Info("reconciling workspace", "workspace", req.NamespacedName, "phase", workspace.Status.Phase) var workspacePods corev1.PodList diff --git a/components/ws-manager-mk2/main.go b/components/ws-manager-mk2/main.go index 654f3cb21e4bc9..0a4899a947c4d2 100644 --- a/components/ws-manager-mk2/main.go +++ b/components/ws-manager-mk2/main.go @@ -18,6 +18,7 @@ import ( "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" _ "k8s.io/client-go/plugin/pkg/client/auth" + "k8s.io/client-go/rest" "github.com/bombsimon/logrusr/v2" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" @@ -43,7 +44,6 @@ import ( workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1" "github.com/gitpod-io/gitpod/ws-manager-mk2/controllers" - "github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/activity" "github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/maintenance" imgproxy "github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/proxy" "github.com/gitpod-io/gitpod/ws-manager-mk2/service" @@ -68,13 +68,9 @@ func init() { } func main() { - var enableLeaderElection bool var configFN string var jsonLog bool var verbose bool - flag.BoolVar(&enableLeaderElection, "leader-elect", false, - "Enable leader election for controller manager. "+ - "Enabling this will ensure there is only one active controller manager.") flag.StringVar(&configFN, "config", "", "Path to the config file") flag.BoolVar(&jsonLog, "json-log", true, "produce JSON log output on verbose level") flag.BoolVar(&verbose, "verbose", false, "Enable verbose logging") @@ -115,25 +111,32 @@ func main() { setupLog.Error(nil, "namespace cannot be empty") os.Exit(1) } + if cfg.Manager.SecretsNamespace == "" { setupLog.Error(nil, "secretsNamespace cannot be empty") os.Exit(1) } mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ - Scheme: scheme, - MetricsBindAddress: cfg.Prometheus.Addr, - Port: 9443, - HealthProbeBindAddress: cfg.Health.Addr, - LeaderElection: enableLeaderElection, - LeaderElectionID: "ws-manager-mk2-leader.gitpod.io", - NewCache: cache.MultiNamespacedCacheBuilder([]string{cfg.Manager.Namespace, cfg.Manager.SecretsNamespace}), + Scheme: scheme, + MetricsBindAddress: cfg.Prometheus.Addr, + Port: 9443, + HealthProbeBindAddress: cfg.Health.Addr, + LeaderElection: true, + LeaderElectionID: "ws-manager-mk2-leader.gitpod.io", + LeaderElectionReleaseOnCancel: true, + NewCache: func(config *rest.Config, opts cache.Options) (cache.Cache, error) { + opts.Namespaces = []string{cfg.Manager.Namespace, cfg.Manager.SecretsNamespace} + return cache.New(config, opts) + }, }) if err != nil { setupLog.Error(err, "unable to start manager") os.Exit(1) } + mgrCtx := ctrl.SetupSignalHandler() + maintenanceReconciler, err := controllers.NewMaintenanceReconciler(mgr.GetClient()) if err != nil { setupLog.Error(err, "unable to create maintenance controller", "controller", "Maintenance") @@ -147,29 +150,42 @@ func main() { os.Exit(1) } - activity := activity.NewWorkspaceActivity() - timeoutReconciler, err := controllers.NewTimeoutReconciler(mgr.GetClient(), mgr.GetEventRecorderFor("workspace"), cfg.Manager, activity, maintenanceReconciler) + timeoutReconciler, err := controllers.NewTimeoutReconciler(mgr.GetClient(), mgr.GetEventRecorderFor("workspace"), cfg.Manager, maintenanceReconciler) if err != nil { setupLog.Error(err, "unable to create timeout controller", "controller", "Timeout") os.Exit(1) } - wsmanService, err := setupGRPCService(cfg, mgr.GetClient(), activity, maintenanceReconciler) + wsmanService, err := setupGRPCService(cfg, mgr.GetClient(), maintenanceReconciler) if err != nil { setupLog.Error(err, "unable to start manager service") os.Exit(1) } - workspaceReconciler.OnReconcile = wsmanService.OnWorkspaceReconcile + subscriberReconciler, err := controllers.NewSubscriberReconciler(mgr.GetClient(), &cfg.Manager) + if err != nil { + setupLog.Error(err, "unable to create subscriber controller", "controller", "Subscribers") + os.Exit(1) + } + + subscriberReconciler.OnReconcile = wsmanService.OnWorkspaceReconcile + + if err = subscriberReconciler.SetupWithManager(mgrCtx, mgr); err != nil { + setupLog.Error(err, "unable to setup workspace controller with manager", "controller", "Subscribers") + os.Exit(1) + } + if err = workspaceReconciler.SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to setup workspace controller with manager", "controller", "Workspace") os.Exit(1) } + if err = timeoutReconciler.SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to setup timeout controller with manager", "controller", "Timeout") os.Exit(1) } - if err = maintenanceReconciler.SetupWithManager(mgr); err != nil { + + if err = maintenanceReconciler.SetupWithManager(mgrCtx, mgr); err != nil { setupLog.Error(err, "unable to setup maintenance controller with manager", "controller", "Maintenance") os.Exit(1) } @@ -191,13 +207,13 @@ func main() { } setupLog.Info("starting manager") - if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + if err := mgr.Start(mgrCtx); err != nil { setupLog.Error(err, "problem running manager") os.Exit(1) } } -func setupGRPCService(cfg *config.ServiceConfiguration, k8s client.Client, activity *activity.WorkspaceActivity, maintenance maintenance.Maintenance) (*service.WorkspaceManagerServer, error) { +func setupGRPCService(cfg *config.ServiceConfiguration, k8s client.Client, maintenance maintenance.Maintenance) (*service.WorkspaceManagerServer, error) { // TODO(cw): remove use of common-go/log if len(cfg.RPCServer.RateLimits) > 0 { @@ -253,7 +269,7 @@ func setupGRPCService(cfg *config.ServiceConfiguration, k8s client.Client, activ imgbldr.RegisterImageBuilderServer(grpcServer, imgproxy.ImageBuilder{D: imgbldr.NewImageBuilderClient(conn)}) } - srv := service.NewWorkspaceManagerServer(k8s, &cfg.Manager, metrics.Registry, activity, maintenance) + srv := service.NewWorkspaceManagerServer(k8s, &cfg.Manager, metrics.Registry, maintenance) grpc_prometheus.Register(grpcServer) wsmanapi.RegisterWorkspaceManagerServer(grpcServer, srv) diff --git a/components/ws-manager-mk2/pkg/activity/activity.go b/components/ws-manager-mk2/pkg/activity/activity.go index 0c6df4157854d9..1c17b5b528f3cb 100644 --- a/components/ws-manager-mk2/pkg/activity/activity.go +++ b/components/ws-manager-mk2/pkg/activity/activity.go @@ -1,47 +1,20 @@ // Copyright (c) 2023 Gitpod GmbH. All rights reserved. // Licensed under the GNU Affero General Public License (AGPL). -// See License.AGPL.txt in the project root for license information. +// See License-AGPL.txt in the project root for license information. package activity import ( - "sync" "time" workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1" ) -// WorkspaceActivity is used to track the last user activity per workspace. This is -// stored in memory instead of on the Workspace resource to limit load on the k8s API, -// as this value will update often for each workspace. -type WorkspaceActivity struct { - ManagerStartedAt time.Time - m sync.Map -} - -func NewWorkspaceActivity() *WorkspaceActivity { - return &WorkspaceActivity{ - ManagerStartedAt: time.Now().UTC(), - } -} - -func (w *WorkspaceActivity) Store(workspaceId string, lastActivity time.Time) { - w.m.Store(workspaceId, &lastActivity) -} - -func (w *WorkspaceActivity) GetLastActivity(ws *workspacev1.Workspace) *time.Time { - lastActivity, ok := w.m.Load(ws.Name) - if ok { - return lastActivity.(*time.Time) - } - - // In case we don't have a record of the workspace's last activity, check for the FirstUserActivity condition - // to see if the lastActivity got lost on a manager restart. - if ws.IsConditionTrue(workspacev1.WorkspaceConditionFirstUserActivity) { - // Manager was restarted, consider the workspace's last activity to be the time the manager restarted. - return &w.ManagerStartedAt +func Last(ws *workspacev1.Workspace) *time.Time { + lastActivity := ws.Status.LastActivity + if lastActivity != nil { + return &lastActivity.Time } - // If the FirstUserActivity condition isn't present we know that the workspace has never had user activity. return nil } diff --git a/components/ws-manager-mk2/service/manager.go b/components/ws-manager-mk2/service/manager.go index 484f05d7351348..7bfc8d9fef7a46 100644 --- a/components/ws-manager-mk2/service/manager.go +++ b/components/ws-manager-mk2/service/manager.go @@ -16,25 +16,13 @@ import ( validation "github.com/go-ozzo/ozzo-validation" "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" + "github.com/sirupsen/logrus" "golang.org/x/xerrors" "google.golang.org/grpc/codes" "google.golang.org/grpc/peer" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" - - wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes" - "github.com/gitpod-io/gitpod/common-go/log" - "github.com/gitpod-io/gitpod/common-go/tracing" - "github.com/gitpod-io/gitpod/common-go/util" - "github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/activity" - "github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/maintenance" - wsmanapi "github.com/gitpod-io/gitpod/ws-manager/api" - "github.com/gitpod-io/gitpod/ws-manager/api/config" - workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1" - - csapi "github.com/gitpod-io/gitpod/content-service/api" - "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -46,6 +34,17 @@ import ( "k8s.io/utils/pointer" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes" + "github.com/gitpod-io/gitpod/common-go/log" + "github.com/gitpod-io/gitpod/common-go/tracing" + "github.com/gitpod-io/gitpod/common-go/util" + csapi "github.com/gitpod-io/gitpod/content-service/api" + "github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/activity" + "github.com/gitpod-io/gitpod/ws-manager-mk2/pkg/maintenance" + wsmanapi "github.com/gitpod-io/gitpod/ws-manager/api" + "github.com/gitpod-io/gitpod/ws-manager/api/config" + workspacev1 "github.com/gitpod-io/gitpod/ws-manager/api/crd/v1" ) const ( @@ -66,15 +65,14 @@ var ( } ) -func NewWorkspaceManagerServer(clnt client.Client, cfg *config.Configuration, reg prometheus.Registerer, activity *activity.WorkspaceActivity, maintenance maintenance.Maintenance) *WorkspaceManagerServer { - metrics := newWorkspaceMetrics(cfg.Namespace, clnt, activity) +func NewWorkspaceManagerServer(clnt client.Client, cfg *config.Configuration, reg prometheus.Registerer, maintenance maintenance.Maintenance) *WorkspaceManagerServer { + metrics := newWorkspaceMetrics(cfg.Namespace, clnt) reg.MustRegister(metrics) return &WorkspaceManagerServer{ Client: clnt, Config: cfg, metrics: metrics, - activity: activity, maintenance: maintenance, subs: subscriptions{ subscribers: make(map[string]chan *wsmanapi.SubscribeResponse), @@ -86,7 +84,6 @@ type WorkspaceManagerServer struct { Client client.Client Config *config.Configuration metrics *workspaceMetrics - activity *activity.WorkspaceActivity maintenance maintenance.Maintenance subs subscriptions @@ -468,7 +465,7 @@ func (wsm *WorkspaceManagerServer) DescribeWorkspace(ctx context.Context, req *w Status: wsm.extractWorkspaceStatus(&ws), } - lastActivity := wsm.activity.GetLastActivity(&ws) + lastActivity := activity.Last(&ws) if lastActivity != nil { result.LastActivity = lastActivity.UTC().Format(time.RFC3339Nano) } @@ -513,10 +510,17 @@ func (wsm *WorkspaceManagerServer) MarkActive(ctx context.Context, req *wsmanapi return &wsmanapi.MarkActiveResponse{}, nil } - // We do not keep the last activity in the workspace resource to limit the load we're placing - // on the K8S master in check. Thus, this state lives locally in a map. now := time.Now().UTC() - wsm.activity.Store(req.Id, now) + lastActivityStatus := metav1.NewTime(now) + ws.Status.LastActivity = &lastActivityStatus + + err = wsm.modifyWorkspace(ctx, req.Id, true, func(ws *workspacev1.Workspace) error { + ws.Status.LastActivity = &lastActivityStatus + return nil + }) + if err != nil { + log.WithError(err).WithFields(log.OWI("", "", workspaceID)).Warn("was unable to update status") + } // We do however maintain the the "closed" flag as condition on the workspace. This flag should not change // very often and provides a better UX if it persists across ws-manager restarts. @@ -1390,7 +1394,7 @@ type workspaceMetrics struct { workspaceActivityVec *workspaceActivityVec } -func newWorkspaceMetrics(namespace string, k8s client.Client, activity *activity.WorkspaceActivity) *workspaceMetrics { +func newWorkspaceMetrics(namespace string, k8s client.Client) *workspaceMetrics { return &workspaceMetrics{ totalStartsCounterVec: prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: "gitpod", @@ -1398,7 +1402,7 @@ func newWorkspaceMetrics(namespace string, k8s client.Client, activity *activity Name: "workspace_starts_total", Help: "total number of workspaces started", }, []string{"type", "class"}), - workspaceActivityVec: newWorkspaceActivityVec(namespace, k8s, activity), + workspaceActivityVec: newWorkspaceActivityVec(namespace, k8s), } } @@ -1430,10 +1434,9 @@ type workspaceActivityVec struct { name string workspaceNamespace string k8s client.Client - activity *activity.WorkspaceActivity } -func newWorkspaceActivityVec(workspaceNamespace string, k8s client.Client, activity *activity.WorkspaceActivity) *workspaceActivityVec { +func newWorkspaceActivityVec(workspaceNamespace string, k8s client.Client) *workspaceActivityVec { opts := prometheus.GaugeOpts{ Namespace: "gitpod", Subsystem: "ws_manager_mk2", @@ -1445,7 +1448,6 @@ func newWorkspaceActivityVec(workspaceNamespace string, k8s client.Client, activ name: prometheus.BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), workspaceNamespace: workspaceNamespace, k8s: k8s, - activity: activity, } } @@ -1484,7 +1486,7 @@ func (wav *workspaceActivityVec) getWorkspaceActivityCounts() (active, notActive continue } - hasActivity := wav.activity.GetLastActivity(&ws) != nil + hasActivity := activity.Last(&ws) != nil if hasActivity { active++ } else { diff --git a/dev/preview/workflow/preview/deploy-gitpod.sh b/dev/preview/workflow/preview/deploy-gitpod.sh index 9ca87157f9f530..fa682395562f79 100755 --- a/dev/preview/workflow/preview/deploy-gitpod.sh +++ b/dev/preview/workflow/preview/deploy-gitpod.sh @@ -302,8 +302,8 @@ yq w -i "${INSTALLER_CONFIG_PATH}" experimental.ide.ideMetrics.enabledErrorRepor # # configureObservability # -TRACING_ENDPOINT="http://otel-collector.monitoring-satellite.svc.cluster.local:14268/api/traces" -yq w -i "${INSTALLER_CONFIG_PATH}" observability.tracing.endpoint "${TRACING_ENDPOINT}" +#TRACING_ENDPOINT="http://otel-collector.monitoring-satellite.svc.cluster.local:14268/api/traces" +#yq w -i "${INSTALLER_CONFIG_PATH}" observability.tracing.endpoint "${TRACING_ENDPOINT}" # # configureAuthProviders diff --git a/install/installer/pkg/components/ws-manager-mk2/deployment.go b/install/installer/pkg/components/ws-manager-mk2/deployment.go index 8fcf3c4507d420..a27581696a3ef8 100644 --- a/install/installer/pkg/components/ws-manager-mk2/deployment.go +++ b/install/installer/pkg/components/ws-manager-mk2/deployment.go @@ -5,10 +5,6 @@ package wsmanagermk2 import ( - "github.com/gitpod-io/gitpod/installer/pkg/cluster" - "github.com/gitpod-io/gitpod/installer/pkg/common" - wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon" - "github.com/gitpod-io/gitpod/installer/pkg/config/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" @@ -16,6 +12,11 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/utils/pointer" + + "github.com/gitpod-io/gitpod/installer/pkg/cluster" + "github.com/gitpod-io/gitpod/installer/pkg/common" + wsdaemon "github.com/gitpod-io/gitpod/installer/pkg/components/ws-daemon" + "github.com/gitpod-io/gitpod/installer/pkg/config/v1" ) func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { @@ -58,7 +59,6 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { Name: Component, Args: []string{ "--config", "/config/config.json", - "--leader-elect", }, Image: ctx.ImageName(ctx.Config.Repository, Component, ctx.VersionManifest.Components.WSManagerMk2.Version), ImagePullPolicy: corev1.PullIfNotPresent, @@ -176,7 +176,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) { }, Spec: appsv1.DeploymentSpec{ Selector: &metav1.LabelSelector{MatchLabels: labels}, - Replicas: common.Replicas(ctx, Component), + Replicas: pointer.Int32(2), Strategy: common.DeploymentStrategy, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ diff --git a/test/go.mod b/test/go.mod index 35ea08a56dfd15..c1669c36bf4cf8 100644 --- a/test/go.mod +++ b/test/go.mod @@ -20,7 +20,7 @@ require ( github.com/prometheus/procfs v0.10.1 github.com/vishvananda/netns v0.0.4 golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e - golang.org/x/oauth2 v0.6.0 + golang.org/x/oauth2 v0.8.0 golang.org/x/sync v0.2.0 golang.org/x/sys v0.11.0 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 @@ -32,7 +32,7 @@ require ( k8s.io/client-go v0.27.3 k8s.io/klog/v2 v2.90.1 k8s.io/kubectl v0.27.3 - sigs.k8s.io/e2e-framework v0.0.7 + sigs.k8s.io/e2e-framework v0.2.0 ) require ( @@ -44,6 +44,7 @@ require ( github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/BurntSushi/toml v0.4.1 // indirect github.com/MakeNowJust/heredoc v1.0.0 // indirect + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aws/aws-sdk-go-v2 v1.17.1 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.9 // indirect github.com/aws/aws-sdk-go-v2/config v1.18.3 // indirect @@ -104,7 +105,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect - github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/hashicorp/golang-lru v0.5.1 // indirect github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect @@ -154,8 +155,8 @@ require ( github.com/xlab/treeprint v1.1.0 // indirect go.opencensus.io v0.24.0 // indirect go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect - go.uber.org/atomic v1.8.0 // indirect - golang.org/x/crypto v0.1.0 // indirect + go.uber.org/atomic v1.7.0 // indirect + golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/term v0.8.0 // indirect @@ -166,7 +167,7 @@ require ( google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230320184635-7606e756e683 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/ini.v1 v1.62.0 // indirect + gopkg.in/ini.v1 v1.57.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect honnef.co/go/tools v0.2.2 // indirect diff --git a/test/go.sum b/test/go.sum index c84bf9cd57a734..642397002db0c4 100644 --- a/test/go.sum +++ b/test/go.sum @@ -21,6 +21,7 @@ github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-sdk-go-v2 v1.17.1 h1:02c72fDJr87N8RAC2s3Qu0YuvMRZKNZJ9F+lAehCazk= github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.9 h1:RKci2D7tMwpvGpDNZnGQw9wk6v7o/xSwFcUAuNPoB8k= @@ -211,8 +212,8 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92Bcuy github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 h1:lLT7ZLSzGLI08vc9cpd+tYmNWjdKDqyr/2L+f6U12Fk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/helloyi/go-sshclient v1.1.1 h1:yRcrc/Q1nJ2hYbtVFfBBTyneQLqr2vy/jD3/GneyirI= github.com/helloyi/go-sshclient v1.1.1/go.mod h1:NrhRWsYJDjoQXTDWZ4YtVk84wZ4LK3NSM9jD2TZDAm8= github.com/heptiolabs/healthcheck v0.0.0-20211123025425-613501dd5deb h1:tsEKRC3PU9rMw18w/uAptoijhgG4EvlA5kfJPtwrMDk= @@ -381,7 +382,7 @@ github.com/uber/jaeger-lib v2.4.1+incompatible h1:td4jdvLcExb4cBISKIpHuGoVXh+dVK github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= -github.com/vladimirvivien/gexe v0.1.1 h1:2A0SBaOSKH+cwLVdt6H+KkHZotZWRNLlWygANGw5DxE= +github.com/vladimirvivien/gexe v0.2.0 h1:nbdAQ6vbZ+ZNsolCgSVb9Fno60kzSuvtzVh6Ytqi/xY= github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -393,8 +394,8 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.8.0 h1:CUhrE4N1rqSE6FM9ecihEjRkLQu8cDfgDyoOs83mEY4= -go.uber.org/atomic v1.8.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= @@ -404,8 +405,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= @@ -445,8 +446,8 @@ golang.org/x/net v0.0.0-20211201190559-0a0e4e1bb54c/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= -golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -565,8 +566,8 @@ gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntN gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/ini.v1 v1.62.0 h1:duBzk771uxoUuOlyRLkHsygud9+5lrlGjdFBb4mSKDU= -gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -602,8 +603,8 @@ k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPB k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= sigs.k8s.io/controller-runtime v0.15.0 h1:ML+5Adt3qZnMSYxZ7gAverBLNPSMQEibtzAgp0UPojU= sigs.k8s.io/controller-runtime v0.15.0/go.mod h1:7ngYvp1MLT+9GeZ+6lH3LOlcHkp/+tzA/fmHa4iq9kk= -sigs.k8s.io/e2e-framework v0.0.7 h1:nMv2oSPBLWARse2aBoqX5Wq3ox67w8jrhTGWGpccWDQ= -sigs.k8s.io/e2e-framework v0.0.7/go.mod h1:hdwYGVQg4bvDAah5eidNf2/qkG35qHjzuyMVr2A3oiY= +sigs.k8s.io/e2e-framework v0.2.0 h1:gD6AWWAHFcHibI69E9TgkNFhh0mVwWtRCHy2RU057jQ= +sigs.k8s.io/e2e-framework v0.2.0/go.mod h1:E6JXj/V4PIlb95jsn2WrNKG+Shb45xaaI7C0+BH4PL8= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kustomize/api v0.13.2 h1:kejWfLeJhUsTGioDoFNJET5LQe/ajzXhJGYoU+pJsiA= diff --git a/test/pkg/integration/setup.go b/test/pkg/integration/setup.go index 0353cfc0acccca..6b31e30ac3146f 100644 --- a/test/pkg/integration/setup.go +++ b/test/pkg/integration/setup.go @@ -79,7 +79,8 @@ func Setup(ctx context.Context) (string, string, env.Environment, bool, string, assess string parallel bool - labels = make(flags.LabelsMap) + labels = make(flags.LabelsMap) + skipLabels = make(flags.LabelsMap) ) flagset := flag.CommandLine @@ -99,6 +100,8 @@ func Setup(ctx context.Context) (string, string, env.Environment, bool, string, flagset.StringVar(&feature, "feature", "", "Regular expression that targets features to test") flagset.StringVar(&assess, "assess", "", "Regular expression that targets assertive steps to run") flagset.Var(&labels, "labels", "Comma-separated key/value pairs to filter tests by labels") + flagset.Var(&skipLabels, "skip-labels", "Comma-separated key/value pairs to skip tests by labels") + if err := flagset.Parse(os.Args[1:]); err != nil { klog.Fatalf("cannot parse flags: %v", err) } @@ -121,6 +124,7 @@ func Setup(ctx context.Context) (string, string, env.Environment, bool, string, e.WithClient(client) e.WithLabels(labels) + e.WithSkipLabels(skipLabels) e.WithNamespace(namespace) // use the namespace from the CurrentContext diff --git a/test/run.sh b/test/run.sh index 25fa46368876a7..8b1b2dc709afac 100755 --- a/test/run.sh +++ b/test/run.sh @@ -14,6 +14,7 @@ # set -euo pipefail +set -x REPORT="" TEST_SUITE=all @@ -114,7 +115,17 @@ if [ "$TEST_SUITE" == "workspace" ]; then set +e # shellcheck disable=SC2086 - go test -p 10 -v $TEST_LIST "${args[@]}" -parallel-features=true 2>&1 | go-junit-report -subtest-mode=exclude-parents -set-exit-code -out "${RESULTS_DIR}/TEST-${TEST_NAME}.xml" -iocopy + go test -p 4 -v $TEST_LIST "${args[@]}" -parallel-features=true -skip-labels="type=maintenance" 2>&1 | go-junit-report -subtest-mode=exclude-parents -set-exit-code -out "${RESULTS_DIR}/TEST-${TEST_NAME}.xml" -iocopy + RC=${PIPESTATUS[0]} + set -e + + if [ "${RC}" -ne "0" ]; then + FAILURE_COUNT=$((FAILURE_COUNT+1)) + fi + + set +e + # shellcheck disable=SC2086 + go test -v $TEST_LIST "${args[@]}" -labels="type=maintenance" 2>&1 RC=${PIPESTATUS[0]} set -e diff --git a/test/tests/components/ws-manager/maintenence_test.go b/test/tests/components/ws-manager/maintenence_test.go new file mode 100644 index 00000000000000..a22a59d5976e90 --- /dev/null +++ b/test/tests/components/ws-manager/maintenence_test.go @@ -0,0 +1,217 @@ +// Copyright (c) 2020 Gitpod GmbH. All rights reserved. +// Licensed under the GNU Affero General Public License (AGPL). +// See License.AGPL.txt in the project root for license information. + +package wsmanager + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/e2e-framework/klient" + "sigs.k8s.io/e2e-framework/pkg/envconf" + "sigs.k8s.io/e2e-framework/pkg/features" + + csapi "github.com/gitpod-io/gitpod/content-service/api" + "github.com/gitpod-io/gitpod/test/pkg/integration" + wsmanapi "github.com/gitpod-io/gitpod/ws-manager/api" + "github.com/gitpod-io/gitpod/ws-manager/api/config" +) + +func TestMaintenance(t *testing.T) { + testRepo := "https://github.com/gitpod-io/empty" + testRepoName := "empty" + + f1 := features.New("maintenance"). + WithLabel("component", "ws-manager"). + WithLabel("type", "maintenance"). + Assess("should display maintenance message", func(testCtx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + kubeClient, err := cfg.NewClient() + if err != nil { + t.Fatal(err) + } + + untilTime := time.Now().Add(1 * time.Hour) + err = configureMaintenanceMode(testCtx, &untilTime, kubeClient) + if err != nil { + t.Fatal(err) + } + + t.Cleanup(func() { + err = configureMaintenanceMode(testCtx, nil, kubeClient) + if err != nil { + t.Error(err) + } + }) + + ctx, cancel := context.WithTimeout(testCtx, time.Duration(5*time.Minute)) + defer cancel() + + api := integration.NewComponentAPI(ctx, cfg.Namespace(), kubeconfig, cfg.Client()) + t.Cleanup(func() { + api.Done(t) + }) + + customizeWorkspace := func(swr *wsmanapi.StartWorkspaceRequest) error { + swr.Spec.Initializer = &csapi.WorkspaceInitializer{ + Spec: &csapi.WorkspaceInitializer_Git{ + Git: &csapi.GitInitializer{ + RemoteUri: testRepo, + CheckoutLocation: testRepoName, + Config: &csapi.GitConfig{}, + }, + }, + } + swr.Spec.WorkspaceLocation = testRepoName + return nil + } + + _, _, err = integration.LaunchWorkspaceDirectly(t, ctx, api, integration.WithRequestModifier(customizeWorkspace)) + if err == nil { + t.Fatalf("expected under maintenance error") + } else { + if !errors.Is(err, status.Error(codes.FailedPrecondition, "under maintenance")) { + t.Fatal(err) + } + } + + return testCtx + }). + Feature() + + f2 := features.New("maintenance-configuration"). + WithLabel("component", "ws-manager"). + WithLabel("type", "maintenance"). + Assess("should display a maintenance message when configured and not when disabled", func(testCtx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + kubeClient, err := cfg.NewClient() + if err != nil { + t.Fatal(err) + } + + defer func() { + err := configureMaintenanceMode(testCtx, nil, kubeClient) + if err != nil { + t.Error(err) + } + }() + + untilTime := time.Now().Add(1 * time.Hour) + err = configureMaintenanceMode(testCtx, &untilTime, kubeClient) + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithTimeout(testCtx, time.Duration(5*time.Minute)) + defer cancel() + + api := integration.NewComponentAPI(ctx, cfg.Namespace(), kubeconfig, cfg.Client()) + t.Cleanup(func() { + api.Done(t) + }) + + _, _, err = integration.LaunchWorkspaceDirectly(t, ctx, api, integration.WithRequestModifier(func(swr *wsmanapi.StartWorkspaceRequest) error { + swr.Spec.Initializer = &csapi.WorkspaceInitializer{ + Spec: &csapi.WorkspaceInitializer_Git{ + Git: &csapi.GitInitializer{ + RemoteUri: testRepo, + CheckoutLocation: testRepoName, + Config: &csapi.GitConfig{}, + }, + }, + } + swr.Spec.WorkspaceLocation = testRepoName + return nil + })) + if err == nil { + t.Fatalf("expected under maintenance error") + } else { + if !errors.Is(err, status.Error(codes.FailedPrecondition, "under maintenance")) { + t.Fatal(err) + } + } + + err = configureMaintenanceMode(testCtx, nil, kubeClient) + if err != nil { + t.Fatal(err) + } + + time.Sleep(1 * time.Second) + + _, stopWs, err := integration.LaunchWorkspaceDirectly(t, ctx, api, integration.WithRequestModifier(func(swr *wsmanapi.StartWorkspaceRequest) error { + swr.Spec.Initializer = &csapi.WorkspaceInitializer{ + Spec: &csapi.WorkspaceInitializer_Git{ + Git: &csapi.GitInitializer{ + RemoteUri: testRepo, + CheckoutLocation: testRepoName, + Config: &csapi.GitConfig{}, + }, + }, + } + swr.Spec.WorkspaceLocation = testRepoName + return nil + })) + if err != nil { + t.Fatal(err) + } + + if err := stopWorkspace(t, cfg, stopWs); err != nil { + t.Errorf("cannot stop workspace: %q", err) + } + + return testCtx + }). + Feature() + + testEnv.Test(t, f1, f2) +} + +func configureMaintenanceMode(ctx context.Context, untilTime *time.Time, kubeClient klient.Client) error { + cmap, err := maintenanceConfigmap(untilTime) + if err != nil { + return err + } + + err = kubeClient.Resources().Create(ctx, cmap) + if err != nil { + if apierrors.IsAlreadyExists(err) { + err = kubeClient.Resources().Update(ctx, cmap) + if err != nil { + return err + } + } + + return err + } + + return nil +} + +func maintenanceConfigmap(untilTime *time.Time) (*corev1.ConfigMap, error) { + mcfg := config.MaintenanceConfig{} + if untilTime != nil { + mcfg.EnabledUntil = untilTime + } + + data, err := json.Marshal(mcfg) + if err != nil { + return nil, err + } + + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ws-manager-mk2-maintenance-mode", + Namespace: "default", + }, + Data: map[string]string{ + "config.json": string(data), + }, + }, nil +}