feat: add rbac for strict job queue control (#927)

implement RBAC for controlling job queue
give permission to ClusterAdmin role to make changes in the strict case
DET-9327

.circleci/devcluster/single-rbac.devcluster.yaml

@@ -25,6 +25,7 @@ stages:
           authz:
             type: rbac
             rbac_ui_enabled: true
+            strict_job_queue_control: true
         scim:
           enabled: true
           auth:

e2e_tests/tests/cluster/test_rbac_ntsc.py

@@ -182,7 +182,7 @@ def can_access_logs(creds: authentication.Credentials, ntsc_id: str) -> bool:
 
             created_id = launch_ntsc(
                 determined_test_session(creds[0]), workspaces[0].id, typ, experiment_id
-            )
+            ).id
 
             # user 0
             assert can_access_logs(
@@ -257,7 +257,7 @@ def can_access_logs(creds: authentication.Credentials, ntsc_id: str) -> bool:
             # test visibility
             created_id2 = launch_ntsc(
                 determined_test_session(creds[0]), workspaces[2].id, typ, experiment_id
-            )
+            ).id
 
             # none of the users should be able to get details
             for cred in [creds[1], creds[2]]:
@@ -325,7 +325,7 @@ def get_proxy(creds: authentication.Credentials, task_id: str) -> Optional[error
 
             created_id = launch_ntsc(
                 determined_test_session(creds[0]), workspaces[0].id, typ, experiment_id
-            )
+            ).id
 
             print(f"created {typ} {created_id}")
             wait_for_ntsc_state(
@@ -380,7 +380,7 @@ def test_tsb_listed() -> None:
                 ["--project_id", str(pid)],
             )
 
-            created_id = launch_ntsc(session, workspace.id, NTSC_Kind.tensorboard, experiment_id)
+            created_id = launch_ntsc(session, workspace.id, NTSC_Kind.tensorboard, experiment_id).id
 
             # list tensorboards and make sure it's included in the response.
             tsbs = bindings.get_GetTensorboards(session, workspaceId=workspace.id).tensorboards

e2e_tests/tests/job/test_rbac.py

@@ -4,11 +4,16 @@
 
 import tests.config as conf
 from determined.common import api
-from determined.common.api import bindings
+from determined.common.api import NTSC_Kind, bindings, errors
 from determined.common.api._util import all_ntsc
 from tests import api_utils
 from tests import experiment as exp
-from tests.cluster.test_rbac import create_workspaces_with_users, rbac_disabled
+from tests.cluster import test_rbac as rbac
+from tests.cluster.test_rbac import (
+    create_users_with_gloabl_roles,
+    create_workspaces_with_users,
+    rbac_disabled,
+)
 from tests.cluster.test_users import ADMIN_CREDENTIALS, det_run, logged_in_user
 
 
@@ -46,6 +51,43 @@ def test_job_global_perm() -> None:
         assert str(experiment_id) in str(output)
 
 
+@pytest.mark.e2e_cpu_rbac
+@pytest.mark.skipif(
+    rbac.strict_q_control_disabled(),
+    reason="ee, rbac, " + "and strict q control are required for this test",
+)
+def test_job_strict_q_control() -> None:
+    [cadmin] = create_users_with_gloabl_roles([["ClusterAdmin"]])
+
+    with create_workspaces_with_users(
+        [
+            [
+                (0, ["Editor"]),
+            ],
+        ]
+    ) as (workspaces, creds):
+        session = api_utils.determined_test_session(creds[0])
+        r = api_utils.launch_ntsc(session, typ=NTSC_Kind.command, workspace_id=workspaces[0].id)
+
+        cases = [
+            rbac.PermCase(creds[0], errors.ForbiddenException),
+            rbac.PermCase(cadmin, None),
+        ]
+
+        def action(cred: api.authentication.Credentials) -> None:
+            session = api_utils.determined_test_session(cred)
+            bindings.post_UpdateJobQueue(
+                session,
+                body=bindings.v1UpdateJobQueueRequest(
+                    updates=[
+                        bindings.v1QueueControl(jobId=r.jobId, priority=3),
+                    ]
+                ),
+            )
+
+        rbac.run_permission_tests(action, cases)
+
+
 @pytest.mark.e2e_cpu_rbac
 @pytest.mark.skipif(rbac_disabled(), reason="ee rbac is required for this test")
 def test_job_filtering() -> None:

master/internal/job/authz_rbac.go

@@ -7,7 +7,9 @@ import (
 
 	"github.com/determined-ai/determined/master/internal/authz"
 	"github.com/determined-ai/determined/master/internal/command"
+	"github.com/determined-ai/determined/master/internal/config"
 	"github.com/determined-ai/determined/master/internal/db"
+	"github.com/determined-ai/determined/master/internal/rbac"
 	"github.com/determined-ai/determined/master/pkg/model"
 	"github.com/determined-ai/determined/proto/pkg/jobv1"
 	"github.com/determined-ai/determined/proto/pkg/rbacv1"
@@ -114,7 +116,11 @@ func (a *JobAuthZRBAC) FilterJobs(
 func (a *JobAuthZRBAC) CanControlJobQueue(
 	ctx context.Context, curUser *model.User,
 ) (permErr error, err error) {
-	return nil, nil
+	if !config.GetMasterConfig().Security.AuthZ.StrictJobQueueControl {
+		return nil, nil
+	}
+	return rbac.CheckForPermission(ctx, "job", curUser, nil,
+		rbacv1.PermissionType_PERMISSION_TYPE_CONTROL_STRICT_JOB_QUEUE)
 }
 
 func init() {

master/static/migrations/20230627154016_precanned-jobq-control-permissions.tx.down.sql

@@ -0,0 +1,3 @@
+DELETE FROM permission_assignments WHERE permission_id = 8101;
+
+DELETE FROM permissions WHERE id = 8101;

master/static/migrations/20230627154016_precanned-jobq-control-permissions.tx.up.sql

@@ -0,0 +1,24 @@
+--  // Ability to control strict job queue.
+--  PERMISSION_TYPE_CONTROL_STRICT_JOB_QUEUE = 8101;
+
+INSERT into permissions(id, name, global_only) VALUES
+    (8101, 'control strict job queue', true);
+
+
+-- determined> select * from roles;
+-- +----+---------------------+----------------------------+
+-- | id | role_name           | created_at                 |
+-- |----+---------------------+----------------------------|
+-- | 1  | ClusterAdmin        | 2023-05-30 16:20:54.825443 |
+-- | 2  | WorkspaceAdmin      | 2023-05-30 16:20:54.825443 |
+-- | 3  | WorkspaceCreator    | 2023-05-30 16:20:54.825443 |
+-- | 4  | Viewer              | 2023-05-30 16:20:54.825443 |
+-- | 5  | Editor              | 2023-05-30 16:20:54.825443 |
+-- | 6  | ModelRegistryViewer | 2023-05-30 16:20:55.136146 |
+-- +----+---------------------+----------------------------+
+-- SELECT 6
+
+INSERT INTO permission_assignments (permission_id, role_id)
+SELECT 8101, roles.id
+FROM roles
+WHERE roles.role_name IN ('ClusterAdmin');