feat: improve users block and allow custom user ACLs (#14)

devshawn · Jun 28, 2020 · 30ab978 · 30ab978
1 parent 25146a7
commit 30ab978
Show file tree

Hide file tree

Showing 18 changed files with 471 additions and 7 deletions.
diff --git a/codecov.yml b/codecov.yml
@@ -0,0 +1,5 @@
+coverage:
+  status:
+    patch:
+      default:
+        enabled: no
diff --git a/docs/specification.md b/docs/specification.md
@@ -9,7 +9,9 @@ The desired state file consists of:
 - **Settings** [Optional]: Specific settings for configuring `kafka-gitops`.
 - **Topics** [Optional]: Topics and topic configuration definitions.
 - **Services** [Optional]: Service definitions for generating ACLs.
+- **Users** [Optional]: User definitions for generating ACLs.
 - **Custom Service ACLs** [Optional]: Definitions for custom, non-generated ACLs.
+- **Custom User ACLs** [Optional]: Definitions for custom, non-generated ACLs.
 
 ## Settings
 
@@ -114,6 +116,31 @@ services:
 
 Under the cover, `kafka-gitops` generates ACLs based on these definitions.
 
+## Users
+
+**Synopsis**: Define the users that will utilize your Kafka cluster. These user definitions allow `kafka-gitops` to generate ACLs for you. Yay!
+
+?> **NOTE**: If using Confluent Cloud, users are service accounts that are prefixed with `user-`.
+
+```yaml
+users:
+  my-user:
+    principal: User:my-user
+    roles:
+      - writer
+      - reader
+      - operator
+```
+
+Currently, three predefined roles exist:
+
+- **writer**: access to write to all topics
+- **reader**: access to read all topics using any consumer group
+- **operator**: access to view topics, topic configs, and to read topics and move their offsets
+
+Outside of these very simple roles, you can define custom ACLs per-user by using the `customUserAcls` block.
+
+
 ## Custom Service ACLs
 
 **Synopsis**: Define custom ACLs for a specific service. 
@@ -130,15 +157,38 @@ customServiceAcls:
       type: TOPIC
       pattern: PREFIXED
       host: "*"
-      principal: 
+      principal: User:my-test-service
       operation: READ
       permission: ALLOW
     read-all-service:
       name: service.
       type: TOPIC
       pattern: PREFIXED
       host: "*"
-      principal: 
+      principal: User:my-test-service
+      operation: READ
+      permission: ALLOW
+```
+
+## Custom User ACLs
+
+**Synopsis**: Define custom ACLs for a specific user. 
+
+For example, if a specific user needs to produce to all topics prefixed with `kafka.` and `service.`, you may not want to define them all in your desired state file. 
+
+If you have a user `my-test-user` defined, you can define custom ACLs as so:
+
+```yaml
+customUserAcls:
+  my-test-user:
+    read-all-kafka:
+      name: kafka.
+      type: TOPIC
+      pattern: PREFIXED
+      host: "*"
       operation: READ
       permission: ALLOW
 ```
+
+?> **NOTE**: The `principal` field can be left out here and it will be inherited from the user definition.
+
diff --git a/src/main/java/com/devshawn/kafka/gitops/StateManager.java b/src/main/java/com/devshawn/kafka/gitops/StateManager.java
@@ -27,7 +27,11 @@
 import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
 import org.slf4j.LoggerFactory;
 
-import java.util.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Optional;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 
@@ -131,6 +135,7 @@ private DesiredState getDesiredState() {
             generateConfluentCloudUserAcls(desiredState, desiredStateFile);
         } else {
             generateServiceAcls(desiredState, desiredStateFile);
+            generateUserAcls(desiredState, desiredStateFile);
         }
 
         return desiredState.build();
@@ -173,6 +178,15 @@ private void generateConfluentCloudUserAcls(DesiredState.Builder desiredState, D
                 List<AclDetails.Builder> acls = roleService.getAcls(role, String.format("User:%s", serviceAccountId));
                 acls.forEach(acl -> desiredState.putAcls(String.format("%s-%s", name, index.getAndSet(index.get() + 1)), acl.build()));
             });
+
+            if (desiredStateFile.getCustomUserAcls().containsKey(name)) {
+                Map<String, CustomAclDetails> customAcls = desiredStateFile.getCustomUserAcls().get(name);
+                customAcls.forEach((aclName, customAcl) -> {
+                    AclDetails.Builder aclDetails = AclDetails.fromCustomAclDetails(customAcl);
+                    aclDetails.setPrincipal(String.format("User:%s", serviceAccountId));
+                    desiredState.putAcls(String.format("%s-%s", name, index.getAndSet(index.get() + 1)), aclDetails.build());
+                });
+            }
         });
     }
 
@@ -188,7 +202,29 @@ private void generateServiceAcls(DesiredState.Builder desiredState, DesiredState
                 customAcls.forEach((aclName, customAcl) -> {
                     AclDetails.Builder aclDetails = AclDetails.fromCustomAclDetails(customAcl);
                     aclDetails.setPrincipal(customAcl.getPrincipal().orElseThrow(() ->
-                            new MissingConfigurationException(String.format("Missing principal for custom ACL %s", aclName))));
+                            new MissingConfigurationException(String.format("Missing principal for custom service ACL %s", aclName))));
+                    desiredState.putAcls(String.format("%s-%s", name, index.getAndSet(index.get() + 1)), aclDetails.build());
+                });
+            }
+        });
+    }
+
+    private void generateUserAcls(DesiredState.Builder desiredState, DesiredStateFile desiredStateFile) {
+        desiredStateFile.getUsers().forEach((name, user) -> {
+            AtomicReference<Integer> index = new AtomicReference<>(0);
+            String userPrincipal = user.getPrincipal()
+                    .orElseThrow(() -> new MissingConfigurationException(String.format("Missing principal for user %s", name)));
+
+            user.getRoles().forEach(role -> {
+                List<AclDetails.Builder> acls = roleService.getAcls(role, userPrincipal);
+                acls.forEach(acl -> desiredState.putAcls(String.format("%s-%s", name, index.getAndSet(index.get() + 1)), acl.build()));
+            });
+
+            if (desiredStateFile.getCustomUserAcls().containsKey(name)) {
+                Map<String, CustomAclDetails> customAcls = desiredStateFile.getCustomUserAcls().get(name);
+                customAcls.forEach((aclName, customAcl) -> {
+                    AclDetails.Builder aclDetails = AclDetails.fromCustomAclDetails(customAcl);
+                    aclDetails.setPrincipal(customAcl.getPrincipal().orElse(userPrincipal));
                     desiredState.putAcls(String.format("%s-%s", name, index.getAndSet(index.get() + 1)), aclDetails.build());
                 });
             }

diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredStateFile.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/DesiredStateFile.java
@@ -22,6 +22,8 @@ public interface DesiredStateFile {
 
     Map<String, Map<String, CustomAclDetails>> getCustomServiceAcls();
 
+    Map<String, Map<String, CustomAclDetails>> getCustomUserAcls();
+
     class Builder extends DesiredStateFile_Builder {
     }
 }
diff --git a/src/main/java/com/devshawn/kafka/gitops/domain/state/UserDetails.java b/src/main/java/com/devshawn/kafka/gitops/domain/state/UserDetails.java
@@ -4,11 +4,14 @@
 import org.inferred.freebuilder.FreeBuilder;
 
 import java.util.List;
+import java.util.Optional;
 
 @FreeBuilder
 @JsonDeserialize(builder = UserDetails.Builder.class)
 public interface UserDetails {
 
+    Optional<String> getPrincipal();
+
     List<String> getRoles();
 
     class Builder extends UserDetails_Builder {

diff --git a/src/test/groovy/com/devshawn/kafka/gitops/ApplyCommandIntegrationSpec.groovy b/src/test/groovy/com/devshawn/kafka/gitops/ApplyCommandIntegrationSpec.groovy
@@ -48,7 +48,10 @@ class ApplyCommandIntegrationSpec extends Specification {
         planFile << [
                 "simple",
                 "application-service",
-                "multi-file"
+                "multi-file",
+                "simple-users",
+                "custom-service-acls",
+                "custom-user-acls"
         ]
     }
 

diff --git a/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy b/src/test/groovy/com/devshawn/kafka/gitops/PlanCommandIntegrationSpec.groovy
@@ -55,7 +55,10 @@ class PlanCommandIntegrationSpec extends Specification {
                 "kafka-connect-service",
                 "kafka-streams-service",
                 "topics-and-services",
-                "multi-file"
+                "multi-file",
+                "simple-users",
+                "custom-service-acls",
+                "custom-user-acls"
         ]
     }
 
@@ -123,7 +126,8 @@ class PlanCommandIntegrationSpec extends Specification {
                 "invalid-missing-principal",
                 "invalid-topic",
                 "unrecognized-property",
-                "invalid-format"
+                "invalid-format",
+                "invalid-missing-user-principal"
         ]
     }
 

diff --git a/src/test/resources/plans/custom-service-acls-apply-output.txt b/src/test/resources/plans/custom-service-acls-apply-output.txt
@@ -0,0 +1,17 @@
+Executing apply...
+
+Applying: [CREATE]
+
++ [ACL] test-service-0
+	 + resource_name: kafka.
+	 + resource_type: TOPIC
+	 + resource_pattern: PREFIXED
+	 + resource_principal: User:test
+	 + host: *
+	 + operation: READ
+	 + permission: ALLOW
+
+
+Successfully applied.
+
+[SUCCESS] Apply complete! Resources: 1 created, 0 updated, 0 deleted.
diff --git a/src/test/resources/plans/custom-service-acls-plan.json b/src/test/resources/plans/custom-service-acls-plan.json
@@ -0,0 +1,18 @@
+{
+    "topicPlans": [],
+    "aclPlans": [
+        {
+            "name": "test-service-0",
+            "aclDetails": {
+                "name": "kafka.",
+                "type": "TOPIC",
+                "pattern": "PREFIXED",
+                "principal": "User:test",
+                "host": "*",
+                "operation": "READ",
+                "permission": "ALLOW"
+            },
+            "action": "ADD"
+        }
+    ]
+}
diff --git a/src/test/resources/plans/custom-service-acls.yaml b/src/test/resources/plans/custom-service-acls.yaml
@@ -0,0 +1,15 @@
+services:
+  test-service:
+    type: application
+    principal: User:test
+
+customServiceAcls:
+  test-service:
+    read-all-kafka:
+      name: kafka.
+      type: TOPIC
+      pattern: PREFIXED
+      host: "*"
+      principal: User:test
+      operation: READ
+      permission: ALLOW
diff --git a/src/test/resources/plans/custom-user-acls-apply-output.txt b/src/test/resources/plans/custom-user-acls-apply-output.txt
@@ -0,0 +1,17 @@
+Executing apply...
+
+Applying: [CREATE]
+
++ [ACL] test-user-0
+	 + resource_name: kafka.
+	 + resource_type: TOPIC
+	 + resource_pattern: PREFIXED
+	 + resource_principal: User:test
+	 + host: *
+	 + operation: READ
+	 + permission: ALLOW
+
+
+Successfully applied.
+
+[SUCCESS] Apply complete! Resources: 1 created, 0 updated, 0 deleted.
diff --git a/src/test/resources/plans/custom-user-acls-plan.json b/src/test/resources/plans/custom-user-acls-plan.json
@@ -0,0 +1,18 @@
+{
+    "topicPlans": [],
+    "aclPlans": [
+        {
+            "name": "test-user-0",
+            "aclDetails": {
+                "name": "kafka.",
+                "type": "TOPIC",
+                "pattern": "PREFIXED",
+                "principal": "User:test",
+                "host": "*",
+                "operation": "READ",
+                "permission": "ALLOW"
+            },
+            "action": "ADD"
+        }
+    ]
+}
diff --git a/src/test/resources/plans/custom-user-acls.yaml b/src/test/resources/plans/custom-user-acls.yaml
@@ -0,0 +1,13 @@
+users:
+  test-user:
+    principal: User:test
+
+customUserAcls:
+  test-user:
+    read-all-kafka:
+      name: kafka.
+      type: TOPIC
+      pattern: PREFIXED
+      host: "*"
+      operation: READ
+      permission: ALLOW
diff --git a/src/test/resources/plans/invalid-missing-user-principal-output.txt b/src/test/resources/plans/invalid-missing-user-principal-output.txt
@@ -0,0 +1,5 @@
+Generating execution plan...
+
+[ERROR] Missing required configuration: Missing principal for user test-user
+
+[ERROR] An error has occurred during the planning process. No plan was created.
diff --git a/src/test/resources/plans/invalid-missing-user-principal.yaml b/src/test/resources/plans/invalid-missing-user-principal.yaml
@@ -0,0 +1,4 @@
+users:
+  test-user:
+    roles:
+      - reader