Add support for NiFi 1.15

CHANGELOG.md

@@ -4,8 +4,14 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+### Added
+- Support for 1.15.0 ([#125])
+- Sensitive property key is setable via a secret ([#125])
+
 ### Changed
 
+- Removed support for 1.13.2 ([#125])
+- Added/removed some default config settings that changed from 1.13 to 1.15 ([#125])
 - `operator-rs` `0.3.0` → `0.4.0` ([#101]).
 - `stackable-zookeeper-crd`: `0.4.1` → `0.5.0` ([#101]).
 - Adapted pod image and container command to docker image ([#101]).

deploy/crd/nificluster.crd.yaml

@@ -52,6 +52,10 @@ spec:
                           minimum: 0.0
                           nullable: true
                           type: integer
+                        sensitivePropertyKeySecret:
+                          type: string
+                      required:
+                        - sensitivePropertyKeySecret
                       type: object
                     configOverrides:
                       additionalProperties:
@@ -91,6 +95,10 @@ spec:
                                 minimum: 0.0
                                 nullable: true
                                 type: integer
+                              sensitivePropertyKeySecret:
+                                type: string
+                            required:
+                              - sensitivePropertyKeySecret
                             type: object
                           configOverrides:
                             additionalProperties:
@@ -147,7 +155,7 @@ spec:
                   type: object
                 version:
                   enum:
-                    - 1.13.2
+                    - 1.15.0
                   type: string
                 zookeeperReference:
                   description: Contains all necessary information identify a Stackable managed ZooKeeper ensemble and build a connection string for it. The main purpose for this struct is for other operators that need to reference a ZooKeeper ensemble to use in their CRDs. This has the benefit of keeping references to Zookeeper ensembles consistent throughout the entire stack.
@@ -224,12 +232,12 @@ spec:
                   properties:
                     current:
                       enum:
-                        - 1.13.2
+                        - 1.15.0
                       nullable: true
                       type: string
                     target:
                       enum:
-                        - 1.13.2
+                        - 1.15.0
                       nullable: true
                       type: string
                   type: object

docs/modules/ROOT/pages/usage.adoc

@@ -4,7 +4,7 @@ After installation, the CRD for this operator must be created:
 
     kubectl apply -f /etc/stackable/nifi-operator/crd/nificluster.crd.yaml
 
-To create a single node Apache NiFi (v1.13.2) cluster with Prometheus metrics exposed on port 8428 and the WebUi on port 10000. As of now only one instance per node is supported:
+To create a single node Apache NiFi (v1.15.0) cluster with Prometheus metrics exposed on port 8428 and the WebUi on port 10000. As of now only one instance per node is supported:
 
 
     cat <<EOF | kubectl apply -f -
@@ -14,7 +14,7 @@ To create a single node Apache NiFi (v1.13.2) cluster with Prometheus metrics ex
       name: simple-nificluster
     spec:
       metricsPort: 8428
-      version: "1.13.2"
+      version: "1.15.0"
       zookeeperReference:
         name: simple
         namespace: default
@@ -30,5 +30,15 @@ To create a single node Apache NiFi (v1.13.2) cluster with Prometheus metrics ex
               httpPort: 10000
               protocolPort: 10443
               loadBalancePort: 6342
+              sensitivePropertyKeySecret: nifi-sensitive-property-key
     EOF
 
+
+    cat <<EOF | kubectl apply -f -
+    apiVersion: v1alpha1
+    kind: Secret
+    metadata:
+      name: nifi-sensitive-property-key
+    stringData:
+      nifiSensitivePropsKey: mYsUp3rS3cr3tk3y
+    EOF

examples/sensitive-property-key-secret.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: nifi-sensitive-property-key
+stringData:
+  nifiSensitivePropsKey: mYsUp3rS3cr3tk3y
+

examples/simple-nificluster.yaml

@@ -4,7 +4,7 @@ metadata:
   name: simple
 spec:
   metricsPort: 8428
-  version: "1.13.2"
+  version: "1.15.0"
   zookeeperReference:
     name: simple
     namespace: default
@@ -19,4 +19,6 @@ spec:
           httpPort: 10000
           protocolPort: 10443
           loadBalancePort: 6342
+          sensitivePropertyKeySecret: nifi-sensitive-property-key
+        replicas: 2
 

rust/crd/src/lib.rs

@@ -21,6 +21,8 @@ pub const NIFI_CLUSTER_NODE_PROTOCOL_PORT: &str = "nifi.cluster.node.protocol.po
 pub const NIFI_CLUSTER_LOAD_BALANCE_PORT: &str = "nifi.cluster.load.balance.port";
 pub const NIFI_CLUSTER_METRICS_PORT: &str = "metricsPort";
 
+pub const NIFI_SENSITIVE_PROPS_KEY: &str = "NIFI_SENSITIVE_PROPS_KEY";
+
 #[derive(Clone, CustomResource, Debug, Deserialize, JsonSchema, Serialize)]
 #[kube(
     group = "nifi.stackable.tech",
@@ -65,16 +67,9 @@ impl Status<NifiStatus> for NifiCluster {
     strum_macros::EnumString,
 )]
 pub enum NifiVersion {
-    #[serde(rename = "1.13.2")]
-    #[strum(serialize = "1.13.2")]
-    v1_13_2,
-
-    // TODO: NiFi 1.14 does not work with this operator yet <https://github.com/stackabletech/nifi-operator/issues/82>
-    //    Therefore we skip using it in the CRD for now. Should be uncommented as soon as #82 is fixed.
-    //#[serde(rename = "1.14.0")]
-    #[serde(skip)]
-    #[strum(serialize = "1.14.0")]
-    v1_14_0,
+    #[serde(rename = "1.15.0")]
+    #[strum(serialize = "1.15.0")]
+    v1_15_0,
 }
 
 impl Versioning for NifiVersion {
@@ -143,6 +138,7 @@ pub struct NifiConfig {
     pub http_port: Option<u16>,
     pub protocol_port: Option<u16>,
     pub load_balance_port: Option<u16>,
+    pub sensitive_property_key_secret: String,
 }
 
 impl Configuration for NifiConfig {
@@ -160,6 +156,10 @@ impl Configuration for NifiConfig {
                 Some(metrics_port.to_string()),
             );
         }
+        result.insert(
+            NIFI_SENSITIVE_PROPS_KEY.to_string(),
+            Some(self.sensitive_property_key_secret.to_string()),
+        );
         Ok(result)
     }
 
@@ -214,23 +214,14 @@ mod tests {
     #[test]
     fn test_zookeeper_version_versioning() {
         assert_eq!(
-            NifiVersion::v1_13_2.versioning_state(&NifiVersion::v1_14_0),
-            VersioningState::ValidUpgrade
-        );
-        assert_eq!(
-            NifiVersion::v1_14_0.versioning_state(&NifiVersion::v1_13_2),
-            VersioningState::ValidDowngrade
-        );
-        assert_eq!(
-            NifiVersion::v1_13_2.versioning_state(&NifiVersion::v1_13_2),
+            NifiVersion::v1_15_0.versioning_state(&NifiVersion::v1_15_0),
             VersioningState::NoOp
         );
     }
 
     #[test]
     fn test_version_conversion() {
-        NifiVersion::from_str("1.13.2").unwrap();
-        NifiVersion::from_str("1.14.0").unwrap();
+        NifiVersion::from_str("1.15.0").unwrap();
         NifiVersion::from_str("1.2.3").unwrap_err();
     }
 }

rust/operator/src/config.rs

@@ -25,6 +25,8 @@ pub fn build_bootstrap_conf() -> String {
     bootstrap.insert("java", "java".to_string());
     // Username to use when running NiFi. This value will be ignored on Windows.
     bootstrap.insert("run.as", "".to_string());
+    // Preserve shell environment while runnning as "run.as" user
+    bootstrap.insert("preserve.environment", "false".to_string());
     // Configure where NiFi's lib and conf directories live
     bootstrap.insert("lib.dir", "./lib".to_string());
     bootstrap.insert("conf.dir", "./conf".to_string());
@@ -213,6 +215,22 @@ pub fn build_nifi_properties(
         ";LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE".to_string(),
     );
 
+    // Repository Encryption properties override individual repository implementation properties
+    properties.insert(
+        "nifi.repository.encryption.protocol.version",
+        "".to_string(),
+    );
+    properties.insert("nifi.repository.encryption.key.id", "".to_string());
+    properties.insert("nifi.repository.encryption.key.provider", "".to_string());
+    properties.insert(
+        "nifi.repository.encryption.key.provider.keystore.location",
+        "".to_string(),
+    );
+    properties.insert(
+        "nifi.repository.encryption.key.provider.keystore.password",
+        "".to_string(),
+    );
+
     // FlowFile Repository
     properties.insert(
         "nifi.flowfile.repository.implementation",
@@ -231,16 +249,6 @@ pub fn build_nifi_properties(
         "20 secs".to_string(),
     );
     properties.insert("nifi.flowfile.repository.always.sync", "false".to_string());
-    properties.insert(
-        "nifi.flowfile.repository.encryption.key.provider.implementation",
-        "".to_string(),
-    );
-    properties.insert(
-        "nifi.flowfile.repository.encryption.key.provider.location",
-        "".to_string(),
-    );
-    properties.insert("nifi.flowfile.repository.encryption.key.id", "".to_string());
-    properties.insert("nifi.flowfile.repository.encryption.key", "".to_string());
     properties.insert(
         "nifi.flowfile.repository.retain.orphaned.flowfiles",
         "true".to_string(),
@@ -279,35 +287,12 @@ pub fn build_nifi_properties(
         "nifi.content.viewer.url",
         "../nifi-content-viewer/".to_string(),
     );
-    properties.insert(
-        "nifi.content.repository.encryption.key.provider.implementation",
-        "".to_string(),
-    );
-    properties.insert(
-        "nifi.content.repository.encryption.key.provider.location",
-        "".to_string(),
-    );
-    properties.insert("nifi.content.repository.encryption.key.id", "".to_string());
-    properties.insert("nifi.content.repository.encryption.key", "".to_string());
 
     // Provenance Repository Properties
     properties.insert(
         "nifi.provenance.repository.implementation",
         "org.apache.nifi.provenance.WriteAheadProvenanceRepository".to_string(),
     );
-    properties.insert(
-        "nifi.provenance.repository.encryption.key.provider.implementation",
-        "".to_string(),
-    );
-    properties.insert(
-        "nifi.provenance.repository.encryption.key.provider.location",
-        "".to_string(),
-    );
-    properties.insert(
-        "nifi.provenance.repository.encryption.key.id",
-        "".to_string(),
-    );
-    properties.insert("nifi.provenance.repository.encryption.key", "".to_string());
 
     // Persistent Provenance Repository Properties
     properties.insert(
@@ -388,6 +373,20 @@ pub fn build_nifi_properties(
         "1 min".to_string(),
     );
 
+    // QuestDB Status History Repository Properties
+    properties.insert(
+        "nifi.status.repository.questdb.persist.node.days",
+        "14".to_string(),
+    );
+    properties.insert(
+        "nifi.status.repository.questdb.persist.component.days",
+        "3".to_string(),
+    );
+    properties.insert(
+        "nifi.status.repository.questdb.persist.location",
+        "./status_repository".to_string(),
+    );
+
     // Site to Site properties
     properties.insert("nifi.remote.input.host", node_name.to_string());
     properties.insert("nifi.remote.input.secure", "false".to_string());
@@ -431,18 +430,29 @@ pub fn build_nifi_properties(
     properties.insert("nifi.web.proxy.host", "".to_string());
     properties.insert("nifi.web.max.content.size", "".to_string());
     properties.insert("nifi.web.max.requests.per.second", "30000".to_string());
+    properties.insert(
+        "nifi.web.max.access.token.requests.per.second",
+        "25".to_string(),
+    );
+    properties.insert("nifi.web.request.timeout", "60 secs".to_string());
+    properties.insert("nifi.web.request.ip.whitelist", "".to_string());
     properties.insert("nifi.web.should.send.server.version", "true".to_string());
 
+    // Include or Exclude TLS Cipher Suites for HTTPS
+    properties.insert("nifi.web.https.ciphersuites.include", "".to_string());
+    properties.insert("nifi.web.https.ciphersuites.exclude", "".to_string());
+
     // security properties
-    properties.insert("nifi.sensitive.props.key", "".to_string());
+    properties.insert("nifi.sensitive.props.key", "".to_string()); // this property is later set from a secret
     properties.insert("nifi.sensitive.props.key.protected", "".to_string());
     properties.insert(
         "nifi.sensitive.props.algorithm",
-        "PBEWITHMD5AND256BITAES-CBC-OPENSSL".to_string(),
+        "NIFI_PBKDF2_AES_GCM_256".to_string(),
     );
-    properties.insert("nifi.sensitive.props.provider", "BC".to_string());
     properties.insert("nifi.sensitive.props.additional.keys", "".to_string());
 
+    properties.insert("nifi.security.autoreload.enabled", "false".to_string());
+    properties.insert("nifi.security.autoreload.interval", "10 secs".to_string());
     properties.insert("nifi.security.keystore", "".to_string());
     properties.insert("nifi.security.keystoreType", "".to_string());
     properties.insert("nifi.security.keystorePasswd", "".to_string());
@@ -459,6 +469,10 @@ pub fn build_nifi_properties(
         "false".to_string(),
     );
     properties.insert("nifi.security.user.login.identity.provider", "".to_string());
+    properties.insert(
+        "nifi.security.user.jws.key.rotation.period",
+        "PT1H".to_string(),
+    );
     properties.insert("nifi.security.ocsp.responder.url", "".to_string());
     properties.insert("nifi.security.ocsp.responder.certificate", "".to_string());
 
@@ -646,6 +660,7 @@ pub fn build_nifi_properties(
     properties.insert("nifi.zookeeper.security.truststore", "".to_string());
     properties.insert("nifi.zookeeper.security.truststoreType", "".to_string());
     properties.insert("nifi.zookeeper.security.truststorePasswd", "".to_string());
+    properties.insert("nifi.zookeeper.jute.maxbuffer", "".to_string());
 
     // Zookeeper properties for the authentication scheme used when creating acls on znodes used for cluster management
     // Values supported for nifi.zookeeper.auth.type are "default", which will apply world/anyone rights on znodes
@@ -699,6 +714,36 @@ pub fn build_nifi_properties(
         ".90".to_string(),
     );
 
+    // runtime monitoring properties
+    properties.insert("nifi.monitor.long.running.task.schedule", "".to_string());
+    properties.insert("nifi.monitor.long.running.task.threshold", "".to_string());
+
+    // Create automatic diagnostics when stopping/restarting NiFi.
+
+    // Enable automatic diagnostic at shutdown.
+    properties.insert("nifi.diagnostics.on.shutdown.enabled", "false".to_string());
+
+    // Include verbose diagnostic information.
+    properties.insert("nifi.diagnostics.on.shutdown.verbose", "false".to_string());
+
+    // The location of the diagnostics folder.
+    properties.insert(
+        "nifi.diagnostics.on.shutdown.directory",
+        "./diagnostics".to_string(),
+    );
+
+    // The maximum number of files permitted in the directory. If the limit is exceeded, the oldest files are deleted.
+    properties.insert(
+        "nifi.diagnostics.on.shutdown.max.filecount",
+        "10".to_string(),
+    );
+
+    // The diagnostics folder's maximum permitted size in bytes. If the limit is exceeded, the oldest files are deleted.
+    properties.insert(
+        "nifi.diagnostics.on.shutdown.max.directory.size",
+        "10 MB".to_string(),
+    );
+
     format_properties(properties)
 }