[SDN-1364] Update CNO for OVN acl audit logging

Changes for [SDN-1364](https://issues.redhat.com/browse/SDN-1364) See [Enhancement](openshift/enhancements#617) for details on the changes Changed `ovn-kubernetes.go` to digest new api fields Added new sidecar container to the ovnkube-node daemonset that handles tailing the logs, and log rotation. add resource limits to the new sidecar container Signed-off-by: Andrew Stoycos <[email protected]>
astoycos · Mar 29, 2021 · ec3b310 · ec3b310
1 parent 47ac328
commit ec3b310
Show file tree

Hide file tree

Showing 27 changed files with 1,764 additions and 293 deletions.
diff --git a/README.md b/README.md
@@ -267,6 +267,24 @@ spec:
       ipsecConfig: {}
 ```
 
+#### Configuring Network Policy audit logging with OVNKubernetes 
+
+OVNKubernetes supports audit logging of network policy traffic events.  Add the following to the `spec:` section of the operator config: 
+
+```yaml
+spec:
+  defaultNetwork:
+    type: OVNKubernetes
+    ovnKubernetesConfig: 
+      policyAuditingConfig:
+        maxFileSize: 1
+        rateLimit: 5
+        destination: libc
+        syslogFacility: local0
+```
+
+To understand more about each field, and to see the default values check out the [Openshift api definition](https://github.com/openshift/api/blob/master/operator/v1/types_network.go#L397)
+
 ### Configuring Kuryr-Kubernetes
 Kuryr-Kubernetes is a CNI plugin that uses OpenStack Neutron to network OpenShift Pods, and OpenStack Octavia to create load balancers for Services. In general it is useful when OpenShift is running on an OpenStack cluster, as you can use the same SDN (OpenStack Neutron) to provide networking for both the VMs OpenShift is running on, and the Pods created by OpenShift. In such case avoidance of double encapsulation gives you two advantages: improved performace (in terms of both latency and throughput) and lower complexity of the networking architecture.
 

diff --git a/bindata/network/ovn-kubernetes/ovnkube-master.yaml b/bindata/network/ovn-kubernetes/ovnkube-master.yaml
@@ -835,7 +835,8 @@ spec:
             --nbctl-daemon-mode \
             --nb-cert-common-name "{{.OVN_CERT_CN}}" \
             --enable-multicast \
-            --disable-snat-multiple-gws
+            --disable-snat-multiple-gws \
+            --acl-logging-rate-limit "{{.OVNPolicyAuditRateLimit}}"
         lifecycle:
           preStop:
             exec:

diff --git a/bindata/network/ovn-kubernetes/ovnkube-node.yaml b/bindata/network/ovn-kubernetes/ovnkube-node.yaml
@@ -48,12 +48,18 @@ spec:
             set -o allexport
             source "/env/${K8S_NODE}"
             set +o allexport
-          fi
+          fi  
+          
           echo "$(date -Iseconds) - starting ovn-controller"
           exec ovn-controller unix:/var/run/openvswitch/db.sock -vfile:off \
             --no-chdir --pidfile=/var/run/ovn/ovn-controller.pid \
+            --syslog-method="{{.OVNPolicyAuditDestination}}" \
+            --log-file=/var/log/ovn/acl-audit-log.log \
+            -vFACILITY:"{{.OVNPolicyAuditSyslogFacility}}" \
             -p /ovn-cert/tls.key -c /ovn-cert/tls.crt -C /ovn-ca/ca-bundle.crt \
-            -vconsole:"${OVN_LOG_LEVEL}"
+            -vconsole:"${OVN_LOG_LEVEL}" -vconsole:"acl_log:off" \
+            -vsyslog:"acl_log:${OVN_LOG_LEVEL}" \
+            -vfile:"acl_log:info"
         securityContext:
           privileged: true
         env:
@@ -80,12 +86,59 @@ spec:
           name: ovn-cert
         - mountPath: /ovn-ca
           name: ovn-ca
+        - mountPath: /var/log/ovn
+          name: node-log
+        - mountPath: /dev/log
+          name: log-socket
         terminationMessagePolicy: FallbackToLogsOnError
         resources:
           requests:
             cpu: 10m
             memory: 300Mi
+      - name: ovn-acl-logging
+        image: "{{.OvnImage}}"
+        command:
+        - /bin/bash
+        - -c
+        - |
+          set -euo pipefail
+
+          # Rotate audit log files when then get to max size (in bytes)
+          MAXFILESIZE=$(( "{{.OVNPolicyAuditMaxFileSize}}"*1000000 )) 
+          LOGFILE=/var/log/ovn/acl-audit-log.log
+          CONTROLLERPID=$(cat /run/ovn/ovn-controller.pid)
 
+          # Redirect err to null so no messages are shown upon rotation
+          tail -F ${LOGFILE} 2> /dev/null &
+
+          while true
+          do
+            # Make sure ovn-controller's logfile exists, and get current size in bytes 
+            if [ -f "$LOGFILE" ]; then 
+              file_size=`du -b ${LOGFILE} | tr -s '\t' ' ' | cut -d' ' -f1`
+            else 
+              ovs-appctl -t /var/run/ovn/ovn-controller.${CONTROLLERPID}.ctl vlog/reopen
+              file_size=`du -b ${LOGFILE} | tr -s '\t' ' ' | cut -d' ' -f1`
+            fi 
+            
+            if [ $file_size -gt $MAXFILESIZE ];then
+              echo "Rotating OVN ACL Log File"
+              timestamp=`date '+%Y-%m-%dT%H-%M-%S'`
+              mv ${LOGFILE} /var/log/ovn/acl-audit-log.$timestamp.log
+              ovs-appctl -t /run/ovn/ovn-controller.${CONTROLLERPID}.ctl vlog/reopen
+              CONTROLLERPID=$(cat /run/ovn/ovn-controller.pid)
+            fi
+          done
+        resources:
+          requests:
+            cpu: 10m
+            memory: 20Mi
+        terminationMessagePolicy: FallbackToLogsOnError
+        volumeMounts:
+        - mountPath: /var/log/ovn
+          name: node-log
+        - mountPath: /run/ovn/
+          name: run-ovn
       - name: kube-rbac-proxy
         image: {{.KubeRBACProxyImage}}
         command:
@@ -138,7 +191,6 @@ spec:
         - name: ovn-node-metrics-cert
           mountPath: /etc/pki/tls/metrics-cert
           readOnly: True
-
       # ovnkube-node: does node-level bookkeeping and configuration
       - name: ovnkube-node
         image: "{{.OvnImage}}"
@@ -306,6 +358,13 @@ spec:
       - name: run-ovn
         hostPath:
           path: /var/run/ovn
+      # Used for placement of ACL audit logs 
+      - name: node-log
+        hostPath: 
+          path: /var/log/ovn
+      - name: log-socket
+        hostPath: 
+          path: /dev/log
       # For CNI server
       - name: host-run-ovn-kubernetes
         hostPath:

diff --git a/go.mod b/go.mod
@@ -17,7 +17,7 @@ require (
 	github.com/mitchellh/copystructure v1.0.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.1 // indirect
 	github.com/onsi/gomega v1.10.2
-	github.com/openshift/api v0.0.0-20210304195353-475eac5412b1
+	github.com/openshift/api v0.0.0-20210317213936-dcbf045ae1b8
 	github.com/openshift/build-machinery-go v0.0.0-20200917070002-f171684f77ab
 	github.com/openshift/client-go v0.0.0-20201214125552-e615e336eb49
 	github.com/openshift/library-go v0.0.0-20201130154959-bd449d1e2e25

diff --git a/go.sum b/go.sum
@@ -462,8 +462,8 @@ github.com/openshift/api v0.0.0-20201019163320-c6a5ec25f267 h1:d6qOoblJz8DjQ44PR
 github.com/openshift/api v0.0.0-20201019163320-c6a5ec25f267/go.mod h1:RDvBcRQMGLa3aNuDuejVBbTEQj/2i14NXdpOLqbNBvM=
 github.com/openshift/api v0.0.0-20201214114959-164a2fb63b5f h1:MhuCP7+M9hmUnZaz6EwOh3+W6FQp+BezIXbL99Q4xq4=
 github.com/openshift/api v0.0.0-20201214114959-164a2fb63b5f/go.mod h1:aqU5Cq+kqKKPbDMqxo9FojgDeSpNJI7iuskjXjtojDg=
-github.com/openshift/api v0.0.0-20210304195353-475eac5412b1 h1:u+npe3/OgwWfttikHLgI2+fnb4BgtL0KIBc7VHvMF0I=
-github.com/openshift/api v0.0.0-20210304195353-475eac5412b1/go.mod h1:aqU5Cq+kqKKPbDMqxo9FojgDeSpNJI7iuskjXjtojDg=
+github.com/openshift/api v0.0.0-20210317213936-dcbf045ae1b8 h1:ZvryFPWTR2B8T1D+BFtnDn4GhTso+vsNdl1Ah0/gP5U=
+github.com/openshift/api v0.0.0-20210317213936-dcbf045ae1b8/go.mod h1:aqU5Cq+kqKKPbDMqxo9FojgDeSpNJI7iuskjXjtojDg=
 github.com/openshift/build-machinery-go v0.0.0-20200211121458-5e3d6e570160/go.mod h1:1CkcsT3aVebzRBzVTSbiKSkJMsC/CASqxesfqEMfJEc=
 github.com/openshift/build-machinery-go v0.0.0-20200424080330-082bf86082cc/go.mod h1:1CkcsT3aVebzRBzVTSbiKSkJMsC/CASqxesfqEMfJEc=
 github.com/openshift/build-machinery-go v0.0.0-20200819073603-48aa266c95f7/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE=

diff --git a/pkg/network/ovn_kubernetes.go b/pkg/network/ovn_kubernetes.go
@@ -81,6 +81,10 @@ func renderOVNKubernetes(conf *operv1.NetworkSpec, bootstrapResult *bootstrap.Bo
 	data.Data["LISTEN_DUAL_STACK"] = listenDualStack(bootstrapResult.OVN.MasterIPs[0])
 	data.Data["OVN_CERT_CN"] = OVN_CERT_CN
 	data.Data["OVN_NORTHD_PROBE_INTERVAL"] = os.Getenv("OVN_NORTHD_PROBE_INTERVAL")
+	data.Data["OVNPolicyAuditRateLimit"] = c.PolicyAuditConfig.RateLimit
+	data.Data["OVNPolicyAuditMaxFileSize"] = c.PolicyAuditConfig.MaxFileSize
+	data.Data["OVNPolicyAuditDestination"] = c.PolicyAuditConfig.Destination
+	data.Data["OVNPolicyAuditSyslogFacility"] = c.PolicyAuditConfig.SyslogFacility
 
 	var ippools string
 	for _, net := range conf.ClusterNetwork {
@@ -238,6 +242,11 @@ func isOVNKubernetesChangeSafe(prev, next *operv1.NetworkSpec) []error {
 			errs = append(errs, errors.Errorf("cannot edit IPsec configuration at runtime"))
 		}
 	}
+	if pn.PolicyAuditConfig != nil && nn.PolicyAuditConfig != nil {
+		if !reflect.DeepEqual(pn.IPsecConfig, nn.IPsecConfig) {
+			errs = append(errs, errors.Errorf("cannot edit Network Policy audit logging configuration at runtime"))
+		}
+	}
 
 	return errs
 }
@@ -275,6 +284,28 @@ func fillOVNKubernetesDefaults(conf, previous *operv1.NetworkSpec, hostMTU int)
 		var geneve uint32 = uint32(6081)
 		sc.GenevePort = &geneve
 	}
+
+	if sc.PolicyAuditConfig == nil {
+		sc.PolicyAuditConfig = &operv1.PolicyAuditConfig{}
+	}
+
+	if sc.PolicyAuditConfig.RateLimit == nil {
+		var ratelimit uint32 = uint32(20)
+		sc.PolicyAuditConfig.RateLimit = &ratelimit
+	}
+	if sc.PolicyAuditConfig.MaxFileSize == nil {
+		var maxfilesize uint32 = uint32(50)
+		sc.PolicyAuditConfig.MaxFileSize = &maxfilesize
+	}
+	if sc.PolicyAuditConfig.Destination == "" {
+		var destination string = string("null")
+		sc.PolicyAuditConfig.Destination = destination
+	}
+	if sc.PolicyAuditConfig.SyslogFacility == "" {
+		var syslogfacility string = string("local0")
+		sc.PolicyAuditConfig.SyslogFacility = syslogfacility
+	}
+
 }
 
 type replicaCountDecoder struct {

diff --git a/pkg/network/ovn_kubernetes_test.go b/pkg/network/ovn_kubernetes_test.go
@@ -365,6 +365,12 @@ func TestFillOVNKubernetesDefaults(t *testing.T) {
 			OVNKubernetesConfig: &operv1.OVNKubernetesConfig{
 				MTU:        ptrToUint32(8900),
 				GenevePort: ptrToUint32(6081),
+				PolicyAuditConfig: &operv1.PolicyAuditConfig{
+					RateLimit:      ptrToUint32(20),
+					MaxFileSize:    ptrToUint32(50),
+					Destination:    "null",
+					SyslogFacility: "local0",
+				},
 			},
 		},
 	}
@@ -400,6 +406,12 @@ func TestFillOVNKubernetesDefaultsIPsec(t *testing.T) {
 				MTU:         ptrToUint32(8854),
 				GenevePort:  ptrToUint32(8061),
 				IPsecConfig: &operv1.IPsecConfig{},
+				PolicyAuditConfig: &operv1.PolicyAuditConfig{
+					RateLimit:      ptrToUint32(20),
+					MaxFileSize:    ptrToUint32(50),
+					Destination:    "null",
+					SyslogFacility: "local0",
+				},
 			},
 		},
 	}