Use a per-node certificate for ovnkube-node pods

On startup each ovnkube-node pod should request a certificate that it will then use to communicate with the API server. This allows to identify which node made a particular request and we will use that to implement an admission webhook for write changes. Additionally add an automatic CSR approver with appropriate checks. Signed-off-by: Patryk Diak <[email protected]>
kyrtapz · Aug 9, 2023 · df6c1b8 · df6c1b8
1 parent 9a8fea3
commit df6c1b8
Show file tree

Hide file tree

Showing 361 changed files with 42,078 additions and 3,489 deletions.
diff --git a/contrib/kind.sh b/contrib/kind.sh
@@ -875,7 +875,8 @@ create_ovn_kube_manifests() {
     --ovnkube-metrics-scale-enable="${OVN_METRICS_SCALE_ENABLE}" \
     --compact-mode="${OVN_COMPACT_MODE}" \
     --enable-interconnect="${OVN_ENABLE_INTERCONNECT}" \
-    --enable-multi-external-gateway=true
+    --enable-multi-external-gateway=true \
+    --enable-per-node-cert=true
   popd
 }
 
@@ -902,6 +903,7 @@ install_ovn_global_zone() {
     run_kubectl apply -f ovnkube-db.yaml
   fi
 
+  run_kubectl apply -f ovnkube-identity.yaml
   run_kubectl apply -f ovnkube-master.yaml
   run_kubectl apply -f ovnkube-node.yaml
 }
@@ -912,6 +914,7 @@ install_ovn_single_node_zones() {
     kubectl label node "${n}" k8s.ovn.org/zone-name=${n} --overwrite
   done
 
+  run_kubectl apply -f ovnkube-identity.yaml
   run_kubectl apply -f ovnkube-control-plane.yaml
   run_kubectl apply -f ovnkube-single-node-zone.yaml
 }
@@ -937,6 +940,7 @@ install_ovn_multiple_nodes_zones() {
     fi
   done
 
+  run_kubectl apply -f ovnkube-identity.yaml
   run_kubectl apply -f ovnkube-control-plane.yaml
   run_kubectl apply -f ovnkube-zone-controller.yaml
   run_kubectl apply -f ovnkube-node.yaml

diff --git a/dist/images/.gitignore b/dist/images/.gitignore
@@ -2,6 +2,7 @@ ovn-k8s-cni-overlay
 ovn-kube-util
 ovnkube
 ovnkube-trace
+ovnkube-identity
 ovndbchecker
 hybrid-overlay-node
 git_info
diff --git a/dist/images/Dockerfile.fedora b/dist/images/Dockerfile.fedora
@@ -44,7 +44,7 @@ RUN rpm -Uhv --nodeps --force *.rpm
 # Built in ../../go_controller, then the binaries are copied here.
 # put things where they are in the pkg
 RUN mkdir -p /usr/libexec/cni/
-COPY ovnkube ovn-kube-util ovndbchecker /usr/bin/
+COPY ovnkube ovn-kube-util ovndbchecker ovnkube-identity /usr/bin/
 COPY ovn-k8s-cni-overlay /usr/libexec/cni/ovn-k8s-cni-overlay
 
 # ovnkube.sh is the entry point. This script examines environment

diff --git a/dist/images/Dockerfile.fedora.dev b/dist/images/Dockerfile.fedora.dev
@@ -85,6 +85,7 @@ RUN mkdir -p /usr/libexec/cni/
 COPY ovnkube /usr/bin/
 COPY ovn-kube-util /usr/bin/
 COPY ovndbchecker /usr/bin/
+COPY ovnkube-identity /usr/bin/
 COPY ovn-k8s-cni-overlay /usr/libexec/cni/ovn-k8s-cni-overlay
 
 # ovnkube.sh is the entry point. This script examines environment

diff --git a/dist/images/daemonset.sh b/dist/images/daemonset.sh
@@ -307,6 +307,9 @@ while [ "$1" != "" ]; do
   --enable-multi-external-gateway)
     OVN_ENABLE_MULTI_EXTERNAL_GATEWAY=$VALUE
     ;;
+  --enable-per-node-cert)
+    OVN_ENABLE_PER_NODE_CERT=$VALUE
+    ;;
   *)
     echo "WARNING: unknown parameter \"$PARAM\""
     exit 1
@@ -467,6 +470,8 @@ ovn_enable_interconnect=${OVN_ENABLE_INTERCONNECT}
 echo "ovn_enable_interconnect: ${ovn_enable_interconnect}"
 ovn_enable_multi_external_gateway=${OVN_ENABLE_MULTI_EXTERNAL_GATEWAY}
 echo "ovn_enable_multi_external_gateway: ${ovn_enable_multi_external_gateway}"
+ovn_enable_per_node_cert=${OVN_ENABLE_PER_NODE_CERT}
+echo "ovn_enable_per_node_cert: ${ovn_enable_per_node_cert}"
 
 ovn_image=${ovnkube_image} \
   ovnkube_compact_mode_enable=${ovnkube_compact_mode_enable} \
@@ -510,6 +515,7 @@ ovn_image=${ovnkube_image} \
   ovnkube_node_mgmt_port_netdev=${ovnkube_node_mgmt_port_netdev} \
   ovn_enable_interconnect=${ovn_enable_interconnect} \
   ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \
+  ovn_enable_per_node_cert=${ovn_enable_per_node_cert} \
   ovnkube_app_name=ovnkube-node \
   j2 ../templates/ovnkube-node.yaml.j2 -o ${output_dir}/ovnkube-node.yaml
 
@@ -586,6 +592,7 @@ ovn_image=${ovnkube_image} \
   ovnkube_compact_mode_enable=${ovnkube_compact_mode_enable} \
   ovn_unprivileged_mode=${ovn_unprivileged_mode} \
   ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \
+  ovn_enable_per_node_cert=${ovn_enable_per_node_cert} \
   j2 ../templates/ovnkube-master.yaml.j2 -o ${output_dir}/ovnkube-master.yaml
 
 ovn_image=${ovnkube_image} \
@@ -618,6 +625,7 @@ ovn_image=${ovnkube_image} \
   ovn_ex_gw_networking_interface=${ovn_ex_gw_networking_interface} \
   ovn_enable_interconnect=${ovn_enable_interconnect} \
   ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \
+  ovn_enable_per_node_cert=${ovn_enable_per_node_cert} \
   j2 ../templates/ovnkube-control-plane.yaml.j2 -o ${output_dir}/ovnkube-control-plane.yaml
 
 ovn_image=${image} \
@@ -700,6 +708,7 @@ ovn_image=${ovnkube_image} \
   ovn_loglevel_nb=${ovn_loglevel_nb} ovn_loglevel_sb=${ovn_loglevel_sb} \
   ovn_enable_interconnect=${ovn_enable_interconnect} \
   ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \
+  ovn_enable_per_node_cert=${ovn_enable_per_node_cert} \
   j2 ../templates/ovnkube-single-node-zone.yaml.j2 -o ${output_dir}/ovnkube-single-node-zone.yaml
 
 ovn_image=${ovnkube_image} \
@@ -753,13 +762,21 @@ ovn_image=${ovnkube_image} \
   ovn_loglevel_nb=${ovn_loglevel_nb} ovn_loglevel_sb=${ovn_loglevel_sb} \
   ovn_enable_interconnect=${ovn_enable_interconnect} \
   ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \
+  ovn_enable_per_node_cert=${ovn_enable_per_node_cert} \
   j2 ../templates/ovnkube-zone-controller.yaml.j2 -o ${output_dir}/ovnkube-zone-controller.yaml
 
 ovn_image=${image} \
   ovn_image_pull_policy=${image_pull_policy} \
   ovn_unprivileged_mode=${ovn_unprivileged_mode} \
   j2 ../templates/ovs-node.yaml.j2 -o ${output_dir}/ovs-node.yaml
 
+ovn_image=${ovnkube_image} \
+  ovn_image=${image} \
+  ovn_image_pull_policy=${image_pull_policy} \
+  ovn_master_count=${ovn_master_count} \
+  ovnkube_master_loglevel=${master_loglevel} \
+  j2 ../templates/ovnkube-identity.yaml.j2 -o ${output_dir}/ovnkube-identity.yaml
+
 if ${enable_ipsec}; then
   ovn_image=${image} \
     j2 ../templates/ovn-ipsec.yaml.j2 -o ${output_dir}/ovn-ipsec.yaml

diff --git a/dist/images/ovnkube.sh b/dist/images/ovnkube.sh
@@ -21,6 +21,7 @@ fi
 #    nb-ovsdb       Runs nb_ovsdb as a process (no detach or monitor) (v3)
 #    sb-ovsdb       Runs sb_ovsdb as a process (no detach or monitor) (v3)
 #    ovn-master     Runs ovnkube in master mode (v3)
+#    ovn-identity   Runs ovnkube-identity (v3)
 #    ovn-controller Runs ovn controller (v3)
 #    ovn-node       Runs ovnkube in node mode (v3)
 #    cleanup-ovn-node   Runs ovnkube to cleanup the node (v3)
@@ -86,6 +87,7 @@ fi
 # OVN_HOST_NETWORK_NAMESPACE - namespace to classify host network traffic for applying network policies
 # OVN_DISABLE_FORWARDING - disable forwarding on OVNK controlled interfaces
 # OVN_ENABLE_MULTI_EXTERNAL_GATEWAY - enable multi external gateway for ovn-kubernetes
+# OVN_ENABLE_PER_NODE_CERT - enable per node certificate ovn-kubernetes
 
 # The argument to the command is the operation to be performed
 # ovn-master ovn-controller ovn-node display display_env ovn_debug
@@ -247,6 +249,8 @@ ovn_stateless_netpol_enable=${OVN_STATELESS_NETPOL_ENABLE:-false}
 ovn_enable_interconnect=${OVN_ENABLE_INTERCONNECT:-false}
 #OVN_ENABLE_MULTI_EXTERNAL_GATEWAY - enable multi external gateway
 ovn_enable_multi_external_gateway=${OVN_ENABLE_MULTI_EXTERNAL_GATEWAY:-false}
+#OVN_ENABLE_PER_NODE_CERT - enable per node cert
+ovn_enable_per_node_cert=${OVN_ENABLE_PER_NODE_CERT:-true}
 
 # OVNKUBE_NODE_MODE - is the mode which ovnkube node operates
 ovnkube_node_mode=${OVNKUBE_NODE_MODE:-"full"}
@@ -995,6 +999,20 @@ run-ovn-northd() {
   exit 8
 }
 
+# v3 -  run ovnkube-identity
+ovnkube-identity() {
+    trap 'kill $(jobs -p); exit 0' TERM
+    check_ovn_daemonset_version "3"
+    rm -f ${OVN_RUNDIR}/ovnkube-identity.pid
+    /usr/bin/ovnkube-identity  --k8s-apiserver=${K8S_APISERVER} --loglevel=${ovnkube_loglevel}
+
+    echo "=============== ovn-identity ========== running"
+    wait_for_event attempts=3 process_ready ovnkube-identity
+
+    process_healthy ovnkube-identity
+    exit 9
+}
+
 # v3 - run ovnkube --master (both cluster-manager and ovnkube-controller)
 ovn-master() {
   trap 'kill $(jobs -p); exit 0' TERM
@@ -1144,6 +1162,12 @@ ovn-master() {
   fi
   echo "ovnkube_enable_multi_external_gateway_flag=${ovnkube_enable_multi_external_gateway_flag}"
 
+  ovnkube_enable_csr_approver_flag=
+  if [[ ${ovn_enable_per_node_cert} == "true" ]]; then
+	  ovnkube_enable_csr_approver_flag="--enable-csr-approver"
+  fi
+  echo "ovnkube_enable_csr_approver_flag: ${ovnkube_enable_csr_approver_flag}"
+
   init_node_flags=
   if [[ ${ovnkube_compact_mode_enable} == "true" ]]; then
     init_node_flags="--init-node ${K8S_NODE} --nodeport"
@@ -1169,6 +1193,7 @@ ovn-master() {
     ${empty_lb_events_flag} \
     ${ovn_v4_join_subnet_opt} \
     ${ovn_v6_join_subnet_opt} \
+    ${ovnkube_enable_csr_approver_flag} \
     --pidfile ${OVN_RUNDIR}/ovnkube-master.pid \
     --logfile /var/log/ovn-kubernetes/ovnkube-master.log \
     ${libovsdb_client_logfile_flag} \
@@ -1485,6 +1510,12 @@ ovn-cluster-manager() {
   fi
   echo "ovnkube_enable_multi_external_gateway_flag=${ovnkube_enable_multi_external_gateway_flag}"
 
+  ovnkube_enable_csr_approver_flag=
+  if [[ ${ovn_enable_per_node_cert} == "true" ]]; then
+	  ovnkube_enable_csr_approver_flag="--enable-csr-approver"
+  fi
+  echo "ovnkube_enable_csr_approver_flag: ${ovnkube_enable_csr_approver_flag}"
+
   echo "=============== ovn-cluster-manager ========== MASTER ONLY"
   /usr/bin/ovnkube \
     --init-cluster-manager ${K8S_NODE} \
@@ -1496,6 +1527,7 @@ ovn-cluster-manager() {
     ${hybrid_overlay_flags} \
     ${ovn_v4_join_subnet_opt} \
     ${ovn_v6_join_subnet_opt} \
+    ${ovnkube_enable_csr_approver_flag} \
     --pidfile ${OVN_RUNDIR}/ovnkube-cluster-manager.pid \
     --logfile /var/log/ovn-kubernetes/ovnkube-cluster-manager.log \
     ${ovnkube_metrics_tls_opts} \
@@ -1780,8 +1812,18 @@ ovn-node() {
       ovn_dbs="${ovn_dbs} --sb-address=${ovn_sbdb}"
   fi
 
+  ovnkube_node_certs_flags=
+  if [[ ${ovn_enable_per_node_cert} == "true" ]]; then
+     ovnkube_node_certs_flags="
+        --bootstrap-kubeconfig /host/etc/kubernetes/kubelet.conf
+        --cert-dir /host/var/run
+     "
+  fi
+  echo "ovnkube_node_certs_flags=${ovnkube_node_certs_flags}"
+
   echo "=============== ovn-node   --init-node"
   /usr/bin/ovnkube --init-node ${K8S_NODE} \
+    ${ovnkube_node_certs_flags} \
     --cluster-subnets ${net_cidr} --k8s-service-cidr=${svc_cidr} \
     $ovn_dbs \
     ${ovn_unprivileged_flag} \
@@ -1898,6 +1940,7 @@ display_version
 # sb-ovsdb       Runs sb_ovsdb as a process (no detach or monitor) (v3)
 # ovn-dbchecker  Runs ovndb checker alongside nb-ovsdb and sb-ovsdb containers (v3)
 # ovn-master     - master only (v3)
+# ovn-identity     - master only (v3)
 # ovn-controller - all nodes (v3)
 # ovn-node       - all nodes (v3)
 # cleanup-ovn-node - all nodes (v3)
@@ -1924,6 +1967,9 @@ case ${cmd} in
 "ovn-master") # pod ovnkube-master container ovnkube-master
   ovn-master
   ;;
+"ovnkube-identity") # pod ovnkube-identity container ovnkube-identity
+  ovnkube-identity
+  ;;
 "ovnkube-controller") # pod ovnkube-master container ovnkube-controller
   ovnkube-controller
   ;;
@@ -1972,7 +2018,7 @@ case ${cmd} in
 *)
   echo "invalid command ${cmd}"
   echo "valid v3 commands: ovs-server nb-ovsdb sb-ovsdb run-ovn-northd ovn-master " \
-    "ovn-controller ovn-node display_env display ovn_debug cleanup-ovs-server " \
+    "ovnkube-identity ovn-controller ovn-node display_env display ovn_debug cleanup-ovs-server " \
     "cleanup-ovn-node nb-ovsdb-raft sb-ovsdb-raft"
   exit 0
   ;;

diff --git a/dist/templates/ovn-setup.yaml.j2 b/dist/templates/ovn-setup.yaml.j2
@@ -132,7 +132,20 @@ subjects:
   namespace: ovn-kubernetes
 
 ---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: ovn-kubernetes-cert
+roleRef:
+  name: ovn-kubernetes
+  kind: ClusterRole
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+- kind: Group
+  name: system:ovn-nodes
+  apiGroup: rbac.authorization.k8s.io
 
+---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
@@ -153,7 +166,49 @@ rules:
   - update
 
 ---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: csr-approver
+rules:
+  - apiGroups:
+      - certificates.k8s.io
+    resources:
+      - certificatesigningrequests
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - certificates.k8s.io
+    resources:
+      - certificatesigningrequests/approval
+    verbs:
+      - update
+  - apiGroups:
+      - certificates.k8s.io
+    resources:
+      - signers
+    resourceNames:
+      - kubernetes.io/kube-apiserver-client
+    verbs:
+      - approve
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: ovn-kubernetes-approver
+roleRef:
+  name: csr-approver
+  kind: ClusterRole
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+  - kind: ServiceAccount
+    name: ovn
+    namespace: ovn-kubernetes
 
+---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
@@ -168,6 +223,21 @@ subjects:
   name: ovn
   namespace: ovn-kubernetes
 
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: only-ovnk8s-configmaps-cert
+  namespace: ovn-kubernetes
+roleRef:
+  name: ovn-k8s-configmap
+  kind: Role
+  apiGroup: rbac.authorization.k8s.io
+subjects:
+- kind: Group
+  name: system:ovn-nodes
+  apiGroup: rbac.authorization.k8s.io
+
 {% if in_upgrade != "true" -%}
 ---
 

diff --git a/dist/templates/ovnkube-control-plane.yaml.j2 b/dist/templates/ovnkube-control-plane.yaml.j2
@@ -162,6 +162,8 @@ spec:
           value: "{{ ovn_enable_interconnect }}"
         - name: OVN_ENABLE_MULTI_EXTERNAL_GATEWAY
           value: "{{ ovn_enable_multi_external_gateway }}"
+        - name: OVN_ENABLE_PER_NODE_CERT
+          value: "{{ ovn_enable_per_node_cert }}"
       # end of container
 
       volumes: