Introduce ovnkube-identity with per-node certs and a validating webhook

On startup each ovnkube-node pod should request a certificate that it will then use to communicate with the API server. This allows to identify which node made a particular request and it is used to implement an admission webhook for write changes. Signed-off-by: Patryk Diak <[email protected]>
kyrtapz · Aug 18, 2023 · 18280b4 · 18280b4
1 parent fb58513
commit 18280b4
Show file tree

Hide file tree

Showing 368 changed files with 43,186 additions and 3,300 deletions.
diff --git a/contrib/kind.sh b/contrib/kind.sh
@@ -168,6 +168,7 @@ usage() {
     echo "-sm  | --scale-metrics              Enable scale metrics"
     echo "-cm  | --compact-mode               Enable compact mode, ovnkube master and node run in the same process."
     echo "-ic  | --enable-interconnect        Enable interconnect with each node as a zone (only valid if OVN_HA is false)"
+    echo "--disable-ovnkube-identity          Disable per-node cert and ovnkube-identity webhook"
     echo "-npz | --nodes-per-zone             If interconnect is enabled, number of nodes per zone (Default 1). If this value > 1, then (total k8s nodes (workers + 1) / num of nodes per zone) should be zero."
     echo "--isolated                          Deploy with an isolated environment (no default gateway)"
     echo "--delete                            Delete current cluster"
@@ -342,6 +343,8 @@ parse_args() {
                                                 ;;
             -ic | --enable-interconnect )       OVN_ENABLE_INTERCONNECT=true
                                                 ;;
+            --disable-ovnkube-identity)         OVN_ENABLE_OVNKUBE_IDENTITY=false
+                                                ;;
             --delete )                          delete
                                                 exit
                                                 ;;
@@ -416,6 +419,7 @@ print_params() {
      if [ "$OVN_ENABLE_INTERCONNECT" == true ]; then
        echo "KIND_NUM_NODES_PER_ZONE = $KIND_NUM_NODES_PER_ZONE"
      fi
+     echo "OVN_ENABLE_OVNKUBE_IDENTITY = $OVN_ENABLE_OVNKUBE_IDENTITY"
      echo "KIND_NUM_WORKER = $KIND_NUM_WORKER"
      echo ""
 }
@@ -562,6 +566,7 @@ set_default_params() {
   MASQUERADE_SUBNET_IPV6=${MASQUERADE_SUBNET_IPV6:-fd69::/125}
   KIND_NUM_MASTER=1
   OVN_ENABLE_INTERCONNECT=${OVN_ENABLE_INTERCONNECT:-false}
+  OVN_ENABLE_OVNKUBE_IDENTITY=${OVN_ENABLE_OVNKUBE_IDENTITY:-true}
 
   if [ "$OVN_HA" == true ] && [ "$OVN_ENABLE_INTERCONNECT" != false ]; then
      echo "HA mode cannot be used together with Interconnect"
@@ -837,6 +842,11 @@ create_ovn_kube_manifests() {
       ovnkube_image=$($OCI_BIN inspect --format='{{index .RepoDigests 0}}' $OVN_IMAGE)
     fi
     pushd ${DIR}/../dist/images
+    enable_multi_node_zone_flag=
+    if [ "$OVN_ENABLE_INTERCONNECT" == true ]; then
+      KIND_NUM_NODES_PER_ZONE=${KIND_NUM_NODES_PER_ZONE:-1}
+      enable_multi_node_zone_flag="--enable-multi-node-zone=$( (( KIND_NUM_NODES_PER_ZONE > 1 )) && echo "true" || echo "false" )"
+    fi
   ./daemonset.sh \
     --output-directory="${MANIFEST_OUTPUT_DIR}"\
     --image="${OVN_IMAGE}" \
@@ -880,7 +890,9 @@ create_ovn_kube_manifests() {
     --ovnkube-metrics-scale-enable="${OVN_METRICS_SCALE_ENABLE}" \
     --compact-mode="${OVN_COMPACT_MODE}" \
     --enable-interconnect="${OVN_ENABLE_INTERCONNECT}" \
-    --enable-multi-external-gateway=true
+    --enable-multi-external-gateway=true \
+    --enable-ovnkube-identity="${OVN_ENABLE_OVNKUBE_IDENTITY}" \
+    "${enable_multi_node_zone_flag}"
   popd
 }
 
@@ -907,6 +919,9 @@ install_ovn_global_zone() {
     run_kubectl apply -f ovnkube-db.yaml
   fi
 
+  if [ "${OVN_ENABLE_OVNKUBE_IDENTITY}" == true ]; then
+    run_kubectl apply -f ovnkube-identity.yaml
+  fi
   run_kubectl apply -f ovnkube-master.yaml
   run_kubectl apply -f ovnkube-node.yaml
 }
@@ -917,6 +932,9 @@ install_ovn_single_node_zones() {
     kubectl label node "${n}" k8s.ovn.org/zone-name=${n} --overwrite
   done
 
+  if [ "${OVN_ENABLE_OVNKUBE_IDENTITY}" == true ]; then
+    run_kubectl apply -f ovnkube-identity.yaml
+  fi
   run_kubectl apply -f ovnkube-control-plane.yaml
   run_kubectl apply -f ovnkube-single-node-zone.yaml
 }
@@ -942,6 +960,9 @@ install_ovn_multiple_nodes_zones() {
     fi
   done
 
+  if [ "${OVN_ENABLE_OVNKUBE_IDENTITY}" == true ]; then
+    run_kubectl apply -f ovnkube-identity.yaml
+  fi
   run_kubectl apply -f ovnkube-control-plane.yaml
   run_kubectl apply -f ovnkube-zone-controller.yaml
   run_kubectl apply -f ovnkube-node.yaml
@@ -958,6 +979,7 @@ install_ovn() {
   run_kubectl apply -f policy.networking.k8s.io_adminnetworkpolicies.yaml
   run_kubectl apply -f policy.networking.k8s.io_baselineadminnetworkpolicies.yaml
   run_kubectl apply -f ovn-setup.yaml
+  run_kubectl apply -f rbac-ovnkube-identity.yaml
   run_kubectl apply -f rbac-ovnkube-cluster-manager.yaml
   run_kubectl apply -f rbac-ovnkube-master.yaml
   run_kubectl apply -f rbac-ovnkube-node.yaml

diff --git a/dist/images/.gitignore b/dist/images/.gitignore
@@ -2,6 +2,7 @@ ovn-k8s-cni-overlay
 ovn-kube-util
 ovnkube
 ovnkube-trace
+ovnkube-identity
 ovndbchecker
 hybrid-overlay-node
 git_info
diff --git a/dist/images/Dockerfile.fedora b/dist/images/Dockerfile.fedora
@@ -44,7 +44,7 @@ RUN rpm -Uhv --nodeps --force *.rpm
 # Built in ../../go_controller, then the binaries are copied here.
 # put things where they are in the pkg
 RUN mkdir -p /usr/libexec/cni/
-COPY ovnkube ovn-kube-util ovndbchecker /usr/bin/
+COPY ovnkube ovn-kube-util ovndbchecker ovnkube-identity /usr/bin/
 COPY ovn-k8s-cni-overlay /usr/libexec/cni/ovn-k8s-cni-overlay
 
 # ovnkube.sh is the entry point. This script examines environment

diff --git a/dist/images/Dockerfile.fedora.dev b/dist/images/Dockerfile.fedora.dev
@@ -85,6 +85,7 @@ RUN mkdir -p /usr/libexec/cni/
 COPY ovnkube /usr/bin/
 COPY ovn-kube-util /usr/bin/
 COPY ovndbchecker /usr/bin/
+COPY ovnkube-identity /usr/bin/
 COPY ovn-k8s-cni-overlay /usr/libexec/cni/ovn-k8s-cni-overlay
 
 # ovnkube.sh is the entry point. This script examines environment

diff --git a/dist/images/daemonset.sh b/dist/images/daemonset.sh
@@ -87,7 +87,8 @@ OVNKUBE_CONFIG_DURATION_ENABLE=
 OVNKUBE_METRICS_SCALE_ENABLE=
 OVN_STATELESS_NETPOL_ENABLE="false"
 OVN_ENABLE_INTERCONNECT=
-
+OVN_ENABLE_OVNKUBE_IDENTITY="true"
+OVN_ENABLE_MULTI_ZONE_NODE="false"
 # IN_UPGRADE is true only if called by upgrade-ovn.sh during the upgrade test,
 # it will render only the parts in ovn-setup.yaml related to RBAC permissions.
 IN_UPGRADE=
@@ -319,6 +320,12 @@ while [ "$1" != "" ]; do
   --enable-multi-external-gateway)
     OVN_ENABLE_MULTI_EXTERNAL_GATEWAY=$VALUE
     ;;
+  --enable-ovnkube-identity)
+    OVN_ENABLE_OVNKUBE_IDENTITY=$VALUE
+    ;;
+  --enable-multi-node-zone)
+    OVN_ENABLE_MULTI_ZONE_NODE=$VALUE
+    ;;
   *)
     echo "WARNING: unknown parameter \"$PARAM\""
     exit 1
@@ -485,6 +492,10 @@ ovn_enable_interconnect=${OVN_ENABLE_INTERCONNECT}
 echo "ovn_enable_interconnect: ${ovn_enable_interconnect}"
 ovn_enable_multi_external_gateway=${OVN_ENABLE_MULTI_EXTERNAL_GATEWAY}
 echo "ovn_enable_multi_external_gateway: ${ovn_enable_multi_external_gateway}"
+ovn_enable_ovnkube_identity=${OVN_ENABLE_OVNKUBE_IDENTITY}
+echo "ovn_enable_ovnkube_identity: ${ovn_enable_ovnkube_identity}"
+ovn_enable_multi_node_zone=${OVN_ENABLE_MULTI_ZONE_NODE}
+echo "ovn_enable_multi_node_zone: ${ovn_enable_multi_node_zone}"
 
 ovn_image=${ovnkube_image} \
   ovnkube_compact_mode_enable=${ovnkube_compact_mode_enable} \
@@ -531,6 +542,7 @@ ovn_image=${ovnkube_image} \
   ovnkube_node_mgmt_port_netdev=${ovnkube_node_mgmt_port_netdev} \
   ovn_enable_interconnect=${ovn_enable_interconnect} \
   ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \
+  ovn_enable_ovnkube_identity=${ovn_enable_ovnkube_identity} \
   ovnkube_app_name=ovnkube-node \
   j2 ../templates/ovnkube-node.yaml.j2 -o ${output_dir}/ovnkube-node.yaml
 
@@ -613,6 +625,7 @@ ovn_image=${ovnkube_image} \
   ovnkube_compact_mode_enable=${ovnkube_compact_mode_enable} \
   ovn_unprivileged_mode=${ovn_unprivileged_mode} \
   ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \
+  ovn_enable_ovnkube_identity=${ovn_enable_ovnkube_identity} \
   j2 ../templates/ovnkube-master.yaml.j2 -o ${output_dir}/ovnkube-master.yaml
 
 ovn_image=${ovnkube_image} \
@@ -648,6 +661,7 @@ ovn_image=${ovnkube_image} \
   ovn_ex_gw_networking_interface=${ovn_ex_gw_networking_interface} \
   ovn_enable_interconnect=${ovn_enable_interconnect} \
   ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \
+  ovn_enable_ovnkube_identity=${ovn_enable_ovnkube_identity} \
   j2 ../templates/ovnkube-control-plane.yaml.j2 -o ${output_dir}/ovnkube-control-plane.yaml
 
 ovn_image=${image} \
@@ -731,6 +745,7 @@ ovn_image=${ovnkube_image} \
   ovn_loglevel_nb=${ovn_loglevel_nb} ovn_loglevel_sb=${ovn_loglevel_sb} \
   ovn_enable_interconnect=${ovn_enable_interconnect} \
   ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \
+  ovn_enable_ovnkube_identity=${ovn_enable_ovnkube_identity} \
   j2 ../templates/ovnkube-single-node-zone.yaml.j2 -o ${output_dir}/ovnkube-single-node-zone.yaml
 
 ovn_image=${ovnkube_image} \
@@ -785,13 +800,36 @@ ovn_image=${ovnkube_image} \
   ovn_loglevel_nb=${ovn_loglevel_nb} ovn_loglevel_sb=${ovn_loglevel_sb} \
   ovn_enable_interconnect=${ovn_enable_interconnect} \
   ovn_enable_multi_external_gateway=${ovn_enable_multi_external_gateway} \
+  ovn_enable_ovnkube_identity=${ovn_enable_ovnkube_identity} \
   j2 ../templates/ovnkube-zone-controller.yaml.j2 -o ${output_dir}/ovnkube-zone-controller.yaml
 
 ovn_image=${image} \
   ovn_image_pull_policy=${image_pull_policy} \
   ovn_unprivileged_mode=${ovn_unprivileged_mode} \
   j2 ../templates/ovs-node.yaml.j2 -o ${output_dir}/ovs-node.yaml
 
+ovnkube_certs_dir="/tmp/ovnkube-certs"
+ovnkube_webhook_name="ovnkube-webhook"
+mkdir -p ${ovnkube_certs_dir}
+path_prefix="${ovnkube_certs_dir}/${ovnkube_webhook_name}"
+
+# Create self signed CA and webhook cert
+openssl req -x509 -newkey rsa:4096 -nodes -keyout "${path_prefix}-ca.key" -out "${path_prefix}-ca.crt" -days 400 -subj "/CN=self-signed-ca"
+openssl req -newkey rsa:4096 -nodes -keyout "${path_prefix}.key" -out "${path_prefix}.csr" -subj "/CN=localhost"
+openssl x509 -req -in "${path_prefix}.csr" -CA "${path_prefix}-ca.crt" -CAkey "${path_prefix}-ca.key" -extfile <(printf "subjectAltName=DNS:localhost") -CAcreateserial -out "${path_prefix}.crt" -days 365
+
+ovn_image=${ovnkube_image} \
+  ovn_image=${image} \
+  ovn_image_pull_policy=${image_pull_policy} \
+  ovn_master_count=${ovn_master_count} \
+  ovnkube_master_loglevel=${master_loglevel} \
+  ovn_enable_interconnect=${ovn_enable_interconnect} \
+  webhook_ca_bundle=$(cat "${path_prefix}-ca.crt" | base64 -w0) \
+  webhook_key=$(cat "${path_prefix}.key" | base64 -w0) \
+  webhook_cert=$(cat "${path_prefix}.crt" | base64 -w0) \
+  ovn_enable_multi_node_zone=${ovn_enable_multi_node_zone} \
+  j2 ../templates/ovnkube-identity.yaml.j2 -o ${output_dir}/ovnkube-identity.yaml
+
 if ${enable_ipsec}; then
   ovn_image=${image} \
     j2 ../templates/ovn-ipsec.yaml.j2 -o ${output_dir}/ovn-ipsec.yaml
@@ -818,8 +856,10 @@ net_cidr=${net_cidr} svc_cidr=${svc_cidr} \
   j2 ../templates/ovn-setup.yaml.j2 -o ${output_dir}/ovn-setup.yaml
 
 ovn_enable_interconnect=${ovn_enable_interconnect} \
+ovn_enable_ovnkube_identity=${ovn_enable_ovnkube_identity} \
   j2 ../templates/rbac-ovnkube-node.yaml.j2 -o ${output_dir}/rbac-ovnkube-node.yaml
 
+cp ../templates/rbac-ovnkube-identity.yaml.j2 ${output_dir}/rbac-ovnkube-identity.yaml
 cp ../templates/rbac-ovnkube-master.yaml.j2 ${output_dir}/rbac-ovnkube-master.yaml
 cp ../templates/rbac-ovnkube-db.yaml.j2 ${output_dir}/rbac-ovnkube-db.yaml
 cp ../templates/rbac-ovnkube-cluster-manager.yaml.j2 ${output_dir}/rbac-ovnkube-cluster-manager.yaml