Skip to content

Commit

Permalink
update node labels/annotations by json merge patch (#4230)
Browse files Browse the repository at this point in the history 
Signed-off-by: zhangzujian <[email protected]>
  • Loading branch information
@zhangzujian
zhangzujian committed Jun 28, 2024
1 parent c893f52 commit 03ce5a3
Show file tree
Hide file tree
Showing 7 changed files with 115 additions and 213 deletions.
36 changes: 5 additions & 31 deletions cmd/daemon/cniserver.go
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package daemon

import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/pprof"
Expand All @@ -13,7 +11,6 @@ import (

"github.com/prometheus/client_golang/prometheus/promhttp"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
kubeinformers "k8s.io/client-go/informers"
"k8s.io/klog/v2"
"k8s.io/sample-controller/pkg/signals"
Expand Down Expand Up @@ -159,41 +156,18 @@ func initChassisAnno(cfg *daemon.Configuration) error {
return err
}

hostname := cfg.NodeName
node, err := cfg.KubeClient.CoreV1().Nodes().Get(context.Background(), hostname, v1.GetOptions{})
if err != nil {
klog.Errorf("failed to get node %s %v", hostname, err)
return err
}

chassistr := string(chassisID)
chassesName := strings.TrimSpace(chassistr)
chassesName := strings.TrimSpace(string(chassisID))
if chassesName == "" {
// not ready yet
err = fmt.Errorf("chassis id is empty")
klog.Error(err)
return err
}
if annoChassesName, ok := node.Annotations[util.ChassisAnnotation]; ok {
if annoChassesName == chassesName {
return nil
}
klog.Infof("chassis id changed, old: %s, new: %s", annoChassesName, chassesName)
}
node.Annotations[util.ChassisAnnotation] = chassesName
patchPayloadTemplate := `[{
"op": "%s",
"path": "/metadata/annotations",
"value": %s
}]`
op := "add"
raw, _ := json.Marshal(node.Annotations)
patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw)
_, err = cfg.KubeClient.CoreV1().Nodes().Patch(context.Background(), hostname, types.JSONPatchType, []byte(patchPayload), v1.PatchOptions{}, "")
if err != nil {
klog.Errorf("patch node %s failed %v", hostname, err)
annotations := map[string]any{util.ChassisAnnotation: chassesName}
if err = util.UpdateNodeAnnotations(cfg.KubeClient.CoreV1().Nodes(), cfg.NodeName, annotations); err != nil {
klog.Errorf("failed to update chassis annotation of node %s: %v", cfg.NodeName, err)
return err
}
klog.Infof("finish adding chassis annotation")

return nil
}
50 changes: 11 additions & 39 deletions pkg/controller/external_gw.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package controller

import (
"context"
"encoding/json"
"fmt"
"reflect"
"strings"
Expand Down Expand Up @@ -80,25 +79,11 @@ func (c *Controller) removeExternalGateway() error {
klog.Errorf("failed to list external gw nodes, %v", err)
return err
}
for _, cachedNode := range nodes {
no := cachedNode.DeepCopy()
patchPayloadTemplate := `[{
"op": "%s",
"path": "/metadata/labels",
"value": %s
}]`
op := "replace"
if len(no.Labels) == 0 {
op = "add"
}
if no.Labels[util.ExGatewayLabel] != "false" {
no.Labels[util.ExGatewayLabel] = "false"
raw, _ := json.Marshal(no.Labels)
patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw)
if _, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), no.Name, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}, ""); err != nil {
klog.Errorf("patch external gw node %s failed %v", no.Name, err)
return err
}
for _, node := range nodes {
labels := map[string]any{util.ExGatewayLabel: "false"}
if err = util.UpdateNodeLabels(c.config.KubeClient.CoreV1().Nodes(), node.Name, labels); err != nil {
klog.Errorf("failed to patch external gw node %s: %v", node.Name, err)
return err
}
}

Expand Down Expand Up @@ -236,30 +221,17 @@ func (c *Controller) getGatewayChassis(config map[string]string) ([]string, erro
}
}
for _, gw := range gwNodes {
cachedNode, err := c.nodesLister.Get(gw)
node, err := c.nodesLister.Get(gw)
if err != nil {
klog.Errorf("failed to get gw node %s, %v", gw, err)
return nil, err
}
node := cachedNode.DeepCopy()
patchPayloadTemplate := `[{
"op": "%s",
"path": "/metadata/labels",
"value": %s
}]`
op := "replace"
if len(node.Labels) == 0 {
op = "add"
}
if node.Labels[util.ExGatewayLabel] != "true" {
node.Labels[util.ExGatewayLabel] = "true"
raw, _ := json.Marshal(node.Labels)
patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw)
if _, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), gw, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}, ""); err != nil {
klog.Errorf("patch external gw node %s failed %v", gw, err)
return nil, err
}
labels := map[string]any{util.ExGatewayLabel: "true"}
if err = util.UpdateNodeLabels(c.config.KubeClient.CoreV1().Nodes(), node.Name, labels); err != nil {
klog.Errorf("failed to update annotations of node %s: %v", node.Name, err)
return nil, err
}

annoChassisName := node.Annotations[util.ChassisAnnotation]
if annoChassisName == "" {
err := fmt.Errorf("node %s has no chassis annotation, kube-ovn-cni not ready", gw)
Expand Down
37 changes: 12 additions & 25 deletions pkg/controller/node.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package controller

import (
"context"
"encoding/json"
"fmt"
"reflect"
"strconv"
Expand Down Expand Up @@ -314,29 +313,17 @@ func (c *Controller) handleAddNode(key string) error {
return err
}

patchPayloadTemplate := `[{
"op": "%s",
"path": "/metadata/annotations",
"value": %s
}]`
op := "replace"
if len(node.Annotations) == 0 {
node.Annotations = map[string]string{}
op = "add"
}

node.Annotations[util.IPAddressAnnotation] = ipStr
node.Annotations[util.MacAddressAnnotation] = mac
node.Annotations[util.CidrAnnotation] = subnet.Spec.CIDRBlock
node.Annotations[util.GatewayAnnotation] = subnet.Spec.Gateway
node.Annotations[util.LogicalSwitchAnnotation] = c.config.NodeSwitch
node.Annotations[util.AllocatedAnnotation] = "true"
node.Annotations[util.PortNameAnnotation] = portName
raw, _ := json.Marshal(node.Annotations)
patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw)
_, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), key, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}, "")
if err != nil {
klog.Errorf("patch node %s failed: %v", key, err)
annotations := map[string]any{
util.IPAddressAnnotation: ipStr,
util.MacAddressAnnotation: mac,
util.CidrAnnotation: subnet.Spec.CIDRBlock,
util.GatewayAnnotation: subnet.Spec.Gateway,
util.LogicalSwitchAnnotation: c.config.NodeSwitch,
util.AllocatedAnnotation: "true",
util.PortNameAnnotation: portName,
}
if err = util.UpdateNodeAnnotations(c.config.KubeClient.CoreV1().Nodes(), node.Name, annotations); err != nil {
klog.Errorf("failed to update annotations of node %s: %v", node.Name, err)
return err
}

Expand All @@ -362,7 +349,7 @@ func (c *Controller) handleAddNode(key string) error {
}

// ovn acl doesn't support address_set name with '-', so replace '-' by '.'
pgName := strings.ReplaceAll(node.Annotations[util.PortNameAnnotation], "-", ".")
pgName := strings.ReplaceAll(portName, "-", ".")
if err = c.OVNNbClient.CreatePortGroup(pgName, map[string]string{networkPolicyKey: "node" + "/" + key}); err != nil {
klog.Errorf("create port group %s for node %s: %v", pgName, key, err)
return err
Expand Down
87 changes: 31 additions & 56 deletions pkg/daemon/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package daemon

import (
"context"
"encoding/json"
"fmt"
"os/exec"
"strconv"
Expand Down Expand Up @@ -279,43 +278,30 @@ func (c *Controller) initProviderNetwork(pn *kubeovnv1.ProviderNetwork, node *v1
}
}

labels := map[string]any{
fmt.Sprintf(util.ProviderNetworkReadyTemplate, pn.Name): nil,
fmt.Sprintf(util.ProviderNetworkInterfaceTemplate, pn.Name): nil,
fmt.Sprintf(util.ProviderNetworkMtuTemplate, pn.Name): nil,
fmt.Sprintf(util.ProviderNetworkExcludeTemplate, pn.Name): nil,
}

var mtu int
var err error
klog.V(3).Infof("ovs init provider network %s", pn.Name)
if mtu, err = c.ovsInitProviderNetwork(pn.Name, nic, pn.Spec.ExchangeLinkName, c.config.MacLearningFallback); err != nil {
if oldLen := len(node.Labels); oldLen != 0 {
delete(node.Labels, fmt.Sprintf(util.ProviderNetworkReadyTemplate, pn.Name))
delete(node.Labels, fmt.Sprintf(util.ProviderNetworkInterfaceTemplate, pn.Name))
delete(node.Labels, fmt.Sprintf(util.ProviderNetworkMtuTemplate, pn.Name))
if len(node.Labels) != oldLen {
raw, _ := json.Marshal(node.Labels)
patchPayload := fmt.Sprintf(`[{ "op": "replace", "path": "/metadata/labels", "value": %s }]`, raw)
_, err1 := c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), node.Name, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{})
if err1 != nil {
klog.Errorf("failed to patch node %s: %v", node.Name, err1)
}
}
delete(labels, fmt.Sprintf(util.ProviderNetworkExcludeTemplate, pn.Name))
if err1 := util.UpdateNodeLabels(c.config.KubeClient.CoreV1().Nodes(), node.Name, labels); err1 != nil {
klog.Errorf("failed to update annotations of node %s: %v", node.Name, err1)
}
c.recordProviderNetworkErr(pn.Name, err.Error())
return err
}

delete(node.Labels, fmt.Sprintf(util.ProviderNetworkExcludeTemplate, pn.Name))
node.Labels[fmt.Sprintf(util.ProviderNetworkReadyTemplate, pn.Name)] = "true"
node.Labels[fmt.Sprintf(util.ProviderNetworkInterfaceTemplate, pn.Name)] = nic
node.Labels[fmt.Sprintf(util.ProviderNetworkMtuTemplate, pn.Name)] = strconv.Itoa(mtu)

patchPayloadTemplate := `[{ "op": "%s", "path": "/metadata/labels", "value": %s }]`
op := "replace"
if len(node.Labels) == 0 {
op = "add"
}

raw, _ := json.Marshal(node.Labels)
patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw)
_, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), node.Name, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{})
if err != nil {
klog.Errorf("failed to patch node %s: %v", node.Name, err)
labels[fmt.Sprintf(util.ProviderNetworkReadyTemplate, pn.Name)] = "true"
labels[fmt.Sprintf(util.ProviderNetworkInterfaceTemplate, pn.Name)] = nic
labels[fmt.Sprintf(util.ProviderNetworkMtuTemplate, pn.Name)] = strconv.Itoa(mtu)
if err = util.UpdateNodeLabels(c.config.KubeClient.CoreV1().Nodes(), node.Name, labels); err != nil {
klog.Errorf("failed to update labels of node %s: %v", node.Name, err)
return err
}
c.recordProviderNetworkErr(pn.Name, "")
Expand Down Expand Up @@ -377,22 +363,14 @@ func (c *Controller) recordProviderNetworkErr(providerNetwork, errMsg string) {
}

func (c *Controller) cleanProviderNetwork(pn *kubeovnv1.ProviderNetwork, node *v1.Node) error {
patchPayloadTemplate := `[{ "op": "%s", "path": "/metadata/labels", "value": %s }]`
op := "replace"
if len(node.Labels) == 0 {
op = "add"
}

var err error
delete(node.Labels, fmt.Sprintf(util.ProviderNetworkReadyTemplate, pn.Name))
delete(node.Labels, fmt.Sprintf(util.ProviderNetworkInterfaceTemplate, pn.Name))
delete(node.Labels, fmt.Sprintf(util.ProviderNetworkMtuTemplate, pn.Name))
node.Labels[fmt.Sprintf(util.ProviderNetworkExcludeTemplate, pn.Name)] = "true"
raw, _ := json.Marshal(node.Labels)
patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw)
_, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), node.Name, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{})
if err != nil {
klog.Errorf("failed to patch node %s: %v", node.Name, err)
labels := map[string]any{
fmt.Sprintf(util.ProviderNetworkReadyTemplate, pn.Name): nil,
fmt.Sprintf(util.ProviderNetworkInterfaceTemplate, pn.Name): nil,
fmt.Sprintf(util.ProviderNetworkMtuTemplate, pn.Name): nil,
fmt.Sprintf(util.ProviderNetworkExcludeTemplate, pn.Name): "true",
}
if err := util.UpdateNodeLabels(c.config.KubeClient.CoreV1().Nodes(), node.Name, labels); err != nil {
klog.Errorf("failed to update labels of node %s: %v", node.Name, err)
return err
}

Expand All @@ -412,17 +390,14 @@ func (c *Controller) handleDeleteProviderNetwork(pn *kubeovnv1.ProviderNetwork)
return nil
}

newNode := node.DeepCopy()
delete(newNode.Labels, fmt.Sprintf(util.ProviderNetworkReadyTemplate, pn.Name))
delete(newNode.Labels, fmt.Sprintf(util.ProviderNetworkExcludeTemplate, pn.Name))
delete(newNode.Labels, fmt.Sprintf(util.ProviderNetworkInterfaceTemplate, pn.Name))
delete(newNode.Labels, fmt.Sprintf(util.ProviderNetworkMtuTemplate, pn.Name))
raw, _ := json.Marshal(newNode.Labels)
patchPayloadTemplate := `[{ "op": "replace", "path": "/metadata/labels", "value": %s }]`
patchPayload := fmt.Sprintf(patchPayloadTemplate, raw)
_, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), node.Name, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{})
if err != nil {
klog.Errorf("failed to patch node %s: %v", node.Name, err)
labels := map[string]any{
fmt.Sprintf(util.ProviderNetworkReadyTemplate, pn.Name): nil,
fmt.Sprintf(util.ProviderNetworkInterfaceTemplate, pn.Name): nil,
fmt.Sprintf(util.ProviderNetworkMtuTemplate, pn.Name): nil,
fmt.Sprintf(util.ProviderNetworkExcludeTemplate, pn.Name): nil,
}
if err = util.UpdateNodeLabels(c.config.KubeClient.CoreV1().Nodes(), node.Name, labels); err != nil {
klog.Errorf("failed to update labels of node %s: %v", node.Name, err)
return err
}

Expand Down
32 changes: 10 additions & 22 deletions pkg/daemon/ovs_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package daemon
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"net"
Expand All @@ -12,6 +11,7 @@ import (
"path"
"path/filepath"
"regexp"
"strconv"
"strings"
"syscall"
"time"
Expand Down Expand Up @@ -742,8 +742,8 @@ func (c *Controller) loopOvnExt0Check() {
klog.Error(err)
return
}
if err = c.patchNodeExternalGwLabel(node.Name, false); err != nil {
klog.Errorf("failed to patch label on node %s, %v", node, err)
if err = c.patchNodeExternalGwLabel(false); err != nil {
klog.Errorf("failed to patch labels of node %s: %v", node.Name, err)
return
}
return
Expand Down Expand Up @@ -791,8 +791,8 @@ func (c *Controller) loopOvnExt0Check() {
klog.Errorf("failed to setup ovnext0, %v", err)
return
}
if err = c.patchNodeExternalGwLabel(portName, true); err != nil {
klog.Errorf("failed to patch label on node %s, %v", node, err)
if err = c.patchNodeExternalGwLabel(true); err != nil {
klog.Errorf("failed to patch labels of node %s: %v", node.Name, err)
return
}
if err = c.patchOvnEipStatus(portName, true); err != nil {
Expand Down Expand Up @@ -828,31 +828,19 @@ func (c *Controller) patchOvnEipStatus(key string, ready bool) error {
return nil
}

func (c *Controller) patchNodeExternalGwLabel(key string, enabled bool) error {
func (c *Controller) patchNodeExternalGwLabel(enabled bool) error {
node, err := c.nodesLister.Get(c.config.NodeName)
if err != nil {
klog.Errorf("failed to get node %s: %v", c.config.NodeName, err)
return err
}

if enabled {
node.Labels[util.NodeExtGwLabel] = "true"
} else {
node.Labels[util.NodeExtGwLabel] = "false"
}

patchPayloadTemplate := `[{ "op": "%s", "path": "/metadata/labels", "value": %s }]`
op := "replace"
if len(node.Labels) == 0 {
op = "add"
}

raw, _ := json.Marshal(node.Labels)
patchPayload := fmt.Sprintf(patchPayloadTemplate, op, raw)
if _, err = c.config.KubeClient.CoreV1().Nodes().Patch(context.Background(), key, types.JSONPatchType, []byte(patchPayload), metav1.PatchOptions{}); err != nil {
klog.Errorf("failed to patch node %s: %v", node.Name, err)
labels := map[string]any{util.NodeExtGwLabel: strconv.FormatBool(enabled)}
if err = util.UpdateNodeLabels(c.config.KubeClient.CoreV1().Nodes(), node.Name, labels); err != nil {
klog.Errorf("failed to update labels of node %s: %v", node.Name, err)
return err
}

return nil
}

Expand Down
Loading

0 comments on commit 03ce5a3

Please sign in to comment.