Skip to content

Commit

Permalink
feat: add editable ovn-ic (#1795)
Browse files Browse the repository at this point in the history
* add editable ovn-ic

* feat: add ovnic e2e for editable ovnic
  • Loading branch information
lut777 authored Aug 16, 2022
1 parent ddcdfb9 commit d41c043
Show file tree
Hide file tree
Showing 6 changed files with 196 additions and 8 deletions.
3 changes: 2 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -289,7 +289,8 @@ kind-install-ic:
@set -e; \
ic_db_host=$$(docker inspect ovn-ic-db -f "{{.NetworkSettings.Networks.kind.IPAddress}}"); \
zone=az0 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn-control-plane j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-0.yaml; \
zone=az1 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn1-control-plane j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml
zone=az1 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn1-control-plane j2 yamls/ovn-ic.yaml.j2 -o ovn-ic-1.yaml; \
zone=az1111 ic_db_host=$$ic_db_host gateway_node_name=kube-ovn1-control-plane j2 yamls/ovn-ic.yaml.j2 -o /tmp/ovn-ic-1-alter.yaml
kubectl config use-context kind-kube-ovn
kubectl apply -f ovn-ic-0.yaml
sleep 6
Expand Down
58 changes: 56 additions & 2 deletions pkg/controller/ovn-ic.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,10 +90,39 @@ func (c *Controller) resyncInterConnection() {
klog.Errorf("failed to config auto route, %v", err)
return
}
if icEnabled == "true" && lastICCM != nil && reflect.DeepEqual(cm.Data, lastICCM) {

isCMEuqal := reflect.DeepEqual(cm.Data, lastICCM)
if icEnabled == "true" && lastICCM != nil && isCMEuqal {
return
}
c.ovnLegacyClient.OVNIcNBAddress = genHostAddress(cm.Data["ic-db-host"], cm.Data["ic-nb-port"])
if icEnabled == "true" && lastICCM != nil && !isCMEuqal {
if err := c.removeInterConnection(lastICCM["az-name"]); err != nil {
klog.Errorf("failed to remove ovn-ic, %v", err)
return
}
if err := c.delLearnedRoute(); err != nil {
klog.Errorf("failed to remove learned static routes, %v", err)
return
}
c.ovnLegacyClient.OvnICSbAddress = genHostAddress(cm.Data["ic-db-host"], cm.Data["ic-sb-port"])

if err := c.RemoveOldChassisInSbDB(); err != nil {
klog.Errorf("failed to remove remote chassis: %v", err)
}

c.ovnLegacyClient.OvnICNbAddress = genHostAddress(cm.Data["ic-db-host"], cm.Data["ic-nb-port"])
klog.Info("start to reestablish ovn-ic")
if err := c.establishInterConnection(cm.Data); err != nil {
klog.Errorf("failed to reestablish ovn-ic, %v", err)
return
}
icEnabled = "true"
lastICCM = cm.Data
klog.Info("finish reestablishing ovn-ic")
return
}

c.ovnLegacyClient.OvnICNbAddress = genHostAddress(cm.Data["ic-db-host"], cm.Data["ic-nb-port"])
klog.Info("start to establish ovn-ic")
if err := c.establishInterConnection(cm.Data); err != nil {
klog.Errorf("failed to establish ovn-ic, %v", err)
Expand Down Expand Up @@ -427,6 +456,31 @@ func (c *Controller) SynRouteToPolicy() {
}
}

func (c *Controller) RemoveOldChassisInSbDB() error {

azUUID, err := c.ovnLegacyClient.GetAZUUID(lastICCM["az-name"])
if err != nil {
klog.Errorf("chassis ungetable %v", err)
}

gateways, err := c.ovnLegacyClient.GetGatewayUUIDsInOneAZ(azUUID)
if err != nil {
klog.Errorf("gateways in as %v ungetable %v", azUUID, err)
}

routes, err := c.ovnLegacyClient.GetRouteUUIDsInOneAZ(azUUID)
if err != nil {
klog.Errorf("routes in as %v ungetable %v", azUUID, err)
}

c.ovnLegacyClient.DestroyGateways(gateways)
c.ovnLegacyClient.DestroyRoutes(routes)
if err := c.ovnLegacyClient.DestroyChassis(azUUID); err != nil {
return err
}
return nil
}

func stripPrefix(policyMatch string) (string, error) {
matches := strings.Split(policyMatch, "==")
if strings.Trim(matches[0], " ") == util.MatchV4Dst {
Expand Down
2 changes: 1 addition & 1 deletion pkg/ovs/ovn-ic-nbctl.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ import (

func (c LegacyClient) ovnIcNbCommand(cmdArgs ...string) (string, error) {
start := time.Now()
cmdArgs = append([]string{fmt.Sprintf("--timeout=%d", c.OvnTimeout), fmt.Sprintf("--db=%s", c.OVNIcNBAddress)}, cmdArgs...)
cmdArgs = append([]string{fmt.Sprintf("--timeout=%d", c.OvnTimeout), fmt.Sprintf("--db=%s", c.OvnICNbAddress)}, cmdArgs...)
raw, err := exec.Command(OVNIcNbCtl, cmdArgs...).CombinedOutput()
elapsed := float64((time.Since(start)) / time.Millisecond)
klog.V(4).Infof("command %s %s in %vms", OVNIcNbCtl, strings.Join(cmdArgs, " "), elapsed)
Expand Down
117 changes: 117 additions & 0 deletions pkg/ovs/ovn-ic-sbctl.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
package ovs

import (
"fmt"
"os/exec"
"strings"
"time"

"k8s.io/klog/v2"
)

func (c LegacyClient) ovnIcSbCommand(cmdArgs ...string) (string, error) {
start := time.Now()
cmdArgs = append([]string{fmt.Sprintf("--timeout=%d", c.OvnTimeout), fmt.Sprintf("--db=%s", c.OvnICSbAddress)}, cmdArgs...)
raw, err := exec.Command(OVNIcSbCtl, cmdArgs...).CombinedOutput()
elapsed := float64((time.Since(start)) / time.Millisecond)
klog.V(4).Infof("command %s %s in %vms", OVNIcSbCtl, strings.Join(cmdArgs, " "), elapsed)
method := ""
for _, arg := range cmdArgs {
if !strings.HasPrefix(arg, "--") {
method = arg
break
}
}
code := "0"
defer func() {
ovsClientRequestLatency.WithLabelValues("ovn-ic-sb", method, code).Observe(elapsed)
}()

if err != nil {
code = "1"
klog.Warningf("ovn-ic-sbctl command error: %s %s in %vms", OVNIcSbCtl, strings.Join(cmdArgs, " "), elapsed)
return "", fmt.Errorf("%s, %q", raw, err)
} else if elapsed > 500 {
klog.Warningf("ovn-ic-sbctl command took too long: %s %s in %vms", OVNIcSbCtl, strings.Join(cmdArgs, " "), elapsed)
}
return trimCommandOutput(raw), nil
}

func (c LegacyClient) FindUUIDWithAttrInTable(attribute, value, table string) ([]string, error) {
key := attribute + "=" + value
output, err := c.ovnIcSbCommand("--format=csv", "--no-heading", "--data=bare", "--columns=_uuid", "find", table, key)
if err != nil {
return nil, fmt.Errorf("failed to find ovn-ic-sb db, %v", err)
}
lines := strings.Split(output, "\n")
result := make([]string, 0, len(lines))
for _, l := range lines {
if len(strings.TrimSpace(l)) == 0 {
continue
}
result = append(result, strings.TrimSpace(l))
}
return result, nil
}

func (c LegacyClient) DestroyTableWithUUID(uuid, table string) error {
_, err := c.ovnIcSbCommand("destroy", table, uuid)
if err != nil {
return fmt.Errorf("failed to destroy table %v with %vcause %v", table, uuid, err)
}
return nil
}

func (c LegacyClient) GetAZUUID(az string) (string, error) {
uuids, err := c.FindUUIDWithAttrInTable("name", az, "availability_zone")
if err != nil {
return "", fmt.Errorf("failed to get ovn-ic-sb availability_zone uuid: %v", err)
}
if len(uuids) == 1 {
return uuids[0], nil
} else if len(uuids) == 0 {
return "", nil
}
return "", fmt.Errorf("two same-name chassises in one db is insane")
}

func (c LegacyClient) GetGatewayUUIDsInOneAZ(uuid string) ([]string, error) {
gateways, err := c.FindUUIDWithAttrInTable("availability_zone", uuid, "gateway")
if err != nil {
return nil, fmt.Errorf("failed to get ovn-ic-sb gateways with uuid %v: %v", uuid, err)
}
return gateways, nil
}

func (c LegacyClient) GetRouteUUIDsInOneAZ(uuid string) ([]string, error) {
routes, err := c.FindUUIDWithAttrInTable("availability_zone", uuid, "route")
if err != nil {
return nil, fmt.Errorf("failed to get ovn-ic-sb routes with uuid %v: %v", uuid, err)
}
return routes, nil
}

func (c LegacyClient) DestroyGateways(uuids []string) {
for _, uuid := range uuids {
if err := c.DestroyTableWithUUID(uuid, "gateway"); err != nil {
klog.Errorf("failed to delete gateway %v: %v", uuid, err)
}
continue
}
}

func (c LegacyClient) DestroyRoutes(uuids []string) {
for _, uuid := range uuids {
if err := c.DestroyTableWithUUID(uuid, "route"); err != nil {
klog.Errorf("failed to delete route %v: %v", uuid, err)
}
continue
}
}

func (c LegacyClient) DestroyChassis(uuid string) error {
if err := c.DestroyTableWithUUID(uuid, "availability_zone"); err != nil {
return fmt.Errorf("failed to delete chassis %v: %v", uuid, err)
}
return nil
}
4 changes: 3 additions & 1 deletion pkg/ovs/ovn.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ type LegacyClient struct {
OvnNbAddress string
OvnTimeout int
OvnSbAddress string
OVNIcNBAddress string
OvnICNbAddress string
OvnICSbAddress string
ClusterRouter string
ClusterTcpLoadBalancer string
ClusterUdpLoadBalancer string
Expand All @@ -47,6 +48,7 @@ const (
OvnNbCtl = "ovn-nbctl"
OvnSbCtl = "ovn-sbctl"
OVNIcNbCtl = "ovn-ic-nbctl"
OVNIcSbCtl = "ovn-ic-sbctl"
OvsVsCtl = "ovs-vsctl"
MayExist = "--may-exist"
IfExists = "--if-exists"
Expand Down
20 changes: 17 additions & 3 deletions test/e2e-ovnic/e2e_suite_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"os/exec"
"strings"
"testing"
"time"

v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -85,13 +86,26 @@ var _ = SynchronizedAfterSuite(func() {}, func() {
checkLSP("ts-az1", pods1.Items[0], f)
checkLSP("ts-az0", pods1.Items[0], f)

output, err = exec.Command("kubectl", "-n", "kube-system", "-l", "app=kube-ovn-pinger", "get", "pod", "-o=jsonpath={.items[0].metadata.name}").CombinedOutput()
pod, err := exec.Command("kubectl", "-n", "kube-system", "-l", "app=kube-ovn-pinger", "get", "pod", "-o=jsonpath={.items[0].metadata.name}").CombinedOutput()
Expect(err).NotTo(HaveOccurred())
Expect(output).ShouldNot(BeEmpty())
Expect(pod).ShouldNot(BeEmpty())

output, err = exec.Command("kubectl", "-n", "kube-system", "exec", "-i", string(output), "--", "/usr/bin/ping", ip0, "-c2").CombinedOutput()
output, err = exec.Command("kubectl", "-n", "kube-system", "exec", "-i", string(pod), "--", "/usr/bin/ping", ip0, "-c2").CombinedOutput()
Expect(err).NotTo(HaveOccurred())
Expect(string(output)).Should(ContainSubstring("0% packet loss"))

output, err = exec.Command("kubectl", "apply", "-f", "/tmp/ovn-ic-1-alter.yaml").CombinedOutput()
Expect(err).NotTo(HaveOccurred())
Expect(string(output)).Should(ContainSubstring("configured"))

time.Sleep(time.Second * 10)

checkLSP("ts-az1111", pods1.Items[0], f)

output, err = exec.Command("kubectl", "-n", "kube-system", "exec", "-i", string(pod), "--", "/usr/bin/ping", ip0, "-c2").CombinedOutput()
Expect(err).NotTo(HaveOccurred())
Expect(string(output)).Should(ContainSubstring("0% packet loss"))

})

func buildConfigFromFlags(context, kubeconfigPath string) (*rest.Config, error) {
Expand Down

0 comments on commit d41c043

Please sign in to comment.