Skip to content

Commit

Permalink
ovn nb and sb can't bind lan ip in ssl (#2200)
Browse files Browse the repository at this point in the history
  • Loading branch information
changluyi authored Dec 30, 2022
1 parent 1489b65 commit c0d76fd
Show file tree
Hide file tree
Showing 8 changed files with 20 additions and 87 deletions.
12 changes: 0 additions & 12 deletions dist/images/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2024,12 +2024,6 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_IPS
valueFrom:
fieldRef:
fieldPath: status.podIPs
- name: ENABLE_BIND_LOCAL_IP
value: "$ENABLE_BIND_LOCAL_IP"
resources:
requests:
cpu: 300m
Expand Down Expand Up @@ -2524,12 +2518,6 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_IPS
valueFrom:
fieldRef:
fieldPath: status.podIPs
- name: ENABLE_BIND_LOCAL_IP
value: "$ENABLE_BIND_LOCAL_IP"
resources:
requests:
cpu: 300m
Expand Down
24 changes: 6 additions & 18 deletions dist/images/ovn-is-leader.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,24 +8,12 @@ ovn-ctl status_northd
ovn-ctl status_ovnnb
ovn-ctl status_ovnsb

BIND_LOCAL_ADDR=127.0.0.1
if [[ $ENABLE_BIND_LOCAL_IP == "true" ]]; then
POD_IPS_LIST=(${POD_IPS//,/ })
if [[ ${#POD_IPS_LIST[@]} == 1 ]]; then
if [[ $POD_IP =~ .*:.* ]]; then
BIND_LOCAL_ADDR=[${POD_IP}] #ipv6
else
BIND_LOCAL_ADDR=${POD_IP} #ipv4
fi
fi
fi

# For data consistency, only store leader address in endpoint
# Store ovn-nb leader to svc kube-system/ovn-nb
if [[ "$ENABLE_SSL" == "false" ]]; then
nb_leader=$(ovsdb-client query tcp:$BIND_LOCAL_ADDR:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
nb_leader=$(ovsdb-client query tcp:127.0.0.1:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
else
nb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:$BIND_LOCAL_ADDR:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
nb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:127.0.0.1:6641 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Northbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
fi

if [[ $nb_leader =~ "true" ]]
Expand All @@ -46,9 +34,9 @@ fi

# Store ovn-sb leader to svc kube-system/ovn-sb
if [[ "$ENABLE_SSL" == "false" ]]; then
sb_leader=$(ovsdb-client query tcp:$BIND_LOCAL_ADDR:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
sb_leader=$(ovsdb-client query tcp:127.0.0.1:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
else
sb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:$BIND_LOCAL_ADDR:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
sb_leader=$(ovsdb-client -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert query ssl:127.0.0.1:6642 "[\"_Server\",{\"table\":\"Database\",\"where\":[[\"name\",\"==\", \"OVN_Southbound\"]],\"columns\": [\"leader\"],\"op\":\"select\"}]")
fi

if [[ $sb_leader =~ "true" ]]
Expand All @@ -63,9 +51,9 @@ then
if [ "$northd_leader" == "" ]; then
# no available northd leader try to release the lock
if [[ "$ENABLE_SSL" == "false" ]]; then
ovsdb-client -v -t 1 steal tcp:$BIND_LOCAL_ADDR:6642 ovn_northd
ovsdb-client -v -t 1 steal tcp:127.0.0.1:6642 ovn_northd
else
ovsdb-client -v -t 1 -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert steal ssl:$BIND_LOCAL_ADDR:6642 ovn_northd
ovsdb-client -v -t 1 -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert steal ssl:127.0.0.1:6642 ovn_northd
fi
fi
fi
Expand Down
28 changes: 10 additions & 18 deletions dist/images/start-db.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,6 @@ DB_NB_PORT=${DB_NB_PORT:-6641}
DB_SB_ADDR=${DB_SB_ADDR:-::}
DB_SB_PORT=${DB_SB_PORT:-6642}
ENABLE_SSL=${ENABLE_SSL:-false}
ENABLE_BIND_LOCAL_IP=${ENABLE_BIND_LOCAL_IP:-false}
BIND_LOCAL_ADDR=[::]
if [[ $ENABLE_BIND_LOCAL_IP == "true" ]]; then
POD_IPS_LIST=(${POD_IPS//,/ })
if [[ ${#POD_IPS_LIST[@]} == 1 ]]; then
BIND_LOCAL_ADDR="[${POD_IP}]"
fi
fi

. /usr/share/openvswitch/scripts/ovs-lib || exit 1

Expand Down Expand Up @@ -185,8 +177,8 @@ if [[ "$ENABLE_SSL" == "false" ]]; then
--db-sb-create-insecure-remote=yes \
--db-nb-cluster-local-addr="[${POD_IP}]" \
--db-sb-cluster-local-addr="[${POD_IP}]" \
--db-nb-addr=$BIND_LOCAL_ADDR \
--db-sb-addr=$BIND_LOCAL_ADDR \
--db-nb-addr=[::] \
--db-sb-addr=[::] \
--ovn-northd-nb-db="$(gen_conn_str 6641)" \
--ovn-northd-sb-db="$(gen_conn_str 6642)" \
start_northd
Expand Down Expand Up @@ -230,8 +222,8 @@ if [[ "$ENABLE_SSL" == "false" ]]; then
--db-sb-cluster-local-addr="[${POD_IP}]" \
--db-nb-cluster-remote-addr="[${nb_leader_ip}]" \
--db-sb-cluster-remote-addr="[${sb_leader_ip}]" \
--db-nb-addr=$BIND_LOCAL_ADDR \
--db-sb-addr=$BIND_LOCAL_ADDR \
--db-nb-addr=[::] \
--db-sb-addr=[::] \
--ovn-northd-nb-db="$(gen_conn_str 6641)" \
--ovn-northd-sb-db="$(gen_conn_str 6642)" \
start_northd
Expand Down Expand Up @@ -285,16 +277,16 @@ else
--ovn-northd-ssl-ca-cert=/var/run/tls/cacert \
--db-nb-cluster-local-addr="[${POD_IP}]" \
--db-sb-cluster-local-addr="[${POD_IP}]" \
--db-nb-addr=$BIND_LOCAL_ADDR \
--db-sb-addr=$BIND_LOCAL_ADDR \
--db-nb-addr=[::] \
--db-sb-addr=[::] \
--ovn-northd-nb-db="$(gen_conn_str 6641)" \
--ovn-northd-sb-db="$(gen_conn_str 6642)" \
start_northd
ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_NB_PORT}":$BIND_LOCAL_ADDR
ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_NB_PORT}":[::]
ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set Connection . inactivity_probe=180000
ovn-nbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set NB_Global . options:use_logical_dp_groups=true

ovn-sbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_SB_PORT}":$BIND_LOCAL_ADDR
ovn-sbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set-connection pssl:"${DB_SB_PORT}":[::]
ovn-sbctl --no-leader-only -p /var/run/tls/key -c /var/run/tls/cert -C /var/run/tls/cacert set Connection . inactivity_probe=180000
else
# get leader if cluster exists
Expand Down Expand Up @@ -336,8 +328,8 @@ else
--db-sb-cluster-local-addr="[${POD_IP}]" \
--db-nb-cluster-remote-addr="[${nb_leader_ip}]" \
--db-sb-cluster-remote-addr="[${sb_leader_ip}]" \
--db-nb-addr=$BIND_LOCAL_ADDR \
--db-sb-addr=$BIND_LOCAL_ADDR \
--db-nb-addr=[::] \
--db-sb-addr=[::] \
--ovn-northd-nb-db="$(gen_conn_str 6641)" \
--ovn-northd-sb-db="$(gen_conn_str 6642)" \
start_northd
Expand Down
6 changes: 0 additions & 6 deletions kubeovn-helm/templates/central-deploy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -65,12 +65,6 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_IPS
valueFrom:
fieldRef:
fieldPath: status.podIPs
- name: ENABLE_BIND_LOCAL_IP
value: "{{- .Values.func.ENABLE_BIND_LOCAL_IP }}"
resources:
requests:
cpu: 300m
Expand Down
25 changes: 4 additions & 21 deletions pkg/ovn_leader_checker/ovn.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"k8s.io/client-go/tools/clientcmd"
"k8s.io/klog/v2"

kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
"github.com/kubeovn/kube-ovn/pkg/util"
)

Expand Down Expand Up @@ -139,18 +138,10 @@ func checkOvnIsAlive() bool {

func checkNbIsLeader() bool {
var command []string
listenIp := "127.0.0.1"
if os.Getenv("ENABLE_BIND_LOCAL_IP") == "true" {
listenIp = os.Getenv("POD_IP")
if util.CheckProtocol(listenIp) == kubeovnv1.ProtocolIPv6 {
listenIp = fmt.Sprintf("[%s]", os.Getenv("POD_IP"))
}
}

if os.Getenv(EnvSSL) == "false" {
command = []string{
"query",
fmt.Sprintf("tcp:%s:6641", listenIp),
"tcp:127.0.0.1:6641",
`["_Server",{"table":"Database","where":[["name","==","OVN_Northbound"]],"columns":["leader"],"op":"select"}]`,
}
} else {
Expand All @@ -162,7 +153,7 @@ func checkNbIsLeader() bool {
"-C",
"/var/run/tls/cacert",
"query",
fmt.Sprintf("ssl:%s:6641", listenIp),
"ssl:127.0.0.1:6641",
`["_Server",{"table":"Database","where":[["name","==","OVN_Northbound"]],"columns":["leader"],"op":"select"}]`,
}
}
Expand All @@ -185,18 +176,10 @@ func checkNbIsLeader() bool {

func checkSbIsLeader() bool {
var command []string
listenIp := "127.0.0.1"
if os.Getenv("ENABLE_BIND_LOCAL_IP") == "true" {
listenIp = os.Getenv("POD_IP")
if util.CheckProtocol(listenIp) == kubeovnv1.ProtocolIPv6 {
listenIp = fmt.Sprintf("[%s]", os.Getenv("POD_IP"))
}
}

if os.Getenv(EnvSSL) == "false" {
command = []string{
"query",
fmt.Sprintf("tcp:%s:6642", listenIp),
"tcp:127.0.0.1:6642",
`["_Server",{"table":"Database","where":[["name","==","OVN_Southbound"]],"columns":["leader"],"op":"select"}]`,
}
} else {
Expand All @@ -208,7 +191,7 @@ func checkSbIsLeader() bool {
"-C",
"/var/run/tls/cacert",
"query",
fmt.Sprintf("ssl:%s:6642", listenIp),
"ssl:127.0.0.1:6642",
`["_Server",{"table":"Database","where":[["name","==","OVN_Southbound"]],"columns":["leader"],"op":"select"}]`,
}
}
Expand Down
4 changes: 0 additions & 4 deletions yamls/ovn-dpdk.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -225,10 +225,6 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_IPS
valueFrom:
fieldRef:
fieldPath: status.podIPs
resources:
requests:
cpu: 500m
Expand Down
4 changes: 0 additions & 4 deletions yamls/ovn-ha.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -244,10 +244,6 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_IPS
valueFrom:
fieldRef:
fieldPath: status.podIPs
resources:
requests:
cpu: 500m
Expand Down
4 changes: 0 additions & 4 deletions yamls/ovn.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -254,10 +254,6 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_IPS
valueFrom:
fieldRef:
fieldPath: status.podIPs
resources:
requests:
cpu: 500m
Expand Down

0 comments on commit c0d76fd

Please sign in to comment.