Skip to content

Commit

Permalink
kubectl-ko: fix destination mac (#1801)
Browse files Browse the repository at this point in the history
  • Loading branch information
@zhangzujian
zhangzujian committed Aug 15, 2022
1 parent 1284f15 commit 5613b63
Show file tree
Hide file tree
Showing 2 changed files with 358 additions and 136 deletions.
247 changes: 179 additions & 68 deletions dist/images/install.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2476,6 +2476,105 @@ showHelp(){
echo " env-check check the environment configuration"
}
# usage: ipv4_to_hex 192.168.0.1
ipv4_to_hex(){
printf "%02x" ${1//./ }
}
# convert hex to dec (portable version)
hex2dec(){
for i in $(echo "$@"); do
printf "%d\n" "$(( 0x$i ))"
done
}
# https://github.com/chmduquesne/wg-ip
# usage: expand_ipv6 2001::1
expand_ipv6(){
local ip=$1
# prepend 0 if we start with :
echo $ip | grep -qs "^:" && ip="0${ip}"
# expand ::
if echo $ip | grep -qs "::"; then
local colons=$(echo $ip | sed 's/[^:]//g')
local missing=$(echo ":::::::::" | sed "s/$colons//")
local expanded=$(echo $missing | sed 's/:/:0/g')
ip=$(echo $ip | sed "s/::/$expanded/")
fi
local blocks=$(echo $ip | grep -o "[0-9a-f]\+")
set $blocks
printf "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n" \
$(hex2dec $@)
}
# convert an IPv6 address to bytes
ipv6_bytes(){
for x in $(expand_ipv6 $1 | tr ':' ' '); do
printf "%d %d " $((0x$x >> 8 & 0xff)) $((0x$x & 0xff))
done
echo
}
# usage: ipIsInCidr 192.168.0.1 192.168.0.0/24
# return: 0 for true, 1 for false
ipIsInCidr(){
local ip=$1
local cidr=$2
if [[ $ip =~ .*:.* ]]; then
# IPv6
cidr=${cidr#*,}
local network=${cidr%/*}
local prefix=${cidr#*/}
local ip_bytes=($(ipv6_bytes $ip))
local network_bytes=($(ipv6_bytes $network))
for ((i=0; i<${#ip_bytes[*]}; i++)); do
if [ ${ip_bytes[$i]} -eq ${network_bytes[$i]} ]; then
continue
fi
if [ $((($i+1)*8)) -le $prefix ]; then
return 1
fi
if [ $(($i*8)) -ge $prefix ]; then
return 0
fi
if [ $((($i+1)*8)) -le $prefix ]; then
return 1
fi
local bits=$(($prefix-$i*8))
local mask=$((0xff<<$bits & 0xff))
# TODO: check whether the IP is network/broadcast address
if [ $((${ip_bytes[$i]} & $mask)) -ne ${network_bytes[$i]} ]; then
return 1
fi
done
return 0
fi
# IPv4
cidr=${cidr%,*}
local network=${cidr%/*}
local prefix=${cidr#*/}
local ip_hex=$(ipv4_to_hex $ip)
local ip_dec=$((0x$ip_hex))
local network_hex=$(ipv4_to_hex $network)
local network_dec=$((0x$network_hex))
local broadcast_dec=$(($network_dec + 2**$prefix - 1))
# TODO: check whether the IP is network/broadcast address
if [ $ip_dec -gt $network_dec -a $ip_dec -lt $broadcast_dec ]; then
return 0
fi
return 1
}
tcpdump(){
namespacedPod="$1"; shift
namespace=$(echo "$namespacedPod" | cut -d "/" -f1)
Expand Down Expand Up @@ -2520,9 +2619,9 @@ tcpdump(){
trace(){
namespacedPod="$1"
namespace=$(echo "$1" | cut -d "/" -f1)
podName=$(echo "$1" | cut -d "/" -f2)
if [ "$podName" = "$1" ]; then
namespace=$(echo "$namespacedPod" | cut -d "/" -f1)
podName=$(echo "$namespacedPod" | cut -d "/" -f2)
if [ "$podName" = "$namespacedPod" ]; then
namespace="default"
fi
Expand All @@ -2532,6 +2631,12 @@ trace(){
exit 1
fi
hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork})
if [ "$hostNetwork" = "true" ]; then
echo "Can not trace host network pod"
exit 1
fi
af="4"
nw="nw"
proto=""
Expand All @@ -2541,18 +2646,6 @@ trace(){
proto="6"
fi
hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork})
if [ "$hostNetwork" = "true" ]; then
echo "Can not trace host network pod"
exit 1
fi
ls=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_switch})
if [ -z "$ls" ]; then
echo "pod address not ready"
exit 1
fi
podIPs=($(kubectl get pod "$podName" -n "$namespace" -o jsonpath="{.status.podIPs[*].ip}"))
if [ ${#podIPs[@]} -eq 0 ]; then
podIPs=($(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/ip_address} | sed 's/,/ /g'))
Expand All @@ -2562,9 +2655,6 @@ trace(){
fi
fi
mac=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address})
nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName})
podIP=""
for ip in ${podIPs[@]}; do
if [ "$af" = "4" ]; then
Expand All @@ -2579,76 +2669,97 @@ trace(){
done
if [ -z "$podIP" ]; then
echo "Pod has no IPv$af address"
echo "Pod $namespacedPod has no IPv$af address"
exit 1
fi
gwMac=""
vlan=$(kubectl get subnet "$ls" -o jsonpath={.spec.vlan})
logicalGateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.logicalGateway})
if [ ! -z "$vlan" -a "$logicalGateway" != "true" ]; then
gateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.gateway})
if [[ "$gateway" =~ .*,.* ]]; then
if [ "$af" = "4" ]; then
gateway=${gateway%%,*}
else
gateway=${gateway##*,}
fi
fi
ls=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_switch})
if [ -z "$ls" ]; then
echo "pod address not ready"
exit 1
fi
ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep -w kube-ovn-cni | grep " $nodeName " | awk '{print $1}')
if [ -z "$ovnCni" ]; then
echo "No kube-ovn-cni Pod running on node $nodeName"
exit 1
fi
local cidr=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/cidr})
mac=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address})
nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName})
nicName=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading --columns=name find interface external-ids:iface-id="$podName"."$namespace" | tr -d '\r')
if [ -z "$nicName" ]; then
echo "nic doesn't exist on node $nodeName"
exit 1
dstMac=""
if ipIsInCidr $dst $cidr; then
set +o pipefail
if [ $af -eq 4 ]; then
dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=addresses list logical_switch_port | grep -w "$(echo $dst | tr . '\.')" | awk '{print $1}')
else
dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=addresses list logical_switch_port | grep -i " $dst\$" | awk '{print $1}')
fi
set -o pipefail
fi
if [ -z "$dstMac" ]; then
vlan=$(kubectl get subnet "$ls" -o jsonpath={.spec.vlan})
logicalGateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.logicalGateway})
if [ ! -z "$vlan" -a "$logicalGateway" != "true" ]; then
gateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.gateway})
if [[ "$gateway" =~ .*,.* ]]; then
if [ "$af" = "4" ]; then
gateway=${gateway%%,*}
else
gateway=${gateway##*,}
fi
fi
podNicType=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/pod_nic_type})
podNetNs=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r' | sed -e 's/^"//' -e 's/"$//')
if [ "$podNicType" != "internal-port" ]; then
nicName="eth0"
fi
ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep -w kube-ovn-cni | grep " $nodeName " | awk '{print $1}')
if [ -z "$ovnCni" ]; then
echo "No kube-ovn-cni Pod running on node $nodeName"
exit 1
fi
if [[ "$gateway" =~ .*:.* ]]; then
cmd="ndisc6 -q $gateway $nicName"
output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ndisc6 -q "$gateway" "$nicName")
else
cmd="arping -c3 -C1 -i1 -I $nicName $gateway"
output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" arping -c3 -C1 -i1 -I "$nicName" "$gateway")
fi
nicName=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading --columns=name find interface external-ids:iface-id="$podName"."$namespace" | tr -d '\r')
if [ -z "$nicName" ]; then
echo "nic doesn't exist on node $nodeName"
exit 1
fi
if [ $? -ne 0 ]; then
echo "failed to run '$cmd' in Pod's netns"
exit 1
fi
gwMac=$(echo "$output" | grep -o -E '([[:xdigit:]]{1,2}:){5}[[:xdigit:]]{1,2}')
else
lr=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_router})
if [ -z "$lr" ]; then
lr=$(kubectl get subnet "$ls" -o jsonpath={.spec.vpc})
podNicType=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/pod_nic_type})
podNetNs=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r' | sed -e 's/^"//' -e 's/"$//')
if [ "$podNicType" != "internal-port" ]; then
nicName="eth0"
fi
if [[ "$gateway" =~ .*:.* ]]; then
cmd="ndisc6 -q $gateway $nicName"
output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ndisc6 -q "$gateway" "$nicName")
else
cmd="arping -c3 -C1 -i1 -I $nicName $gateway"
output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" arping -c3 -C1 -i1 -I "$nicName" "$gateway")
fi
if [ $? -ne 0 ]; then
echo "failed to run '$cmd' in Pod's netns"
exit 1
fi
dstMac=$(echo "$output" | grep -o -E '([[:xdigit:]]{1,2}:){5}[[:xdigit:]]{1,2}')
else
lr=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_router})
if [ -z "$lr" ]; then
lr=$(kubectl get subnet "$ls" -o jsonpath={.spec.vpc})
fi
dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=mac find logical_router_port name="$lr"-"$ls" | tr -d '\r')
fi
gwMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=mac find logical_router_port name="$lr"-"$ls" | tr -d '\r')
fi
if [ -z "$gwMac" ]; then
echo "get gw mac failed"
if [ -z "$dstMac" ]; then
echo "failed to get destination mac"
exit 1
fi
type="$3"
case $type in
icmp)
set -x
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && icmp && eth.src == $mac && ip$af.src == $podIP && eth.dst == $gwMac && ip$af.dst == $dst"
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && icmp && eth.src == $mac && ip$af.src == $podIP && eth.dst == $dstMac && ip$af.dst == $dst"
;;
tcp|udp)
set -x
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && eth.src == $mac && ip$af.src == $podIP && eth.dst == $gwMac && ip$af.dst == $dst && $type.src == 10000 && $type.dst == $4"
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && eth.src == $mac && ip$af.src == $podIP && eth.dst == $dstMac && ip$af.dst == $dst && $type.src == 10000 && $type.dst == $4"
;;
*)
echo "type $type not supported"
Expand All @@ -2673,11 +2784,11 @@ trace(){
case $type in
icmp)
set -x
kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,icmp$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$gwMac"
kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,icmp$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$dstMac"
;;
tcp|udp)
set -x
kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,$type$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$gwMac,${type}_src=1000,${type}_dst=$4"
kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,$type$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$dstMac,${type}_src=1000,${type}_dst=$4"
;;
*)
echo "type $type not supported"
Expand Down
Loading

0 comments on commit 5613b63

Please sign in to comment.