Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

kubectl-ko: fix destination mac #1801

Merged
merged 2 commits into from
Aug 15, 2022
Merged

kubectl-ko: fix destination mac #1801

Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
26cd3a6
kubectl-ko: fix destination mac
zhangzujian Aug 13, 2022
a46a18a
some optimization
zhangzujian Aug 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
249 changes: 181 additions & 68 deletions dist/images/kubectl-ko
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,105 @@ showHelp(){
echo " reload restart all kube-ovn components"
}

# usage: ipv4_to_hex 192.168.0.1
ipv4_to_hex(){
printf "%02x" ${1//./ }
}

# convert hex to dec (portable version)
hex2dec(){
for i in $(echo "$@"); do
printf "%d\n" "$(( 0x$i ))"
done
}

# https://github.com/chmduquesne/wg-ip
# usage: expand_ipv6 2001::1
expand_ipv6(){
local ip=$1

# prepend 0 if we start with :
echo $ip | grep -qs "^:" && ip="0${ip}"

# expand ::
if echo $ip | grep -qs "::"; then
local colons=$(echo $ip | sed 's/[^:]//g')
local missing=$(echo ":::::::::" | sed "s/$colons//")
local expanded=$(echo $missing | sed 's/:/:0/g')
ip=$(echo $ip | sed "s/::/$expanded/")
fi

local blocks=$(echo $ip | grep -o "[0-9a-f]\+")
set $blocks

printf "%04x:%04x:%04x:%04x:%04x:%04x:%04x:%04x\n" \
$(hex2dec $@)
}

# convert an IPv6 address to bytes
ipv6_bytes(){
for x in $(expand_ipv6 $1 | tr ':' ' '); do
printf "%d %d " $((0x$x >> 8 & 0xff)) $((0x$x & 0xff))
done
echo
}

# usage: ipIsInCidr 192.168.0.1 192.168.0.0/24
# return: 0 for true, 1 for false
ipIsInCidr(){
local ip=$1
local cidr=$2

if [[ $ip =~ .*:.* ]]; then
# IPv6
cidr=${cidr#*,}
local network=${cidr%/*}
local prefix=${cidr#*/}
local ip_bytes=($(ipv6_bytes $ip))
local network_bytes=($(ipv6_bytes $network))
for ((i=0; i<${#ip_bytes[*]}; i++)); do
if [ ${ip_bytes[$i]} -eq ${network_bytes[$i]} ]; then
continue
fi

if [ $((($i+1)*8)) -le $prefix ]; then
return 1
fi
if [ $(($i*8)) -ge $prefix ]; then
return 0
fi
if [ $((($i+1)*8)) -le $prefix ]; then
return 1
fi

local bits=$(($prefix-$i*8))
local mask=$((0xff<<$bits & 0xff))
# TODO: check whether the IP is network/broadcast address
if [ $((${ip_bytes[$i]} & $mask)) -ne ${network_bytes[$i]} ]; then
return 1
fi
done

return 0
fi

# IPv4
cidr=${cidr%,*}
local network=${cidr%/*}
local prefix=${cidr#*/}
local ip_hex=$(ipv4_to_hex $ip)
local ip_dec=$((0x$ip_hex))
local network_hex=$(ipv4_to_hex $network)
local network_dec=$((0x$network_hex))
local broadcast_dec=$(($network_dec + 2**$prefix - 1))
# TODO: check whether the IP is network/broadcast address
if [ $ip_dec -gt $network_dec -a $ip_dec -lt $broadcast_dec ]; then
return 0
fi

return 1
}

tcpdump(){
namespacedPod="$1"; shift
namespace=$(echo "$namespacedPod" | cut -d "/" -f1)
Expand Down Expand Up @@ -69,12 +168,18 @@ tcpdump(){

trace(){
namespacedPod="$1"
namespace=$(echo "$1" | cut -d "/" -f1)
podName=$(echo "$1" | cut -d "/" -f2)
if [ "$podName" = "$1" ]; then
namespace=$(echo "$namespacedPod" | cut -d "/" -f1)
podName=$(echo "$namespacedPod" | cut -d "/" -f2)
if [ "$podName" = "$namespacedPod" ]; then
namespace="default"
fi

hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork})
if [ "$hostNetwork" = "true" ]; then
echo "Can not trace host network pod"
exit 1
fi

dst="$2"
if [ -z "$dst" ]; then
echo "need a target ip address"
Expand All @@ -90,18 +195,6 @@ trace(){
proto="6"
fi

hostNetwork=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.hostNetwork})
if [ "$hostNetwork" = "true" ]; then
echo "Can not trace host network pod"
exit 1
fi

ls=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_switch})
if [ -z "$ls" ]; then
echo "pod address not ready"
exit 1
fi

podIPs=($(kubectl get pod "$podName" -n "$namespace" -o jsonpath="{.status.podIPs[*].ip}"))
if [ ${#podIPs[@]} -eq 0 ]; then
podIPs=($(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/ip_address} | sed 's/,/ /g'))
Expand All @@ -111,9 +204,6 @@ trace(){
fi
fi

mac=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address})
nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName})

podIP=""
for ip in ${podIPs[@]}; do
if [ "$af" = "4" ]; then
Expand All @@ -128,76 +218,99 @@ trace(){
done

if [ -z "$podIP" ]; then
echo "Pod has no IPv$af address"
echo "Pod $namespacedPod has no IPv$af address"
exit 1
fi

gwMac=""
vlan=$(kubectl get subnet "$ls" -o jsonpath={.spec.vlan})
logicalGateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.logicalGateway})
if [ ! -z "$vlan" -a "$logicalGateway" != "true" ]; then
gateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.gateway})
if [[ "$gateway" =~ .*,.* ]]; then
if [ "$af" = "4" ]; then
gateway=${gateway%%,*}
else
gateway=${gateway##*,}
fi
fi
ls=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_switch})
if [ -z "$ls" ]; then
echo "pod address not ready"
exit 1
fi

ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep -w kube-ovn-cni | grep " $nodeName " | awk '{print $1}')
if [ -z "$ovnCni" ]; then
echo "No kube-ovn-cni Pod running on node $nodeName"
exit 1
fi
local cidr=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/cidr})
mac=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/mac_address})
nodeName=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.spec.nodeName})

nicName=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading --columns=name find interface external-ids:iface-id="$podName"."$namespace" | tr -d '\r')
if [ -z "$nicName" ]; then
echo "nic doesn't exist on node $nodeName"
exit 1
dstMac=""
if ipIsInCidr $dst $cidr; then
if [ $af -eq 4 ]; then
set +o pipefail
dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=addresses list logical_switch_port | grep -w "$(echo $dst | tr . '\.')" | awk '{print $1}')
set -o pipefail
else
set +o pipefail
dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=addresses list logical_switch_port | grep -i " $dst\$" | awk '{print $1}')
set -o pipefail
fi
fi
if [ -z "$dstMac" ]; then
vlan=$(kubectl get subnet "$ls" -o jsonpath={.spec.vlan})
logicalGateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.logicalGateway})
if [ ! -z "$vlan" -a "$logicalGateway" != "true" ]; then
gateway=$(kubectl get subnet "$ls" -o jsonpath={.spec.gateway})
if [[ "$gateway" =~ .*,.* ]]; then
if [ "$af" = "4" ]; then
gateway=${gateway%%,*}
else
gateway=${gateway##*,}
fi
fi

podNicType=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/pod_nic_type})
podNetNs=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r' | sed -e 's/^"//' -e 's/"$//')
if [ "$podNicType" != "internal-port" ]; then
nicName="eth0"
fi
ovnCni=$(kubectl get pod -n $KUBE_OVN_NS -o wide | grep -w kube-ovn-cni | grep " $nodeName " | awk '{print $1}')
if [ -z "$ovnCni" ]; then
echo "No kube-ovn-cni Pod running on node $nodeName"
exit 1
fi

if [[ "$gateway" =~ .*:.* ]]; then
cmd="ndisc6 -q $gateway $nicName"
output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ndisc6 -q "$gateway" "$nicName")
else
cmd="arping -c3 -C1 -i1 -I $nicName $gateway"
output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" arping -c3 -C1 -i1 -I "$nicName" "$gateway")
fi
nicName=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading --columns=name find interface external-ids:iface-id="$podName"."$namespace" | tr -d '\r')
if [ -z "$nicName" ]; then
echo "nic doesn't exist on node $nodeName"
exit 1
fi

if [ $? -ne 0 ]; then
echo "failed to run '$cmd' in Pod's netns"
exit 1
fi
gwMac=$(echo "$output" | grep -o -E '([[:xdigit:]]{1,2}:){5}[[:xdigit:]]{1,2}')
else
lr=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_router})
if [ -z "$lr" ]; then
lr=$(kubectl get subnet "$ls" -o jsonpath={.spec.vpc})
podNicType=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/pod_nic_type})
podNetNs=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- ovs-vsctl --data=bare --no-heading get interface "$nicName" external-ids:pod_netns | tr -d '\r' | sed -e 's/^"//' -e 's/"$//')
if [ "$podNicType" != "internal-port" ]; then
nicName="eth0"
fi

if [[ "$gateway" =~ .*:.* ]]; then
cmd="ndisc6 -q $gateway $nicName"
output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" ndisc6 -q "$gateway" "$nicName")
else
cmd="arping -c3 -C1 -i1 -I $nicName $gateway"
output=$(kubectl exec "$ovnCni" -n $KUBE_OVN_NS -- nsenter --net="$podNetNs" arping -c3 -C1 -i1 -I "$nicName" "$gateway")
fi

if [ $? -ne 0 ]; then
echo "failed to run '$cmd' in Pod's netns"
exit 1
fi
dstMac=$(echo "$output" | grep -o -E '([[:xdigit:]]{1,2}:){5}[[:xdigit:]]{1,2}')
else
lr=$(kubectl get pod "$podName" -n "$namespace" -o jsonpath={.metadata.annotations.ovn\\.kubernetes\\.io/logical_router})
if [ -z "$lr" ]; then
lr=$(kubectl get subnet "$ls" -o jsonpath={.spec.vpc})
fi
dstMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=mac find logical_router_port name="$lr"-"$ls" | tr -d '\r')
fi
gwMac=$(kubectl exec $OVN_NB_POD -n $KUBE_OVN_NS -c ovn-central -- ovn-nbctl --data=bare --no-heading --columns=mac find logical_router_port name="$lr"-"$ls" | tr -d '\r')
fi

if [ -z "$gwMac" ]; then
echo "get gw mac failed"
if [ -z "$dstMac" ]; then
echo "failed to get destination mac"
exit 1
fi

type="$3"
case $type in
icmp)
set -x
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && icmp && eth.src == $mac && ip$af.src == $podIP && eth.dst == $gwMac && ip$af.dst == $dst"
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && icmp && eth.src == $mac && ip$af.src == $podIP && eth.dst == $dstMac && ip$af.dst == $dst"
;;
tcp|udp)
set -x
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && eth.src == $mac && ip$af.src == $podIP && eth.dst == $gwMac && ip$af.dst == $dst && $type.src == 10000 && $type.dst == $4"
kubectl exec "$OVN_SB_POD" -n $KUBE_OVN_NS -c ovn-central -- ovn-trace --ct=new "$ls" "inport == \"$podName.$namespace\" && ip.ttl == 64 && eth.src == $mac && ip$af.src == $podIP && eth.dst == $dstMac && ip$af.dst == $dst && $type.src == 10000 && $type.dst == $4"
;;
*)
echo "type $type not supported"
Expand All @@ -222,11 +335,11 @@ trace(){
case $type in
icmp)
set -x
kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,icmp$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$gwMac"
kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,icmp$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$dstMac"
;;
tcp|udp)
set -x
kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,$type$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$gwMac,${type}_src=1000,${type}_dst=$4"
kubectl exec "$ovsPod" -n $KUBE_OVN_NS -- ovs-appctl ofproto/trace br-int "in_port=$inPort,$type$proto,nw_ttl=64,${nw}_src=$podIP,${nw}_dst=$dst,dl_src=$mac,dl_dst=$dstMac,${type}_src=1000,${type}_dst=$4"
;;
*)
echo "type $type not supported"
Expand Down