diff --git a/test/860_weave_removes_dead_nodes_from_ipam_3_test.sh b/test/860_weave_removes_dead_nodes_from_ipam_3_test.sh new file mode 100755 index 0000000000..374cb2b9c9 --- /dev/null +++ b/test/860_weave_removes_dead_nodes_from_ipam_3_test.sh @@ -0,0 +1,100 @@ +#! /bin/bash + +. "$(dirname "$0")/config.sh" + +function howmany { echo $#; } + +# +# Test vars +# +TOKEN=112233.4455667788990000 +HOST1IP=$($SSH $HOST1 "getent hosts $HOST1 | cut -f 1 -d ' '") +NUM_HOSTS=$(howmany $HOSTS) +SUCCESS="$(( $NUM_HOSTS * ($NUM_HOSTS-1) )) established" +KUBECTL="sudo kubectl --kubeconfig /etc/kubernetes/admin.conf" +KUBE_PORT=6443 +IMAGE=weaveworks/network-tester:latest + +if [ -n "$COVERAGE" ]; then + COVERAGE_ARGS="env:\\n - name: EXTRA_ARGS\\n value: \"-test.coverprofile=/home/weave/cover.prof --\"" +else + COVERAGE_ARGS="env:" +fi + +# +# Utility functions +# +function teardown_kubernetes_cluster { + greyly echo "Tearing down kubernetes cluster" + for host in $HOSTS; do + run_on $host "sudo kubeadm reset && sudo rm -r -f /opt/cni/bin/*weave*" + done +} + +function setup_kubernetes_cluster { + teardown_kubernetes_cluster; + + greyly echo "Setting up kubernetes cluster" + + # kubeadm init upgrades to latest Kubernetes version by default, therefore we try to lock the version using the below option: + k8s_version="$(run_on $HOST1 "kubelet --version" | grep -oP "(?<=Kubernetes )v[\d\.\-beta]+")" + k8s_version_option="$([[ "$k8s_version" > "v1.6" ]] && echo "kubernetes-version" || echo "use-kubernetes-version")" + + for host in $HOSTS; do + if [ $host = $HOST1 ] ; then + run_on $host "sudo systemctl start kubelet && sudo kubeadm init --$k8s_version_option=$k8s_version --token=$TOKEN" + else + run_on $host "sudo systemctl start kubelet && sudo kubeadm join --token=$TOKEN $HOST1IP:$KUBE_PORT" + fi + done + + # Ensure Kubernetes uses locally built container images and inject code coverage environment variable (or do nothing depending on $COVERAGE): + sed -e "s%imagePullPolicy: Always%imagePullPolicy: Never%" \ + -e "s%env:%$COVERAGE_ARGS%" \ + "$(dirname "$0")/../prog/weave-kube/weave-daemonset-k8s-1.7.yaml" | run_on "$HOST1" "$KUBECTL apply -n kube-system -f -" +} + +function force_drop_node { + greyly echo "Resetting node $1 with 'sudo kubeadm reset'" + run_on $1 "sudo kubeadm reset" +} + +function weave_connected { + run_on $HOST1 "curl -sS http://127.0.0.1:6784/status | grep \"$SUCCESS\"" +} + +function unreachable_ip_addresses_count { + host=$1 + run_on $host "sudo weave status ipam | grep unreachable | wc -l" +} + +# +# Suite +# +function main { + WEAVE_IPAM_RECOVERY_DELAY_SECONDS=30 + + start_suite "Test weave-net deallocates from IPAM on node failure"; + + setup_kubernetes_cluster; + + # Need to wait until all pods have come up + assert_raises "wait_for_x weave_connected 'pods to be running weave net' $POD_SETUP_TIMEOUT_SECONDS" + + assert "unreachable_ip_addresses_count $HOST1" "0"; + assert "unreachable_ip_addresses_count $HOST2" "0"; + assert "unreachable_ip_addresses_count $HOST3" "0"; + + force_drop_node $HOST2; + + sleep $WEAVE_IPAM_RECOVERY_DELAY_SECONDS; + + assert "unreachable_ip_addresses_count $HOST1" "0"; + assert "unreachable_ip_addresses_count $HOST3" "0"; + + teardown_kubernetes_cluster; + + end_suite; +} + +main diff --git a/test/README.md b/test/README.md index a3e29f1cf4..10660a0c10 100644 --- a/test/README.md +++ b/test/README.md @@ -11,6 +11,7 @@ The `Vagrantfile` in this directory constructs three such VMs. If you are [building Weave Net using Vagrant](https://www.weave.works/docs/net/latest/building/), it is recommended to run the tests from the build VM and not the host. +> **NOTE WELL** The number of hosts required by a test must be specified in the test name. For example, `000_some_test.sh` will use one host, while `001_some_other_3_test.sh` will use three hosts. The number of hosts must be the last part of the test name before the `_test.sh` prefix. See `/tools/runner/runner.go` for details ## Running tests