Add support for testing code changes in kubernetes (#105)

automated via Vagrant provisioned local VMs

fixes #105

.dockerignore

@@ -0,0 +1 @@
+**/_cache

.gitignore

@@ -1,2 +1,4 @@
 /kube-router
 /gobgp
+_output
+_cache

Documentation/developing.md

@@ -49,6 +49,76 @@ kube-router images, cloudnativelabs/kube-router. You can push to a different
 repository by changing a couple settings, as described in [Image Options](#image-options)
 below.
 
+## Testing Code Changes
+
+### Running Your Code On A Local VM Cluster
+
+Running your code changes in a real Kubernetes cluster is easy. Just make sure
+you have Virtualbox, VMware Fusion, or VMware Workstation installed and run:
+```
+make vagrant-up-single-node
+```
+
+Alternatively if you have 6GB RAM for the VMs, you can run a multi-node cluster
+that consists of a dedicated etcd node, a controller node, and a worker node:
+```
+make vagrant-up-multi-node
+```
+
+You will see lots of output as the VMs are provisioned, and the first run may
+take some time as VM and container images are downloaded. After the cluster is
+up you will recieve instructions for using kubectl and gaining ssh access:
+```
+  SUCCESS! The local cluster is ready.
+
+  ### kubectl usage ###
+  # Quickstart - Use this kubeconfig for individual commands
+  KUBECONFIG=/tmp/kr-vagrant-shortcut/cluster/auth/kubeconfig kubectl get pods --all-namespaces -o wide
+  #
+  ## OR ##
+  #
+  # Use this kubeconfig for the current terminal session
+  KUBECONFIG=/tmp/kr-vagrant-shortcut/cluster/auth/kubeconfig
+  export KUBECONFIG
+  kubectl get pods --all-namespaces -o wide
+  #
+  ## OR ##
+  #
+  # Backup and replace your default kubeconfig
+  # Note: This will continue to work on recreated local clusters
+  mv ~/.kube/config ~/.kube/config-backup
+  ln -s /tmp/kr-vagrant-shortcut/cluster/auth/kubeconfig ~/.kube/config
+
+  ### SSH ###
+  # Get node names
+  make vagrant status
+  # SSH into a the controller node (c1)
+  make vagrant ssh c1
+```
+
+#### Managing A Local VM Cluster
+
+You can use [Vagrant](https://www.vagrantup.com/docs/cli/) commands against the
+running cluster with `make vagrant COMMANDS`.
+
+For example, `make vagrant status` outputs:
+```
+Current machine states:
+
+e1                        not created (virtualbox)
+c1                        not created (virtualbox)
+w1                        not created (virtualbox)
+
+This environment represents multiple VMs. The VMs are all listed
+above with their current state. For more information about a specific
+VM, run `vagrant status NAME`.
+```
+
+With this information you can ssh into any of the VMs listed:
+```
+make vagrant ssh c1
+```
+
 ### Makefile Options
 
 There are several variables which can be modified in the Makefile to customize

Makefile

@@ -23,6 +23,22 @@ kube-router: $(shell find . -name \*.go) ## Builds kube-router.
 
 test: gofmt ## Runs code quality pipelines (gofmt, tests, coverage, lint, etc)
 
+vagrant-up-single-node: all vagrant-destroy ## Test the current codebase in a local VM single-node cluster
+	@docker="$(DOCKER)" hack/vagrant-up.sh
+
+vagrant-up-multi-node: all vagrant-destroy ## Test the current codebase in a local VM multi-node cluster
+	@docker="$(DOCKER)" HACK_MULTI_NODE="true" hack/vagrant-up.sh
+
+vagrant: ## Run vagrant against a previously up'd cluster. Example: make vagrant status
+	@hack/vagrant.sh $(VAGRANT_RUN_ARGS)
+
+vagrant-destroy: ## Destroy a previously created local VM cluster
+	@hack/vagrant-destroy.sh
+	@HACK_MULTI_NODE="true" hack/vagrant-destroy.sh
+
+vagrant-clean: vagrant-destroy ## Destroy a previously created local VM cluster and remove all downloaded/generated assets
+	@rm -rf hack/_output
+
 run: kube-router ## Runs "kube-router --help".
 	./kube-router --help
 
@@ -75,7 +91,6 @@ release: push-release github-release ## Pushes a release to DockerHub and GitHub
 clean: ## Removes the kube-router binary and Docker images
 	rm -f kube-router
 	$(DOCKER) rmi $(REGISTRY_DEV)
-	$(DOCKER) rmi $(REGISTRY)
 
 gofmt: ## Tells you what files need to be gofmt'd.
 	@build/verify-gofmt.sh
@@ -131,7 +146,7 @@ gobgp: vendor/github.com/osrg/gobgp/gobgp
 # http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
 help:
 	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | \
-	  awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-15s\033[0m %s\n", $$1, $$2}'
+	  awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-22s\033[0m %s\n", $$1, $$2}'
 
 # TODO: Uncomment this target when all deps are version-pinned in glide.yaml
 # update-glide:
@@ -140,8 +155,17 @@ help:
 # 	# go get -d -u github.com/sgotti/glide-vc
 # 	glide vc --only-code --no-tests
 
+# If the first argument is "vagrant"...
+ifeq (vagrant,$(firstword $(MAKECMDGOALS)))
+  # use the rest as arguments for "vagrant"
+  VAGRANT_RUN_ARGS := $(wordlist 2,$(words $(MAKECMDGOALS)),$(MAKECMDGOALS))
+  # ...and turn them into do-nothing targets
+  $(eval $(VAGRANT_RUN_ARGS):;@:)
+endif
+
 .PHONY: build clean container run release goreleaser push gofmt gofmt-fix
 .PHONY: update-glide test docker-login push-release github-release help
-.PHONY: gopath gopath-fix
+.PHONY: gopath gopath-fix vagrant-up-single-node
+.PHONY: vagrant-up-multi-node vagrant-destroy vagrant-clean vagrant
 
 .DEFAULT: all

hack/sync-image-cache.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env sh
+# vim: noai:ts=2:sw=2:set expandtab
+set -e
+
+HACK_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)"
+export HACK_DIR
+
+# shellcheck source=vagrant-common.sh
+. "${HACK_DIR}/vagrant-common.sh"
+
+command -v rkt >/dev/null 2>&1    && RKT_INSTALLED=1
+command -v docker >/dev/null 2>&1 && DOCKER_INSTALLED=1
+
+if [ -z "${RKT_INSTALLED}" ]; then
+  echo "WARN: rkt not found. Skipping rkt ACI image caching."
+else
+  if [ -f "${HACK_ACI_CACHE_FILE}" ]; then
+    echo "INFO: Cached hyperkube ACI already exists."
+    echo "INFO: Location: ${HACK_ACI_CACHE_FILE}"
+  else
+    echo "INFO: Fetching ${HYPERKUBE_IMG_URL} ACI."
+    sudo rkt --trust-keys-from-https fetch "${HYPERKUBE_IMG_URL}"
+
+    HYPERKUBE_ACI_ID="$(sudo rkt image list | grep -F "${HYPERKUBE_IMG_URL}" | awk '{print $1}')"
+
+    echo "INFO: Saving ${HYPERKUBE_IMG_URL} ACI to cache directory."
+    sudo rkt image export "${HYPERKUBE_ACI_ID}" "${HACK_ACI_CACHE_FILE}"
+  fi
+fi
+
+if [ -z "${DOCKER_INSTALLED}" ]; then
+  echo "WARN: docker not found. Skipping docker image caching."
+else
+  if [ -f "${HACK_DOCKER_CACHE_FILE}" ]; then
+    echo "INFO: Cached hyperkube Docker image already exists."
+    echo "INFO: Location: ${HACK_DOCKER_CACHE_FILE}"
+  else
+    echo "INFO: Fetching ${HYPERKUBE_IMG_URL} Docker image."
+    "${docker}" pull "${HYPERKUBE_IMG_URL}"
+
+    echo "INFO: Saving ${HYPERKUBE_IMG_URL} Docker image to cache directory."
+    "${docker}" save "${HYPERKUBE_IMG_URL}" -o "${HACK_DOCKER_CACHE_FILE}"
+  fi
+fi

hack/vagrant-common.sh

@@ -0,0 +1,51 @@
+#!/usr/bin/env sh
+# vim: noai:ts=2:sw=2:set expandtab
+set -e
+
+if [ -z "${HACK_DIR}" ]; then
+  echo "ERROR: HACK_DIR must be specified."
+  echo
+  echo "## Example"
+  echo "HACK_DIR=\"\$\(CDPATH=i\'\' cd -- \"\$\(dirname -- \"\$0\"\)\" \&\& pwd -P\)\""
+  echo "export HACK_DIR"
+  echo ". ${0}"
+  echo
+  exit 1
+fi
+
+REPO_DIR="$(dirname "${HACK_DIR}")"
+HACK_TMP_DIR="${HACK_DIR}/_cache"
+HACK_MANIFEST_DIRS="${REPO_DIR}/contrib/bootkube"
+export HACK_MANIFEST_DIRS
+
+[ -z "${BK_VERSION}" ]   && BK_VERSION="v0.6.0_kube-router"
+[ -z "${BK_CLONE_URL}" ] && BK_CLONE_URL="https://github.com/bzub/bootkube.git"
+[ -z "${BK_CLONE_DIR}" ] && BK_CLONE_DIR="${HACK_TMP_DIR}/bootkube"
+
+if [ -z "${HACK_MULTI_NODE}" ]; then
+  BK_HACK_DIR="${BK_CLONE_DIR}/hack/single-node"
+else
+  BK_HACK_DIR="${BK_CLONE_DIR}/hack/multi-node"
+fi
+export BK_HACK_DIR
+
+BK_SHORTCUT_DIR="/tmp/kr-vagrant-shortcut"
+export BK_SHORTCUT_DIR
+
+[ -z "${KR_IMAGE_TAG}" ] && KR_IMAGE_TAG="test.kube-router.io"
+[ -z "${KR_MANIFEST}" ]  && KR_MANIFEST="kube-router.yaml"
+[ -z "${docker}" ] && docker="sudo docker"
+KR_MANIFEST_PATH="${BK_CLONE_DIR}/hack/custom-manifests/${KR_MANIFEST}"
+export KR_MANIFEST_PATH
+
+# TODO: Dynamically determine this from Bootkube version/source
+[ -z "${HYPERKUBE_IMG}" ]     && HYPERKUBE_IMG="quay.io/coreos/hyperkube"
+[ -z "${HYPERKUBE_IMG_TAG}" ] && HYPERKUBE_IMG_TAG="v1.7.1_coreos.0"
+HYPERKUBE_IMG_URL="${HYPERKUBE_IMG}:${HYPERKUBE_IMG_TAG}"
+export HYPERKUBE_IMG_URL
+HACK_IMG_CACHE_DIR="${BK_CLONE_DIR}/hack/local-images"
+export HACK_IMG_CACHE_DIR
+HACK_ACI_CACHE_FILE="${HACK_IMG_CACHE_DIR}/hyperkube-${HYPERKUBE_IMG_TAG}.aci"
+export HACK_ACI_CACHE_FILE
+HACK_DOCKER_CACHE_FILE="${HACK_IMG_CACHE_DIR}/hyperkube-${HYPERKUBE_IMG_TAG}.docker"
+export HACK_DOCKER_CACHE_FILE

hack/vagrant-destroy.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env sh
+# vim: noai:ts=2:sw=2:set expandtab
+set -e
+
+HACK_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)"
+export HACK_DIR
+
+# shellcheck source=vagrant-common.sh
+. "${HACK_DIR}/vagrant-common.sh"
+
+if [ ! -d "${BK_SHORTCUT_DIR}" ]; then
+  echo "INFO: bootkube hack shortcut not found."
+  exit 0
+fi
+
+echo "INFO: Running vagrant destroy -f"
+cd "${BK_SHORTCUT_DIR}"
+vagrant destroy -f
+
+echo "INFO: Removing cluster assets."
+rm -rf "${BK_SHORTCUT_DIR}/cluster"
+
+echo "INFO: Removing symbolic link to Bootkube hack directory"
+rm -rf "${BK_SHORTCUT_DIR}"

hack/vagrant-up.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env sh
+# vim: noai:ts=2:sw=2:set expandtab
+set -e
+
+HACK_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)"
+export HACK_DIR
+
+# shellcheck source=vagrant-common.sh
+. "${HACK_DIR}/vagrant-common.sh"
+
+if [ -n "${1}" ]; then
+  echo "### Usage ###"
+  echo "# Single node cluster."
+  echo "${0}"
+  echo
+  echo "# Multi node cluster."
+  echo "HACK_MULTI_NODE=\"true\" ${0}"
+  echo
+  echo "# Use custom k8s manifests."
+  echo "HACK_MANIFEST_DIRS=\"/path/to/yaml/files\" ${0}"
+  echo
+  echo "# HACK_MANIFEST_DIRS is one or more space separated directories and"
+  echo "# should at least include kube-router.yaml and kube-router-cfg.yaml"
+  echo "# or equivalent."
+  echo
+  exit 0
+fi
+
+# Get bootkube
+if [ -d "${BK_CLONE_DIR}/.git" ]; then
+  echo "INFO: Bootkube repo already cloned."
+  echo "INFO: Checking out version ${BK_VERSION}."
+  cd "${BK_CLONE_DIR}"
+  git fetch
+  git checkout "${BK_VERSION}"
+else
+  echo "INFO: Bootkube repo not found."
+  echo "INFO: Cloning bootkube version ${BK_VERSION}."
+  git clone --depth=1 --branch "${BK_VERSION}" "${BK_CLONE_URL}" "${BK_CLONE_DIR}"
+fi
+
+# Export the kube-router container image
+echo "INFO: Exporting your kube-router container image."
+mkdir -p "${HACK_IMG_CACHE_DIR}"
+"${docker}" tag cloudnativelabs/kube-router-git:latest "${KR_IMAGE_TAG}"
+"${docker}" save "${KR_IMAGE_TAG}" -o "${HACK_IMG_CACHE_DIR}/kube-router.docker"
+
+# Copy cached images to Bootkube local-images directory
+echo "INFO: Caching hyperkube images to Bootkube local-images directory."
+"${HACK_DIR}/sync-image-cache.sh"
+
+# Copy custom manifests for Bootkube to use
+echo "INFO: Using custom manifests from ${HACK_MANIFEST_DIRS}"
+mkdir -p "${BK_CLONE_DIR}/hack/custom-manifests"
+for i in ${HACK_MANIFEST_DIRS}
+do
+  cp -f "${i}"/*.yaml "${BK_CLONE_DIR}/hack/custom-manifests" \
+    || echo "INFO: No custom .yaml files found."
+  cp -f "${i}"/*.yml "${BK_CLONE_DIR}/hack/custom-manifests" \
+    || echo "INFO: No custom .yml files found."
+
+  if [ -f "${KR_MANIFEST_PATH}" ]; then
+    echo "Modifying image attribute in ${KR_MANIFEST_PATH}"
+    sed -i -e "s/image: cloudnativelabs\/kube-router/image: ${KR_IMAGE_TAG}/" \
+      "${KR_MANIFEST_PATH}"
+    sed -i -e "s/imagePullPolicy: Always/imagePullPolicy: Never/" "${KR_MANIFEST_PATH}"
+    echo "Verify modification:"
+    grep -F "image: " "${KR_MANIFEST_PATH}"
+    grep -F "imagePullPolicy: " "${KR_MANIFEST_PATH}"
+  else
+    echo "kube-router manifest not found at ${KR_MANIFEST_PATH}"
+    echo "Couldn't modify."
+  fi
+done
+
+# Build Bootkube
+echo "INFO: Building Bootkube"
+make -C "${BK_CLONE_DIR}"
+
+# Start cluster
+echo "INFO: Starting VM(s) and cluster"
+cd "${BK_HACK_DIR}"
+KUBE_ROUTER="true" ./bootkube-up
+
+# Create symlink to bootkube hack dir
+ln -sf "${BK_HACK_DIR}" "${BK_SHORTCUT_DIR}"
+
+echo
+echo "SUCCESS! The local cluster is ready."
+echo
+echo "### kubectl usage ###"
+echo "# Quickstart - Use this kubeconfig for individual commands"
+echo "KUBECONFIG=${BK_SHORTCUT_DIR}/cluster/auth/kubeconfig kubectl get pods --all-namespaces -o wide"
+echo "#"
+echo "## OR ##"
+echo "#"
+echo "# Use this kubeconfig for the current terminal session"
+echo "KUBECONFIG=${BK_SHORTCUT_DIR}/cluster/auth/kubeconfig"
+echo "export KUBECONFIG"
+echo "kubectl get pods --all-namespaces -o wide"
+echo "#"
+echo "## OR ##"
+echo "#"
+echo "# Backup and replace your default kubeconfig"
+echo "# Note: This will continue to work on recreated local clusters"
+echo "mv ~/.kube/config ~/.kube/config-backup"
+echo "ln -s ${BK_SHORTCUT_DIR}/cluster/auth/kubeconfig ~/.kube/config"
+echo
+echo "### SSH ###"
+echo "# Get node names"
+echo "make vagrant status"
+echo "# SSH into a the controller node (c1)"
+echo "make vagrant ssh c1"
+echo
+echo "Enjoy!"

hack/vagrant.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env sh
+# vim: noai:ts=2:sw=2:set expandtab
+set -e
+
+HACK_DIR="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)"
+export HACK_DIR
+
+# shellcheck source=vagrant-common.sh
+. "${HACK_DIR}/vagrant-common.sh"
+
+if [ ! -d "${BK_SHORTCUT_DIR}" ]; then
+  echo "INFO: bootkube hack shortcut is not initialized."
+  echo "INFO: \"vagrant up\" has not been run yet."
+  exit 0
+fi
+
+cd "${BK_SHORTCUT_DIR}"
+vagrant "${@}"