Add locust and locust kind deploy

.devcontainer/Dockerfile

@@ -3,7 +3,7 @@
 # Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information.
 #-------------------------------------------------------------------------------------------------------------
 
-FROM golang:1.12.5-stretch
+FROM golang:1.12-buster
 
 ARG KubectlVersion=v1.16.2
 ARG HelmVersion=v3.0.3
@@ -16,6 +16,10 @@ RUN apt-get update \
     && apt-get -y install --no-install-recommends apt-utils 2>&1 \
     # Verify git, process tools, lsb-release (common in install instructions for CLIs) installed
     && apt-get -y install git procps lsb-release \
+    # Install python
+    && apt-get -y install --no-install-recommends git openssl build-essential ca-certificates nano curl python python3-dev python3-pip python3-venv python3-setuptools python3-wheel\
+    # Install pylint
+    && pip3 --disable-pip-version-check --no-cache-dir install pylint \
     # Install Editor
     && apt-get install vim -y \
     # Clean up
@@ -105,3 +109,8 @@ RUN git clone https://github.com/magicmonty/bash-git-prompt.git ~/.bash-git-prom
     && echo "if [ -f \"$HOME/.bash-git-prompt/gitprompt.sh\" ]; then GIT_PROMPT_ONLY_IN_REPO=1 && source $HOME/.bash-git-prompt/gitprompt.sh; fi" >> "/root/.bashrc"
 
 ENV PATH="/root/.kubectx:${PATH}"
+
+COPY ./locust/requirements.* ./
+COPY ./.devcontainer/scripts/python_venv.sh ./
+
+RUN   bash -f ./python_venv.sh

.devcontainer/devcontainer.json

@@ -5,7 +5,7 @@
 	"workspaceFolder": "/workspace",
 	"context": "..",
 	"workspaceMount": "src=${localWorkspaceFolder},dst=/workspace,type=bind",
-    "runArgs": [
+	"runArgs": [
 		// Mount the env file
 		"--env-file", "${localWorkspaceFolder}/.devcontainer/.env",
 		// Mount go mod cache
@@ -25,11 +25,13 @@
 		// Optionally mount k8s auth to existing cluster
 		// "-v", "${env:HOME}${env:USERPROFILE}/.kube:/home/vscode/.kube",
 	],
-	"forwardPorts": [8085],
+	"forwardPorts": [8085, 8089, 9090],
 	"extensions": [
 		"ms-azuretools.vscode-docker",
+		"ms-vscode.go",
+		"ms-python.python",
 		"humao.rest-client",
-		"ms-vscode.go"
+		"yzhang.markdown-all-in-one"
 	],
 	"settings": {
 		"go.gopath": "/go",
@@ -58,9 +60,19 @@
 		"go.lintFlags": [
 		"--fast"
 		],
+		"python.pythonPath": "/usr/local/bin/python",
+		"python.linting.enabled": true,
+		"python.linting.pylintEnabled": true,
+		"python.testing.pytestEnabled": true,
+		"python.linting.flake8Enabled": true,
+		"python.linting.lintOnSave": true,
+		"python.testing.promptToConfigure": false,
+		"python.testing.unittestEnabled": false,
+		"python.testing.nosetestsEnabled": false,
+		"python.formatting.provider": "black",
+		"python.linting.pylintPath": "/usr/local/bin/pylint",
 		"remote.extensionKind": {
 			"ms-azuretools.vscode-docker": "workspace"
 		}
 	}
-
-}
+}

.devcontainer/scripts/python_venv.sh

@@ -0,0 +1,10 @@
+#! /bin/bash 
+set -e
+set -x
+
+mkdir /python_venv
+python3 -m venv /python_venv/venv 
+source /python_venv/venv/bin/activate # You can also tell VSCode to use the interpretter in this location 
+pip3 install -r requirements.dev.txt 
+pip3 install -r requirements.txt 
+echo "source /python_venv/venv/bin/activate" >> "$HOME/.bashrc"

Makefile

@@ -1,3 +1,4 @@
+# Timestamp for image tags
 timestamp := $(shell /bin/date "+%Y%m%d-%H%M%S")
 
 # Image URL to use all building/pushing image targets
@@ -6,6 +7,12 @@ IMG ?= controller:latest
 # MockAPI image URL to use all building/pushing image targets
 MOCKAPI_IMG ?= mockapi:${timestamp}
 
+# MockAPI image URL to use all building/pushing image targets
+LOCUST_IMG ?= locust:${timestamp}
+
+# Default namespace for the installation
+LOCUST_FILE ?= "behaviours/scenario1_run_submit_delete.py"
+
 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
 CRD_OPTIONS ?= "crd:trivialVersions=true"
 
@@ -18,6 +25,7 @@ KIND_CLUSTER_NAME ?= "azure-databricks-operator"
 # Default namespace for the installation
 OPERATOR_NAMESPACE ?= "azure-databricks-operator-system"
 
+
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
 GOBIN=$(shell go env GOPATH)/bin
@@ -244,4 +252,14 @@ kind-load-image-mock-api: create-kindcluster  docker-build-mock-api install-prom
 
 kind-deploy-mock-api: kind-load-image-mock-api apply-manifests-mock-api
 
-aks-deploy-mock-api: docker-push-mock-api install-prometheus apply-manifests-mock-api
+kind-deploy-locust: create-kindcluster install-prometheus
+	docker build -t ${LOCUST_IMG} -f locust/Dockerfile .
+	kind load docker-image ${LOCUST_IMG} --name ${KIND_CLUSTER_NAME} -v 1
+	cat ./locust/manifests/deployment.yaml | sed "s|locust:latest|${LOCUST_IMG}|" | sed "s|behaviours/scenario1_run_submit_delete.py|${LOCUST_FILE}|" | kubectl apply -f -
+
+format-locust:
+	black .
+
+test-locust: 
+	pip install -e ./locust -q
+	pytest

docs/locust.md

@@ -0,0 +1,193 @@
+# Load testing with locust
+
+The load testing project for the [azure-databricks-operator](https://github.com/microsoft/azure-databricks-operator/) can be found under `/locust`. Tests are built and run using the python [Locust load testing framework](https://docs.locust.io/en/stable/index.html).
+
+## Table of contents <!-- omit in toc -->
+
+- [Load testing with locust](#load-testing-with-locust)
+  - [Deploying dependencies](#deploying-dependencies)
+  - [Build and Test](#build-and-test)
+    - [Deploy to kind](#deploy-to-kind)
+    - [Run tests](#run-tests)
+    - [Adding tests](#adding-tests)
+  - [Contribute](#contribute)
+    - [Extending the supported Databricks functionality](#extending-the-supported-databricks-functionality)
+  - [Prometheus Endpoint](#prometheus-endpoint)
+  - [Running test under docker](#running-test-under-docker)
+    - [Test locally against cluster](#test-locally-against-cluster)
+    - [Deploy into the cluster and run](#deploy-into-the-cluster-and-run)
+      - [How do I update a dashboard](#how-do-i-update-a-dashboard)
+      - [How do I set error conditions](#how-do-i-set-error-conditions)
+  - [Known issues](#known-issues)
+
+## Deploying dependencies
+
+For documentation on deploying the `azure-databricks-operator` and `databricks-mock-api` for testing see [deploy/README.md](deploy/README.md)
+
+## Build and Test
+
+Everything needed to build and test the project is set up in the dev container.
+
+To run the project without the dev container you need:
+
+- Python 3
+- Pip
+- Set up your python environment
+
+    ```bash
+    python -m venv venv
+    source venv/bin/activate # You can also tell VSCode to use the interpretter in this location
+    pip install -r requirements.dev.txt
+    pip install -r requirements.txt
+    ```
+
+### Deploy to kind
+
+> Before proceeding make sure your container or environment is up and running
+
+1. Deploy locust to local KIND instance. Set `LOCUST_FILE` to the  the locust scenario you'd like to run from  `locust/behaviours`.
+
+    ```bash
+    make kind-deploy-locust LOCUST_FILE="behaviours/scenario1_run_submit_delete.py"
+    ```
+
+2. Start the test server
+
+    ```bash
+    locust -f behaviours/<my_locust_file>.py
+    ```
+
+### Run tests
+
+Tests are written using `pytest`. More information [is available here](https://docs.pytest.org/en/latest/).
+
+> Before proceeding make sure your container or environment is up and running
+
+1. Run the tests from the root of the project
+
+    ```bash
+    make test-locust
+    ```
+
+2. All being well you should see the following output:
+
+    ```bash
+    ============================================================================================= test session starts ==============================================================================================
+    platform linux -- Python 3.7.4, pytest-5.3.2, py-1.8.0, pluggy-0.13.1
+    rootdir: /workspace
+    plugins: mock-1.13.0
+    collected 8 items
+
+
+    test/unit/db_run_client_test.py ........
+    ```
+
+### Adding tests
+
+The project is setup to automatically discover any tests under the `locust/test` folder. Provided the following criteria are met:
+
+- your test `.py` file follows the naming convention `<something>_test.py`
+- within your test file your methods follow the naming convention `def test_<what you want to test>()`
+
+## Contribute
+
+- Test files are added to the `/behaviours` directory
+- These files take the recommended format described by the Locust documentation representing the behvaiour of a single (or set of) users
+
+### Extending the supported Databricks functionality
+
+- `/locust_files/db_locust` contains all files related to how Locust can interact with Databricks using K8s via the [azure-databricks-operator](https://github.com/microsoft/azure-databricks-operator/)
+  - `db_locust`: The brain of the behaviour driven tests. Inherits from the default `Locust`, read more [here](https://docs.locust.io/en/stable/testing-other-systems.html)
+  - `db_client.py`: Core client used by the `db_locust`. It is a wrapper of "sub" clients that interface to specific databricks operator Kinds
+    - `db_run_client.py`: all actions relating to `run` api interfaces
+    - More clients to be added - ***this is where the majority of contributions will be made***
+  - `db_decorator.py`: A simple decorator for Databricks operations that gives basic metric logging and error handling
+
+## Prometheus Endpoint
+
+This suite of Locust tests exposes stats to Prometheus via a web endpoint.
+
+The endpoint is exposed at `/export/prometheus`. When running the tests with the web endpoints enabled, you can visit <http://localhost:8089/export/prometheus> to see the exported stats.
+
+## Running test under docker
+
+This guide assumes you have used `./deploy/README.md` to deploy an AKS Engine cluster and have the `KUBECONFG` set correctly and also used `./deploy/prometheus-grafana` to setup the `prometheus` operator.
+
+### Test locally against cluster
+
+To build and test the locust image locally againt the cluster you can run:
+
+```bash
+make docker-run-local
+```
+
+This will build a docker image **which contains the kubeconfig** file.
+
+> Why put the file in the docker image? As we're using a devcontainer the normal approach of mounting a file doesn't work as the path on the host to the file (which is what the dameon uses) isn't the same as the path in the devcontainer so the file is never mounted.
+
+### Deploy into the cluster and run
+
+1. To deploy into the cluster run and port forward:
+
+    ```bash
+    CONTAINER_REGISTRY=$ACR_LOGIN_SERVER make deploy-loadtest
+    k port-forward service/locust-loadtest 8089:8089 9090:9090 -n locust
+    ```
+
+2. Visit `http://localhost:8089` to start the loadtest from the locust web UI.
+
+3. View stats on the test
+
+      ```bash
+      kubectl port-forward service/prom-azure-databricks-operator-grafana 8080:80
+      ```
+
+      ```text
+      Username: admin
+      Password: prom-operator
+      http://localhost:8080
+      ```
+
+    Then navigate to the locust dashboard to view the results.
+
+If you want to setup port forwards for all the things then do the following:
+
+```bash
+k port-forward service/prom-azure-databricks-oper-prometheus 9091:9090 &
+k port-forward service/locust-loadtest 8089:8089 9090:9090 &
+k port-forward service/prom-azure-databricks-operator-grafana 8080:80 &
+
+Browse to locust webui   -> http://localhost:8089/
+Browse to locust metrics -> http://localhost:9090/
+Browse to Prometheus     -> http://localhost:9091/targets
+Browse to Grafana        -> http://localhost:8080/
+```
+
+> Note: If one of these port forwards stops working use `ps aux | grep kubectl` and look for the process id of the one thats broken then use `kill 21283` (your id in there) to stop it. Then rerun the port forward command.
+
+#### How do I update a dashboard
+
+Best way I've found is to import the JSON for the board into the grafana instance, edit it using the UI then export it back to JSON and update the file in the repo.
+
+#### How do I set error conditions
+
+For some of the load test scenarios we want to trigger error behaviour in the mock-api during a test run.
+
+First step for this is to port-forward the mock-api service
+
+```bash
+# port-forward to localhost:8085
+kubectl port-forward -n databricks-mock-api svc/databricks-mock-api 8085:8080
+```
+
+Next we can issue a `PATCH` request to update the error rate, e.g. to set 20% probability for status code 500 responses
+
+```bash
+curl --request PATCH \
+     --url http://localhost:8085/config \
+     --header 'content-type: application/json'   \
+     --data '{"DATABRICKS_MOCK_API_ERROR_500_PROBABILITY":20}'
+```
+## Known issues
+
+- When the port you're forwarding your Locust server to is not exposed from the container, you cannot hit it from your localhost machine. Use the [VSCode temporary port forwarding](https://code.visualstudio.com/docs/remote/containers#_temporarily-forwarding-a-port) to resolve this.