From 13f1239a3e5736e298975fd27161b90f1fb26723 Mon Sep 17 00:00:00 2001 From: Tom Wilkie Date: Sun, 12 Jul 2015 09:34:52 +0000 Subject: [PATCH] Various improvements to unit test running - units.sh should exit on error. - Add race detector to unit test runner. - Use local testing/cover/cover tool in gen_coverage_report.sh and units.sh - Use the scheduler to parallelise the unit tests. - Make test run id a string and differentiate between unit runs and integration runs in the scheduler. - Pipe circle env vars through to build container so we can detect when to use the scheduler - Add python requests to the weave build container. - Quote test names in urls, make flask handle url segments with slashes in. --- .gitignore | 1 + Makefile | 12 ++++--- build/Dockerfile | 2 +- circle.yml | 5 +-- test/gen_coverage_reports.sh | 3 +- test/run_all.sh | 2 +- test/sched | 8 ++--- test/scheduler/.gitignore | 7 ++++ test/scheduler/README.md | 6 ++++ test/scheduler/main.py | 6 ++-- test/scheduler/requirements.txt | 1 + test/units.sh | 62 ++++++++++++++++++++++++--------- 12 files changed, 82 insertions(+), 33 deletions(-) create mode 100644 test/scheduler/.gitignore create mode 100644 test/scheduler/README.md create mode 100644 test/scheduler/requirements.txt diff --git a/.gitignore b/.gitignore index 36eeb6b23d..b79063526a 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,7 @@ prog/weaveproxy/weaveproxy prog/sigproxy/sigproxy prog/weavewait/weavewait prog/netcheck/netcheck +testing/cover/cover tools/bin tools/build releases diff --git a/Makefile b/Makefile index ac9515bcf3..1a20388402 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,9 @@ WEAVEPROXY_EXE=prog/weaveproxy/weaveproxy SIGPROXY_EXE=prog/sigproxy/sigproxy WEAVEWAIT_EXE=prog/weavewait/weavewait NETCHECK_EXE=prog/netcheck/netcheck +COVER_EXE=testing/cover/cover -EXES=$(WEAVER_EXE) $(WEAVEDNS_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(NETCHECK_EXE) +EXES=$(WEAVER_EXE) $(WEAVEDNS_EXE) $(SIGPROXY_EXE) $(WEAVEPROXY_EXE) $(WEAVEWAIT_EXE) $(NETCHECK_EXE) $(COVER_EXE) WEAVER_UPTODATE=.weaver.uptodate WEAVEDNS_UPTODATE=.weavedns.uptodate @@ -75,11 +76,12 @@ $(NETCHECK_EXE): prog/netcheck/netcheck.go # Sigproxy and weavewait need separate rules as they fail the netgo check in # the main build stanza due to not importing net package - $(SIGPROXY_EXE): prog/sigproxy/main.go - go build -o $@ ./$(@D) - $(WEAVEWAIT_EXE): prog/weavewait/main.go +$(COVER_EXE): testing/cover/cover.go + +$(WEAVEWAIT_EXE) $(SIGPROXY_EXE) $(COVER_EXE): + go get ./$(@D) go build -o $@ ./$(@D) $(WEAVER_UPTODATE): prog/weaver/Dockerfile $(WEAVER_EXE) @@ -106,7 +108,7 @@ $(WEAVE_EXPORT): $(IMAGES_UPTODATE) $(DOCKER_DISTRIB): curl -o $(DOCKER_DISTRIB) $(DOCKER_DISTRIB_URL) -tests: +tests: $(COVER_EXE) @test/units.sh $(PUBLISH): publish_%: diff --git a/build/Dockerfile b/build/Dockerfile index 23861bdfcd..08df1d9d97 100644 --- a/build/Dockerfile +++ b/build/Dockerfile @@ -3,7 +3,7 @@ FROM ubuntu RUN apt-get -y update && apt-get -y install --no-install-recommends ca-certificates apt-transport-https RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 RUN echo deb https://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list -RUN apt-get -y update && apt-get -y install --no-install-recommends build-essential git lxc-docker-1.3.1 mercurial libpcap-dev curl make pkg-config gcc bison flex +RUN apt-get -y update && apt-get -y install --no-install-recommends build-essential git lxc-docker-1.3.1 mercurial libpcap-dev curl make pkg-config gcc bison flex python-requests # When doing a build in a container, "apt-get update" happens twice, # which can be a very significant overhead for incremental builds. diff --git a/circle.yml b/circle.yml index 0d1298f81a..7b23b9f136 100644 --- a/circle.yml +++ b/circle.yml @@ -35,7 +35,7 @@ dependencies: test: override: - - test "$CIRCLE_NODE_INDEX" != "0" || (docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go -e "COVERDIR=test/coverage" weaveworks/weave-build tests): + - docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go -e COVERDIR=test/coverage -e SLOW=true -e CIRCLECI -e CIRCLE_BUILD_NUM -e CIRCLE_NODE_TOTAL -e CIRCLE_NODE_INDEX weaveworks/weave-build tests: parallel: true - docker run -v /var/run/docker.sock:/run/docker.sock -v /home/ubuntu:/home/go -e COVERAGE=true weaveworks/weave-build: parallel: true @@ -54,6 +54,7 @@ teardown: pre: - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; ./gen_coverage_reports.sh): parallel: true - - test "$CIRCLE_NODE_INDEX" != "0" || (go get github.com/mattn/goveralls && goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/test/profile.cov -service=circleci || true) + - test "$CIRCLE_NODE_INDEX" != "0" || (go get github.com/mattn/goveralls && goveralls -repotoken $COVERALLS_REPO_TOKEN -coverprofile=$SRCDIR/test/profile.cov -service=circleci || true): + parallel: true - test "$CIRCLE_NODE_INDEX" != "0" || (cd $SRCDIR/test; cp coverage.* $CIRCLE_ARTIFACTS): parallel: true diff --git a/test/gen_coverage_reports.sh b/test/gen_coverage_reports.sh index 523d09d17a..567cb0f33d 100755 --- a/test/gen_coverage_reports.sh +++ b/test/gen_coverage_reports.sh @@ -1,6 +1,7 @@ #!/bin/bash set -ex +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" if [ -n "$CIRCLECI" ]; then for i in $(seq 1 $(($CIRCLE_NODE_TOTAL - 1))); do @@ -8,7 +9,7 @@ if [ -n "$CIRCLECI" ]; then done fi -go get github.com/weaveworks/weave/testing/cover +$DIR/../testing/cover/cover cover ./coverage/* >profile.cov go tool cover -html=profile.cov -o coverage.html go tool cover -func=profile.cov -o coverage.txt diff --git a/test/run_all.sh b/test/run_all.sh index 4afd13d58a..ac1d114340 100755 --- a/test/run_all.sh +++ b/test/run_all.sh @@ -29,7 +29,7 @@ TESTS="${@:-*_test.sh}" # If running on circle, use the scheduler to work out what tests to run if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" ]; then - TESTS=$(echo $TESTS | "$DIR/sched" sched $CIRCLE_BUILD_NUM $CIRCLE_NODE_TOTAL $CIRCLE_NODE_INDEX) + TESTS=$(echo $TESTS | "$DIR/sched" sched integration-$CIRCLE_BUILD_NUM $CIRCLE_NODE_TOTAL $CIRCLE_NODE_INDEX) fi echo Running $TESTS diff --git a/test/sched b/test/sched index 4a7c7021a1..e94e8af8f1 100755 --- a/test/sched +++ b/test/sched @@ -1,17 +1,17 @@ #!/usr/bin/python -import sys, string, json +import sys, string, json, urllib import requests BASE_URL="http://positive-cocoa-90213.appspot.com" def test_time(test_name, runtime): - r = requests.post(BASE_URL + "/record/%s/%f" % (test_name, runtime)) + r = requests.post(BASE_URL + "/record/%s/%f" % (urllib.quote(test_name, safe=""), runtime)) print r.text assert r.status_code == 204 def test_sched(test_run, shard_count, shard_id): tests = json.dumps({'tests': string.split(sys.stdin.read())}) - r = requests.post(BASE_URL + "/schedule/%d/%d/%d" % (test_run, shard_count, shard_id), data=tests) + r = requests.post(BASE_URL + "/schedule/%s/%d/%d" % (test_run, shard_count, shard_id), data=tests) assert r.status_code == 200 result = r.json() for test in sorted(result['tests']): @@ -30,7 +30,7 @@ def main(): if sys.argv[1] == "time": test_time(sys.argv[2], float(sys.argv[3])) elif sys.argv[1] == "sched": - test_sched(int(sys.argv[2]), int(sys.argv[3]), int(sys.argv[4])) + test_sched(sys.argv[2], int(sys.argv[3]), int(sys.argv[4])) else: usage() diff --git a/test/scheduler/.gitignore b/test/scheduler/.gitignore new file mode 100644 index 0000000000..8531a10cc0 --- /dev/null +++ b/test/scheduler/.gitignore @@ -0,0 +1,7 @@ +*.egg-info +*.dist-info +flask +jinja2 +markupsafe +werkzeug +itsdangerous.* diff --git a/test/scheduler/README.md b/test/scheduler/README.md new file mode 100644 index 0000000000..df2b472634 --- /dev/null +++ b/test/scheduler/README.md @@ -0,0 +1,6 @@ +To upload newer version: + +``` +pip install -r requirements.txt -t . +appcfg.py update . +``` diff --git a/test/scheduler/main.py b/test/scheduler/main.py index a2cf113b4d..35d824abc3 100644 --- a/test/scheduler/main.py +++ b/test/scheduler/main.py @@ -19,7 +19,7 @@ class Test(ndb.Model): class Schedule(ndb.Model): shards = ndb.JsonProperty() -@app.route('/record//', methods=['POST']) +@app.route('/record//', methods=['POST']) @ndb.transactional def record(test_name, runtime): test = Test.get_by_id(test_name) @@ -30,13 +30,13 @@ def record(test_name, runtime): test.put() return ('', 204) -@app.route('/schedule///', methods=['POST']) +@app.route('/schedule///', methods=['POST']) def schedule(test_run, shard_count, shard): # read tests from body test_names = flask.request.get_json(force=True)['tests'] # first see if we have a scedule already - schedule_id = "%d-%d" % (test_run, shard_count) + schedule_id = "%s-%d" % (test_run, shard_count) schedule = Schedule.get_by_id(schedule_id) if schedule is not None: return flask.json.jsonify(tests=schedule.shards[str(shard)]) diff --git a/test/scheduler/requirements.txt b/test/scheduler/requirements.txt new file mode 100644 index 0000000000..7e1060246f --- /dev/null +++ b/test/scheduler/requirements.txt @@ -0,0 +1 @@ +flask diff --git a/test/units.sh b/test/units.sh index 2a383c7a78..d388b23e0d 100755 --- a/test/units.sh +++ b/test/units.sh @@ -1,28 +1,58 @@ #!/bin/bash -go get github.com/weaveworks/weave/testing/cover +set -e -if [ -n "$COVERDIR" ] ; then - coverdir="$COVERDIR" -else - coverdir=$(mktemp -d coverage.XXXXXXXXXX) +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SLOW=${SLOW-} +GO_TEST_ARGS="-tags netgo -cpu 4" +if [ -n "$SLOW" -o "$1" = "-slow" ]; then + GO_TEST_ARGS="$GO_TEST_ARGS -race -covermode=atomic" + + if [ -n "$COVERDIR" ] ; then + coverdir="$COVERDIR" + else + coverdir=$(mktemp -d coverage.XXXXXXXXXX) + fi + + mkdir -p $coverdir fi -mkdir -p $coverdir fail=0 -for dir in $(find . -type f -name '*_test.go' | xargs -n1 dirname | grep -v prog | sort -u); do - go get -t -tags netgo $dir - output=$(mktemp $coverdir/unit.XXXXXXXXXX) - if ! go test -cpu 4 -tags netgo -covermode=count -coverprofile=$output $dir ; then - fail=1 - fi +TESTDIRS=$(find . -type f -name '*_test.go' | xargs -n1 dirname | grep -v prog | sort -u) + +# If running on circle, use the scheduler to work out what tests to run on what shard +if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" ]; then + TESTDIRS=$(echo $TESTDIRS | "$DIR/sched" sched units-$CIRCLE_BUILD_NUM $CIRCLE_NODE_TOTAL $CIRCLE_NODE_INDEX) + echo $TESTDIRS +fi + +for dir in $TESTDIRS; do + + GO_TEST_ARGS_RUN="$GO_TEST_ARGS" + if [ -n "$SLOW" ]; then + go get -t -tags netgo $dir + output=$(mktemp $coverdir/unit.XXXXXXXXXX) + GO_TEST_ARGS_RUN="$GO_TEST_ARGS -coverprofile=$output" + fi + + START=$(date +%s) + if ! go test $GO_TEST_ARGS_RUN $dir ; then + fail=1 + fi + RUNTIME=$(( $(date +%s) - $START )) + + # Report test runtime when running on circle, to help scheduler + if [ -n "$CIRCLECI" -a -z "$NO_SCHEDULER" ]; then + "$DIR/sched" time $dir $RUNTIME + fi done -if [ -z "$COVERDIR" ] ; then - cover $coverdir/* >profile.cov - rm -rf $coverdir - go tool cover -html=profile.cov -o=coverage.html +if [ -n "$SLOW" -a -z "$COVERDIR" ] ; then + $DIR/../testing/cover/cover $coverdir/* >profile.cov + rm -rf $coverdir + go tool cover -html=profile.cov -o=coverage.html + go tool cover -func=profile.cov | tail -n1 fi exit $fail