From f92d45ae90d38a13647211f8644d9e36a458aca4 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Wed, 30 Oct 2019 15:27:16 -0600 Subject: [PATCH 01/25] WIP Signed-off-by: Morgan Tocker --- Vagrantfile | 4 +- bootstrap.sh | 57 ++++++++----------- build.env | 10 +--- dev.env | 31 ++-------- doc/DockerBuild.md | 2 +- docker/base/Dockerfile | 2 +- docker/base/Dockerfile.mariadb | 2 +- docker/base/Dockerfile.mariadb103 | 2 +- docker/base/Dockerfile.mysql56 | 2 +- docker/base/Dockerfile.mysql80 | 2 +- docker/base/Dockerfile.percona | 2 +- docker/base/Dockerfile.percona57 | 2 +- docker/base/Dockerfile.percona80 | 2 +- docker/bootstrap/Dockerfile.common | 17 +++--- docker/bootstrap/Dockerfile.mariadb | 3 +- docker/bootstrap/Dockerfile.mariadb103 | 3 +- docker/bootstrap/Dockerfile.mysql56 | 3 +- docker/bootstrap/Dockerfile.mysql57 | 4 +- docker/bootstrap/Dockerfile.mysql80 | 4 +- docker/bootstrap/Dockerfile.percona | 3 +- docker/bootstrap/Dockerfile.percona57 | 3 +- docker/bootstrap/Dockerfile.percona80 | 3 +- docker/bootstrap/build.sh | 2 +- docker/k8s/Dockerfile | 7 +-- docker/lite/Dockerfile | 12 ++-- docker/lite/Dockerfile.alpine | 12 ++-- docker/lite/Dockerfile.mariadb | 12 ++-- docker/lite/Dockerfile.mariadb103 | 12 ++-- docker/lite/Dockerfile.mysql56 | 11 ++-- docker/lite/Dockerfile.mysql57 | 12 ++-- docker/lite/Dockerfile.mysql80 | 12 ++-- docker/lite/Dockerfile.percona | 12 ++-- docker/lite/Dockerfile.percona57 | 12 ++-- docker/lite/Dockerfile.percona80 | 12 ++-- docker/packaging/package_vitess.sh | 8 +-- docker/publish-site/Dockerfile | 34 ----------- docker/test/run.sh | 2 +- examples/compose/docker-compose.beginners.yml | 4 +- examples/compose/vtcompose/vtcompose.go | 4 +- examples/demo/run.py | 8 +-- examples/kubernetes/guestbook/extract.sh | 2 +- .../vtctld-controller-template.yaml | 4 +- examples/local/env.sh | 3 - examples/local/vtctld-up.sh | 4 +- go.mod | 3 - go/vt/mysqlctl/mycnf_test.go | 8 +-- .../vtgate/endtoend/deletetest/delete_test.go | 2 +- go/vt/vtgate/endtoend/main_test.go | 2 +- .../tabletserver/vstreamer/testenv/testenv.go | 2 +- go/vt/vttest/environment.go | 4 +- .../test/java/io/vitess/client/TestEnv.java | 16 +++--- misc/git/hooks/pylint | 4 +- py/vttest/mysql_db_mysqlctl.py | 2 +- py/vttest/mysql_flavor.py | 24 ++++---- py/vttest/run_local_database.py | 2 +- test.go | 8 +-- test/backup.py | 2 +- test/backup_only.py | 2 +- test/backup_transform.py | 2 +- test/client_test.sh | 2 +- test/cluster/k8s_environment.py | 2 +- test/cluster/keytar/README.md | 6 +- test/cluster/keytar/config/vitess_config.yaml | 6 +- test/cluster/keytar/keytar_web_test.py | 2 +- test/cluster/keytar/test_config.yaml | 1 - test/cluster/sandbox/create_schema.py | 2 +- test/cluster/sandbox/kubernetes_components.py | 2 +- .../sandbox/vitess_kubernetes_sandbox.py | 4 +- test/cluster/sandbox/vtctl_sandbox.py | 2 +- test/environment.py | 10 +--- test/initial_sharding_multi.py | 2 +- test/local_example.sh | 2 +- test/mysql_flavor.py | 8 +-- test/tablet.py | 8 +-- test/utils.py | 4 +- test/vtbackup.py | 2 +- test/vtctld_web_test.py | 8 +-- test/vttest_sample_test.py | 4 +- tools/bootstrap_web.sh | 4 +- tools/generate_web_artifacts.sh | 2 +- vagrant-scripts/bootstrap_vm.sh | 4 +- vagrant-scripts/vagrant-bashrc | 1 - 82 files changed, 207 insertions(+), 328 deletions(-) delete mode 100644 docker/publish-site/Dockerfile diff --git a/Vagrantfile b/Vagrantfile index 3b38717c90c..5f727efed91 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -44,8 +44,8 @@ Vagrant.configure("2") do |config| # Additional details in https://github.com/vitessio/vitess/issues/4396 # # To enable, use: - # config.vm.synced_folder ".", "/vagrant/src/vitess.io/vitess", type: "nfs" - config.vm.synced_folder ".", "/vagrant/src/vitess.io/vitess" + # config.vm.synced_folder ".", "/vagrant/vt", type: "nfs" + config.vm.synced_folder ".", "/vagrant/vt" config.vm.provider :virtualbox do |vb| vb.name = "vitess" diff --git a/bootstrap.sh b/bootstrap.sh index 124332cc516..181da11e707 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -35,47 +35,40 @@ function fail() { [[ "$(dirname "$0")" = "." ]] || fail "bootstrap.sh must be run from its current directory" -go version &>/dev/null || fail "Go is not installed or is not on \$PATH" -[[ "$(go version 2>&1)" =~ go1\.[1-9][1-9] ]] || fail "Go is not version 1.11+" - # Create main directories. -mkdir -p "$VTROOT/dist" -mkdir -p "$VTROOT/bin" -mkdir -p "$VTROOT/lib" -mkdir -p "$VTROOT/vthook" + +mkdir -p "dist" +mkdir -p "bin" +mkdir -p "lib" +mkdir -p "vthook" + +# temporary +rm -rf lib/* +rm -rf vthook/* # This is required for VIRTUALENV # Used by Python below -if [ "$BUILD_TESTS" == 1 ] ; then - source ./dev.env -else - source ./build.env -fi +source ./dev.env -if [ "$BUILD_TESTS" == 1 ] ; then - # Set up required soft links. - # TODO(mberlin): Which of these can be deleted? - ln -snf "$VTTOP/config" "$VTROOT/config" - ln -snf "$VTTOP/data" "$VTROOT/data" - ln -snf "$VTTOP/py" "$VTROOT/py-vtdb" - ln -snf "$VTTOP/go/vt/zkctl/zksrv.sh" "$VTROOT/bin/zksrv.sh" - ln -snf "$VTTOP/test/vthook-test.sh" "$VTROOT/vthook/test.sh" - ln -snf "$VTTOP/test/vthook-test_backup_error" "$VTROOT/vthook/test_backup_error" - ln -snf "$VTTOP/test/vthook-test_backup_transform" "$VTROOT/vthook/test_backup_transform" -else - ln -snf "$VTTOP/config" "$VTROOT/config" - ln -snf "$VTTOP/data" "$VTROOT/data" - ln -snf "$VTTOP/go/vt/zkctl/zksrv.sh" "$VTROOT/bin/zksrv.sh" -fi +go version &>/dev/null || fail "Go is not installed or is not on \$PATH" +goversion_min 1.12 || fail "Go is not version 1.12+" + +# Set up required soft links. +# TODO(mberlin): Which of these can be deleted? +ln -snf "$VTROOT/py" "$VTROOT/py-vtdb" +ln -snf "$VTROOT/go/vt/zkctl/zksrv.sh" "$VTROOT/bin/zksrv.sh" +ln -snf "$VTROOT/test/vthook-test.sh" "$VTROOT/vthook/test.sh" +ln -snf "$VTROOT/test/vthook-test_backup_error" "$VTROOT/vthook/test_backup_error" +ln -snf "$VTROOT/test/vthook-test_backup_transform" "$VTROOT/vthook/test_backup_transform" # git hooks are only required if someone intends to contribute. echo "creating git hooks" -mkdir -p "$VTTOP/.git/hooks" -ln -sf "$VTTOP/misc/git/pre-commit" "$VTTOP/.git/hooks/pre-commit" -ln -sf "$VTTOP/misc/git/commit-msg" "$VTTOP/.git/hooks/commit-msg" -(cd "$VTTOP" && git config core.hooksPath "$VTTOP/.git/hooks") +mkdir -p "$VTROOT/.git/hooks" +ln -sf "$VTROOT/misc/git/pre-commit" "$VTROOT/.git/hooks/pre-commit" +ln -sf "$VTROOT/misc/git/commit-msg" "$VTROOT/.git/hooks/commit-msg" +(cd "$VTROOT" && git config core.hooksPath "$VTROOT/.git/hooks") # install_dep is a helper function to generalize the download and installation of dependencies. # @@ -264,7 +257,7 @@ function install_pymock() { PYTHONPATH=$(prepend_path "$PYTHONPATH" "$dist/lib/python2.7/site-packages") export PYTHONPATH - pushd "$VTTOP/third_party/py" >/dev/null + pushd "$VTROOT/third_party/py" >/dev/null tar -xzf "mock-$version.tar.gz" cd "mock-$version" $PYTHON ./setup.py install --prefix="$dist" diff --git a/build.env b/build.env index 6bc36b42206..7920a58288f 100644 --- a/build.env +++ b/build.env @@ -28,13 +28,5 @@ if ! source "${dir}/tools/shell_functions.inc"; then return 1 fi -VTTOP=$(pwd) -export VTTOP -VTROOT="${VTROOT:-${VTTOP/\/src\/vitess.io\/vitess/}}" -export VTROOT -# VTTOP sanity check -if [[ "$VTTOP" == "${VTTOP/\/src\/vitess.io\/vitess/}" ]]; then - echo "WARNING: VTTOP($VTTOP) does not contain src/vitess.io/vitess" -fi - +export VTROOT=$PWD export GO111MODULE=on diff --git a/dev.env b/dev.env index e185c40c019..07899bee179 100644 --- a/dev.env +++ b/dev.env @@ -22,8 +22,8 @@ source ./build.env -export GOTOP=$VTTOP/go -export PYTOP=$VTTOP/py +export GOTOP=$VTROOT/go +export PYTOP=$VTROOT/py export VTDATAROOT="${VTDATAROOT:-${VTROOT}/vtdataroot}" mkdir -p "$VTDATAROOT" @@ -43,8 +43,8 @@ IFS="$BACKUP_IFS" PYTHONPATH=$(prepend_path "$PYTHONPATH" "$VTROOT/py-vtdb") PYTHONPATH=$(prepend_path "$PYTHONPATH" "$VTROOT/dist/selenium") -PYTHONPATH=$(prepend_path "$PYTHONPATH" "$VTTOP/test") -PYTHONPATH=$(prepend_path "$PYTHONPATH" "$VTTOP/test/cluster/sandbox") +PYTHONPATH=$(prepend_path "$PYTHONPATH" "$VTROOT/test") +PYTHONPATH=$(prepend_path "$PYTHONPATH" "$VTROOT/test/cluster/sandbox") export PYTHONPATH # Ensure bootstrap.sh uses python2 on systems which default to python3. @@ -55,17 +55,6 @@ export PIP command -v virtualenv2 >/dev/null && VIRTUALENV=virtualenv2 || VIRTUALENV=virtualenv export VIRTUALENV -# Add the current GOBIN -if [ "$GOBIN" ]; then - PATH=$(prepend_path "$PATH" "$GOBIN") -fi - -# Many tests rely on "go install" and assume GOBIN is really $VTROOT/bin. -# Make sure these take precedence. -GOBIN=$VTROOT/bin -export GOBIN -PATH=$(prepend_path "$PATH" "$GOBIN") - # Add chromedriver to path for Selenium tests. PATH=$(prepend_path "$PATH" "$VTROOT/dist/chromedriver") @@ -73,16 +62,6 @@ PATH=$(prepend_path "$PATH" "$VTROOT/dist/chromedriver") PATH=$(prepend_path "$PATH" "$VTROOT/dist/node/bin") export PATH -# GOROOT sanity -go_bin=$(which go) -go_env=$(go env | grep GOROOT | cut -f 2 -d\") -if [[ "$go_bin" != "" && "$go_bin" != "$go_env/bin/go" ]]; then - echo "WARNING: \$GOROOT may not be compatible with the used go binary" - echo "Please make sure 'go' comes from \$GOROOT/bin" - echo "go_env: $go_env" - echo "go_bin: $go_bin" -fi - # mysql install location. Please set based on your environment. # Build will not work if this is incorrect. @@ -101,4 +80,4 @@ export PKG_CONFIG_PATH # Useful aliases. Remove if inconvenient. alias gt='cd $GOTOP' alias pt='cd $PYTOP' -alias vt='cd $VTTOP' +alias vt='cd $VTROOT' diff --git a/doc/DockerBuild.md b/doc/DockerBuild.md index 986b8e672c6..5b97b1e19e4 100644 --- a/doc/DockerBuild.md +++ b/doc/DockerBuild.md @@ -20,7 +20,7 @@ Then you can run our build script for the `lite` image which extracts the Vitess 1. Create an account on [Docker Hub](https://docs.docker.com/docker-hub/) and then `docker login` to it. -1. Go to your `src/vitess.io/vitess` directory. +1. Go to your the vitess source directory. 1. Usually, you won't need to [build your own bootstrap image](https://github.com/vitessio/vitess/blob/master/docker/bootstrap/README.md) unless you edit [bootstrap.sh](https://github.com/vitessio/vitess/blob/master/bootstrap.sh) diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index ecab9ab6c42..20f0052b154 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -27,7 +27,7 @@ ARG CGO_ENABLED=0 # Re-copy sources from working tree USER root -COPY . /vt/src/vitess.io/vitess +COPY . /vt/ # Build Vitess RUN make build diff --git a/docker/base/Dockerfile.mariadb b/docker/base/Dockerfile.mariadb index a3f9771334f..1c7d1633a87 100644 --- a/docker/base/Dockerfile.mariadb +++ b/docker/base/Dockerfile.mariadb @@ -2,7 +2,7 @@ FROM vitess/bootstrap:mariadb # Re-copy sources from working tree USER root -COPY . /vt/src/vitess.io/vitess +COPY . /vt/ # Build Vitess RUN make build diff --git a/docker/base/Dockerfile.mariadb103 b/docker/base/Dockerfile.mariadb103 index 75811dc68ba..cb3bdfc8513 100644 --- a/docker/base/Dockerfile.mariadb103 +++ b/docker/base/Dockerfile.mariadb103 @@ -2,7 +2,7 @@ FROM vitess/bootstrap:mariadb103 # Re-copy sources from working tree USER root -COPY . /vt/src/vitess.io/vitess +COPY . /vt/ # Build Vitess RUN make build diff --git a/docker/base/Dockerfile.mysql56 b/docker/base/Dockerfile.mysql56 index 5d1f55079fa..be632e6aff8 100644 --- a/docker/base/Dockerfile.mysql56 +++ b/docker/base/Dockerfile.mysql56 @@ -2,7 +2,7 @@ FROM vitess/bootstrap:mysql56 # Re-copy sources from working tree USER root -COPY . /vt/src/vitess.io/vitess +COPY . /vt/ # Build Vitess RUN make build diff --git a/docker/base/Dockerfile.mysql80 b/docker/base/Dockerfile.mysql80 index 5b2fc5c91ae..36ab976ba2a 100644 --- a/docker/base/Dockerfile.mysql80 +++ b/docker/base/Dockerfile.mysql80 @@ -2,7 +2,7 @@ FROM vitess/bootstrap:mysql80 # Re-copy sources from working tree USER root -COPY . /vt/src/vitess.io/vitess +COPY . /vt/ # Build Vitess RUN make build diff --git a/docker/base/Dockerfile.percona b/docker/base/Dockerfile.percona index 93a218ff059..a2e9ba0f781 100644 --- a/docker/base/Dockerfile.percona +++ b/docker/base/Dockerfile.percona @@ -2,7 +2,7 @@ FROM vitess/bootstrap:percona # Re-copy sources from working tree USER root -COPY . /vt/src/vitess.io/vitess +COPY . /vt/ # Fix permissions RUN chown -R vitess:vitess /vt diff --git a/docker/base/Dockerfile.percona57 b/docker/base/Dockerfile.percona57 index 4f396c9c86b..92d03255f4c 100644 --- a/docker/base/Dockerfile.percona57 +++ b/docker/base/Dockerfile.percona57 @@ -2,7 +2,7 @@ FROM vitess/bootstrap:percona57 # Re-copy sources from working tree USER root -COPY . /vt/src/vitess.io/vitess +COPY . /vt/ # Build Vitess RUN make build diff --git a/docker/base/Dockerfile.percona80 b/docker/base/Dockerfile.percona80 index 83672111166..67515df1840 100644 --- a/docker/base/Dockerfile.percona80 +++ b/docker/base/Dockerfile.percona80 @@ -2,7 +2,7 @@ FROM vitess/bootstrap:percona80 # Re-copy sources from working tree USER root -COPY . /vt/src/vitess.io/vitess +COPY . /vt/ # Fix permissions RUN chown -R vitess:vitess /vt diff --git a/docker/bootstrap/Dockerfile.common b/docker/bootstrap/Dockerfile.common index 4b6a21beb49..d4b6eea1f35 100644 --- a/docker/bootstrap/Dockerfile.common +++ b/docker/bootstrap/Dockerfile.common @@ -38,10 +38,9 @@ RUN mkdir -p /vt/dist && \ mv apache-maven-3.3.9 maven # Set up Vitess environment (equivalent to '. dev.env') -ENV VTTOP /vt/src/vitess.io/vitess ENV VTROOT /vt -ENV GOTOP $VTTOP/go -ENV PYTOP $VTTOP/py +ENV GOTOP $VTROOT/go +ENV PYTOP $VTROOT/py ENV VTDATAROOT $VTROOT/vtdataroot ENV VTPORTSTART 15000 ENV PYTHONPATH $VTROOT/dist/grpc/usr/local/lib/python2.7/site-packages:$VTROOT/dist/py-mock-1.0.1/lib/python2.7/site-packages:$VTROOT/py-vtdb:$VTROOT/dist/selenium/lib/python2.7/site-packages @@ -53,11 +52,11 @@ ENV USER vitess ENV GO111MODULE on # Copy files needed for bootstrap -COPY bootstrap.sh dev.env build.env go.mod go.sum /vt/src/vitess.io/vitess/ -COPY config /vt/src/vitess.io/vitess/config -COPY third_party /vt/src/vitess.io/vitess/third_party -COPY tools /vt/src/vitess.io/vitess/tools -COPY travis /vt/src/vitess.io/vitess/travis +COPY bootstrap.sh dev.env build.env go.mod go.sum /vt/ +COPY config /vt/config +COPY third_party /vt/third_party +COPY tools /vt/tools +COPY travis /vt/travis # Create vitess user RUN groupadd -r vitess && useradd -r -g vitess vitess && \ @@ -65,7 +64,7 @@ RUN groupadd -r vitess && useradd -r -g vitess vitess && \ chown -R vitess:vitess /vt /home/vitess # Download vendored Go dependencies -RUN cd /vt/src/vitess.io/vitess && \ +RUN cd /vt && \ su vitess -c "/usr/local/go/bin/go mod download" # Create mount point for actual data (e.g. MySQL data dir) diff --git a/docker/bootstrap/Dockerfile.mariadb b/docker/bootstrap/Dockerfile.mariadb index 19d5c9973d0..9f1ee44d7b2 100644 --- a/docker/bootstrap/Dockerfile.mariadb +++ b/docker/bootstrap/Dockerfile.mariadb @@ -20,8 +20,7 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r && rm -rf /var/lib/apt/lists/* # Bootstrap Vitess -WORKDIR /vt/src/vitess.io/vitess +WORKDIR /vt -ENV MYSQL_FLAVOR MariaDB USER vitess RUN ./bootstrap.sh diff --git a/docker/bootstrap/Dockerfile.mariadb103 b/docker/bootstrap/Dockerfile.mariadb103 index c2828ddd25d..a7d3408751b 100644 --- a/docker/bootstrap/Dockerfile.mariadb103 +++ b/docker/bootstrap/Dockerfile.mariadb103 @@ -9,8 +9,7 @@ RUN apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 0xF1656F24 && rm -rf /var/lib/apt/lists/* # Bootstrap Vitess -WORKDIR /vt/src/vitess.io/vitess +WORKDIR /vt -ENV MYSQL_FLAVOR MariaDB103 USER vitess RUN ./bootstrap.sh diff --git a/docker/bootstrap/Dockerfile.mysql56 b/docker/bootstrap/Dockerfile.mysql56 index b07fddedb61..3726eefc229 100644 --- a/docker/bootstrap/Dockerfile.mysql56 +++ b/docker/bootstrap/Dockerfile.mysql56 @@ -15,8 +15,7 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver pool.s rm -rf /var/lib/apt/lists/* # Bootstrap Vitess -WORKDIR /vt/src/vitess.io/vitess +WORKDIR /vt -ENV MYSQL_FLAVOR MySQL56 USER vitess RUN ./bootstrap.sh diff --git a/docker/bootstrap/Dockerfile.mysql57 b/docker/bootstrap/Dockerfile.mysql57 index 2fb1f4e1aa5..37aa75d85a0 100644 --- a/docker/bootstrap/Dockerfile.mysql57 +++ b/docker/bootstrap/Dockerfile.mysql57 @@ -16,9 +16,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins rm -rf /var/lib/apt/lists/* # Bootstrap Vitess -WORKDIR /vt/src/vitess.io/vitess +WORKDIR /vt - -ENV MYSQL_FLAVOR MySQL56 USER vitess RUN ./bootstrap.sh \ No newline at end of file diff --git a/docker/bootstrap/Dockerfile.mysql80 b/docker/bootstrap/Dockerfile.mysql80 index ec53895165f..b833e9f036d 100644 --- a/docker/bootstrap/Dockerfile.mysql80 +++ b/docker/bootstrap/Dockerfile.mysql80 @@ -15,9 +15,7 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver ha.poo rm -rf /var/lib/apt/lists/* # Bootstrap Vitess -WORKDIR /vt/src/vitess.io/vitess +WORKDIR /vt - -ENV MYSQL_FLAVOR MySQL80 USER vitess RUN ./bootstrap.sh \ No newline at end of file diff --git a/docker/bootstrap/Dockerfile.percona b/docker/bootstrap/Dockerfile.percona index 6d13fa4dfb0..ba18fb8afc7 100644 --- a/docker/bootstrap/Dockerfile.percona +++ b/docker/bootstrap/Dockerfile.percona @@ -14,8 +14,7 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r rm -rf /var/lib/apt/lists/* # Bootstrap Vitess -WORKDIR /vt/src/vitess.io/vitess +WORKDIR /vt -ENV MYSQL_FLAVOR MySQL56 USER vitess RUN ./bootstrap.sh diff --git a/docker/bootstrap/Dockerfile.percona57 b/docker/bootstrap/Dockerfile.percona57 index 6ed54c76923..1ddad7b6bd4 100644 --- a/docker/bootstrap/Dockerfile.percona57 +++ b/docker/bootstrap/Dockerfile.percona57 @@ -15,8 +15,7 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r rm -rf /var/lib/apt/lists/* # Bootstrap Vitess -WORKDIR /vt/src/vitess.io/vitess +WORKDIR /vt -ENV MYSQL_FLAVOR MySQL56 USER vitess RUN ./bootstrap.sh diff --git a/docker/bootstrap/Dockerfile.percona80 b/docker/bootstrap/Dockerfile.percona80 index 1ce9c52103f..5fde3ee5310 100644 --- a/docker/bootstrap/Dockerfile.percona80 +++ b/docker/bootstrap/Dockerfile.percona80 @@ -30,8 +30,7 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r && rm -rf /var/lib/apt/lists/* # Bootstrap Vitess -WORKDIR /vt/src/vitess.io/vitess +WORKDIR /vt -ENV MYSQL_FLAVOR MySQL80 USER vitess RUN ./bootstrap.sh diff --git a/docker/bootstrap/build.sh b/docker/bootstrap/build.sh index 0a3a48c1253..36fb4633148 100755 --- a/docker/bootstrap/build.sh +++ b/docker/bootstrap/build.sh @@ -22,7 +22,7 @@ if [[ -z "$flavor" ]]; then fi if [[ ! -f bootstrap.sh ]]; then - echo "This script should be run from the root of the Vitess source tree - e.g. ~/src/vitess.io/vitess" + echo "This script should be run from the root of the Vitess source tree" exit 1 fi diff --git a/docker/k8s/Dockerfile b/docker/k8s/Dockerfile index ddafcd95af7..5ced6f87c35 100644 --- a/docker/k8s/Dockerfile +++ b/docker/k8s/Dockerfile @@ -27,7 +27,6 @@ RUN apt-get update && \ # Set up Vitess environment (just enough to run pre-built Go binaries) ENV VTROOT /vt ENV VTDATAROOT /vtdataroot -ENV VTTOP /vt/src/vitess.io/vitess # Prepare directory structure. RUN mkdir -p /vt && \ @@ -50,13 +49,13 @@ COPY --from=base /vt/bin/vtworker /vt/bin/ COPY --from=base /vt/bin/vtbackup /vt/bin/ # copy web admin files -COPY --from=base $VTTOP/web /vt/web/ +COPY --from=base $VTROOT/web /vt/web/ # copy vitess config -COPY --from=base $VTTOP/config/init_db.sql /vt/config/ +COPY --from=base $VTROOT/config/init_db.sql /vt/config/ # my.cnf include files -COPY --from=base $VTTOP/config/mycnf /vt/config/mycnf +COPY --from=base $VTROOT/config/mycnf /vt/config/mycnf # add vitess user and add permissions RUN groupadd -r --gid 2000 vitess && useradd -r -g vitess --uid 1000 vitess && \ diff --git a/docker/lite/Dockerfile b/docker/lite/Dockerfile index 268a68fffce..2a799266ad9 100644 --- a/docker/lite/Dockerfile +++ b/docker/lite/Dockerfile @@ -17,12 +17,12 @@ FROM debian:stretch-slim AS staging RUN mkdir -p /vt/vtdataroot/ \ && mkdir -p /vt/bin \ - && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ + && mkdir -p /vt/web/vtctld2 \ && groupadd -r vitess && useradd -r -g vitess vitess -COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld -COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app -COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config +COPY --from=builder /vt/web/vtctld /vt/web/vtctld +COPY --from=builder /vt/web/vtctld2/app /vt/web/vtctld2/app +COPY --from=builder /vt/config /vt/config COPY --from=builder /vt/bin/mysqlctld /vt/bin/ COPY --from=builder /vt/bin/vtctld /vt/bin/ COPY --from=builder /vt/bin/vtctlclient /vt/bin/ @@ -54,16 +54,14 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTTOP /vt/src/vitess.io/vitess ENV VTROOT /vt -ENV GOTOP $VTTOP/go +ENV GOTOP $VTROOT/go ENV VTDATAROOT $VTROOT/vtdataroot ENV GOBIN $VTROOT/bin ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib -ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.alpine b/docker/lite/Dockerfile.alpine index f3d282163a9..cde81ae854c 100644 --- a/docker/lite/Dockerfile.alpine +++ b/docker/lite/Dockerfile.alpine @@ -2,11 +2,11 @@ FROM vitess/base AS builder FROM alpine:3.8 AS staging -RUN mkdir -p /vt/vtdataroot/ && mkdir -p /vt/bin && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 +RUN mkdir -p /vt/vtdataroot/ && mkdir -p /vt/bin && mkdir -p /vt/web/vtctld2 -COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld -COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app -COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config +COPY --from=builder /vt/web/vtctld /vt/web/vtctld +COPY --from=builder /vt/web/vtctld2/app /vt/web/vtctld2/app +COPY --from=builder /vt/config /vt/config COPY --from=builder /vt/bin/mysqlctld /vt/bin/ COPY --from=builder /vt/bin/vtctld /vt/bin/ COPY --from=builder /vt/bin/vtctlclient /vt/bin/ @@ -22,16 +22,14 @@ RUN echo '@edge http://nl.alpinelinux.org/alpine/edge/main' >> /etc/apk/reposito apk add --no-cache mariadb@edge mariadb-client@edge bzip2 bash # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTTOP /vt/src/vitess.io/vitess ENV VTROOT /vt -ENV GOTOP $VTTOP/go +ENV GOTOP $VTROOT/go ENV VTDATAROOT $VTROOT/vtdataroot ENV GOBIN $VTROOT/bin ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib -ENV MYSQL_FLAVOR MariaDB103 # Create vitess user RUN addgroup -S vitess && adduser -S -G vitess vitess && mkdir -p /vt diff --git a/docker/lite/Dockerfile.mariadb b/docker/lite/Dockerfile.mariadb index cea94d615e5..9622a2cf158 100644 --- a/docker/lite/Dockerfile.mariadb +++ b/docker/lite/Dockerfile.mariadb @@ -3,12 +3,12 @@ FROM debian:stretch-slim AS staging RUN mkdir -p /vt/vtdataroot/ \ && mkdir -p /vt/bin \ - && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ + && mkdir -p /vt/web/vtctld2 \ && groupadd -r vitess && useradd -r -g vitess vitess -COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld -COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app -COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config +COPY --from=builder /vt/web/vtctld /vt/web/vtctld +COPY --from=builder /vt/web/vtctld2/app /vt/web/vtctld2/app +COPY --from=builder /vt/config /vt/config COPY --from=builder /vt/bin/mysqlctld /vt/bin/ COPY --from=builder /vt/bin/vtctld /vt/bin/ COPY --from=builder /vt/bin/vtctlclient /vt/bin/ @@ -33,16 +33,14 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTTOP /vt/src/vitess.io/vitess ENV VTROOT /vt -ENV GOTOP $VTTOP/go +ENV GOTOP $VTROOT/go ENV VTDATAROOT $VTROOT/vtdataroot ENV GOBIN $VTROOT/bin ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib -ENV MYSQL_FLAVOR MariaDB # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.mariadb103 b/docker/lite/Dockerfile.mariadb103 index 4ff440d3d86..1c5f02ae927 100644 --- a/docker/lite/Dockerfile.mariadb103 +++ b/docker/lite/Dockerfile.mariadb103 @@ -3,12 +3,12 @@ FROM debian:stretch-slim AS staging RUN mkdir -p /vt/vtdataroot/ \ && mkdir -p /vt/bin \ - && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ + && mkdir -p /vt/web/vtctld2 \ && groupadd -r vitess && useradd -r -g vitess vitess -COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld -COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app -COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config +COPY --from=builder /vt/web/vtctld /vt/web/vtctld +COPY --from=builder /vt/web/vtctld2/app /vt/web/vtctld2/app +COPY --from=builder /vt/config /vt/config COPY --from=builder /vt/bin/mysqlctld /vt/bin/ COPY --from=builder /vt/bin/vtctld /vt/bin/ COPY --from=builder /vt/bin/vtctlclient /vt/bin/ @@ -32,16 +32,14 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTTOP /vt/src/vitess.io/vitess ENV VTROOT /vt -ENV GOTOP $VTTOP/go +ENV GOTOP $VTROOT/go ENV VTDATAROOT $VTROOT/vtdataroot ENV GOBIN $VTROOT/bin ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib -ENV MYSQL_FLAVOR MariaDB103 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.mysql56 b/docker/lite/Dockerfile.mysql56 index f3d6b3dcb7a..8275a1f407a 100644 --- a/docker/lite/Dockerfile.mysql56 +++ b/docker/lite/Dockerfile.mysql56 @@ -3,12 +3,12 @@ FROM debian:stretch-slim AS staging RUN mkdir -p /vt/vtdataroot/ \ && mkdir -p /vt/bin \ - && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ + && mkdir -p /vt/web/vtctld2 \ && groupadd -r vitess && useradd -r -g vitess vitess -COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld -COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app -COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config +COPY --from=builder /vt/web/vtctld /vt/web/vtctld +COPY --from=builder /vt/web/vtctld2/app /vt/web/vtctld2/app +COPY --from=builder /vt/config /vt/config COPY --from=builder /vt/bin/mysqlctld /vt/bin/ COPY --from=builder /vt/bin/vtctld /vt/bin/ COPY --from=builder /vt/bin/vtctlclient /vt/bin/ @@ -36,9 +36,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTTOP /vt/src/vitess.io/vitess ENV VTROOT /vt -ENV GOTOP $VTTOP/go +ENV GOTOP $VTROOT/go ENV VTDATAROOT $VTROOT/vtdataroot ENV GOBIN $VTROOT/bin ENV GOPATH $VTROOT diff --git a/docker/lite/Dockerfile.mysql57 b/docker/lite/Dockerfile.mysql57 index 78318e5c652..8ce9582b059 100644 --- a/docker/lite/Dockerfile.mysql57 +++ b/docker/lite/Dockerfile.mysql57 @@ -3,12 +3,12 @@ FROM debian:stretch-slim AS staging RUN mkdir -p /vt/vtdataroot/ \ && mkdir -p /vt/bin \ - && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ + && mkdir -p /vt/web/vtctld2 \ && groupadd -r vitess && useradd -r -g vitess vitess -COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld -COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app -COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config +COPY --from=builder /vt/web/vtctld /vt/web/vtctld +COPY --from=builder /vt/web/vtctld2/app /vt/web/vtctld2/app +COPY --from=builder /vt/config /vt/config COPY --from=builder /vt/bin/mysqlctld /vt/bin/ COPY --from=builder /vt/bin/vtctld /vt/bin/ COPY --from=builder /vt/bin/vtctlclient /vt/bin/ @@ -36,16 +36,14 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTTOP /vt/src/vitess.io/vitess ENV VTROOT /vt -ENV GOTOP $VTTOP/go +ENV GOTOP $VTROOT/go ENV VTDATAROOT $VTROOT/vtdataroot ENV GOBIN $VTROOT/bin ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib -ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.mysql80 b/docker/lite/Dockerfile.mysql80 index fb71b6f6b56..775e9880a59 100644 --- a/docker/lite/Dockerfile.mysql80 +++ b/docker/lite/Dockerfile.mysql80 @@ -3,12 +3,12 @@ FROM debian:stretch-slim AS staging RUN mkdir -p /vt/vtdataroot/ \ && mkdir -p /vt/bin \ - && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ + && mkdir -p /vt/web/vtctld2 \ && groupadd -r vitess && useradd -r -g vitess vitess -COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld -COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app -COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config +COPY --from=builder /vt/web/vtctld /vt/web/vtctld +COPY --from=builder /vt/web/vtctld2/app /vt/web/vtctld2/app +COPY --from=builder /vt/config /vt/config COPY --from=builder /vt/bin/mysqlctld /vt/bin/ COPY --from=builder /vt/bin/vtctld /vt/bin/ COPY --from=builder /vt/bin/vtctlclient /vt/bin/ @@ -36,16 +36,14 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTTOP /vt/src/vitess.io/vitess ENV VTROOT /vt -ENV GOTOP $VTTOP/go +ENV GOTOP $VTROOT/go ENV VTDATAROOT $VTROOT/vtdataroot ENV GOBIN $VTROOT/bin ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib -ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.percona b/docker/lite/Dockerfile.percona index e8d127dc56f..c0406b8e0bf 100644 --- a/docker/lite/Dockerfile.percona +++ b/docker/lite/Dockerfile.percona @@ -3,12 +3,12 @@ FROM debian:stretch-slim AS staging RUN mkdir -p /vt/vtdataroot/ \ && mkdir -p /vt/bin \ - && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ + && mkdir -p /vt/web/vtctld2 \ && groupadd -r vitess && useradd -r -g vitess vitess -COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld -COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app -COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config +COPY --from=builder /vt/web/vtctld /vt/web/vtctld +COPY --from=builder /vt/web/vtctld2/app /vt/web/vtctld2/app +COPY --from=builder /vt/config /vt/config COPY --from=builder /vt/bin/mysqlctld /vt/bin/ COPY --from=builder /vt/bin/vtctld /vt/bin/ COPY --from=builder /vt/bin/vtctlclient /vt/bin/ @@ -38,16 +38,14 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTTOP /vt/src/vitess.io/vitess ENV VTROOT /vt -ENV GOTOP $VTTOP/go +ENV GOTOP $VTROOT/go ENV VTDATAROOT $VTROOT/vtdataroot ENV GOBIN $VTROOT/bin ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib -ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.percona57 b/docker/lite/Dockerfile.percona57 index 30472af2a60..50d35d7b316 100644 --- a/docker/lite/Dockerfile.percona57 +++ b/docker/lite/Dockerfile.percona57 @@ -3,12 +3,12 @@ FROM debian:stretch-slim AS staging RUN mkdir -p /vt/vtdataroot/ \ && mkdir -p /vt/bin \ - && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ + && mkdir -p /vt/web/vtctld2 \ && groupadd -r vitess && useradd -r -g vitess vitess -COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld -COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app -COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config +COPY --from=builder /vt/web/vtctld /vt/web/vtctld +COPY --from=builder /vt/web/vtctld2/app /vt/web/vtctld2/app +COPY --from=builder /vt/config /vt/config COPY --from=builder /vt/bin/mysqlctld /vt/bin/ COPY --from=builder /vt/bin/vtctld /vt/bin/ COPY --from=builder /vt/bin/vtctlclient /vt/bin/ @@ -39,16 +39,14 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTTOP /vt/src/vitess.io/vitess ENV VTROOT /vt -ENV GOTOP $VTTOP/go +ENV GOTOP $VTROOT/go ENV VTDATAROOT $VTROOT/vtdataroot ENV GOBIN $VTROOT/bin ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib -ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.percona80 b/docker/lite/Dockerfile.percona80 index 31182d277d6..cc68f9ca08e 100644 --- a/docker/lite/Dockerfile.percona80 +++ b/docker/lite/Dockerfile.percona80 @@ -3,12 +3,12 @@ FROM debian:stretch-slim AS staging RUN mkdir -p /vt/vtdataroot/ \ && mkdir -p /vt/bin \ - && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ + && mkdir -p /vt/web/vtctld2 \ && groupadd -r vitess && useradd -r -g vitess vitess -COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld -COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app -COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config +COPY --from=builder /vt/web/vtctld /vt/web/vtctld +COPY --from=builder /vt/web/vtctld2/app /vt/web/vtctld2/app +COPY --from=builder /vt/config /vt/config COPY --from=builder /vt/bin/mysqlctld /vt/bin/ COPY --from=builder /vt/bin/vtctld /vt/bin/ COPY --from=builder /vt/bin/vtctlclient /vt/bin/ @@ -41,16 +41,14 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTTOP /vt/src/vitess.io/vitess ENV VTROOT /vt -ENV GOTOP $VTTOP/go +ENV GOTOP $VTROOT/go ENV VTDATAROOT $VTROOT/vtdataroot ENV GOBIN $VTROOT/bin ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib -ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/packaging/package_vitess.sh b/docker/packaging/package_vitess.sh index 609fd1447d5..bbc55603267 100755 --- a/docker/packaging/package_vitess.sh +++ b/docker/packaging/package_vitess.sh @@ -33,10 +33,10 @@ cat <> "${inputs_file}" /vt/bin/vtgate=${PREFIX}/bin/vtgate /vt/bin/vttablet=${PREFIX}/bin/vttablet /vt/bin/vtworker=${PREFIX}/bin/vtworker -/vt/src/vitess.io/vitess/config/=/etc/vitess -/vt/src/vitess.io/vitess/web/vtctld2/app=${PREFIX}/lib/vitess/web/vtcld2 -/vt/src/vitess.io/vitess/web/vtctld=${PREFIX}/lib/vitess/web -/vt/src/vitess.io/vitess/examples/local/=${PREFIX}/share/vitess/examples +/vt/config/=/etc/vitess +/vt/web/vtctld2/app=${PREFIX}/lib/vitess/web/vtcld2 +/vt/web/vtctld=${PREFIX}/lib/vitess/web +/vt/examples/local/=${PREFIX}/share/vitess/examples EOF description='A database clustering system for horizontal scaling of MySQL diff --git a/docker/publish-site/Dockerfile b/docker/publish-site/Dockerfile deleted file mode 100644 index e0a1946996f..00000000000 --- a/docker/publish-site/Dockerfile +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This image should be built with $VTTOP as the context dir. -# For example: -# vitess$ docker build -f docker/publish-site/Dockerfile -t vitess/publish-site . -FROM ruby:2.3 - -# Install apt dependencies. -RUN apt-get update && \ - apt-get install -y --no-install-recommends \ - nodejs && \ - rm -rf /var/lib/apt/lists/* - -# Install ruby dependencies. -COPY vitess.io/Gemfile /vitess.io/Gemfile -RUN cd /vitess.io && \ - gem install bundler && \ - bundle install - -# Expose port for preview-site.sh. -EXPOSE 4000 - diff --git a/docker/test/run.sh b/docker/test/run.sh index 147663d028b..b1080be2ba0 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -111,7 +111,7 @@ if [[ -z "$cmd" ]]; then fi if [[ ! -f bootstrap.sh ]]; then - echo "This script should be run from the root of the Vitess source tree - e.g. ~/src/vitess.io/vitess" + echo "This script should be run from the root of the Vitess source tree" exit 1 fi diff --git a/examples/compose/docker-compose.beginners.yml b/examples/compose/docker-compose.beginners.yml index 4f23b688f1a..67294b90c69 100644 --- a/examples/compose/docker-compose.beginners.yml +++ b/examples/compose/docker-compose.beginners.yml @@ -37,8 +37,8 @@ services: command: ["sh", "-c", " $$VTROOT/bin/vtctld \ $TOPOLOGY_FLAGS \ -cell $CELL \ - -web_dir $$VTTOP/web/vtctld \ - -web_dir2 $$VTTOP/web/vtctld2/app \ + -web_dir $$VTROOT/web/vtctld \ + -web_dir2 $$VTROOT/web/vtctld2/app \ -workflow_manager_init \ -workflow_manager_use_election \ -service_map 'grpc-vtctl' \ diff --git a/examples/compose/vtcompose/vtcompose.go b/examples/compose/vtcompose/vtcompose.go index 489de6ff51a..ebda64081de 100644 --- a/examples/compose/vtcompose/vtcompose.go +++ b/examples/compose/vtcompose/vtcompose.go @@ -466,8 +466,8 @@ func generateVtctld() string { command: ["sh", "-c", " $$VTROOT/bin/vtctld \ %[3]s \ -cell %[4]s \ - -web_dir $$VTTOP/web/vtctld \ - -web_dir2 $$VTTOP/web/vtctld2/app \ + -web_dir $$VTROOT/web/vtctld \ + -web_dir2 $$VTROOT/web/vtctld2/app \ -workflow_manager_init \ -workflow_manager_use_election \ -service_map 'grpc-vtctl' \ diff --git a/examples/demo/run.py b/examples/demo/run.py index 45eb8267ce2..47f35f26ef3 100755 --- a/examples/demo/run.py +++ b/examples/demo/run.py @@ -51,13 +51,13 @@ def start_vitess(): keyspace = topology.keyspaces.add(name='lookup') keyspace.shards.add(name='0') - vttop = os.environ['VTTOP'] - args = [os.path.join(vttop, 'py/vttest/run_local_database.py'), + vtroot = os.environ['VTROOT'] + args = [os.path.join(vtroot, 'py/vttest/run_local_database.py'), '--port', '12345', '--proto_topo', text_format.MessageToString(topology, as_one_line=True), - '--web_dir', os.path.join(vttop, 'web/vtctld'), - '--schema_dir', os.path.join(vttop, 'examples/demo/schema')] + '--web_dir', os.path.join(vtroot, 'web/vtctld'), + '--schema_dir', os.path.join(vtroot, 'examples/demo/schema')] sp = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE) # This load will make us wait for vitess to come up. diff --git a/examples/kubernetes/guestbook/extract.sh b/examples/kubernetes/guestbook/extract.sh index e029df3fc77..04499c87397 100644 --- a/examples/kubernetes/guestbook/extract.sh +++ b/examples/kubernetes/guestbook/extract.sh @@ -17,7 +17,7 @@ set -e # Collect all the local Python libs we need. mkdir -p /out/pkg/py-vtdb -cp -R $VTTOP/py/* /out/pkg/py-vtdb/ +cp -R $VTROOT/py/* /out/pkg/py-vtdb/ cp -R /usr/local/lib/python2.7/dist-packages /out/pkg/ cp -R /vt/dist/py-* /out/pkg/ diff --git a/examples/kubernetes/vtctld-controller-template.yaml b/examples/kubernetes/vtctld-controller-template.yaml index 2ca6f6d80c1..4e28aa8bfc4 100644 --- a/examples/kubernetes/vtctld-controller-template.yaml +++ b/examples/kubernetes/vtctld-controller-template.yaml @@ -41,8 +41,8 @@ spec: chown -R vitess /vt && su -p -c "/vt/bin/vtctld -cell {{cell}} - -web_dir $VTTOP/web/vtctld - -web_dir2 $VTTOP/web/vtctld2/app + -web_dir $VTROOT/web/vtctld + -web_dir2 $VTROOT/web/vtctld2/app -workflow_manager_init -workflow_manager_use_election -log_dir $VTDATAROOT/tmp diff --git a/examples/local/env.sh b/examples/local/env.sh index ef9a0c8538a..9ad2bd5c93c 100644 --- a/examples/local/env.sh +++ b/examples/local/env.sh @@ -17,9 +17,6 @@ hostname=`hostname -f` vtctld_web_port=15000 -# Set up environment. -export VTTOP=${VTTOP-$VTROOT/src/vitess.io/vitess} - # Try to find mysqld on PATH. if [ -z "$VT_MYSQL_ROOT" ]; then mysql_path=`which mysqld` diff --git a/examples/local/vtctld-up.sh b/examples/local/vtctld-up.sh index dbd16a8de66..128152b837d 100755 --- a/examples/local/vtctld-up.sh +++ b/examples/local/vtctld-up.sh @@ -36,8 +36,8 @@ echo "Starting vtctld..." $VTROOT/bin/vtctld \ $TOPOLOGY_FLAGS \ -cell $cell \ - -web_dir $VTTOP/web/vtctld \ - -web_dir2 $VTTOP/web/vtctld2/app \ + -web_dir $VTROOT/web/vtctld \ + -web_dir2 $VTROOT/web/vtctld2/app \ -workflow_manager_init \ -workflow_manager_use_election \ -service_map 'grpc-vtctl' \ diff --git a/go.mod b/go.mod index e3081b34798..ae3b4d9aee4 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,6 @@ require ( github.com/golang/mock v1.3.1 github.com/golang/protobuf v1.3.2 github.com/golang/snappy v0.0.0-20170215233205-553a64147049 - github.com/google/btree v1.0.0 // indirect github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf // indirect github.com/gorilla/websocket v0.0.0-20160912153041-2d1e4548da23 github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 @@ -49,8 +48,6 @@ require ( github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1 github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/mitchellh/mapstructure v1.1.2 // indirect - github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect github.com/olekukonko/tablewriter v0.0.0-20160115111002-cca8bbc07984 github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02 github.com/opentracing/opentracing-go v1.1.0 diff --git a/go/vt/mysqlctl/mycnf_test.go b/go/vt/mysqlctl/mycnf_test.go index e7215f894fe..b478ade3db1 100644 --- a/go/vt/mysqlctl/mycnf_test.go +++ b/go/vt/mysqlctl/mycnf_test.go @@ -42,9 +42,9 @@ func TestMycnf(t *testing.T) { t.Errorf("err: %v", err) } cnfTemplatePaths := []string{ - path.Join(root, "src/vitess.io/vitess/config/mycnf/default.cnf"), - path.Join(root, "src/vitess.io/vitess/config/mycnf/replica.cnf"), - path.Join(root, "src/vitess.io/vitess/config/mycnf/master.cnf"), + path.Join(root, "config/mycnf/default.cnf"), + path.Join(root, "config/mycnf/replica.cnf"), + path.Join(root, "config/mycnf/master.cnf"), } data, err := cnf.makeMycnf(cnfTemplatePaths) if err != nil { @@ -81,7 +81,7 @@ func TestMycnf(t *testing.T) { // Run this test if any changes are made to hook handling / make_mycnf hook // other tests fail if we keep the hook around -// 1. ln -snf $VTTOP/test/vthook-make_mycnf $VTROOT/vthook/make_mycnf +// 1. ln -snf $VTROOT/test/vthook-make_mycnf $VTROOT/vthook/make_mycnf // 2. Remove "No" prefix from func name // 3. go test // 4. \rm $VTROOT/vthook/make_mycnf diff --git a/go/vt/vtgate/endtoend/deletetest/delete_test.go b/go/vt/vtgate/endtoend/deletetest/delete_test.go index 9f0cb6ceb79..de6614ed7c9 100644 --- a/go/vt/vtgate/endtoend/deletetest/delete_test.go +++ b/go/vt/vtgate/endtoend/deletetest/delete_test.go @@ -127,7 +127,7 @@ func TestMain(m *testing.M) { }}, }}, } - cfg.ExtraMyCnf = []string{path.Join(os.Getenv("VTTOP"), "config/mycnf/rbr.cnf")} + cfg.ExtraMyCnf = []string{path.Join(os.Getenv("VTROOT"), "config/mycnf/rbr.cnf")} if err := cfg.InitSchemas("ks", schema, vschema); err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) os.RemoveAll(cfg.SchemaDir) diff --git a/go/vt/vtgate/endtoend/main_test.go b/go/vt/vtgate/endtoend/main_test.go index da52636dcec..a3b69cdb6ea 100644 --- a/go/vt/vtgate/endtoend/main_test.go +++ b/go/vt/vtgate/endtoend/main_test.go @@ -171,7 +171,7 @@ func TestMain(m *testing.M) { }}, }}, } - cfg.ExtraMyCnf = []string{path.Join(os.Getenv("VTTOP"), "config/mycnf/rbr.cnf")} + cfg.ExtraMyCnf = []string{path.Join(os.Getenv("VTROOT"), "config/mycnf/rbr.cnf")} if err := cfg.InitSchemas("ks", schema, vschema); err != nil { fmt.Fprintf(os.Stderr, "%v\n", err) os.RemoveAll(cfg.SchemaDir) diff --git a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go index be2eb3070fe..ba48990b5b2 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go +++ b/go/vt/vttablet/tabletserver/vstreamer/testenv/testenv.go @@ -92,7 +92,7 @@ func Init() (*Env, error) { }, }, }, - ExtraMyCnf: []string{path.Join(os.Getenv("VTTOP"), "config/mycnf/rbr.cnf")}, + ExtraMyCnf: []string{path.Join(os.Getenv("VTROOT"), "config/mycnf/rbr.cnf")}, OnlyMySQL: true, } te.cluster = &vttest.LocalCluster{ diff --git a/go/vt/vttest/environment.go b/go/vt/vttest/environment.go index 336e92c3b06..8a53c0d9191 100644 --- a/go/vt/vttest/environment.go +++ b/go/vt/vttest/environment.go @@ -134,7 +134,7 @@ func GetMySQLOptions(flavor string) (string, []string, error) { } for i, cnf := range mycnf { - mycnf[i] = path.Join(os.Getenv("VTTOP"), cnf) + mycnf[i] = path.Join(os.Getenv("VTROOT"), cnf) } return flavor, mycnf, nil @@ -154,7 +154,7 @@ func (env *LocalTestEnv) BinaryPath(binary string) string { func (env *LocalTestEnv) MySQLManager(mycnf []string, snapshot string) (MySQLManager, error) { return &Mysqlctl{ Binary: env.BinaryPath("mysqlctl"), - InitFile: path.Join(os.Getenv("VTTOP"), "config/init_db.sql"), + InitFile: path.Join(os.Getenv("VTROOT"), "config/init_db.sql"), Directory: env.TmpPath, Port: env.PortForProtocol("mysql", ""), MyCnf: append(env.DefaultMyCnf, mycnf...), diff --git a/java/client/src/test/java/io/vitess/client/TestEnv.java b/java/client/src/test/java/io/vitess/client/TestEnv.java index 1e631e8bfb4..1b12162fcce 100644 --- a/java/client/src/test/java/io/vitess/client/TestEnv.java +++ b/java/client/src/test/java/io/vitess/client/TestEnv.java @@ -72,13 +72,13 @@ public void setPythonScriptProcess(Process process) { * Get setup command to launch a cluster. */ public List getSetupCommand(int port) { - String vtTop = System.getenv("VTTOP"); - if (vtTop == null) { - throw new RuntimeException("cannot find env variable: VTTOP"); + String vtRoot = System.getenv("VTROOT"); + if (vtRoot == null) { + throw new RuntimeException("cannot find env variable: VTROOT"); } String schemaDir = getTestDataPath() + "/schema"; List command = new ArrayList(); - command.add(vtTop + "/py/vttest/run_local_database.py"); + command.add(vtRoot + "/py/vttest/run_local_database.py"); command.add("--port"); command.add(Integer.toString(port)); command.add("--proto_topo"); @@ -89,11 +89,11 @@ public List getSetupCommand(int port) { } public String getTestDataPath() { - String vtTop = System.getenv("VTTOP"); - if (vtTop == null) { - throw new RuntimeException("cannot find env variable: VTTOP"); + String vtRoot = System.getenv("VTROOT"); + if (vtRoot == null) { + throw new RuntimeException("cannot find env variable: VTROOT"); } - return vtTop + "/data/test"; + return vtRoot + "/data/test"; } public String getTestOutputPath() { diff --git a/misc/git/hooks/pylint b/misc/git/hooks/pylint index 4452647f6f9..35f04cee37c 100755 --- a/misc/git/hooks/pylint +++ b/misc/git/hooks/pylint @@ -26,7 +26,7 @@ function msg() { } PYLINT=${PYLINT:-/usr/bin/gpylint} -pylint_script=$VTTOP/tools/pylint.sh +pylint_script=$VTROOT/tools/pylint.sh # This script does not handle file names that contain spaces. pyfiles=$(git diff --cached --name-only --diff-filter=ACM | grep '.*\.py$' | grep -v '^py/vtproto/') @@ -70,7 +70,7 @@ if [[ $? -eq 0 ]]; then do echo msg "Press enter to show the warnings for $pyfile:" - read -p " \$VTTOP/tools/pylint.sh $pyfile" + read -p " \$VTROOT/tools/pylint.sh $pyfile" $pylint_script $pyfile done read -r -p \ diff --git a/py/vttest/mysql_db_mysqlctl.py b/py/vttest/mysql_db_mysqlctl.py index f79f5c6504f..c1ecc8d049f 100644 --- a/py/vttest/mysql_db_mysqlctl.py +++ b/py/vttest/mysql_db_mysqlctl.py @@ -40,7 +40,7 @@ def setup(self): '-mysql_port', str(self._port), 'init', '-init_db_sql_file', - os.path.join(os.environ['VTTOP'], 'config/init_db.sql'), + os.path.join(os.environ['VTROOT'], 'config/init_db.sql'), ] env = os.environ env['VTDATAROOT'] = self._directory diff --git a/py/vttest/mysql_flavor.py b/py/vttest/mysql_flavor.py index 26e8aaf3d49..336f8604ef4 100644 --- a/py/vttest/mysql_flavor.py +++ b/py/vttest/mysql_flavor.py @@ -23,17 +23,15 @@ import sys -# For now, vttop is only used in this module. If other people +# For now, vtroot is only used in this module. If other people # need this, we should move it to environment. -if "VTTOP" not in os.environ: +if "VTROOT" not in os.environ: sys.stderr.write( "ERROR: Vitess environment not set up. " 'Please run "source dev.env" first.\n') sys.exit(1) -# vttop is the toplevel of the vitess source tree -vttop = os.environ["VTTOP"] - +vtroot = os.environ["VTROOT"] class MysqlFlavor(object): """Base class with default SQL statements.""" @@ -48,8 +46,8 @@ class MariaDB(MysqlFlavor): def my_cnf(self): files = [ - os.path.join(vttop, "config/mycnf/default-fast.cnf"), - os.path.join(vttop, "config/mycnf/master_mariadb100.cnf"), + os.path.join(vtroot, "config/mycnf/default-fast.cnf"), + os.path.join(vtroot, "config/mycnf/master_mariadb100.cnf"), ] return ":".join(files) @@ -58,8 +56,8 @@ class MariaDB103(MysqlFlavor): def my_cnf(self): files = [ - os.path.join(vttop, "config/mycnf/default-fast.cnf"), - os.path.join(vttop, "config/mycnf/master_mariadb103.cnf"), + os.path.join(vtroot, "config/mycnf/default-fast.cnf"), + os.path.join(vtroot, "config/mycnf/master_mariadb103.cnf"), ] return ":".join(files) @@ -68,8 +66,8 @@ class MySQL56(MysqlFlavor): def my_cnf(self): files = [ - os.path.join(vttop, "config/mycnf/default-fast.cnf"), - os.path.join(vttop, "config/mycnf/master_mysql56.cnf"), + os.path.join(vtroot, "config/mycnf/default-fast.cnf"), + os.path.join(vtroot, "config/mycnf/master_mysql56.cnf"), ] return ":".join(files) @@ -78,8 +76,8 @@ class MySQL80(MysqlFlavor): def my_cnf(self): files = [ - os.path.join(vttop, "config/mycnf/default-fast.cnf"), - os.path.join(vttop, "config/mycnf/master_mysql80.cnf"), + os.path.join(vtroot, "config/mycnf/default-fast.cnf"), + os.path.join(vtroot, "config/mycnf/master_mysql80.cnf"), ] return ":".join(files) diff --git a/py/vttest/run_local_database.py b/py/vttest/run_local_database.py index 8301a4e2b42..63b0da5f164 100755 --- a/py/vttest/run_local_database.py +++ b/py/vttest/run_local_database.py @@ -98,7 +98,7 @@ def main(cmdline_options): init_data_opts.max_table_shard_size = cmdline_options.max_table_shard_size init_data_opts.null_probability = cmdline_options.null_probability - extra_my_cnf = os.path.join(os.environ['VTTOP'], 'config/mycnf/vtcombo.cnf') + extra_my_cnf = os.path.join(os.environ['VTROOT'], 'config/mycnf/vtcombo.cnf') if cmdline_options.extra_my_cnf: extra_my_cnf += ':' + cmdline_options.extra_my_cnf diff --git a/test.go b/test.go index 73b75e9df30..3266001c0cb 100755 --- a/test.go +++ b/test.go @@ -27,7 +27,7 @@ run against a given flavor, it may take some time for the corresponding bootstrap image (vitess/bootstrap:) to be downloaded. It is meant to be run from the Vitess root, like so: - ~/src/vitess.io/vitess$ go run test.go [args] + $ go run test.go [args] For a list of options, run: $ go run test.go --help @@ -370,7 +370,7 @@ func main() { } tests = dup - vtTop := "." + vtRoot := "." tmpDir := "" if *docker { // Copy working repo to tmpDir. @@ -387,7 +387,7 @@ func main() { if out, err := exec.Command("chmod", "-R", "go=u", tmpDir).CombinedOutput(); err != nil { log.Printf("Can't set permissions on temp dir %v: %v: %s", tmpDir, err, out) } - vtTop = tmpDir + vtRoot = tmpDir } else { // Since we're sharing the working dir, do the build once for all tests. log.Printf("Running make build...") @@ -473,7 +473,7 @@ func main() { // Run the test. start := time.Now() - output, err := test.run(vtTop, dataDir) + output, err := test.run(vtRoot, dataDir) duration := time.Since(start) // Save/print test output. diff --git a/test/backup.py b/test/backup.py index 823e2a2ba2a..51b7a94eef4 100755 --- a/test/backup.py +++ b/test/backup.py @@ -95,7 +95,7 @@ def setUpModule(): # Create a new init_db.sql file that sets up passwords for all users. # Then we use a db-credentials-file with the passwords. new_init_db = environment.tmproot + '/init_db_with_passwords.sql' - with open(environment.vttop + '/config/init_db.sql') as fd: + with open(environment.vtroot + '/config/init_db.sql') as fd: init_db = fd.read() with open(new_init_db, 'w') as fd: fd.write(init_db) diff --git a/test/backup_only.py b/test/backup_only.py index dd9b547dacf..04f2db4dfdc 100755 --- a/test/backup_only.py +++ b/test/backup_only.py @@ -95,7 +95,7 @@ def setUpModule(): # Create a new init_db.sql file that sets up passwords for all users. # Then we use a db-credentials-file with the passwords. new_init_db = environment.tmproot + '/init_db_with_passwords.sql' - with open(environment.vttop + '/config/init_db.sql') as fd: + with open(environment.vtroot + '/config/init_db.sql') as fd: init_db = fd.read() with open(new_init_db, 'w') as fd: fd.write(init_db) diff --git a/test/backup_transform.py b/test/backup_transform.py index b0e665b6fa8..00f3ef9223e 100755 --- a/test/backup_transform.py +++ b/test/backup_transform.py @@ -79,7 +79,7 @@ def setUpModule(): # Create a new init_db.sql file that sets up passwords for all users. # Then we use a db-credentials-file with the passwords. new_init_db = environment.tmproot + '/init_db_with_passwords.sql' - with open(environment.vttop + '/config/init_db.sql') as fd: + with open(environment.vtroot + '/config/init_db.sql') as fd: init_db = fd.read() with open(new_init_db, 'w') as fd: fd.write(init_db) diff --git a/test/client_test.sh b/test/client_test.sh index 82be7996c8d..b9d70253e5f 100755 --- a/test/client_test.sh +++ b/test/client_test.sh @@ -18,7 +18,7 @@ # but has been moved to its own test. It hijacks the public examples scripts set -xe -cd "$VTTOP/examples/local" +cd "$VTROOT/examples/local" CELL=test ./etcd-up.sh CELL=test ./vtctld-up.sh diff --git a/test/cluster/k8s_environment.py b/test/cluster/k8s_environment.py index 0951ae58a09..d0026fb70b5 100644 --- a/test/cluster/k8s_environment.py +++ b/test/cluster/k8s_environment.py @@ -121,7 +121,7 @@ def create(self, **kwargs): if 'VITESS_NAME' not in kwargs: kwargs['VITESS_NAME'] = getpass.getuser() kwargs['TEST_MODE'] = '1' - self.script_dir = os.path.join(os.environ['VTTOP'], 'examples/kubernetes') + self.script_dir = os.path.join(os.environ['VTROOT'], 'examples/kubernetes') try: subprocess.check_output(['gcloud', 'config', 'list']) except OSError: diff --git a/test/cluster/keytar/README.md b/test/cluster/keytar/README.md index 081957d192a..b9279dfc235 100644 --- a/test/cluster/keytar/README.md +++ b/test/cluster/keytar/README.md @@ -7,8 +7,8 @@ Keytar is an internally used Vitess system for continuous execution of cluster t How to set up Keytar for Vitess: * Create service account keys with GKE credentials on the account to run the tests on. Follow [step 1 from the GKE developers page](https://developers.google.com/identity/protocols/application-default-credentials?hl=en_US#howtheywork). -* Move the generated keyfile to `$VTTOP/test/cluster/keytar/config`. -* Create or modify the test configuration file (`$VTTOP/test/cluster/keytar/config/vitess_config.yaml`). +* Move the generated keyfile to `$VTROOT/test/cluster/keytar/config`. +* Create or modify the test configuration file (`$VTROOT/test/cluster/keytar/config/vitess_config.yaml`). * Ensure the configuration has the correct values for GKE project name and keyfile: ``` cluster_setup: @@ -18,7 +18,7 @@ How to set up Keytar for Vitess: ``` * Then run the following commands: ``` - > cd $VTTOP/test/cluster/keytar + > cd $VTROOT/test/cluster/keytar > KEYTAR_PASSWORD= KEYTAR_PORT= KEYTAR_CONFIG= ./keytar-up.sh ``` * Add a Docker Hub webhook pointing to the Keytar service. The webhook URL should be in the form: diff --git a/test/cluster/keytar/config/vitess_config.yaml b/test/cluster/keytar/config/vitess_config.yaml index a8b0e8a995b..34213cfaeb9 100644 --- a/test/cluster/keytar/config/vitess_config.yaml +++ b/test/cluster/keytar/config/vitess_config.yaml @@ -20,17 +20,15 @@ config: - docker_image: vitess/root github: repo: vitessio/vitess - repo_prefix: src/vitess.io/vitess environment: sandbox: test/cluster/sandbox/vitess_kubernetes_sandbox.py config: test/cluster/sandbox/example_sandbox.yaml cluster_type: gke application_type: k8s before_test: - - export VTTOP=$(pwd) - - export VTROOT="${VTROOT:-${VTTOP/\/src\/github.com\/youtube\/vitess/}}" + - export VTROOT=$(pwd) - export GOPATH=$VTROOT - - export PYTHONPATH=$VTTOP/py:$VTTOP/test:$VTTOP/test/cluster/sandbox:/usr/lib/python2.7/dist-packages:/env/lib/python2.7/site-packages + - export PYTHONPATH=$VTROOT/py:$VTROOT/test:$VTROOT/test/cluster/sandbox:/usr/lib/python2.7/dist-packages:/env/lib/python2.7/site-packages - go get vitess.io/vitess/go/cmd/vtctlclient - export PATH=$GOPATH/bin:$PATH tests: diff --git a/test/cluster/keytar/keytar_web_test.py b/test/cluster/keytar/keytar_web_test.py index 5784d3335f5..247cec4478a 100755 --- a/test/cluster/keytar/keytar_web_test.py +++ b/test/cluster/keytar/keytar_web_test.py @@ -35,7 +35,7 @@ class TestKeytarWeb(unittest.TestCase): def setUpClass(cls): cls.driver = environment.create_webdriver() port = environment.reserve_ports(1) - keytar_folder = os.path.join(environment.vttop, 'test/cluster/keytar') + keytar_folder = os.path.join(environment.vtroot, 'test/cluster/keytar') cls.flask_process = subprocess.Popen( [os.path.join(keytar_folder, 'keytar.py'), '--config_file=%s' % os.path.join(keytar_folder, 'test_config.yaml'), diff --git a/test/cluster/keytar/test_config.yaml b/test/cluster/keytar/test_config.yaml index 308c6de7026..8eaad2c36b1 100644 --- a/test/cluster/keytar/test_config.yaml +++ b/test/cluster/keytar/test_config.yaml @@ -5,7 +5,6 @@ config: - docker_image: test/image github: repo: vitessio/vitess - repo_prefix: src/vitess.io/vitess before_test: - touch /tmp/test_file environment: diff --git a/test/cluster/sandbox/create_schema.py b/test/cluster/sandbox/create_schema.py index 198858eeffe..d60839bd013 100755 --- a/test/cluster/sandbox/create_schema.py +++ b/test/cluster/sandbox/create_schema.py @@ -33,7 +33,7 @@ def main(): parser.add_option( '-s', '--sql_file', help='File containing sql schema', default=os.path.join( - os.environ['VTTOP'], 'examples/kubernetes/create_test_table.sql')) + os.environ['VTROOT'], 'examples/kubernetes/create_test_table.sql')) logging.getLogger().setLevel(logging.INFO) options, _ = parser.parse_args() diff --git a/test/cluster/sandbox/kubernetes_components.py b/test/cluster/sandbox/kubernetes_components.py index e67c4bf379c..94d3ac5e947 100755 --- a/test/cluster/sandbox/kubernetes_components.py +++ b/test/cluster/sandbox/kubernetes_components.py @@ -81,7 +81,7 @@ def start(self): logging.info('Installing helm.') try: subprocess.check_output( - ['helm', 'install', os.path.join(os.environ['VTTOP'], 'helm/vitess'), + ['helm', 'install', os.path.join(os.environ['VTROOT'], 'helm/vitess'), '-n', self.sandbox_name, '--namespace', self.sandbox_name, '--replace', '--values', self.helm_config], stderr=subprocess.STDOUT) diff --git a/test/cluster/sandbox/vitess_kubernetes_sandbox.py b/test/cluster/sandbox/vitess_kubernetes_sandbox.py index 734f8d1ac46..016a8d99916 100755 --- a/test/cluster/sandbox/vitess_kubernetes_sandbox.py +++ b/test/cluster/sandbox/vitess_kubernetes_sandbox.py @@ -46,7 +46,7 @@ def generate_guestbook_sandlet(self): """Creates a sandlet encompassing the guestbook app built on Vitess.""" guestbook_sandlet = sandlet.Sandlet('guestbook') guestbook_sandlet.dependencies = ['helm'] - template_dir = os.path.join(os.environ['VTTOP'], 'examples/kubernetes') + template_dir = os.path.join(os.environ['VTROOT'], 'examples/kubernetes') guestbook_sandlet.components.add_component( self.cluster_env.Port('%s-guestbook' % self.name, 80)) for keyspace in self.app_options.keyspaces: @@ -54,7 +54,7 @@ def generate_guestbook_sandlet(self): 'create_schema_%s' % keyspace['name'], self.name, 'create_schema.py', self.log_dir, namespace=self.name, keyspace=keyspace['name'], drop_table='messages', sql_file=os.path.join( - os.environ['VTTOP'], 'examples/kubernetes/create_test_table.sql')) + os.environ['VTROOT'], 'examples/kubernetes/create_test_table.sql')) guestbook_sandlet.components.add_component(create_schema_subprocess) guestbook_sandlet.components.add_component( kubernetes_components.KubernetesResource( diff --git a/test/cluster/sandbox/vtctl_sandbox.py b/test/cluster/sandbox/vtctl_sandbox.py index ef0f1d978ed..3e495b414ab 100755 --- a/test/cluster/sandbox/vtctl_sandbox.py +++ b/test/cluster/sandbox/vtctl_sandbox.py @@ -68,7 +68,7 @@ def execute_vtctl_command(vtctl_args, namespace='default', timeout_s=180): # Default to trying to use kvtctl.sh if a forwarded port cannot be found. os.environ['VITESS_NAME'] = namespace vtctl_cmd_args = ( - [os.path.join(os.environ['VTTOP'], 'examples/kubernetes/kvtctl.sh')] + [os.path.join(os.environ['VTROOT'], 'examples/kubernetes/kvtctl.sh')] + vtctl_args) start_time = time.time() diff --git a/test/environment.py b/test/environment.py index ce3b58db683..eee51cc4523 100644 --- a/test/environment.py +++ b/test/environment.py @@ -53,14 +53,6 @@ 'ERROR: Vitess and mysqld ' 'should not be run as root.\n') sys.exit(1) -if 'VTTOP' not in os.environ: - sys.stderr.write( - 'ERROR: Vitess environment not set up. ' - 'Please run "source dev.env" first.\n') - sys.exit(1) - -# vttop is the toplevel of the vitess source tree -vttop = os.environ['VTTOP'] # vtroot is where everything gets installed vtroot = os.environ['VTROOT'] @@ -162,7 +154,7 @@ def prog_compile(name): return compiled_progs.append(name) logging.debug('Compiling %s', name) - run(['go', 'install'], cwd=os.path.join(vttop, 'go', 'cmd', name)) + run(['go', 'install'], cwd=os.path.join(vtroot, 'go', 'cmd', name)) # binary management: returns the full path for a binary this should diff --git a/test/initial_sharding_multi.py b/test/initial_sharding_multi.py index af0b167ed2d..8edad5a73e5 100755 --- a/test/initial_sharding_multi.py +++ b/test/initial_sharding_multi.py @@ -127,7 +127,7 @@ def setUpModule(): # Create a new init_db.sql file that sets up passwords for all users. # Then we use a db-credentials-file with the passwords. new_init_db = environment.tmproot + '/init_db_with_passwords.sql' - with open(environment.vttop + '/config/init_db.sql') as fd: + with open(environment.vtroot + '/config/init_db.sql') as fd: init_db = fd.read() with open(new_init_db, 'w') as fd: fd.write(init_db) diff --git a/test/local_example.sh b/test/local_example.sh index f0ba0f4278b..6b9658b0068 100755 --- a/test/local_example.sh +++ b/test/local_example.sh @@ -20,7 +20,7 @@ set -xe -cd "$VTTOP/examples/local" +cd "$VTROOT/examples/local" ./101_initial_cluster.sh diff --git a/test/mysql_flavor.py b/test/mysql_flavor.py index 9cf057982ea..3ad970bcf38 100644 --- a/test/mysql_flavor.py +++ b/test/mysql_flavor.py @@ -126,7 +126,7 @@ def reset_replication_commands(self): ] def extra_my_cnf(self): - return environment.vttop + "/config/mycnf/master_mariadb100.cnf" + return environment.vtroot + "/config/mycnf/master_mariadb100.cnf" def master_position(self, tablet): gtid = tablet.mquery("", "SELECT @@GLOBAL.gtid_binlog_pos")[0][0] @@ -152,7 +152,7 @@ class MariaDB103(MariaDB): """Overrides specific to MariaDB 10.3+.""" def extra_my_cnf(self): - return environment.vttop + "/config/mycnf/master_mariadb103.cnf" + return environment.vtroot + "/config/mycnf/master_mariadb103.cnf" class MySQL56(MysqlFlavor): """Overrides specific to MySQL 5.6/5.7""" @@ -172,7 +172,7 @@ def position_at_least(self, a, b): ]).strip() == "true" def extra_my_cnf(self): - return environment.vttop + "/config/mycnf/master_mysql56.cnf" + return environment.vtroot + "/config/mycnf/master_mysql56.cnf" def change_master_commands(self, host, port, pos): gtid = pos.split("/")[1] @@ -186,7 +186,7 @@ def change_master_commands(self, host, port, pos): class MySQL80(MySQL56): """Overrides specific to MySQL 8.0.""" def extra_my_cnf(self): - return environment.vttop + "/config/mycnf/master_mysql80.cnf" + return environment.vtroot + "/config/mycnf/master_mysql80.cnf" def change_passwords(self, password_col): """set real passwords for all users""" return ''' diff --git a/test/tablet.py b/test/tablet.py index 88c791c8245..b5ac0641f1e 100644 --- a/test/tablet.py +++ b/test/tablet.py @@ -55,7 +55,7 @@ def get_backup_storage_flags(): def get_all_extra_my_cnf(extra_my_cnf): - all_extra_my_cnf = [environment.vttop + '/config/mycnf/default-fast.cnf'] + all_extra_my_cnf = [environment.vtroot + '/config/mycnf/default-fast.cnf'] flavor_my_cnf = mysql_flavor().extra_my_cnf() if flavor_my_cnf: all_extra_my_cnf.append(flavor_my_cnf) @@ -186,12 +186,12 @@ def init_mysql(self, extra_my_cnf=None, init_db=None, extra_args=None, """ if use_rbr: if extra_my_cnf: - extra_my_cnf += ':' + environment.vttop + '/config/mycnf/rbr.cnf' + extra_my_cnf += ':' + environment.vtroot + '/config/mycnf/rbr.cnf' else: - extra_my_cnf = environment.vttop + '/config/mycnf/rbr.cnf' + extra_my_cnf = environment.vtroot + '/config/mycnf/rbr.cnf' if not init_db: - init_db = environment.vttop + '/config/init_db.sql' + init_db = environment.vtroot + '/config/init_db.sql' if self.use_mysqlctld: self.mysqlctld_process = self.mysqlctld(['-init_db_sql_file', init_db], diff --git a/test/utils.py b/test/utils.py index fd73e2ef331..c0949c62bd1 100644 --- a/test/utils.py +++ b/test/utils.py @@ -1214,8 +1214,8 @@ def start(self, enable_schema_change_dir=False, extra_flags=None): args = environment.binary_args('vtctld') + [ '-enable_queries', '-cell', 'test_nj', - '-web_dir', environment.vttop + '/web/vtctld', - '-web_dir2', environment.vttop + '/web/vtctld2', + '-web_dir', environment.vtroot + '/web/vtctld', + '-web_dir2', environment.vtroot + '/web/vtctld2', '--log_dir', environment.vtlogroot, '--port', str(self.port), '-tablet_manager_protocol', diff --git a/test/vtbackup.py b/test/vtbackup.py index f42831ea4e0..fecaca6612d 100644 --- a/test/vtbackup.py +++ b/test/vtbackup.py @@ -35,7 +35,7 @@ def get_backup_storage_flags(): os.path.join(environment.tmproot, 'backupstorage')] def get_all_extra_my_cnf(extra_my_cnf): - all_extra_my_cnf = [environment.vttop + '/config/mycnf/default-fast.cnf'] + all_extra_my_cnf = [environment.vtroot + '/config/mycnf/default-fast.cnf'] flavor_my_cnf = mysql_flavor().extra_my_cnf() if flavor_my_cnf: all_extra_my_cnf.append(flavor_my_cnf) diff --git a/test/vtctld_web_test.py b/test/vtctld_web_test.py index 19738abf77f..33fd8e00861 100755 --- a/test/vtctld_web_test.py +++ b/test/vtctld_web_test.py @@ -86,12 +86,12 @@ def setUpClass(cls): cls.db = local_database.LocalDatabase( topology, - os.path.join(environment.vttop, 'test/vttest_schema'), + os.path.join(environment.vtroot, 'test/vttest_schema'), False, None, - web_dir=os.path.join(environment.vttop, 'web/vtctld'), + web_dir=os.path.join(environment.vtroot, 'web/vtctld'), default_schema_dir=os.path.join( - environment.vttop, 'test/vttest_schema/default'), - web_dir2=os.path.join(environment.vttop, 'web/vtctld2/app')) + environment.vtroot, 'test/vttest_schema/default'), + web_dir2=os.path.join(environment.vtroot, 'web/vtctld2/app')) cls.db.setup() cls.vtctld_addr = 'http://localhost:%d' % cls.db.config()['port'] utils.pause('Paused test after vtcombo was started.\n' diff --git a/test/vttest_sample_test.py b/test/vttest_sample_test.py index 98888a97777..a4a7de4639f 100755 --- a/test/vttest_sample_test.py +++ b/test/vttest_sample_test.py @@ -82,9 +82,9 @@ def test_standalone(self): '--port', str(port), '--proto_topo', text_format.MessageToString(topology, as_one_line=True), - '--schema_dir', os.path.join(environment.vttop, 'test', + '--schema_dir', os.path.join(environment.vtroot, 'test', 'vttest_schema'), - '--web_dir', environment.vttop + '/web/vtctld', + '--web_dir', environment.vtroot + '/web/vtctld', ] sp = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE) config = json.loads(sp.stdout.readline()) diff --git a/tools/bootstrap_web.sh b/tools/bootstrap_web.sh index aecc89b8f80..cc0e40966c9 100755 --- a/tools/bootstrap_web.sh +++ b/tools/bootstrap_web.sh @@ -39,12 +39,12 @@ else # Add the node directory to PATH to make sure that the Angular # installation below can find the "node" binary. # (dev.env does actually append it to PATH.) - source $VTTOP/dev.env + source $VTROOT/dev.env fi echo "Installing dependencies for building web UI" angular_cli_dir=$VTROOT/dist/angular-cli -web_dir2=$VTTOP/web/vtctld2 +web_dir2=$VTROOT/web/vtctld2 angular_cli_commit=cacaa4eff10e135016ef81076fab1086a3bce92f if [[ -d $angular_cli_dir && `cd $angular_cli_dir && git rev-parse HEAD` == "$angular_cli_commit" ]]; then echo "skipping angular cli download. remove $angular_cli_dir to force download." diff --git a/tools/generate_web_artifacts.sh b/tools/generate_web_artifacts.sh index d21da7e84e6..a7080db7828 100755 --- a/tools/generate_web_artifacts.sh +++ b/tools/generate_web_artifacts.sh @@ -20,7 +20,7 @@ set -e -vtctld2_dir=$VTTOP/web/vtctld2 +vtctld2_dir=$VTROOT/web/vtctld2 if [[ -d $vtctld2_dir/app ]]; then rm -rf $vtctld2_dir/app fi diff --git a/vagrant-scripts/bootstrap_vm.sh b/vagrant-scripts/bootstrap_vm.sh index bea07394dea..1c152ea3184 100755 --- a/vagrant-scripts/bootstrap_vm.sh +++ b/vagrant-scripts/bootstrap_vm.sh @@ -61,7 +61,7 @@ printf "\nSetting /etc/environment\n" echo "GOROOT=${GOROOT}" echo "GOPATH=${GOPATH}" echo "PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin" - echo "VITESS_WORKSPACE=/vagrant/src/vitess.io/vitess" + echo "VITESS_WORKSPACE=/vagrant/vt" } >> /etc/environment # shellcheck disable=SC2013 # shellcheck disable=SC2163 @@ -72,7 +72,7 @@ echo "fs.file-max = 10000" >> /etc/sysctl.conf sysctl -p # Set vitess env in .bashrc -cat /vagrant/src/vitess.io/vitess/vagrant-scripts/vagrant-bashrc >> /home/vagrant/.bashrc +cat /vagrant/vt/vagrant-scripts/vagrant-bashrc >> /home/vagrant/.bashrc # Provisioning completed touch $SEED_FILE diff --git a/vagrant-scripts/vagrant-bashrc b/vagrant-scripts/vagrant-bashrc index 27b1828450c..08654d625f2 100644 --- a/vagrant-scripts/vagrant-bashrc +++ b/vagrant-scripts/vagrant-bashrc @@ -9,7 +9,6 @@ then fi ulimit -n 10000 -export MYSQL_FLAVOR=MySQL56 export VT_MYSQL_ROOT=/usr # This is just to make sure the vm can write into these directories sudo chown "$(whoami)":"$(whoami)" /vagrant From dbeffa08d2f0ee812c438cc1aa250f20c5513a03 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Wed, 30 Oct 2019 17:09:32 -0600 Subject: [PATCH 02/25] Revert change of docker working dir Remove mysql_flavor settings (do in different pr) Signed-off-by: Morgan Tocker --- Vagrantfile | 4 ++-- doc/DockerBuild.md | 2 +- docker/base/Dockerfile | 2 +- docker/base/Dockerfile.mariadb | 2 +- docker/base/Dockerfile.mariadb103 | 2 +- docker/base/Dockerfile.mysql56 | 2 +- docker/base/Dockerfile.mysql80 | 2 +- docker/base/Dockerfile.percona | 2 +- docker/base/Dockerfile.percona57 | 2 +- docker/base/Dockerfile.percona80 | 2 +- docker/bootstrap/Dockerfile.common | 14 +++++++------- docker/bootstrap/Dockerfile.mariadb | 3 ++- docker/bootstrap/Dockerfile.mariadb103 | 3 ++- docker/bootstrap/Dockerfile.mysql56 | 3 ++- docker/bootstrap/Dockerfile.mysql57 | 4 +++- docker/bootstrap/Dockerfile.mysql80 | 4 +++- docker/bootstrap/Dockerfile.percona | 3 ++- docker/bootstrap/Dockerfile.percona57 | 3 ++- docker/bootstrap/Dockerfile.percona80 | 3 ++- docker/bootstrap/build.sh | 2 +- docker/k8s/Dockerfile | 2 +- docker/lite/Dockerfile | 13 +++++++------ docker/lite/Dockerfile.alpine | 11 ++++++----- docker/lite/Dockerfile.mariadb | 11 ++++++----- docker/lite/Dockerfile.mariadb103 | 13 +++++++------ docker/lite/Dockerfile.mysql56 | 10 +++++----- docker/lite/Dockerfile.mysql57 | 11 ++++++----- docker/lite/Dockerfile.mysql80 | 11 ++++++----- docker/lite/Dockerfile.percona | 11 ++++++----- docker/lite/Dockerfile.percona57 | 11 ++++++----- docker/lite/Dockerfile.percona80 | 11 ++++++----- docker/packaging/package_vitess.sh | 8 ++++---- docker/test/run.sh | 2 +- go.mod | 3 +++ vagrant-scripts/bootstrap_vm.sh | 4 ++-- vagrant-scripts/vagrant-bashrc | 1 + 36 files changed, 110 insertions(+), 87 deletions(-) diff --git a/Vagrantfile b/Vagrantfile index 5f727efed91..3b38717c90c 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -44,8 +44,8 @@ Vagrant.configure("2") do |config| # Additional details in https://github.com/vitessio/vitess/issues/4396 # # To enable, use: - # config.vm.synced_folder ".", "/vagrant/vt", type: "nfs" - config.vm.synced_folder ".", "/vagrant/vt" + # config.vm.synced_folder ".", "/vagrant/src/vitess.io/vitess", type: "nfs" + config.vm.synced_folder ".", "/vagrant/src/vitess.io/vitess" config.vm.provider :virtualbox do |vb| vb.name = "vitess" diff --git a/doc/DockerBuild.md b/doc/DockerBuild.md index 5b97b1e19e4..986b8e672c6 100644 --- a/doc/DockerBuild.md +++ b/doc/DockerBuild.md @@ -20,7 +20,7 @@ Then you can run our build script for the `lite` image which extracts the Vitess 1. Create an account on [Docker Hub](https://docs.docker.com/docker-hub/) and then `docker login` to it. -1. Go to your the vitess source directory. +1. Go to your `src/vitess.io/vitess` directory. 1. Usually, you won't need to [build your own bootstrap image](https://github.com/vitessio/vitess/blob/master/docker/bootstrap/README.md) unless you edit [bootstrap.sh](https://github.com/vitessio/vitess/blob/master/bootstrap.sh) diff --git a/docker/base/Dockerfile b/docker/base/Dockerfile index 20f0052b154..ecab9ab6c42 100644 --- a/docker/base/Dockerfile +++ b/docker/base/Dockerfile @@ -27,7 +27,7 @@ ARG CGO_ENABLED=0 # Re-copy sources from working tree USER root -COPY . /vt/ +COPY . /vt/src/vitess.io/vitess # Build Vitess RUN make build diff --git a/docker/base/Dockerfile.mariadb b/docker/base/Dockerfile.mariadb index 1c7d1633a87..a3f9771334f 100644 --- a/docker/base/Dockerfile.mariadb +++ b/docker/base/Dockerfile.mariadb @@ -2,7 +2,7 @@ FROM vitess/bootstrap:mariadb # Re-copy sources from working tree USER root -COPY . /vt/ +COPY . /vt/src/vitess.io/vitess # Build Vitess RUN make build diff --git a/docker/base/Dockerfile.mariadb103 b/docker/base/Dockerfile.mariadb103 index cb3bdfc8513..75811dc68ba 100644 --- a/docker/base/Dockerfile.mariadb103 +++ b/docker/base/Dockerfile.mariadb103 @@ -2,7 +2,7 @@ FROM vitess/bootstrap:mariadb103 # Re-copy sources from working tree USER root -COPY . /vt/ +COPY . /vt/src/vitess.io/vitess # Build Vitess RUN make build diff --git a/docker/base/Dockerfile.mysql56 b/docker/base/Dockerfile.mysql56 index be632e6aff8..5d1f55079fa 100644 --- a/docker/base/Dockerfile.mysql56 +++ b/docker/base/Dockerfile.mysql56 @@ -2,7 +2,7 @@ FROM vitess/bootstrap:mysql56 # Re-copy sources from working tree USER root -COPY . /vt/ +COPY . /vt/src/vitess.io/vitess # Build Vitess RUN make build diff --git a/docker/base/Dockerfile.mysql80 b/docker/base/Dockerfile.mysql80 index 36ab976ba2a..5b2fc5c91ae 100644 --- a/docker/base/Dockerfile.mysql80 +++ b/docker/base/Dockerfile.mysql80 @@ -2,7 +2,7 @@ FROM vitess/bootstrap:mysql80 # Re-copy sources from working tree USER root -COPY . /vt/ +COPY . /vt/src/vitess.io/vitess # Build Vitess RUN make build diff --git a/docker/base/Dockerfile.percona b/docker/base/Dockerfile.percona index a2e9ba0f781..93a218ff059 100644 --- a/docker/base/Dockerfile.percona +++ b/docker/base/Dockerfile.percona @@ -2,7 +2,7 @@ FROM vitess/bootstrap:percona # Re-copy sources from working tree USER root -COPY . /vt/ +COPY . /vt/src/vitess.io/vitess # Fix permissions RUN chown -R vitess:vitess /vt diff --git a/docker/base/Dockerfile.percona57 b/docker/base/Dockerfile.percona57 index 92d03255f4c..4f396c9c86b 100644 --- a/docker/base/Dockerfile.percona57 +++ b/docker/base/Dockerfile.percona57 @@ -2,7 +2,7 @@ FROM vitess/bootstrap:percona57 # Re-copy sources from working tree USER root -COPY . /vt/ +COPY . /vt/src/vitess.io/vitess # Build Vitess RUN make build diff --git a/docker/base/Dockerfile.percona80 b/docker/base/Dockerfile.percona80 index 67515df1840..83672111166 100644 --- a/docker/base/Dockerfile.percona80 +++ b/docker/base/Dockerfile.percona80 @@ -2,7 +2,7 @@ FROM vitess/bootstrap:percona80 # Re-copy sources from working tree USER root -COPY . /vt/ +COPY . /vt/src/vitess.io/vitess # Fix permissions RUN chown -R vitess:vitess /vt diff --git a/docker/bootstrap/Dockerfile.common b/docker/bootstrap/Dockerfile.common index d4b6eea1f35..3b08b423bda 100644 --- a/docker/bootstrap/Dockerfile.common +++ b/docker/bootstrap/Dockerfile.common @@ -38,7 +38,7 @@ RUN mkdir -p /vt/dist && \ mv apache-maven-3.3.9 maven # Set up Vitess environment (equivalent to '. dev.env') -ENV VTROOT /vt +ENV VTROOT /vt/src/vitess.io/vitess ENV GOTOP $VTROOT/go ENV PYTOP $VTROOT/py ENV VTDATAROOT $VTROOT/vtdataroot @@ -52,11 +52,11 @@ ENV USER vitess ENV GO111MODULE on # Copy files needed for bootstrap -COPY bootstrap.sh dev.env build.env go.mod go.sum /vt/ -COPY config /vt/config -COPY third_party /vt/third_party -COPY tools /vt/tools -COPY travis /vt/travis +COPY bootstrap.sh dev.env build.env go.mod go.sum /vt/src/vitess.io/vitess/ +COPY config /vt/src/vitess.io/vitess/config +COPY third_party /vt/src/vitess.io/vitess/third_party +COPY tools /vt/src/vitess.io/vitess/tools +COPY travis /vt/src/vitess.io/vitess/travis # Create vitess user RUN groupadd -r vitess && useradd -r -g vitess vitess && \ @@ -64,7 +64,7 @@ RUN groupadd -r vitess && useradd -r -g vitess vitess && \ chown -R vitess:vitess /vt /home/vitess # Download vendored Go dependencies -RUN cd /vt && \ +RUN cd /vt/src/vitess.io/vitess && \ su vitess -c "/usr/local/go/bin/go mod download" # Create mount point for actual data (e.g. MySQL data dir) diff --git a/docker/bootstrap/Dockerfile.mariadb b/docker/bootstrap/Dockerfile.mariadb index 9f1ee44d7b2..19d5c9973d0 100644 --- a/docker/bootstrap/Dockerfile.mariadb +++ b/docker/bootstrap/Dockerfile.mariadb @@ -20,7 +20,8 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r && rm -rf /var/lib/apt/lists/* # Bootstrap Vitess -WORKDIR /vt +WORKDIR /vt/src/vitess.io/vitess +ENV MYSQL_FLAVOR MariaDB USER vitess RUN ./bootstrap.sh diff --git a/docker/bootstrap/Dockerfile.mariadb103 b/docker/bootstrap/Dockerfile.mariadb103 index a7d3408751b..c2828ddd25d 100644 --- a/docker/bootstrap/Dockerfile.mariadb103 +++ b/docker/bootstrap/Dockerfile.mariadb103 @@ -9,7 +9,8 @@ RUN apt-key adv --no-tty --recv-keys --keyserver keyserver.ubuntu.com 0xF1656F24 && rm -rf /var/lib/apt/lists/* # Bootstrap Vitess -WORKDIR /vt +WORKDIR /vt/src/vitess.io/vitess +ENV MYSQL_FLAVOR MariaDB103 USER vitess RUN ./bootstrap.sh diff --git a/docker/bootstrap/Dockerfile.mysql56 b/docker/bootstrap/Dockerfile.mysql56 index 3726eefc229..b07fddedb61 100644 --- a/docker/bootstrap/Dockerfile.mysql56 +++ b/docker/bootstrap/Dockerfile.mysql56 @@ -15,7 +15,8 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver pool.s rm -rf /var/lib/apt/lists/* # Bootstrap Vitess -WORKDIR /vt +WORKDIR /vt/src/vitess.io/vitess +ENV MYSQL_FLAVOR MySQL56 USER vitess RUN ./bootstrap.sh diff --git a/docker/bootstrap/Dockerfile.mysql57 b/docker/bootstrap/Dockerfile.mysql57 index 37aa75d85a0..2fb1f4e1aa5 100644 --- a/docker/bootstrap/Dockerfile.mysql57 +++ b/docker/bootstrap/Dockerfile.mysql57 @@ -16,7 +16,9 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins rm -rf /var/lib/apt/lists/* # Bootstrap Vitess -WORKDIR /vt +WORKDIR /vt/src/vitess.io/vitess + +ENV MYSQL_FLAVOR MySQL56 USER vitess RUN ./bootstrap.sh \ No newline at end of file diff --git a/docker/bootstrap/Dockerfile.mysql80 b/docker/bootstrap/Dockerfile.mysql80 index b833e9f036d..ec53895165f 100644 --- a/docker/bootstrap/Dockerfile.mysql80 +++ b/docker/bootstrap/Dockerfile.mysql80 @@ -15,7 +15,9 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --recv-keys --keyserver ha.poo rm -rf /var/lib/apt/lists/* # Bootstrap Vitess -WORKDIR /vt +WORKDIR /vt/src/vitess.io/vitess + +ENV MYSQL_FLAVOR MySQL80 USER vitess RUN ./bootstrap.sh \ No newline at end of file diff --git a/docker/bootstrap/Dockerfile.percona b/docker/bootstrap/Dockerfile.percona index ba18fb8afc7..6d13fa4dfb0 100644 --- a/docker/bootstrap/Dockerfile.percona +++ b/docker/bootstrap/Dockerfile.percona @@ -14,7 +14,8 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r rm -rf /var/lib/apt/lists/* # Bootstrap Vitess -WORKDIR /vt +WORKDIR /vt/src/vitess.io/vitess +ENV MYSQL_FLAVOR MySQL56 USER vitess RUN ./bootstrap.sh diff --git a/docker/bootstrap/Dockerfile.percona57 b/docker/bootstrap/Dockerfile.percona57 index 1ddad7b6bd4..6ed54c76923 100644 --- a/docker/bootstrap/Dockerfile.percona57 +++ b/docker/bootstrap/Dockerfile.percona57 @@ -15,7 +15,8 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r rm -rf /var/lib/apt/lists/* # Bootstrap Vitess -WORKDIR /vt +WORKDIR /vt/src/vitess.io/vitess +ENV MYSQL_FLAVOR MySQL56 USER vitess RUN ./bootstrap.sh diff --git a/docker/bootstrap/Dockerfile.percona80 b/docker/bootstrap/Dockerfile.percona80 index 5fde3ee5310..1ce9c52103f 100644 --- a/docker/bootstrap/Dockerfile.percona80 +++ b/docker/bootstrap/Dockerfile.percona80 @@ -30,7 +30,8 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r && rm -rf /var/lib/apt/lists/* # Bootstrap Vitess -WORKDIR /vt +WORKDIR /vt/src/vitess.io/vitess +ENV MYSQL_FLAVOR MySQL80 USER vitess RUN ./bootstrap.sh diff --git a/docker/bootstrap/build.sh b/docker/bootstrap/build.sh index 36fb4633148..0a3a48c1253 100755 --- a/docker/bootstrap/build.sh +++ b/docker/bootstrap/build.sh @@ -22,7 +22,7 @@ if [[ -z "$flavor" ]]; then fi if [[ ! -f bootstrap.sh ]]; then - echo "This script should be run from the root of the Vitess source tree" + echo "This script should be run from the root of the Vitess source tree - e.g. ~/src/vitess.io/vitess" exit 1 fi diff --git a/docker/k8s/Dockerfile b/docker/k8s/Dockerfile index 5ced6f87c35..391c352e4f8 100644 --- a/docker/k8s/Dockerfile +++ b/docker/k8s/Dockerfile @@ -25,7 +25,7 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTROOT /vt +ENV VTROOT /vt/src/vitess.io/vitess ENV VTDATAROOT /vtdataroot # Prepare directory structure. diff --git a/docker/lite/Dockerfile b/docker/lite/Dockerfile index 2a799266ad9..128c4e232c8 100644 --- a/docker/lite/Dockerfile +++ b/docker/lite/Dockerfile @@ -17,12 +17,12 @@ FROM debian:stretch-slim AS staging RUN mkdir -p /vt/vtdataroot/ \ && mkdir -p /vt/bin \ - && mkdir -p /vt/web/vtctld2 \ + && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ && groupadd -r vitess && useradd -r -g vitess vitess -COPY --from=builder /vt/web/vtctld /vt/web/vtctld -COPY --from=builder /vt/web/vtctld2/app /vt/web/vtctld2/app -COPY --from=builder /vt/config /vt/config +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app +COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config COPY --from=builder /vt/bin/mysqlctld /vt/bin/ COPY --from=builder /vt/bin/vtctld /vt/bin/ COPY --from=builder /vt/bin/vtctlclient /vt/bin/ @@ -54,14 +54,15 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTROOT /vt -ENV GOTOP $VTROOT/go +ENV VTROOT /vt/src/vitess.io/vitess +ENV GOTOP $VTTOP/go ENV VTDATAROOT $VTROOT/vtdataroot ENV GOBIN $VTROOT/bin ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib +ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.alpine b/docker/lite/Dockerfile.alpine index cde81ae854c..d3f1a915545 100644 --- a/docker/lite/Dockerfile.alpine +++ b/docker/lite/Dockerfile.alpine @@ -2,11 +2,11 @@ FROM vitess/base AS builder FROM alpine:3.8 AS staging -RUN mkdir -p /vt/vtdataroot/ && mkdir -p /vt/bin && mkdir -p /vt/web/vtctld2 +RUN mkdir -p /vt/vtdataroot/ && mkdir -p /vt/bin && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 -COPY --from=builder /vt/web/vtctld /vt/web/vtctld -COPY --from=builder /vt/web/vtctld2/app /vt/web/vtctld2/app -COPY --from=builder /vt/config /vt/config +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app +COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config COPY --from=builder /vt/bin/mysqlctld /vt/bin/ COPY --from=builder /vt/bin/vtctld /vt/bin/ COPY --from=builder /vt/bin/vtctlclient /vt/bin/ @@ -22,7 +22,7 @@ RUN echo '@edge http://nl.alpinelinux.org/alpine/edge/main' >> /etc/apk/reposito apk add --no-cache mariadb@edge mariadb-client@edge bzip2 bash # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTROOT /vt +ENV VTROOT /vt/src/vitess.io/vitess ENV GOTOP $VTROOT/go ENV VTDATAROOT $VTROOT/vtdataroot ENV GOBIN $VTROOT/bin @@ -30,6 +30,7 @@ ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib +ENV MYSQL_FLAVOR MariaDB103 # Create vitess user RUN addgroup -S vitess && adduser -S -G vitess vitess && mkdir -p /vt diff --git a/docker/lite/Dockerfile.mariadb b/docker/lite/Dockerfile.mariadb index 9622a2cf158..8415b460a66 100644 --- a/docker/lite/Dockerfile.mariadb +++ b/docker/lite/Dockerfile.mariadb @@ -3,12 +3,12 @@ FROM debian:stretch-slim AS staging RUN mkdir -p /vt/vtdataroot/ \ && mkdir -p /vt/bin \ - && mkdir -p /vt/web/vtctld2 \ + && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ && groupadd -r vitess && useradd -r -g vitess vitess -COPY --from=builder /vt/web/vtctld /vt/web/vtctld -COPY --from=builder /vt/web/vtctld2/app /vt/web/vtctld2/app -COPY --from=builder /vt/config /vt/config +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app +COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config COPY --from=builder /vt/bin/mysqlctld /vt/bin/ COPY --from=builder /vt/bin/vtctld /vt/bin/ COPY --from=builder /vt/bin/vtctlclient /vt/bin/ @@ -33,7 +33,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTROOT /vt +ENV VTROOT /vt/src/vitess.io/vitess ENV GOTOP $VTROOT/go ENV VTDATAROOT $VTROOT/vtdataroot ENV GOBIN $VTROOT/bin @@ -41,6 +41,7 @@ ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib +ENV MYSQL_FLAVOR MariaDB # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.mariadb103 b/docker/lite/Dockerfile.mariadb103 index 1c5f02ae927..a474d2495fc 100644 --- a/docker/lite/Dockerfile.mariadb103 +++ b/docker/lite/Dockerfile.mariadb103 @@ -3,12 +3,12 @@ FROM debian:stretch-slim AS staging RUN mkdir -p /vt/vtdataroot/ \ && mkdir -p /vt/bin \ - && mkdir -p /vt/web/vtctld2 \ + && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ && groupadd -r vitess && useradd -r -g vitess vitess -COPY --from=builder /vt/web/vtctld /vt/web/vtctld -COPY --from=builder /vt/web/vtctld2/app /vt/web/vtctld2/app -COPY --from=builder /vt/config /vt/config +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app +COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config COPY --from=builder /vt/bin/mysqlctld /vt/bin/ COPY --from=builder /vt/bin/vtctld /vt/bin/ COPY --from=builder /vt/bin/vtctlclient /vt/bin/ @@ -32,14 +32,15 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTROOT /vt -ENV GOTOP $VTROOT/go +ENV VTROOT /vt/src/vitess.io/vitess +ENV GOTOP $VROOT/go ENV VTDATAROOT $VTROOT/vtdataroot ENV GOBIN $VTROOT/bin ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib +ENV MYSQL_FLAVOR MariaDB103 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.mysql56 b/docker/lite/Dockerfile.mysql56 index 8275a1f407a..28e8ccac4d3 100644 --- a/docker/lite/Dockerfile.mysql56 +++ b/docker/lite/Dockerfile.mysql56 @@ -3,12 +3,12 @@ FROM debian:stretch-slim AS staging RUN mkdir -p /vt/vtdataroot/ \ && mkdir -p /vt/bin \ - && mkdir -p /vt/web/vtctld2 \ + && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ && groupadd -r vitess && useradd -r -g vitess vitess -COPY --from=builder /vt/web/vtctld /vt/web/vtctld -COPY --from=builder /vt/web/vtctld2/app /vt/web/vtctld2/app -COPY --from=builder /vt/config /vt/config +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app +COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config COPY --from=builder /vt/bin/mysqlctld /vt/bin/ COPY --from=builder /vt/bin/vtctld /vt/bin/ COPY --from=builder /vt/bin/vtctlclient /vt/bin/ @@ -36,7 +36,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTROOT /vt +ENV VTROOT /vt/src/vitess.io/vitess ENV GOTOP $VTROOT/go ENV VTDATAROOT $VTROOT/vtdataroot ENV GOBIN $VTROOT/bin diff --git a/docker/lite/Dockerfile.mysql57 b/docker/lite/Dockerfile.mysql57 index 8ce9582b059..89211c2011b 100644 --- a/docker/lite/Dockerfile.mysql57 +++ b/docker/lite/Dockerfile.mysql57 @@ -3,12 +3,12 @@ FROM debian:stretch-slim AS staging RUN mkdir -p /vt/vtdataroot/ \ && mkdir -p /vt/bin \ - && mkdir -p /vt/web/vtctld2 \ + && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ && groupadd -r vitess && useradd -r -g vitess vitess -COPY --from=builder /vt/web/vtctld /vt/web/vtctld -COPY --from=builder /vt/web/vtctld2/app /vt/web/vtctld2/app -COPY --from=builder /vt/config /vt/config +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app +COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config COPY --from=builder /vt/bin/mysqlctld /vt/bin/ COPY --from=builder /vt/bin/vtctld /vt/bin/ COPY --from=builder /vt/bin/vtctlclient /vt/bin/ @@ -36,7 +36,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTROOT /vt +ENV VTROOT /vt/src/vitess.io/vitess ENV GOTOP $VTROOT/go ENV VTDATAROOT $VTROOT/vtdataroot ENV GOBIN $VTROOT/bin @@ -44,6 +44,7 @@ ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib +ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.mysql80 b/docker/lite/Dockerfile.mysql80 index 775e9880a59..0edb3b4aca6 100644 --- a/docker/lite/Dockerfile.mysql80 +++ b/docker/lite/Dockerfile.mysql80 @@ -3,12 +3,12 @@ FROM debian:stretch-slim AS staging RUN mkdir -p /vt/vtdataroot/ \ && mkdir -p /vt/bin \ - && mkdir -p /vt/web/vtctld2 \ + && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ && groupadd -r vitess && useradd -r -g vitess vitess -COPY --from=builder /vt/web/vtctld /vt/web/vtctld -COPY --from=builder /vt/web/vtctld2/app /vt/web/vtctld2/app -COPY --from=builder /vt/config /vt/config +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app +COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config COPY --from=builder /vt/bin/mysqlctld /vt/bin/ COPY --from=builder /vt/bin/vtctld /vt/bin/ COPY --from=builder /vt/bin/vtctlclient /vt/bin/ @@ -36,7 +36,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTROOT /vt +ENV VTROOT /vt/src/vitess.io/vitess ENV GOTOP $VTROOT/go ENV VTDATAROOT $VTROOT/vtdataroot ENV GOBIN $VTROOT/bin @@ -44,6 +44,7 @@ ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib +ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.percona b/docker/lite/Dockerfile.percona index c0406b8e0bf..5f0eb6438a1 100644 --- a/docker/lite/Dockerfile.percona +++ b/docker/lite/Dockerfile.percona @@ -3,12 +3,12 @@ FROM debian:stretch-slim AS staging RUN mkdir -p /vt/vtdataroot/ \ && mkdir -p /vt/bin \ - && mkdir -p /vt/web/vtctld2 \ + && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ && groupadd -r vitess && useradd -r -g vitess vitess -COPY --from=builder /vt/web/vtctld /vt/web/vtctld -COPY --from=builder /vt/web/vtctld2/app /vt/web/vtctld2/app -COPY --from=builder /vt/config /vt/config +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app +COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config COPY --from=builder /vt/bin/mysqlctld /vt/bin/ COPY --from=builder /vt/bin/vtctld /vt/bin/ COPY --from=builder /vt/bin/vtctlclient /vt/bin/ @@ -38,7 +38,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTROOT /vt +ENV VTROOT /vt/src/vitess.io/vitess ENV GOTOP $VTROOT/go ENV VTDATAROOT $VTROOT/vtdataroot ENV GOBIN $VTROOT/bin @@ -46,6 +46,7 @@ ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib +ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.percona57 b/docker/lite/Dockerfile.percona57 index 50d35d7b316..94e6161cf1e 100644 --- a/docker/lite/Dockerfile.percona57 +++ b/docker/lite/Dockerfile.percona57 @@ -3,12 +3,12 @@ FROM debian:stretch-slim AS staging RUN mkdir -p /vt/vtdataroot/ \ && mkdir -p /vt/bin \ - && mkdir -p /vt/web/vtctld2 \ + && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ && groupadd -r vitess && useradd -r -g vitess vitess -COPY --from=builder /vt/web/vtctld /vt/web/vtctld -COPY --from=builder /vt/web/vtctld2/app /vt/web/vtctld2/app -COPY --from=builder /vt/config /vt/config +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app +COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config COPY --from=builder /vt/bin/mysqlctld /vt/bin/ COPY --from=builder /vt/bin/vtctld /vt/bin/ COPY --from=builder /vt/bin/vtctlclient /vt/bin/ @@ -39,7 +39,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTROOT /vt +ENV VTROOT /vt/src/vitess.io/vitess ENV GOTOP $VTROOT/go ENV VTDATAROOT $VTROOT/vtdataroot ENV GOBIN $VTROOT/bin @@ -47,6 +47,7 @@ ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib +ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/lite/Dockerfile.percona80 b/docker/lite/Dockerfile.percona80 index cc68f9ca08e..84042202523 100644 --- a/docker/lite/Dockerfile.percona80 +++ b/docker/lite/Dockerfile.percona80 @@ -3,12 +3,12 @@ FROM debian:stretch-slim AS staging RUN mkdir -p /vt/vtdataroot/ \ && mkdir -p /vt/bin \ - && mkdir -p /vt/web/vtctld2 \ + && mkdir -p /vt/src/vitess.io/vitess/web/vtctld2 \ && groupadd -r vitess && useradd -r -g vitess vitess -COPY --from=builder /vt/web/vtctld /vt/web/vtctld -COPY --from=builder /vt/web/vtctld2/app /vt/web/vtctld2/app -COPY --from=builder /vt/config /vt/config +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld /vt/src/vitess.io/vitess/web/vtctld +COPY --from=builder /vt/src/vitess.io/vitess/web/vtctld2/app /vt/src/vitess.io/vitess/web/vtctld2/app +COPY --from=builder /vt/src/vitess.io/vitess/config /vt/config COPY --from=builder /vt/bin/mysqlctld /vt/bin/ COPY --from=builder /vt/bin/vtctld /vt/bin/ COPY --from=builder /vt/bin/vtctlclient /vt/bin/ @@ -41,7 +41,7 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && groupadd -r vitess && useradd -r -g vitess vitess # Set up Vitess environment (just enough to run pre-built Go binaries) -ENV VTROOT /vt +ENV VTROOT /vt/src/vitess.io/vitess ENV GOTOP $VTROOT/go ENV VTDATAROOT $VTROOT/vtdataroot ENV GOBIN $VTROOT/bin @@ -49,6 +49,7 @@ ENV GOPATH $VTROOT ENV PATH $VTROOT/bin:$PATH ENV VT_MYSQL_ROOT /usr ENV PKG_CONFIG_PATH $VTROOT/lib +ENV MYSQL_FLAVOR MySQL56 # Copy binaries (placed by build.sh) COPY --from=staging /vt/ /vt/ diff --git a/docker/packaging/package_vitess.sh b/docker/packaging/package_vitess.sh index bbc55603267..609fd1447d5 100755 --- a/docker/packaging/package_vitess.sh +++ b/docker/packaging/package_vitess.sh @@ -33,10 +33,10 @@ cat <> "${inputs_file}" /vt/bin/vtgate=${PREFIX}/bin/vtgate /vt/bin/vttablet=${PREFIX}/bin/vttablet /vt/bin/vtworker=${PREFIX}/bin/vtworker -/vt/config/=/etc/vitess -/vt/web/vtctld2/app=${PREFIX}/lib/vitess/web/vtcld2 -/vt/web/vtctld=${PREFIX}/lib/vitess/web -/vt/examples/local/=${PREFIX}/share/vitess/examples +/vt/src/vitess.io/vitess/config/=/etc/vitess +/vt/src/vitess.io/vitess/web/vtctld2/app=${PREFIX}/lib/vitess/web/vtcld2 +/vt/src/vitess.io/vitess/web/vtctld=${PREFIX}/lib/vitess/web +/vt/src/vitess.io/vitess/examples/local/=${PREFIX}/share/vitess/examples EOF description='A database clustering system for horizontal scaling of MySQL diff --git a/docker/test/run.sh b/docker/test/run.sh index b1080be2ba0..147663d028b 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -111,7 +111,7 @@ if [[ -z "$cmd" ]]; then fi if [[ ! -f bootstrap.sh ]]; then - echo "This script should be run from the root of the Vitess source tree" + echo "This script should be run from the root of the Vitess source tree - e.g. ~/src/vitess.io/vitess" exit 1 fi diff --git a/go.mod b/go.mod index ae3b4d9aee4..e3081b34798 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/golang/mock v1.3.1 github.com/golang/protobuf v1.3.2 github.com/golang/snappy v0.0.0-20170215233205-553a64147049 + github.com/google/btree v1.0.0 // indirect github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf // indirect github.com/gorilla/websocket v0.0.0-20160912153041-2d1e4548da23 github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 @@ -48,6 +49,8 @@ require ( github.com/minio/minio-go v0.0.0-20190131015406-c8a261de75c1 github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/mitchellh/mapstructure v1.1.2 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/olekukonko/tablewriter v0.0.0-20160115111002-cca8bbc07984 github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02 github.com/opentracing/opentracing-go v1.1.0 diff --git a/vagrant-scripts/bootstrap_vm.sh b/vagrant-scripts/bootstrap_vm.sh index 1c152ea3184..bea07394dea 100755 --- a/vagrant-scripts/bootstrap_vm.sh +++ b/vagrant-scripts/bootstrap_vm.sh @@ -61,7 +61,7 @@ printf "\nSetting /etc/environment\n" echo "GOROOT=${GOROOT}" echo "GOPATH=${GOPATH}" echo "PATH=${PATH}:${GOROOT}/bin:${GOPATH}/bin" - echo "VITESS_WORKSPACE=/vagrant/vt" + echo "VITESS_WORKSPACE=/vagrant/src/vitess.io/vitess" } >> /etc/environment # shellcheck disable=SC2013 # shellcheck disable=SC2163 @@ -72,7 +72,7 @@ echo "fs.file-max = 10000" >> /etc/sysctl.conf sysctl -p # Set vitess env in .bashrc -cat /vagrant/vt/vagrant-scripts/vagrant-bashrc >> /home/vagrant/.bashrc +cat /vagrant/src/vitess.io/vitess/vagrant-scripts/vagrant-bashrc >> /home/vagrant/.bashrc # Provisioning completed touch $SEED_FILE diff --git a/vagrant-scripts/vagrant-bashrc b/vagrant-scripts/vagrant-bashrc index 08654d625f2..27b1828450c 100644 --- a/vagrant-scripts/vagrant-bashrc +++ b/vagrant-scripts/vagrant-bashrc @@ -9,6 +9,7 @@ then fi ulimit -n 10000 +export MYSQL_FLAVOR=MySQL56 export VT_MYSQL_ROOT=/usr # This is just to make sure the vm can write into these directories sudo chown "$(whoami)":"$(whoami)" /vagrant From 1ffe42327ae26e0543634761898d6e702483a8f3 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Tue, 5 Nov 2019 18:35:24 -0700 Subject: [PATCH 03/25] Test CI Signed-off-by: Morgan Tocker --- bootstrap.sh | 19 ++++++++++--------- docker/test/run.sh | 11 +++++++++-- 2 files changed, 19 insertions(+), 11 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index 22a23fa28ff..b7efb5d752c 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -37,15 +37,16 @@ function fail() { # Create main directories. -VTROOT="${VTROOT:-${PWD/\/src\/vitess.io\/vitess/}}" -mkdir -p "dist" -mkdir -p "bin" -mkdir -p "lib" -mkdir -p "vthook" - -# temporary -rm -rf lib/* -rm -rf vthook/* +VTROOT="$PWD" +LEGACYVTROOT="${VTROOT/\/src\/vitess.io\/vitess/}" + +# This might be the testsuite running from an older bootstrap. +# To support an upgrade, move the older directories to the new location + +[ -d "$LEGACYVTROOT/dist" ] && mv "$LEGACYVTROOT/dist" . || mkdir -p dist +[ -d "$LEGACYVTROOT/bin" ] && mv "$LEGACYVTROOT/bin" . || mkdir -p bin +[ -d "$LEGACYVTROOT/lib" ] && mv "$LEGACYVTROOT/lib" . || mkdir -p lib +[ -d "$LEGACYVTROOT/vthook" ] && mv "$LEGACYVTROOT/vthook" . || mkdir -p vthook # This is required for VIRTUALENV # Used by Python below diff --git a/docker/test/run.sh b/docker/test/run.sh index 147663d028b..c20e3de68b1 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -179,11 +179,18 @@ copy_src_cmd=$(append_cmd "$copy_src_cmd" "cp -R /tmp/src/.git .") # Enable gomodules run_bootstrap_cmd="export GO111MODULE=on" # Copy bootstrap.sh if it changed -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ \$(diff -w bootstrap.sh /tmp/src/bootstrap.sh) ]]; then cp -f /tmp/src/bootstrap.sh .; bootstrap=1; fi") +# run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ \$(diff -w bootstrap.sh /tmp/src/bootstrap.sh) ]]; then cp -f /tmp/src/bootstrap.sh .; bootstrap=1; fi") # run bootstrap.sh if necessary -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ -n \$bootstrap ]]; then ./bootstrap.sh; fi") +# run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ -n \$bootstrap ]]; then ./bootstrap.sh; fi") +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "cp /tmp/src/bootstrap.sh .;./bootstrap.sh") copy_src_cmd=$(append_cmd "$copy_src_cmd" "$run_bootstrap_cmd") +# Create symlinks to support legacy VTROOT/VTTOP structure +#copy_src_cmd=$(append_cmd "$copy_src_cmd" "ln -s /vt/dist /vt/src/vitess.io/vitess/dist") +#copy_src_cmd=$(append_cmd "$copy_src_cmd" "ln -s /vt/bin /vt/src/vitess.io/vitess/bin") +#copy_src_cmd=$(append_cmd "$copy_src_cmd" "ln -s /vt/lib /vt/src/vitess.io/vitess/lib") +#copy_src_cmd=$(append_cmd "$copy_src_cmd" "ln -s /vt/vthook /vt/src/vitess.io/vitess/vthook") + # Construct the command we will actually run. # # Uncomment the next line if you need to debug "bashcmd". From 96c3ecea841cc2dfb6cb8601617766d24b08836a Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Tue, 5 Nov 2019 19:09:17 -0700 Subject: [PATCH 04/25] Test ci with re-sourcing dev.env file Signed-off-by: Morgan Tocker --- bootstrap.sh | 11 ++++------- docker/test/run.sh | 14 ++++++++------ 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index b7efb5d752c..4f630610101 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -40,13 +40,10 @@ function fail() { VTROOT="$PWD" LEGACYVTROOT="${VTROOT/\/src\/vitess.io\/vitess/}" -# This might be the testsuite running from an older bootstrap. -# To support an upgrade, move the older directories to the new location - -[ -d "$LEGACYVTROOT/dist" ] && mv "$LEGACYVTROOT/dist" . || mkdir -p dist -[ -d "$LEGACYVTROOT/bin" ] && mv "$LEGACYVTROOT/bin" . || mkdir -p bin -[ -d "$LEGACYVTROOT/lib" ] && mv "$LEGACYVTROOT/lib" . || mkdir -p lib -[ -d "$LEGACYVTROOT/vthook" ] && mv "$LEGACYVTROOT/vthook" . || mkdir -p vthook +mkdir -p dist +mkdir -p bin +mkdir -p lib +mkdir -p vthook # This is required for VIRTUALENV # Used by Python below diff --git a/docker/test/run.sh b/docker/test/run.sh index c20e3de68b1..f3621c01cf7 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -182,14 +182,16 @@ run_bootstrap_cmd="export GO111MODULE=on" # run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ \$(diff -w bootstrap.sh /tmp/src/bootstrap.sh) ]]; then cp -f /tmp/src/bootstrap.sh .; bootstrap=1; fi") # run bootstrap.sh if necessary # run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ -n \$bootstrap ]]; then ./bootstrap.sh; fi") + +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/dist ] && mv /vt/dist .") +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/bin ] && mv /vt/bin .") +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/lib ] && mv /vt/lib .") +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/vthook ] && mv /vt/vthook .") + run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "cp /tmp/src/bootstrap.sh .;./bootstrap.sh") -copy_src_cmd=$(append_cmd "$copy_src_cmd" "$run_bootstrap_cmd") +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "source ./dev.env") -# Create symlinks to support legacy VTROOT/VTTOP structure -#copy_src_cmd=$(append_cmd "$copy_src_cmd" "ln -s /vt/dist /vt/src/vitess.io/vitess/dist") -#copy_src_cmd=$(append_cmd "$copy_src_cmd" "ln -s /vt/bin /vt/src/vitess.io/vitess/bin") -#copy_src_cmd=$(append_cmd "$copy_src_cmd" "ln -s /vt/lib /vt/src/vitess.io/vitess/lib") -#copy_src_cmd=$(append_cmd "$copy_src_cmd" "ln -s /vt/vthook /vt/src/vitess.io/vitess/vthook") +copy_src_cmd=$(append_cmd "$copy_src_cmd" "$run_bootstrap_cmd") # Construct the command we will actually run. # From d4a2e83d3dd0bb38a4050d5278c14b3644956b63 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Tue, 5 Nov 2019 20:05:42 -0700 Subject: [PATCH 05/25] test with destroy local and then link Signed-off-by: Morgan Tocker --- docker/test/run.sh | 28 ++++++++++------------------ 1 file changed, 10 insertions(+), 18 deletions(-) diff --git a/docker/test/run.sh b/docker/test/run.sh index f3621c01cf7..acf68b73128 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -160,18 +160,8 @@ case "$mode" in "create_cache") echo "Creating cache image $cache_image ..." ;; esac -# Construct "cp" command to copy the source code. -# -# Copy the full source tree except: -# - vendor -# That's because these directories are already part of the image. -# -# Note that we're using the Bash extended Glob support "!(vendor)" on -# purpose here to minimize the size of the cache image: With this trick, -# we do not move or overwrite the existing files while copying the other -# directories. Therefore, the existing files do not count as changed and will -# not be part of the new Docker layer of the cache image. -copy_src_cmd="cp -R /tmp/src/!(vendor|bootstrap.sh) ." +# "cp" command to copy the source code. +copy_src_cmd="cp -R /tmp/src/* ." # Copy the .git directory because travis/check_make_proto.sh needs a working # Git repository. copy_src_cmd=$(append_cmd "$copy_src_cmd" "cp -R /tmp/src/.git .") @@ -183,12 +173,14 @@ run_bootstrap_cmd="export GO111MODULE=on" # run bootstrap.sh if necessary # run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ -n \$bootstrap ]]; then ./bootstrap.sh; fi") -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/dist ] && mv /vt/dist .") -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/bin ] && mv /vt/bin .") -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/lib ] && mv /vt/lib .") -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/vthook ] && mv /vt/vthook .") +# If an original location dir exists, move everything from it to here. + +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/dist ] && rm -rf dist && ln -s /vt/dist /vt/src/vitess.io/vitess/dist") +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/bin ] && rm -rf bin && ln -s /vt/bin /vt/src/vitess.io/vitess/bin") +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/lib ] && rm -rf lib && ln -s /vt/lib /vt/src/vitess.io/vitess/lib") +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/vthook ] && rm -rf vthook && ln -s /vt/vthook /vt/src/vitess.io/vitess/vthook") -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "cp /tmp/src/bootstrap.sh .;./bootstrap.sh") +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "./bootstrap.sh") run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "source ./dev.env") copy_src_cmd=$(append_cmd "$copy_src_cmd" "$run_bootstrap_cmd") @@ -196,7 +188,7 @@ copy_src_cmd=$(append_cmd "$copy_src_cmd" "$run_bootstrap_cmd") # Construct the command we will actually run. # # Uncomment the next line if you need to debug "bashcmd". -#bashcmd="set -x" +bashcmd="set -x" if [[ -z "$existing_cache_image" ]]; then bashcmd=$(append_cmd "$bashcmd" "$copy_src_cmd") fi From eb5b8da833be77ded219ae0aa92ce263e9bde721 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Tue, 5 Nov 2019 20:36:43 -0700 Subject: [PATCH 06/25] Attempt to fix vtroot Signed-off-by: Morgan Tocker --- .travis.yml | 2 +- docker/test/run.sh | 31 ++++++++++++++++++++----------- test.go | 1 + test/config.json | 20 ++++++++++---------- 4 files changed, 32 insertions(+), 22 deletions(-) diff --git a/.travis.yml b/.travis.yml index 733f8213a21..6233e64621c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,7 @@ env: # See: https://docs.travis-ci.com/user/customizing-the-build#Build-Timeouts # To diagnose stuck tests, add "-follow" to TEST_FLAGS below. Then test.go # will print the test's output. - - TEST_FLAGS="-docker -use_docker_cache -timeout=8m -print-log" + - TEST_FLAGS="-docker -use_docker_cache=false -timeout=12m -print-log" matrix: # NOTE: Travis CI schedules up to 5 tests simultaneously. # All our tests should be spread out as evenly as possible across these 5 slots. diff --git a/docker/test/run.sh b/docker/test/run.sh index acf68b73128..da4c8d93ab5 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -160,35 +160,44 @@ case "$mode" in "create_cache") echo "Creating cache image $cache_image ..." ;; esac -# "cp" command to copy the source code. -copy_src_cmd="cp -R /tmp/src/* ." +# Construct "cp" command to copy the source code. +# +# Copy the full source tree except: +# - vendor +# That's because these directories are already part of the image. +# +# Note that we're using the Bash extended Glob support "!(vendor)" on +# purpose here to minimize the size of the cache image: With this trick, +# we do not move or overwrite the existing files while copying the other +# directories. Therefore, the existing files do not count as changed and will +# not be part of the new Docker layer of the cache image. +copy_src_cmd="cp -R /tmp/src/!(vendor|bootstrap.sh) ." # Copy the .git directory because travis/check_make_proto.sh needs a working # Git repository. copy_src_cmd=$(append_cmd "$copy_src_cmd" "cp -R /tmp/src/.git .") # Enable gomodules run_bootstrap_cmd="export GO111MODULE=on" -# Copy bootstrap.sh if it changed -# run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ \$(diff -w bootstrap.sh /tmp/src/bootstrap.sh) ]]; then cp -f /tmp/src/bootstrap.sh .; bootstrap=1; fi") -# run bootstrap.sh if necessary -# run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ -n \$bootstrap ]]; then ./bootstrap.sh; fi") - -# If an original location dir exists, move everything from it to here. run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/dist ] && rm -rf dist && ln -s /vt/dist /vt/src/vitess.io/vitess/dist") run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/bin ] && rm -rf bin && ln -s /vt/bin /vt/src/vitess.io/vitess/bin") run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/lib ] && rm -rf lib && ln -s /vt/lib /vt/src/vitess.io/vitess/lib") run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/vthook ] && rm -rf vthook && ln -s /vt/vthook /vt/src/vitess.io/vitess/vthook") -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "./bootstrap.sh") -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "source ./dev.env") +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "env") +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "export VTROOT=/vt/src/vitess.io/vitess && unset VTTOP") +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "env") +# Copy bootstrap.sh if it changed +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ \$(diff -w bootstrap.sh /tmp/src/bootstrap.sh) ]]; then cp -f /tmp/src/bootstrap.sh .; bootstrap=1; fi") +# run bootstrap.sh if necessary +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ -n \$bootstrap ]]; then ./bootstrap.sh; fi") copy_src_cmd=$(append_cmd "$copy_src_cmd" "$run_bootstrap_cmd") # Construct the command we will actually run. # # Uncomment the next line if you need to debug "bashcmd". -bashcmd="set -x" +#bashcmd="set -x" if [[ -z "$existing_cache_image" ]]; then bashcmd=$(append_cmd "$bashcmd" "$copy_src_cmd") fi diff --git a/test.go b/test.go index 3266001c0cb..846cb0c8755 100755 --- a/test.go +++ b/test.go @@ -195,6 +195,7 @@ func (t *Test) run(dir, dataDir string) ([]byte, error) { // Also try to make them use different port ranges // to mitigate failures due to zombie processes. cmd.Env = updateEnv(os.Environ(), map[string]string{ + "VTROOT": "/vt/src/vitess.io/vitess", "VTDATAROOT": dataDir, "VTPORTSTART": strconv.FormatInt(int64(getPortStart(100)), 10), }) diff --git a/test/config.json b/test/config.json index c303b4a6398..9521369c7a9 100644 --- a/test/config.json +++ b/test/config.json @@ -50,7 +50,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 0, "RetryMax": 0, "Tags": [ "worker_test" @@ -61,7 +61,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 0, "RetryMax": 0, "Tags": [] }, @@ -70,7 +70,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 0, "RetryMax": 0, "Tags": [ "site_test" @@ -81,7 +81,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 0, "RetryMax": 0, "Tags": [ "site_test" @@ -94,7 +94,7 @@ "tools/check_make_parser.sh" ], "Manual": false, - "Shard": 4, + "Shard": 1, "RetryMax": 1, "Tags": [] }, @@ -103,7 +103,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 1, "RetryMax": 0, "Tags": [ "worker_test" @@ -114,7 +114,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 0, "RetryMax": 0, "Tags": [] }, @@ -123,7 +123,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 1, "RetryMax": 0, "Tags": [] }, @@ -166,7 +166,7 @@ "java_test" ], "Manual": false, - "Shard": 4, + "Shard": 1, "RetryMax": 0, "Tags": [] }, @@ -188,7 +188,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 3, "RetryMax": 0, "Tags": [ "site_test" From 79ad6c1810960ce6860f336ea5589b7a4d4567ec Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Tue, 5 Nov 2019 20:36:43 -0700 Subject: [PATCH 07/25] Attempt to fix vtroot Signed-off-by: Morgan Tocker --- .github/workflows/local_example.yml | 34 +++++++++++++++++++++++++++++ .travis.yml | 2 +- docker/test/run.sh | 31 ++++++++++++++++---------- test.go | 1 + test/config.json | 20 ++++++++--------- 5 files changed, 66 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/local_example.yml diff --git a/.github/workflows/local_example.yml b/.github/workflows/local_example.yml new file mode 100644 index 00000000000..c3dc1151433 --- /dev/null +++ b/.github/workflows/local_example.yml @@ -0,0 +1,34 @@ +name: Go +on: [push] +jobs: + + build: + name: Build + runs-on: ubuntu-latest + steps: + + - name: Set up Go 1.13 + uses: actions/setup-go@v1 + with: + go-version: 1.13 + id: local-example + + - name: Check out code into the Go module directory + uses: actions/checkout@v1 + + - name: Get dependencies + run: | + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget + sudo service mysql stop + sudo service etcd stop + sudo systemctl disable mysql + sudo systemctl disable etcd + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + - name: Build + run: make build + + - name: Run Local Example + run: | + VTROOT=$PWD test/local_example.sh + diff --git a/.travis.yml b/.travis.yml index 733f8213a21..6233e64621c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,7 @@ env: # See: https://docs.travis-ci.com/user/customizing-the-build#Build-Timeouts # To diagnose stuck tests, add "-follow" to TEST_FLAGS below. Then test.go # will print the test's output. - - TEST_FLAGS="-docker -use_docker_cache -timeout=8m -print-log" + - TEST_FLAGS="-docker -use_docker_cache=false -timeout=12m -print-log" matrix: # NOTE: Travis CI schedules up to 5 tests simultaneously. # All our tests should be spread out as evenly as possible across these 5 slots. diff --git a/docker/test/run.sh b/docker/test/run.sh index acf68b73128..da4c8d93ab5 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -160,35 +160,44 @@ case "$mode" in "create_cache") echo "Creating cache image $cache_image ..." ;; esac -# "cp" command to copy the source code. -copy_src_cmd="cp -R /tmp/src/* ." +# Construct "cp" command to copy the source code. +# +# Copy the full source tree except: +# - vendor +# That's because these directories are already part of the image. +# +# Note that we're using the Bash extended Glob support "!(vendor)" on +# purpose here to minimize the size of the cache image: With this trick, +# we do not move or overwrite the existing files while copying the other +# directories. Therefore, the existing files do not count as changed and will +# not be part of the new Docker layer of the cache image. +copy_src_cmd="cp -R /tmp/src/!(vendor|bootstrap.sh) ." # Copy the .git directory because travis/check_make_proto.sh needs a working # Git repository. copy_src_cmd=$(append_cmd "$copy_src_cmd" "cp -R /tmp/src/.git .") # Enable gomodules run_bootstrap_cmd="export GO111MODULE=on" -# Copy bootstrap.sh if it changed -# run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ \$(diff -w bootstrap.sh /tmp/src/bootstrap.sh) ]]; then cp -f /tmp/src/bootstrap.sh .; bootstrap=1; fi") -# run bootstrap.sh if necessary -# run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ -n \$bootstrap ]]; then ./bootstrap.sh; fi") - -# If an original location dir exists, move everything from it to here. run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/dist ] && rm -rf dist && ln -s /vt/dist /vt/src/vitess.io/vitess/dist") run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/bin ] && rm -rf bin && ln -s /vt/bin /vt/src/vitess.io/vitess/bin") run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/lib ] && rm -rf lib && ln -s /vt/lib /vt/src/vitess.io/vitess/lib") run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/vthook ] && rm -rf vthook && ln -s /vt/vthook /vt/src/vitess.io/vitess/vthook") -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "./bootstrap.sh") -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "source ./dev.env") +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "env") +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "export VTROOT=/vt/src/vitess.io/vitess && unset VTTOP") +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "env") +# Copy bootstrap.sh if it changed +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ \$(diff -w bootstrap.sh /tmp/src/bootstrap.sh) ]]; then cp -f /tmp/src/bootstrap.sh .; bootstrap=1; fi") +# run bootstrap.sh if necessary +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ -n \$bootstrap ]]; then ./bootstrap.sh; fi") copy_src_cmd=$(append_cmd "$copy_src_cmd" "$run_bootstrap_cmd") # Construct the command we will actually run. # # Uncomment the next line if you need to debug "bashcmd". -bashcmd="set -x" +#bashcmd="set -x" if [[ -z "$existing_cache_image" ]]; then bashcmd=$(append_cmd "$bashcmd" "$copy_src_cmd") fi diff --git a/test.go b/test.go index 3266001c0cb..846cb0c8755 100755 --- a/test.go +++ b/test.go @@ -195,6 +195,7 @@ func (t *Test) run(dir, dataDir string) ([]byte, error) { // Also try to make them use different port ranges // to mitigate failures due to zombie processes. cmd.Env = updateEnv(os.Environ(), map[string]string{ + "VTROOT": "/vt/src/vitess.io/vitess", "VTDATAROOT": dataDir, "VTPORTSTART": strconv.FormatInt(int64(getPortStart(100)), 10), }) diff --git a/test/config.json b/test/config.json index c303b4a6398..9521369c7a9 100644 --- a/test/config.json +++ b/test/config.json @@ -50,7 +50,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 0, "RetryMax": 0, "Tags": [ "worker_test" @@ -61,7 +61,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 0, "RetryMax": 0, "Tags": [] }, @@ -70,7 +70,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 0, "RetryMax": 0, "Tags": [ "site_test" @@ -81,7 +81,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 0, "RetryMax": 0, "Tags": [ "site_test" @@ -94,7 +94,7 @@ "tools/check_make_parser.sh" ], "Manual": false, - "Shard": 4, + "Shard": 1, "RetryMax": 1, "Tags": [] }, @@ -103,7 +103,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 1, "RetryMax": 0, "Tags": [ "worker_test" @@ -114,7 +114,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 0, "RetryMax": 0, "Tags": [] }, @@ -123,7 +123,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 1, "RetryMax": 0, "Tags": [] }, @@ -166,7 +166,7 @@ "java_test" ], "Manual": false, - "Shard": 4, + "Shard": 1, "RetryMax": 0, "Tags": [] }, @@ -188,7 +188,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 3, "RetryMax": 0, "Tags": [ "site_test" From a897c1781626ec7352bdaa9d464a4a6ffed8c65a Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Sun, 1 Dec 2019 10:21:39 -0700 Subject: [PATCH 08/25] Add make tools Change parts of bootstrap to run in build.sh instead Always execute build.sh as part of make build. Signed-off-by: Morgan Tocker --- Makefile | 7 ++++++- bootstrap.sh | 32 ++------------------------------ build.env | 44 ++++++++++++++++++++++++++++---------------- 3 files changed, 36 insertions(+), 47 deletions(-) mode change 100644 => 100755 build.env diff --git a/Makefile b/Makefile index 24eab88c988..60db25e518e 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ MAKEFLAGS = -s # Since we are not using this Makefile for compilation, limiting parallelism will not increase build time. .NOTPARALLEL: -.PHONY: all build build_web test clean unit_test unit_test_cover unit_test_race integration_test proto proto_banner site_test site_integration_test docker_bootstrap docker_test docker_unit_test java_test reshard_tests e2e_test e2e_test_race +.PHONY: all build build_web test clean unit_test unit_test_cover unit_test_race integration_test proto proto_banner site_test site_integration_test docker_bootstrap docker_test docker_unit_test java_test reshard_tests e2e_test e2e_test_race tools all: build @@ -50,6 +50,7 @@ build: ifndef NOBANNER echo $$(date): Building source tree endif + bash ./build.env go install $(EXTRA_BUILD_FLAGS) $(VT_GO_PARALLEL) -ldflags "$(shell tools/build_version_flags.sh)" ./go/... parser: @@ -289,3 +290,7 @@ packages: docker_base docker build -f docker/packaging/Dockerfile -t vitess/packaging . docker run --rm -v ${PWD}/releases:/vt/releases --env VERSION=$(VERSION) vitess/packaging --package /vt/releases -t deb --deb-no-default-config-files docker run --rm -v ${PWD}/releases:/vt/releases --env VERSION=$(VERSION) vitess/packaging --package /vt/releases -t rpm + +tools: + echo $$(date): Installing dependencies + BUILD_PYTHON=0 BUILD_JAVA=0 ./bootstrap.sh \ No newline at end of file diff --git a/bootstrap.sh b/bootstrap.sh index c91b4ddb1d1..91b312ee926 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -15,6 +15,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +### This file is executed by 'make tools'. You do not need to execute it directly. # Outline of this file. # 0. Initialization and helper methods. @@ -38,37 +39,8 @@ function fail() { # Create main directories. VTROOT="$PWD" -LEGACYVTROOT="${VTROOT/\/src\/vitess.io\/vitess/}" - -mkdir -p dist -mkdir -p bin -mkdir -p lib -mkdir -p vthook - -# This is required for VIRTUALENV -# Used by Python below - source ./dev.env -go version &>/dev/null || fail "Go is not installed or is not on \$PATH" -goversion_min 1.12 || fail "Go is not version 1.12+" - -# Set up required soft links. -# TODO(mberlin): Which of these can be deleted? -ln -snf "$VTROOT/py" "$VTROOT/py-vtdb" -ln -snf "$VTROOT/go/vt/zkctl/zksrv.sh" "$VTROOT/bin/zksrv.sh" -ln -snf "$VTROOT/test/vthook-test.sh" "$VTROOT/vthook/test.sh" -ln -snf "$VTROOT/test/vthook-test_backup_error" "$VTROOT/vthook/test_backup_error" -ln -snf "$VTROOT/test/vthook-test_backup_transform" "$VTROOT/vthook/test_backup_transform" - -# git hooks are only required if someone intends to contribute. - -echo "creating git hooks" -mkdir -p "$VTROOT/.git/hooks" -ln -sf "$VTROOT/misc/git/pre-commit" "$VTROOT/.git/hooks/pre-commit" -ln -sf "$VTROOT/misc/git/commit-msg" "$VTROOT/.git/hooks/commit-msg" -(cd "$VTROOT" && git config core.hooksPath "$VTROOT/.git/hooks") - # install_dep is a helper function to generalize the download and installation of dependencies. # # If the installation is successful, it puts the installed version string into @@ -326,4 +298,4 @@ if [ "$BUILD_PYTHON" == 1 ] ; then fi echo -echo "bootstrap finished - run 'source dev.env' or 'source build.env' in your shell before building." +echo "bootstrap finished - run 'make build' to compile" diff --git a/build.env b/build.env old mode 100644 new mode 100755 index 7920a58288f..ec3eb989550 --- a/build.env +++ b/build.env @@ -14,19 +14,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Plese ensure dev.env is written in a way which is POSIX (bourne) -# shell compatible. -# - Some build systems like rpm require the different scriptlets used -# to build a package to be run under a POSIX shell so non-POSIX -# syntax will break that as dev.env will not be sourced by bash.. - -# Import prepend_path function. -dir="$(dirname "${BASH_SOURCE[0]}")" -# shellcheck source=tools/shell_functions.inc -if ! source "${dir}/tools/shell_functions.inc"; then - echo "failed to load tools/shell_functions.inc" - return 1 -fi - -export VTROOT=$PWD -export GO111MODULE=on +[[ "$(dirname "$0")" = "." ]] || fail "build.sh must be run from its current directory" + +source ./tools/shell_functions.inc + +mkdir -p dist +mkdir -p bin +mkdir -p lib +mkdir -p vthook + +go version &>/dev/null || fail "Go is not installed or is not on \$PATH" +goversion_min 1.12 || fail "Go is not version 1.12+" + +# Set up required soft links. +# TODO(mberlin): Which of these can be deleted? +ln -snf py py-vtdb +ln -snf go/vt/zkctl/zksrv.sh bin/zksrv.sh +ln -snf test/vthook-test.sh vthook/test.sh +ln -snf test/vthook-test_backup_error vthook/test_backup_error +ln -snf test/vthook-test_backup_transform vthook/test_backup_transform + +# install git hooks + +mkdir -p .git/hooks +ln -sf misc/git/pre-commit .git/hooks/pre-commit +ln -sf misc/git/commit-msg .git/hooks/commit-msg +git config core.hooksPath .git/hooks + +export VTROOT="$PWD" \ No newline at end of file From f90cc827a10f43c6967be7969b8b1706426d9d88 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Sun, 1 Dec 2019 12:10:30 -0700 Subject: [PATCH 09/25] Make 'make test' work Add dependency checking support. Signed-off-by: Morgan Tocker --- Makefile | 21 ++++++++++++--------- bootstrap.sh | 1 - build.env | 16 +++++++--------- tools/unit_test_runner.sh | 4 ++++ 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index 60db25e518e..a4caa4dfc5f 100644 --- a/Makefile +++ b/Makefile @@ -14,6 +14,9 @@ MAKEFLAGS = -s +export GOBIN=$(PWD)/bin +export GO111MODULE=on + # Disabled parallel processing of target prerequisites to avoid that integration tests are racing each other (e.g. for ports) and may fail. # Since we are not using this Makefile for compilation, limiting parallelism will not increase build time. .NOTPARALLEL: @@ -59,8 +62,9 @@ parser: # To pass extra flags, run test.go manually. # For example: go run test.go -docker=false -- --extra-flag # For more info see: go run test.go -help -test: - go run test.go -docker=false +test: build dependency_check + echo $$(date): Running unit tests + tools/unit_test_runner.sh site_test: unit_test site_integration_test @@ -83,11 +87,7 @@ cleanall: # - exclude vtdataroot and vthook as they may have data we want rm -rf ../../../../bin ../../../../dist ../../../../lib ../../../../pkg # Remind people to run bootstrap.sh again - echo "Please run bootstrap.sh again to setup your environment" - -unit_test: build - echo $$(date): Running unit tests - go test $(VT_GO_PARALLEL) ./go/... + echo "Please run 'make tools' again to setup your environment" e2e_test: build echo $$(date): Running endtoend tests @@ -99,7 +99,7 @@ e2e_test: build unit_test_cover: build go test $(VT_GO_PARALLEL) -cover ./go/... | misc/parse_cover.py -unit_test_race: build +unit_test_race: build dependency_check tools/unit_test_race.sh e2e_test_race: build @@ -293,4 +293,7 @@ packages: docker_base tools: echo $$(date): Installing dependencies - BUILD_PYTHON=0 BUILD_JAVA=0 ./bootstrap.sh \ No newline at end of file + BUILD_PYTHON=0 ./bootstrap.sh + +dependency_check: + ./tools/dependency_check.sh \ No newline at end of file diff --git a/bootstrap.sh b/bootstrap.sh index 91b312ee926..4b8b99e7e05 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -21,7 +21,6 @@ # 0. Initialization and helper methods. # 1. Installation of dependencies. -BUILD_TESTS=${BUILD_TESTS:-1} BUILD_PYTHON=${BUILD_PYTHON:-1} BUILD_JAVA=${BUILD_JAVA:-1} diff --git a/build.env b/build.env index ec3eb989550..700feb1a3cb 100755 --- a/build.env +++ b/build.env @@ -14,8 +14,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -[[ "$(dirname "$0")" = "." ]] || fail "build.sh must be run from its current directory" - source ./tools/shell_functions.inc mkdir -p dist @@ -28,17 +26,17 @@ goversion_min 1.12 || fail "Go is not version 1.12+" # Set up required soft links. # TODO(mberlin): Which of these can be deleted? -ln -snf py py-vtdb -ln -snf go/vt/zkctl/zksrv.sh bin/zksrv.sh -ln -snf test/vthook-test.sh vthook/test.sh -ln -snf test/vthook-test_backup_error vthook/test_backup_error -ln -snf test/vthook-test_backup_transform vthook/test_backup_transform +ln -snf "$PWD/py" py-vtdb +ln -snf "$PWD/go/vt/zkctl/zksrv.sh" bin/zksrv.sh +ln -snf "$PWD/test/vthook-test.sh" vthook/test.sh +ln -snf "$PWD/test/vthook-test_backup_error" vthook/test_backup_error +ln -snf "$PWD/test/vthook-test_backup_transform" vthook/test_backup_transform # install git hooks mkdir -p .git/hooks -ln -sf misc/git/pre-commit .git/hooks/pre-commit -ln -sf misc/git/commit-msg .git/hooks/commit-msg +ln -sf "$PWD/misc/git/pre-commit" .git/hooks/pre-commit +ln -sf "$PWD/misc/git/commit-msg" .git/hooks/commit-msg git config core.hooksPath .git/hooks export VTROOT="$PWD" \ No newline at end of file diff --git a/tools/unit_test_runner.sh b/tools/unit_test_runner.sh index 382dd2f0b76..cc20f52c77b 100755 --- a/tools/unit_test_runner.sh +++ b/tools/unit_test_runner.sh @@ -29,6 +29,10 @@ # Set VT_GO_PARALLEL variable in the same way as the Makefile does. # We repeat this here because this script is called directly by test.go # and not via the Makefile. + +source build.env +export VTDATAROOT=`mktemp -d` + if [[ -z $VT_GO_PARALLEL && -n $VT_GO_PARALLEL_VALUE ]]; then VT_GO_PARALLEL="-p $VT_GO_PARALLEL_VALUE" fi From 9f5861adca706420e93d66a1c094131f6a636f20 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Mon, 2 Dec 2019 11:42:56 -0700 Subject: [PATCH 10/25] Attempt unit test in new build style Signed-off-by: Morgan Tocker --- .github/bootstrap.sh | 175 ------------------------ .github/workflows/check_make_parser.yml | 9 +- .github/workflows/cluster_endtoend.yml | 14 +- .github/workflows/e2e_race.yml | 14 +- .github/workflows/endtoend.yml | 13 +- .github/workflows/local_example.yml | 11 +- .github/workflows/unit.yml | 13 +- .github/workflows/unit_race.yml | 12 +- .gitignore | 3 + .travis.yml | 2 +- Makefile | 12 +- bootstrap.sh | 5 +- build.env | 12 +- dev.env | 3 - examples/local/env.sh | 5 + examples/local/etcd-up.sh | 3 + test/client_test.sh | 2 + test/config.json | 26 ++-- test/local_example.sh | 2 + tools/check_make_parser.sh | 2 + tools/dependency_check.sh | 6 +- tools/e2e_test_cluster.sh | 3 + tools/e2e_test_race.sh | 5 +- tools/e2e_test_runner.sh | 3 + tools/unit_test_race.sh | 32 ++--- tools/unit_test_runner.sh | 1 - 26 files changed, 94 insertions(+), 294 deletions(-) delete mode 100755 .github/bootstrap.sh diff --git a/.github/bootstrap.sh b/.github/bootstrap.sh deleted file mode 100755 index f02bffd1f1b..00000000000 --- a/.github/bootstrap.sh +++ /dev/null @@ -1,175 +0,0 @@ -#!/bin/bash -# shellcheck disable=SC2164 - -# Copyright 2019 The Vitess Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This is a next-gen bootstrap which skips Python and Java tests, -# and does not use the VTROOT/VTTOP layout. -# -# My original intention was to use the same bootstrap.sh and gate -# for new features, but it has turned out to be difficult to do, -# due to the way that Docker cache works in the CI environment. - -function fail() { - echo "ERROR: $1" - exit 1 -} - -[[ "$(dirname "$0")" = "." ]] || fail "bootstrap.sh must be run from its current directory" - -# Create main directories. - -VTROOT="$PWD" - -mkdir -p dist -mkdir -p bin -mkdir -p lib -mkdir -p vthook - -source ./dev.env - -go version &>/dev/null || fail "Go is not installed or is not on \$PATH" -goversion_min 1.12 || fail "Go is not version 1.12+" - -# Set up required soft links. -# TODO(mberlin): Which of these can be deleted? -ln -snf "$VTROOT/py" "$VTROOT/py-vtdb" -ln -snf "$VTROOT/go/vt/zkctl/zksrv.sh" "$VTROOT/bin/zksrv.sh" -ln -snf "$VTROOT/test/vthook-test.sh" "$VTROOT/vthook/test.sh" -ln -snf "$VTROOT/test/vthook-test_backup_error" "$VTROOT/vthook/test_backup_error" -ln -snf "$VTROOT/test/vthook-test_backup_transform" "$VTROOT/vthook/test_backup_transform" - -# git hooks are only required if someone intends to contribute. - -echo "creating git hooks" -mkdir -p "$VTROOT/.git/hooks" -ln -sf "$VTROOT/misc/git/pre-commit" "$VTROOT/.git/hooks/pre-commit" -ln -sf "$VTROOT/misc/git/commit-msg" "$VTROOT/.git/hooks/commit-msg" -git config core.hooksPath "$VTROOT/.git/hooks" - -# install_dep is a helper function to generalize the download and installation of dependencies. -# -# If the installation is successful, it puts the installed version string into -# the $dist/.installed_version file. If the version has not changed, bootstrap -# will skip future installations. -function install_dep() { - if [[ $# != 4 ]]; then - fail "install_dep function requires exactly 4 parameters (and not $#). Parameters: $*" - fi - local name="$1" - local version="$2" - local dist="$3" - local install_func="$4" - - version_file="$dist/.installed_version" - if [[ -f "$version_file" && "$(cat "$version_file")" == "$version" ]]; then - echo "skipping $name install. remove $dist to force re-install." - return - fi - - echo "installing $name $version" - - # shellcheck disable=SC2064 - trap "fail '$name build failed'; exit 1" ERR - - # Cleanup any existing data and re-create the directory. - rm -rf "$dist" - mkdir -p "$dist" - - # Change $CWD to $dist before calling "install_func". - pushd "$dist" >/dev/null - # -E (same as "set -o errtrace") makes sure that "install_func" inherits the - # trap. If here's an error, the trap will be called which will exit this - # script. - set -E - $install_func "$version" "$dist" - set +E - popd >/dev/null - - trap - ERR - - echo "$version" > "$version_file" -} - - -# -# 1. Installation of dependencies. -# - -# Wrapper around the `arch` command which plays nice with OS X -function get_arch() { - case $(uname) in - Linux) arch;; - Darwin) uname -m;; - esac -} - -# Install protoc. -function install_protoc() { - local version="$1" - local dist="$2" - - case $(uname) in - Linux) local platform=linux;; - Darwin) local platform=osx;; - esac - - case $(get_arch) in - aarch64) local target=aarch_64;; - x86_64) local target=x86_64;; - *) echo "ERROR: unsupported architecture"; exit 1;; - esac - - wget https://github.com/protocolbuffers/protobuf/releases/download/v$version/protoc-$version-$platform-${target}.zip - unzip "protoc-$version-$platform-${target}.zip" - ln -snf "$dist/bin/protoc" "$VTROOT/bin/protoc" -} -protoc_ver=3.6.1 -install_dep "protoc" "$protoc_ver" "$VTROOT/dist/vt-protoc-$protoc_ver" install_protoc - -# Download and install etcd, link etcd binary into our root. -function install_etcd() { - local version="$1" - local dist="$2" - - case $(uname) in - Linux) local platform=linux; local ext=tar.gz;; - Darwin) local platform=darwin; local ext=zip;; - esac - - case $(get_arch) in - aarch64) local target=arm64;; - x86_64) local target=amd64;; - *) echo "ERROR: unsupported architecture"; exit 1;; - esac - - download_url=https://github.com/coreos/etcd/releases/download - file="etcd-${version}-${platform}-${target}.${ext}" - - wget "$download_url/$version/$file" - if [ "$ext" = "tar.gz" ]; then - tar xzf "$file" - else - unzip "$file" - fi - rm "$file" - ln -snf "$dist/etcd-${version}-${platform}-${target}/etcd" "$VTROOT/bin/etcd" -} - -# Install etcd if not detected -which etcd || install_dep "etcd" "v3.3.10" "$VTROOT/dist/etcd" install_etcd - -echo -echo "bootstrap finished" diff --git a/.github/workflows/check_make_parser.yml b/.github/workflows/check_make_parser.yml index 1cf47a6d4a7..0657236fc38 100644 --- a/.github/workflows/check_make_parser.yml +++ b/.github/workflows/check_make_parser.yml @@ -24,14 +24,11 @@ jobs: sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download - - name: Run bootstrap.sh + - name: Run make minimaltools run: | - echo "Copying new bootstrap over location of legacy one." - cp .github/bootstrap.sh . - ./bootstrap.sh + make minimaltools - name: check_make_parser run: | - export PATH=$PWD/bin:$PATH - VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD tools/check_make_parser.sh + tools/check_make_parser.sh diff --git a/.github/workflows/cluster_endtoend.yml b/.github/workflows/cluster_endtoend.yml index 61ddea9a5b9..a2f3f9d7d3d 100644 --- a/.github/workflows/cluster_endtoend.yml +++ b/.github/workflows/cluster_endtoend.yml @@ -24,18 +24,10 @@ jobs: sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download - - name: Run bootstrap.sh + - name: Run make minimaltools run: | - echo "Copying new bootstrap over location of legacy one." - cp .github/bootstrap.sh . - ./bootstrap.sh - - - name: Build - run: | - VTROOT=$PWD VTTOP=$PWD make build + make minimaltools - name: cluster_endtoend run: | - export PATH=$PWD/bin:$PATH - source ./dev.env - VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD make e2e_test_cluster + make e2e_test_cluster diff --git a/.github/workflows/e2e_race.yml b/.github/workflows/e2e_race.yml index a0404cfefb6..0439d067116 100644 --- a/.github/workflows/e2e_race.yml +++ b/.github/workflows/e2e_race.yml @@ -24,18 +24,10 @@ jobs: sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download - - name: Run bootstrap.sh + - name: Run make minimaltools run: | - echo "Copying new bootstrap over location of legacy one." - cp .github/bootstrap.sh . - ./bootstrap.sh - - - name: Build - run: | - VTROOT=$PWD VTTOP=$PWD make build + make minimaltools - name: e2e_race run: | - export PATH=$PWD/bin:$PATH - source ./dev.env - VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD make e2e_test_race + make e2e_test_race diff --git a/.github/workflows/endtoend.yml b/.github/workflows/endtoend.yml index a83ad499040..6ebec31c3b5 100644 --- a/.github/workflows/endtoend.yml +++ b/.github/workflows/endtoend.yml @@ -24,19 +24,14 @@ jobs: sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download - - name: Run bootstrap.sh + - name: Run make minimaltools run: | - echo "Copying new bootstrap over location of legacy one." - cp .github/bootstrap.sh . - ./bootstrap.sh + make minimaltools - name: Build run: | - VTROOT=$PWD VTTOP=$PWD make build + make build - name: endtoend run: | - export PATH=$PWD/bin:$PATH - source ./dev.env - mkdir -p /tmp/vtdataroot - VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD tools/e2e_test_runner.sh + tools/e2e_test_runner.sh diff --git a/.github/workflows/local_example.yml b/.github/workflows/local_example.yml index 9e9467a7f92..fe0035ec10e 100644 --- a/.github/workflows/local_example.yml +++ b/.github/workflows/local_example.yml @@ -24,18 +24,15 @@ jobs: sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download - - name: Run bootstrap.sh + - name: Run make minimaltools run: | - echo "Copying new bootstrap over location of legacy one." - cp .github/bootstrap.sh . - ./bootstrap.sh + make minimaltools - name: Build run: | - VTROOT=$PWD VTTOP=$PWD make build + make build - name: local_example run: | - export PATH=$PWD/bin:$PATH - VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD test/local_example.sh + test/local_example.sh diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 774bdbd7365..dd53b59a1d6 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -24,17 +24,10 @@ jobs: sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download - - name: Run bootstrap.sh + - name: Run make tools run: | - VTTOP=$PWD VTROOT=$PWD BUILD_PYTHON=0 ./bootstrap.sh - - - name: Build - run: | - VTROOT=$PWD VTTOP=$PWD make build + make tools - name: unit run: | - export PATH=$PWD/bin:$PATH - source ./dev.env - mkdir -p /tmp/vtdataroot - VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD tools/unit_test_runner.sh + make test diff --git a/.github/workflows/unit_race.yml b/.github/workflows/unit_race.yml index fd2e7ecd671..4cada70f482 100644 --- a/.github/workflows/unit_race.yml +++ b/.github/workflows/unit_race.yml @@ -24,16 +24,10 @@ jobs: sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld go mod download - - name: Run bootstrap.sh + - name: Run make tools run: | - VTTOP=$PWD VTROOT=$PWD BUILD_PYTHON=0 ./bootstrap.sh - - - name: Build - run: | - VTROOT=$PWD VTTOP=$PWD make build + make tools - name: unit_race run: | - export PATH=$PWD/bin:$PATH - source ./dev.env - VTDATAROOT=/tmp/vtdataroot VTTOP=$PWD VTROOT=$PWD make unit_test_race + make unit_test_race diff --git a/.gitignore b/.gitignore index 5bd875db51e..2561bdd9379 100644 --- a/.gitignore +++ b/.gitignore @@ -83,3 +83,6 @@ releases dist/* py-vtdb* vthook* +bin* + +vtdataroot* diff --git a/.travis.yml b/.travis.yml index 6233e64621c..733f8213a21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,7 @@ env: # See: https://docs.travis-ci.com/user/customizing-the-build#Build-Timeouts # To diagnose stuck tests, add "-follow" to TEST_FLAGS below. Then test.go # will print the test's output. - - TEST_FLAGS="-docker -use_docker_cache=false -timeout=12m -print-log" + - TEST_FLAGS="-docker -use_docker_cache -timeout=8m -print-log" matrix: # NOTE: Travis CI schedules up to 5 tests simultaneously. # All our tests should be spread out as evenly as possible across these 5 slots. diff --git a/Makefile b/Makefile index 80540efb1f0..48ad2e27475 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ export GO111MODULE=on # Since we are not using this Makefile for compilation, limiting parallelism will not increase build time. .NOTPARALLEL: -.PHONY: all build build_web test clean unit_test unit_test_cover unit_test_race integration_test proto proto_banner site_test site_integration_test docker_bootstrap docker_test docker_unit_test java_test reshard_tests e2e_test e2e_test_race tools +.PHONY: all build build_web test clean unit_test unit_test_cover unit_test_race integration_test proto proto_banner site_test site_integration_test docker_bootstrap docker_test docker_unit_test java_test reshard_tests e2e_test e2e_test_race minimaltools tools all: build @@ -91,7 +91,7 @@ e2e_test: build unit_test_cover: build go test $(VT_GO_PARALLEL) -cover ./go/... | misc/parse_cover.py -unit_test_race: build dependency_check +unit_test_race: build dependency_check tools/unit_test_race.sh e2e_test_race: build @@ -113,7 +113,7 @@ site_integration_test: java_test: go install ./go/cmd/vtgateclienttest ./go/cmd/vtcombo - mvn -f java/pom.xml -B clean verify + VTROOT=${PWD} mvn -f java/pom.xml -B clean verify install_protoc-gen-go: go install github.com/golang/protobuf/protoc-gen-go @@ -284,5 +284,9 @@ tools: echo $$(date): Installing dependencies BUILD_PYTHON=0 ./bootstrap.sh +minimaltools: + echo $$(date): Installing minimal dependencies + BUILD_PYTHON=0 BULD_JAVA=0 BUILD_CONSUL=0 ./bootstrap.sh + dependency_check: - ./tools/dependency_check.sh \ No newline at end of file + ./tools/dependency_check.sh diff --git a/bootstrap.sh b/bootstrap.sh index ecbb1e6ebb0..4c1cc3dee3c 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -23,6 +23,7 @@ BUILD_PYTHON=${BUILD_PYTHON:-1} BUILD_JAVA=${BUILD_JAVA:-1} +BUILD_CONSUL=${BUILD_CONSUL:-1} # # 0. Initialization and helper methods. @@ -221,8 +222,10 @@ function install_consul() { unzip "consul_${version}_${platform}_${target}.zip" ln -snf "$dist/consul" "$VTROOT/bin/consul" } -install_dep "Consul" "1.4.0" "$VTROOT/dist/consul" install_consul +if [ "$BUILD_CONSUL" == 1 ] ; then + install_dep "Consul" "1.4.0" "$VTROOT/dist/consul" install_consul +fi # Install py-mock. function install_pymock() { diff --git a/build.env b/build.env index 131852a7405..6fb37f47a62 100755 --- a/build.env +++ b/build.env @@ -16,13 +16,19 @@ source ./tools/shell_functions.inc +go version &>/dev/null || fail "Go is not installed or is not on \$PATH" +goversion_min 1.12 || fail "Go is not version 1.12+" + mkdir -p dist mkdir -p bin mkdir -p lib mkdir -p vthook -go version &>/dev/null || fail "Go is not installed or is not on \$PATH" -goversion_min 1.12 || fail "Go is not version 1.12+" +export VTROOT="$PWD" +export VTDATAROOT="${VTDATAROOT:-${VTROOT}/vtdataroot}" +export PATH="$PWD/bin:$PATH" + +mkdir -p "$VTDATAROOT" # Set up required soft links. # TODO(mberlin): Which of these can be deleted? @@ -38,5 +44,3 @@ mkdir -p .git/hooks ln -sf "$PWD/misc/git/pre-commit" .git/hooks/pre-commit ln -sf "$PWD/misc/git/commit-msg" .git/hooks/commit-msg git config core.hooksPath .git/hooks - -export VTROOT="$PWD" diff --git a/dev.env b/dev.env index f7d9e28a1a7..f256abbd93a 100644 --- a/dev.env +++ b/dev.env @@ -22,9 +22,6 @@ source ./build.env -export VTDATAROOT="${VTDATAROOT:-${VTROOT}/vtdataroot}" -mkdir -p "$VTDATAROOT" - export VTPORTSTART=15000 # Add all site-packages or dist-packages directories below $VTROOT/dist to $PYTHONPATH. diff --git a/examples/local/env.sh b/examples/local/env.sh index e6ab57254b2..b1833e997f5 100644 --- a/examples/local/env.sh +++ b/examples/local/env.sh @@ -17,6 +17,11 @@ hostname=`hostname -f` vtctld_web_port=15000 +function fail() { + echo "ERROR: $1" + exit 1 +} + if [ "${TOPO}" = "zk2" ]; then # Each ZooKeeper server needs a list of all servers in the quorum. # Since we're running them all locally, we need to give them unique ports. diff --git a/examples/local/etcd-up.sh b/examples/local/etcd-up.sh index 12207e23b86..36cd4269565 100755 --- a/examples/local/etcd-up.sh +++ b/examples/local/etcd-up.sh @@ -26,6 +26,9 @@ export ETCDCTL_API=2 # shellcheck disable=SC1091 source "${script_root}/env.sh" +# Check that etcd is not already running +curl "http://${ETCD_SERVER}" > /dev/null 2>&1 && fail "etcd is already running. Exiting." + etcd --enable-v2=true --data-dir "${VTDATAROOT}/etcd/" --listen-client-urls "http://${ETCD_SERVER}" --advertise-client-urls "http://${ETCD_SERVER}" > "${VTDATAROOT}"/tmp/etcd.out 2>&1 & PID=$! echo $PID > "${VTDATAROOT}/tmp/etcd.pid" diff --git a/test/client_test.sh b/test/client_test.sh index b9d70253e5f..6ad69b2023f 100755 --- a/test/client_test.sh +++ b/test/client_test.sh @@ -17,6 +17,8 @@ # This runs client tests. It used to be part of local_example, # but has been moved to its own test. It hijacks the public examples scripts +source build.env + set -xe cd "$VTROOT/examples/local" diff --git a/test/config.json b/test/config.json index 91d3de790b9..273fa24bd1a 100644 --- a/test/config.json +++ b/test/config.json @@ -50,7 +50,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 0, + "Shard": 4, "RetryMax": 0, "Tags": [ "worker_test" @@ -61,7 +61,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 0, + "Shard": 4, "RetryMax": 0, "Tags": [] }, @@ -70,7 +70,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 0, + "Shard": 4, "RetryMax": 0, "Tags": [ "site_test" @@ -81,7 +81,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 0, + "Shard": 4, "RetryMax": 0, "Tags": [ "site_test" @@ -103,7 +103,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 1, + "Shard": 4, "RetryMax": 0, "Tags": [ "worker_test" @@ -114,7 +114,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 1, + "Shard": 4, "RetryMax": 0, "Tags": [] }, @@ -123,7 +123,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 3, + "Shard": 4, "RetryMax": 0, "Tags": [] }, @@ -166,7 +166,7 @@ "java_test" ], "Manual": false, - "Shard": 3, + "Shard": 4, "RetryMax": 0, "Tags": [] }, @@ -188,7 +188,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 3, + "Shard": 4, "RetryMax": 0, "Tags": [ "site_test" @@ -199,7 +199,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 3, + "Shard": 4, "RetryMax": 0, "Tags": [ "worker_test" @@ -232,7 +232,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 3, + "Shard": 4, "RetryMax": 0, "Tags": [ "worker_test" @@ -243,7 +243,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 1, + "Shard": 4, "RetryMax": 0, "Tags": [ "worker_test" @@ -263,7 +263,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 1, + "Shard": 4, "RetryMax": 0, "Tags": [] }, diff --git a/test/local_example.sh b/test/local_example.sh index 6b9658b0068..26c41abf15f 100755 --- a/test/local_example.sh +++ b/test/local_example.sh @@ -18,6 +18,8 @@ # It should be kept in sync with the steps in https://vitess.io/docs/get-started/local/ # So we can detect if a regression affecting a tutorial is introduced. +source build.env + set -xe cd "$VTROOT/examples/local" diff --git a/tools/check_make_parser.sh b/tools/check_make_parser.sh index d28d4f18f09..ef59eb67a9f 100755 --- a/tools/check_make_parser.sh +++ b/tools/check_make_parser.sh @@ -6,6 +6,8 @@ # This is used in Travis to verify that the currently committed version was # generated with the proper version of goyacc. +source build.env + CUR="sql.go" TMP="/tmp/sql.$$.go" diff --git a/tools/dependency_check.sh b/tools/dependency_check.sh index 63f4ad092dc..33b3f1ecb24 100755 --- a/tools/dependency_check.sh +++ b/tools/dependency_check.sh @@ -14,13 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +source build.env + function fail() { echo "ERROR: $1" exit 1 } -# TODO: Add zksrv.sh - -for binary in mysqld consul etcd etctctl; do +for binary in mysqld consul etcd etcdctl zksrv.sh; do command -v "$binary" > /dev/null || fail "${binary} is not installed in PATH. Run 'make tools' to install dependencies." done; diff --git a/tools/e2e_test_cluster.sh b/tools/e2e_test_cluster.sh index d9ca94f6557..991cd29ce53 100755 --- a/tools/e2e_test_cluster.sh +++ b/tools/e2e_test_cluster.sh @@ -19,6 +19,9 @@ # All Go packages with test files. # Output per line: * + +source build.env + packages_with_tests=$(go list -f '{{if len .TestGoFiles}}{{.ImportPath}} {{join .TestGoFiles " "}}{{end}}' ./go/.../endtoend/... | sort) cluster_tests=$(echo "$packages_with_tests" | grep -E "go/test/endtoend" | cut -d" " -f1) diff --git a/tools/e2e_test_race.sh b/tools/e2e_test_race.sh index f3a31ac3af8..32374a25c17 100755 --- a/tools/e2e_test_race.sh +++ b/tools/e2e_test_race.sh @@ -14,12 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +source build.env + temp_log_file="$(mktemp --suffix .unit_test_race.log)" trap '[ -f "$temp_log_file" ] && rm $temp_log_file' EXIT -# This can be removed once the docker images are rebuilt -export GO111MODULE=on - # Wrapper around go test -race. # This script exists because the -race test doesn't allow to distinguish diff --git a/tools/e2e_test_runner.sh b/tools/e2e_test_runner.sh index e2b9da256ad..c581957a366 100755 --- a/tools/e2e_test_runner.sh +++ b/tools/e2e_test_runner.sh @@ -29,6 +29,9 @@ # Set VT_GO_PARALLEL variable in the same way as the Makefile does. # We repeat this here because this script is called directly by test.go # and not via the Makefile. + +source build.env + if [[ -z $VT_GO_PARALLEL && -n $VT_GO_PARALLEL_VALUE ]]; then VT_GO_PARALLEL="-p $VT_GO_PARALLEL_VALUE" fi diff --git a/tools/unit_test_race.sh b/tools/unit_test_race.sh index fef312ff340..d31a1410cf6 100755 --- a/tools/unit_test_race.sh +++ b/tools/unit_test_race.sh @@ -14,19 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -temp_log_file="$(mktemp --suffix .unit_test_race.log)" -trap '[ -f "$temp_log_file" ] && rm $temp_log_file' EXIT +source build.env -# Wrapper around go test -race. - -# This script exists because the -race test doesn't allow to distinguish -# between a failed (e.g. flaky) unit test and a found data race. -# Although Go 1.5 says 'exit status 66' in case of a race, it exits with 1. -# Therefore, we manually check the output of 'go test' for data races and -# exit with an error if one was found. -# TODO(mberlin): Test all packages (go/... instead of go/vt/...) once -# go/cgzip is moved into a separate repository. We currently -# skip the cgzip package because -race takes >30 sec for it. +if [[ -z $VT_GO_PARALLEL && -n $VT_GO_PARALLEL_VALUE ]]; then + VT_GO_PARALLEL="-p $VT_GO_PARALLEL_VALUE" +fi # All Go packages with test files. # Output per line: * @@ -36,17 +28,11 @@ packages_with_tests=$(go list -f '{{if len .TestGoFiles}}{{.ImportPath}} {{join all_except_e2e_tests=$(echo "$packages_with_tests" | cut -d" " -f1 | grep -v "endtoend") # Run non endtoend tests. -echo "$all_except_e2e_tests" | xargs go test $VT_GO_PARALLEL -race 2>&1 | tee $temp_log_file -if [ ${PIPESTATUS[0]} -ne 0 ]; then - if grep "WARNING: DATA RACE" -q $temp_log_file; then - echo - echo "ERROR: go test -race found a data race. See log above." - exit 2 - fi +echo "$all_except_e2e_tests" | xargs go test $VT_GO_PARALLEL -race - echo "ERROR: go test -race found NO data race, but failed. See log above." +if [ $? -ne 0 ]; then + echo "WARNING: POSSIBLE DATA RACE" + echo + echo "ERROR: go test -race failed. See log above." exit 1 fi - -echo -echo "SUCCESS: No data race was found." diff --git a/tools/unit_test_runner.sh b/tools/unit_test_runner.sh index cc20f52c77b..4cffb3eb913 100755 --- a/tools/unit_test_runner.sh +++ b/tools/unit_test_runner.sh @@ -31,7 +31,6 @@ # and not via the Makefile. source build.env -export VTDATAROOT=`mktemp -d` if [[ -z $VT_GO_PARALLEL && -n $VT_GO_PARALLEL_VALUE ]]; then VT_GO_PARALLEL="-p $VT_GO_PARALLEL_VALUE" From abf6492dab88355cfef12fd05f5fb880ee89753f Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Mon, 2 Dec 2019 17:46:45 -0700 Subject: [PATCH 11/25] Simplify docker/test/run.sh Signed-off-by: Morgan Tocker --- docker/test/run.sh | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/docker/test/run.sh b/docker/test/run.sh index da4c8d93ab5..f5b07d015c0 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -161,37 +161,17 @@ case "$mode" in esac # Construct "cp" command to copy the source code. -# -# Copy the full source tree except: -# - vendor -# That's because these directories are already part of the image. -# -# Note that we're using the Bash extended Glob support "!(vendor)" on -# purpose here to minimize the size of the cache image: With this trick, -# we do not move or overwrite the existing files while copying the other -# directories. Therefore, the existing files do not count as changed and will -# not be part of the new Docker layer of the cache image. -copy_src_cmd="cp -R /tmp/src/!(vendor|bootstrap.sh) ." +copy_src_cmd="cp -R /tmp/src/ ." # Copy the .git directory because travis/check_make_proto.sh needs a working # Git repository. copy_src_cmd=$(append_cmd "$copy_src_cmd" "cp -R /tmp/src/.git .") -# Enable gomodules -run_bootstrap_cmd="export GO111MODULE=on" - run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/dist ] && rm -rf dist && ln -s /vt/dist /vt/src/vitess.io/vitess/dist") run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/bin ] && rm -rf bin && ln -s /vt/bin /vt/src/vitess.io/vitess/bin") run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/lib ] && rm -rf lib && ln -s /vt/lib /vt/src/vitess.io/vitess/lib") run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/vthook ] && rm -rf vthook && ln -s /vt/vthook /vt/src/vitess.io/vitess/vthook") - -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "env") run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "export VTROOT=/vt/src/vitess.io/vitess && unset VTTOP") -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "env") - -# Copy bootstrap.sh if it changed -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ \$(diff -w bootstrap.sh /tmp/src/bootstrap.sh) ]]; then cp -f /tmp/src/bootstrap.sh .; bootstrap=1; fi") -# run bootstrap.sh if necessary -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "if [[ -n \$bootstrap ]]; then ./bootstrap.sh; fi") +run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "./bootstrap.sh") copy_src_cmd=$(append_cmd "$copy_src_cmd" "$run_bootstrap_cmd") # Construct the command we will actually run. From 2f04bd83bf05ffe6dd3ea2e872fe66314cc104f9 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Mon, 2 Dec 2019 18:07:19 -0700 Subject: [PATCH 12/25] Turn caching off again Signed-off-by: Morgan Tocker --- .travis.yml | 2 +- docker/bootstrap/Dockerfile.common | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 733f8213a21..6233e64621c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,7 @@ env: # See: https://docs.travis-ci.com/user/customizing-the-build#Build-Timeouts # To diagnose stuck tests, add "-follow" to TEST_FLAGS below. Then test.go # will print the test's output. - - TEST_FLAGS="-docker -use_docker_cache -timeout=8m -print-log" + - TEST_FLAGS="-docker -use_docker_cache=false -timeout=12m -print-log" matrix: # NOTE: Travis CI schedules up to 5 tests simultaneously. # All our tests should be spread out as evenly as possible across these 5 slots. diff --git a/docker/bootstrap/Dockerfile.common b/docker/bootstrap/Dockerfile.common index 539e0af4847..d75bdc32c63 100644 --- a/docker/bootstrap/Dockerfile.common +++ b/docker/bootstrap/Dockerfile.common @@ -30,8 +30,8 @@ RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-ins && rm -rf /var/lib/apt/lists/* # Install Maven 3.1+ -RUN mkdir -p /vt/dist && \ - cd /vt/dist && \ +RUN mkdir -p /vt/src/vitess.io/vitess/dist && \ + cd /vt/src/vitess.io/vitess/dist && \ curl -sL --connect-timeout 10 --retry 3 \ http://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz | tar -xz && \ mv apache-maven-3.3.9 maven From 1df5999aa08a1b0684c28926cf369f7dd6bd5942 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Thu, 5 Dec 2019 11:08:35 -0700 Subject: [PATCH 13/25] Change bootstrap to always run Signed-off-by: Morgan Tocker --- docker/test/run.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/docker/test/run.sh b/docker/test/run.sh index f5b07d015c0..936015a74e2 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -161,16 +161,22 @@ case "$mode" in esac # Construct "cp" command to copy the source code. -copy_src_cmd="cp -R /tmp/src/ ." +# +# Copy the full source tree except: +# - vendor +# That's because these directories are already part of the image. +# +# Note that we're using the Bash extended Glob support "!(vendor)" on +# purpose here to minimize the size of the cache image: With this trick, +# we do not move or overwrite the existing files while copying the other +# directories. Therefore, the existing files do not count as changed and will +# not be part of the new Docker layer of the cache image. +copy_src_cmd="cp -R /tmp/src/!(vtdataroot|lib|vthook|py-vtdb) ." # Copy the .git directory because travis/check_make_proto.sh needs a working # Git repository. copy_src_cmd=$(append_cmd "$copy_src_cmd" "cp -R /tmp/src/.git .") -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/dist ] && rm -rf dist && ln -s /vt/dist /vt/src/vitess.io/vitess/dist") -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/bin ] && rm -rf bin && ln -s /vt/bin /vt/src/vitess.io/vitess/bin") -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/lib ] && rm -rf lib && ln -s /vt/lib /vt/src/vitess.io/vitess/lib") -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "[ -d /vt/vthook ] && rm -rf vthook && ln -s /vt/vthook /vt/src/vitess.io/vitess/vthook") -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "export VTROOT=/vt/src/vitess.io/vitess && unset VTTOP") +# run bootstrap.sh run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "./bootstrap.sh") copy_src_cmd=$(append_cmd "$copy_src_cmd" "$run_bootstrap_cmd") From f8a0b6d3d4f9847b2c9ce229045b0996cd30e048 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Thu, 5 Dec 2019 15:46:39 -0700 Subject: [PATCH 14/25] Updated detection of old bootstrap Signed-off-by: Morgan Tocker --- .travis.yml | 4 ++-- docker/test/run.sh | 42 +++++++++++++++++------------------------- 2 files changed, 19 insertions(+), 27 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6233e64621c..d94a532e712 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,7 @@ env: # See: https://docs.travis-ci.com/user/customizing-the-build#Build-Timeouts # To diagnose stuck tests, add "-follow" to TEST_FLAGS below. Then test.go # will print the test's output. - - TEST_FLAGS="-docker -use_docker_cache=false -timeout=12m -print-log" + - TEST_FLAGS="-docker -use_docker_cache=true -timeout=8m -print-log" matrix: # NOTE: Travis CI schedules up to 5 tests simultaneously. # All our tests should be spread out as evenly as possible across these 5 slots. @@ -53,7 +53,7 @@ env: - TEST_MATRIX="-shard 3" - TEST_MATRIX="-shard 4" script: - - go run test.go $TEST_FLAGS $TEST_MATRIX + - VTDATAROOT=/tmp go run test.go $TEST_FLAGS $TEST_MATRIX # Uncomment the next line to verify the GOMAXPROCS value (should be 2 as of 09/2017). # - ./docker/test/run.sh mysql57 'go run travis/log_gomaxprocs.go' notifications: diff --git a/docker/test/run.sh b/docker/test/run.sh index 936015a74e2..7d2c5fe763f 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -160,33 +160,25 @@ case "$mode" in "create_cache") echo "Creating cache image $cache_image ..." ;; esac -# Construct "cp" command to copy the source code. -# -# Copy the full source tree except: -# - vendor -# That's because these directories are already part of the image. -# -# Note that we're using the Bash extended Glob support "!(vendor)" on -# purpose here to minimize the size of the cache image: With this trick, -# we do not move or overwrite the existing files while copying the other -# directories. Therefore, the existing files do not count as changed and will -# not be part of the new Docker layer of the cache image. -copy_src_cmd="cp -R /tmp/src/!(vtdataroot|lib|vthook|py-vtdb) ." -# Copy the .git directory because travis/check_make_proto.sh needs a working -# Git repository. -copy_src_cmd=$(append_cmd "$copy_src_cmd" "cp -R /tmp/src/.git .") - -# run bootstrap.sh -run_bootstrap_cmd=$(append_cmd "$run_bootstrap_cmd" "./bootstrap.sh") -copy_src_cmd=$(append_cmd "$copy_src_cmd" "$run_bootstrap_cmd") - -# Construct the command we will actually run. -# -# Uncomment the next line if you need to debug "bashcmd". -#bashcmd="set -x" +bashcmd="" + if [[ -z "$existing_cache_image" ]]; then - bashcmd=$(append_cmd "$bashcmd" "$copy_src_cmd") + + # Construct "cp" command to copy the source code. + bashcmd=$(append_cmd "$bashcmd" "cp -R /tmp/src/!(vtdataroot|lib|vthook|py-vtdb) . && cp -R /tmp/src/.git .") + fi + +# Reset the environment if this was an old bootstrap. We can detect this from VTTOP presence. +bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && export VTROOT=/vt/src/vitess.io/vitess && export VTDATAROOT=/vt/vtdataroot") +bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && export PYTHONPATH=/vt/src/vitess.io/vitess/dist/grpc/usr/local/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/dist/py-mock-1.0.1/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/py-vtdb:/vt/src/vitess.io/vitess/dist/selenium/lib/python2.7/site-packages") +bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && export PATH=/vt/src/vitess.io/vitess/bin:/vt/src/vitess.io/vitess/dist/maven/bin:/vt/src/vitess.io/vitess/dist/chromedriver:\$PATH") +bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && unset VTTOP") + + +# Run bootstrap every time now +bashcmd=$(append_cmd "$bashcmd" "./bootstrap.sh") + # At last, append the user's command. bashcmd=$(append_cmd "$bashcmd" "$cmd") From c94430b4463ac7b453be6e9ca2348c6b4418f2c4 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Thu, 5 Dec 2019 15:56:12 -0700 Subject: [PATCH 15/25] revert travis config Signed-off-by: Morgan Tocker --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d94a532e712..733f8213a21 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,7 @@ env: # See: https://docs.travis-ci.com/user/customizing-the-build#Build-Timeouts # To diagnose stuck tests, add "-follow" to TEST_FLAGS below. Then test.go # will print the test's output. - - TEST_FLAGS="-docker -use_docker_cache=true -timeout=8m -print-log" + - TEST_FLAGS="-docker -use_docker_cache -timeout=8m -print-log" matrix: # NOTE: Travis CI schedules up to 5 tests simultaneously. # All our tests should be spread out as evenly as possible across these 5 slots. @@ -53,7 +53,7 @@ env: - TEST_MATRIX="-shard 3" - TEST_MATRIX="-shard 4" script: - - VTDATAROOT=/tmp go run test.go $TEST_FLAGS $TEST_MATRIX + - go run test.go $TEST_FLAGS $TEST_MATRIX # Uncomment the next line to verify the GOMAXPROCS value (should be 2 as of 09/2017). # - ./docker/test/run.sh mysql57 'go run travis/log_gomaxprocs.go' notifications: From 170f8c1e4eae1bc9f30fba11d321cd3780500a57 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Fri, 6 Dec 2019 09:09:40 -0700 Subject: [PATCH 16/25] Add symlinks Signed-off-by: Morgan Tocker --- bootstrap.sh | 2 +- docker/bootstrap/Dockerfile.common | 4 ++++ docker/test/run.sh | 14 ++++++++++- tools/unit_test_race.sh | 37 +++++++++++------------------- 4 files changed, 31 insertions(+), 26 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index beadc40de47..639f406357a 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -199,7 +199,7 @@ function install_etcd() { ln -snf "$dist/etcd-${version}-${platform}-${target}/etcd" "$VTROOT/bin/etcd" ln -snf "$dist/etcd-${version}-${platform}-${target}/etcdctl" "$VTROOT/bin/etcdctl" } -which etcd || install_dep "etcd" "v3.3.10" "$VTROOT/dist/etcd" install_etcd +command -v etcdctl || install_dep "etcd" "v3.3.10" "$VTROOT/dist/etcd" install_etcd # Download and install consul, link consul binary into our root. diff --git a/docker/bootstrap/Dockerfile.common b/docker/bootstrap/Dockerfile.common index d75bdc32c63..6a9255b38e7 100644 --- a/docker/bootstrap/Dockerfile.common +++ b/docker/bootstrap/Dockerfile.common @@ -63,6 +63,10 @@ RUN cd /vt/src/vitess.io/vitess && \ # Create mount point for actual data (e.g. MySQL data dir) VOLUME /vt/vtdataroot +# The docker lite images copy from the builder in /vt/bin +# Add compatibility to the previous layout for now +RUN su vitess -c "mkdir -p /vt/src/vitess.io/vitess/bin && rm -rf /vt/bin && ln -s /vt/src/vitess.io/vitess/bin /vt/bin" + # If the user doesn't specify a command, load a shell. CMD ["/bin/bash"] diff --git a/docker/test/run.sh b/docker/test/run.sh index 7d2c5fe763f..e15a150a655 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -171,10 +171,22 @@ fi # Reset the environment if this was an old bootstrap. We can detect this from VTTOP presence. bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && export VTROOT=/vt/src/vitess.io/vitess && export VTDATAROOT=/vt/vtdataroot") + +# Maven is setup in /vt/dist, need to move it. +bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && mkdir -p /vt/src/vitess.io/vitess/dist && mv /vt/dist/* /vt/src/vitess.io/vitess/dist") +bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && ln -s /vt/src/vitess.io/vitess/dist /vt/dist") + +bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && rm -rf /vt/bin && ln -s /vt/src/vitess.io/vitess/bin /vt/bin") +bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && rm -rf /vt/lib && ln -s /vt/src/vitess.io/vitess/lib /vt/lib") +bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && rm -rf /vt/vthook && ln -s /vt/src/vitess.io/vitess/vthook /vt/vthook") bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && export PYTHONPATH=/vt/src/vitess.io/vitess/dist/grpc/usr/local/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/dist/py-mock-1.0.1/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/py-vtdb:/vt/src/vitess.io/vitess/dist/selenium/lib/python2.7/site-packages") bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && export PATH=/vt/src/vitess.io/vitess/bin:/vt/src/vitess.io/vitess/dist/maven/bin:/vt/src/vitess.io/vitess/dist/chromedriver:\$PATH") -bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && unset VTTOP") +# These were broken somehow +bashcmd=$(append_cmd "$bashcmd" "which protoc || rm -rf /vt/src/vitess.io/vitess/dist/vt-protoc-3.6.1") +bashcmd=$(append_cmd "$bashcmd" "which consul || rm -rf /vt/src/vitess.io/vitess/dist/consul") + +bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && unset VTTOP") # Run bootstrap every time now bashcmd=$(append_cmd "$bashcmd" "./bootstrap.sh") diff --git a/tools/unit_test_race.sh b/tools/unit_test_race.sh index fef312ff340..2253cab77bc 100755 --- a/tools/unit_test_race.sh +++ b/tools/unit_test_race.sh @@ -14,39 +14,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -temp_log_file="$(mktemp --suffix .unit_test_race.log)" -trap '[ -f "$temp_log_file" ] && rm $temp_log_file' EXIT +source build.env -# Wrapper around go test -race. - -# This script exists because the -race test doesn't allow to distinguish -# between a failed (e.g. flaky) unit test and a found data race. -# Although Go 1.5 says 'exit status 66' in case of a race, it exits with 1. -# Therefore, we manually check the output of 'go test' for data races and -# exit with an error if one was found. -# TODO(mberlin): Test all packages (go/... instead of go/vt/...) once -# go/cgzip is moved into a separate repository. We currently -# skip the cgzip package because -race takes >30 sec for it. +if [[ -z $VT_GO_PARALLEL && -n $VT_GO_PARALLEL_VALUE ]]; then + VT_GO_PARALLEL="-p $VT_GO_PARALLEL_VALUE" +fi # All Go packages with test files. # Output per line: * +# TODO: This tests ./go/vt/... instead of ./go/... due to a historical reason. +# When https://github.com/vitessio/vitess/issues/5493 is closed, we should change it. + packages_with_tests=$(go list -f '{{if len .TestGoFiles}}{{.ImportPath}} {{join .TestGoFiles " "}}{{end}}' ./go/vt/... | sort) # endtoend tests should be in a directory called endtoend all_except_e2e_tests=$(echo "$packages_with_tests" | cut -d" " -f1 | grep -v "endtoend") # Run non endtoend tests. -echo "$all_except_e2e_tests" | xargs go test $VT_GO_PARALLEL -race 2>&1 | tee $temp_log_file -if [ ${PIPESTATUS[0]} -ne 0 ]; then - if grep "WARNING: DATA RACE" -q $temp_log_file; then - echo - echo "ERROR: go test -race found a data race. See log above." - exit 2 - fi - - echo "ERROR: go test -race found NO data race, but failed. See log above." +echo "$all_except_e2e_tests" | xargs go test $VT_GO_PARALLEL -race + +if [ $? -ne 0 ]; then + echo "WARNING: POSSIBLE DATA RACE" + echo + echo "ERROR: go test -race failed. See log above." exit 1 fi - -echo -echo "SUCCESS: No data race was found." From ff17696f20ba212032e597bcdba70a7eea3f8f2d Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Fri, 6 Dec 2019 11:48:00 -0700 Subject: [PATCH 17/25] Fix PATH Signed-off-by: Morgan Tocker --- bootstrap.sh | 2 +- docker/test/run.sh | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index 639f406357a..ba16923a957 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -199,7 +199,7 @@ function install_etcd() { ln -snf "$dist/etcd-${version}-${platform}-${target}/etcd" "$VTROOT/bin/etcd" ln -snf "$dist/etcd-${version}-${platform}-${target}/etcdctl" "$VTROOT/bin/etcdctl" } -command -v etcdctl || install_dep "etcd" "v3.3.10" "$VTROOT/dist/etcd" install_etcd +command -v etcd && echo "etcd already installed" || install_dep "etcd" "v3.3.10" "$VTROOT/dist/etcd" install_etcd # Download and install consul, link consul binary into our root. diff --git a/docker/test/run.sh b/docker/test/run.sh index e15a150a655..aacf3c0176b 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -180,7 +180,9 @@ bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && rm -rf /vt/bin && ln -s /vt/sr bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && rm -rf /vt/lib && ln -s /vt/src/vitess.io/vitess/lib /vt/lib") bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && rm -rf /vt/vthook && ln -s /vt/src/vitess.io/vitess/vthook /vt/vthook") bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && export PYTHONPATH=/vt/src/vitess.io/vitess/dist/grpc/usr/local/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/dist/py-mock-1.0.1/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/py-vtdb:/vt/src/vitess.io/vitess/dist/selenium/lib/python2.7/site-packages") -bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && export PATH=/vt/src/vitess.io/vitess/bin:/vt/src/vitess.io/vitess/dist/maven/bin:/vt/src/vitess.io/vitess/dist/chromedriver:\$PATH") + +# etcd is broken in the bootstrap image. +bashcmd=$(append_cmd "$bashcmd" "which etcdctl || rm -rf /vt/src/vitess.io/vitess/dist/etcd* && rm -rf /vt/src/vitess.io/vitess/bin/etcd*") # These were broken somehow bashcmd=$(append_cmd "$bashcmd" "which protoc || rm -rf /vt/src/vitess.io/vitess/dist/vt-protoc-3.6.1") @@ -188,6 +190,9 @@ bashcmd=$(append_cmd "$bashcmd" "which consul || rm -rf /vt/src/vitess.io/vitess bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && unset VTTOP") +# Need to re-add PATH for now as well +bashcmd=$(append_cmd "$bashcmd" "export PATH=/vt/src/vitess.io/vitess/bin:/vt/src/vitess.io/vitess/dist/maven/bin:/vt/src/vitess.io/vitess/dist/chromedriver:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") + # Run bootstrap every time now bashcmd=$(append_cmd "$bashcmd" "./bootstrap.sh") From 7ddd68e193ff45cb7c6caa93e52be6f77ff3ad93 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Fri, 6 Dec 2019 12:27:57 -0700 Subject: [PATCH 18/25] Fix docker test run Signed-off-by: Morgan Tocker --- docker/test/run.sh | 21 +++++++++------------ 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/docker/test/run.sh b/docker/test/run.sh index aacf3c0176b..7d1b9acf61e 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -170,16 +170,17 @@ if [[ -z "$existing_cache_image" ]]; then fi # Reset the environment if this was an old bootstrap. We can detect this from VTTOP presence. -bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && export VTROOT=/vt/src/vitess.io/vitess && export VTDATAROOT=/vt/vtdataroot") +bashcmd=$(append_cmd "$bashcmd" "export VTROOT=/vt/src/vitess.io/vitess") +bashcmd=$(append_cmd "$bashcmd" "export VTDATAROOT=/vt/vtdataroot") +bashcmd=$(append_cmd "$bashcmd" "export PYTHONPATH=/vt/src/vitess.io/vitess/dist/grpc/usr/local/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/dist/py-mock-1.0.1/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/py-vtdb:/vt/src/vitess.io/vitess/dist/selenium/lib/python2.7/site-packages") # Maven is setup in /vt/dist, need to move it. -bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && mkdir -p /vt/src/vitess.io/vitess/dist && mv /vt/dist/* /vt/src/vitess.io/vitess/dist") -bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && ln -s /vt/src/vitess.io/vitess/dist /vt/dist") +bashcmd=$(append_cmd "$bashcmd" "mkdir -p /vt/src/vitess.io/vitess/dist; mv /vt/dist/* /vt/src/vitess.io/vitess/dist") -bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && rm -rf /vt/bin && ln -s /vt/src/vitess.io/vitess/bin /vt/bin") -bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && rm -rf /vt/lib && ln -s /vt/src/vitess.io/vitess/lib /vt/lib") -bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && rm -rf /vt/vthook && ln -s /vt/src/vitess.io/vitess/vthook /vt/vthook") -bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && export PYTHONPATH=/vt/src/vitess.io/vitess/dist/grpc/usr/local/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/dist/py-mock-1.0.1/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/py-vtdb:/vt/src/vitess.io/vitess/dist/selenium/lib/python2.7/site-packages") +bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/dist; ln -s /vt/src/vitess.io/vitess/dist /vt/dist") +bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/bin; ln -s /vt/src/vitess.io/vitess/bin /vt/bin") +bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/lib; ln -s /vt/src/vitess.io/vitess/lib /vt/lib") +bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/vthook; ln -s /vt/src/vitess.io/vitess/vthook /vt/vthook") # etcd is broken in the bootstrap image. bashcmd=$(append_cmd "$bashcmd" "which etcdctl || rm -rf /vt/src/vitess.io/vitess/dist/etcd* && rm -rf /vt/src/vitess.io/vitess/bin/etcd*") @@ -187,11 +188,7 @@ bashcmd=$(append_cmd "$bashcmd" "which etcdctl || rm -rf /vt/src/vitess.io/vites # These were broken somehow bashcmd=$(append_cmd "$bashcmd" "which protoc || rm -rf /vt/src/vitess.io/vitess/dist/vt-protoc-3.6.1") bashcmd=$(append_cmd "$bashcmd" "which consul || rm -rf /vt/src/vitess.io/vitess/dist/consul") - -bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && unset VTTOP") - -# Need to re-add PATH for now as well -bashcmd=$(append_cmd "$bashcmd" "export PATH=/vt/src/vitess.io/vitess/bin:/vt/src/vitess.io/vitess/dist/maven/bin:/vt/src/vitess.io/vitess/dist/chromedriver:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") +bashcmd=$(append_cmd "$bashcmd" "unset VTTOP") # Run bootstrap every time now bashcmd=$(append_cmd "$bashcmd" "./bootstrap.sh") From 14850f77a4b489752cc87bf74ee0c12b19175529 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Fri, 6 Dec 2019 12:27:57 -0700 Subject: [PATCH 19/25] Fix docker test run Signed-off-by: Morgan Tocker --- bootstrap.sh | 1 - docker/test/run.sh | 22 +++++++++------------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/bootstrap.sh b/bootstrap.sh index ba16923a957..c0a1af875ba 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -38,7 +38,6 @@ function fail() { # Create main directories. -VTROOT="$PWD" source ./dev.env # install_dep is a helper function to generalize the download and installation of dependencies. diff --git a/docker/test/run.sh b/docker/test/run.sh index aacf3c0176b..ee35ccd1931 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -170,16 +170,16 @@ if [[ -z "$existing_cache_image" ]]; then fi # Reset the environment if this was an old bootstrap. We can detect this from VTTOP presence. -bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && export VTROOT=/vt/src/vitess.io/vitess && export VTDATAROOT=/vt/vtdataroot") +bashcmd=$(append_cmd "$bashcmd" "export VTROOT=/vt/src/vitess.io/vitess") +bashcmd=$(append_cmd "$bashcmd" "export VTDATAROOT=/vt/vtdataroot") +bashcmd=$(append_cmd "$bashcmd" "export PYTHONPATH=/vt/src/vitess.io/vitess/dist/grpc/usr/local/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/dist/py-mock-1.0.1/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/py-vtdb:/vt/src/vitess.io/vitess/dist/selenium/lib/python2.7/site-packages") # Maven is setup in /vt/dist, need to move it. -bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && mkdir -p /vt/src/vitess.io/vitess/dist && mv /vt/dist/* /vt/src/vitess.io/vitess/dist") -bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && ln -s /vt/src/vitess.io/vitess/dist /vt/dist") - -bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && rm -rf /vt/bin && ln -s /vt/src/vitess.io/vitess/bin /vt/bin") -bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && rm -rf /vt/lib && ln -s /vt/src/vitess.io/vitess/lib /vt/lib") -bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && rm -rf /vt/vthook && ln -s /vt/src/vitess.io/vitess/vthook /vt/vthook") -bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && export PYTHONPATH=/vt/src/vitess.io/vitess/dist/grpc/usr/local/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/dist/py-mock-1.0.1/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/py-vtdb:/vt/src/vitess.io/vitess/dist/selenium/lib/python2.7/site-packages") +bashcmd=$(append_cmd "$bashcmd" "[[ -L "/vt/dist" && -d "/vt/dist" ]] || mkdir -p /vt/src/vitess.io/vitess/dist && mv /vt/dist/* /vt/src/vitess.io/vitess/dist") +bashcmd=$(append_cmd "$bashcmd" "[[ -L "/vt/dist" && -d "/vt/dist" ]] || rm -rf /vt/dist && ln -s /vt/src/vitess.io/vitess/dist /vt/dist") +bashcmd=$(append_cmd "$bashcmd" "[[ -L "/vt/bin" && -d "/vt/bin" ]] || rm -rf /vt/bin && mkdir -p /vt/src/vitess.io/vitess/bin ln -s /vt/src/vitess.io/vitess/bin /vt/bin") +bashcmd=$(append_cmd "$bashcmd" "[[ -L "/vt/lib" && -d "/vt/lib" ]] || rm -rf /vt/lib && mkdir -p /vt/src/vitess.io/vitess/lib && ln -s /vt/src/vitess.io/vitess/lib /vt/lib") +bashcmd=$(append_cmd "$bashcmd" "[[ -L "/vt/vthook" && -d "/vt/vthook" ]] || rm -rf /vt/vthook && mkdir -p /vt/src/vitess.io/vitess/vthook & ln -s /vt/src/vitess.io/vitess/vthook /vt/vthook") # etcd is broken in the bootstrap image. bashcmd=$(append_cmd "$bashcmd" "which etcdctl || rm -rf /vt/src/vitess.io/vitess/dist/etcd* && rm -rf /vt/src/vitess.io/vitess/bin/etcd*") @@ -187,11 +187,7 @@ bashcmd=$(append_cmd "$bashcmd" "which etcdctl || rm -rf /vt/src/vitess.io/vites # These were broken somehow bashcmd=$(append_cmd "$bashcmd" "which protoc || rm -rf /vt/src/vitess.io/vitess/dist/vt-protoc-3.6.1") bashcmd=$(append_cmd "$bashcmd" "which consul || rm -rf /vt/src/vitess.io/vitess/dist/consul") - -bashcmd=$(append_cmd "$bashcmd" "[ -v VTTOP ] && unset VTTOP") - -# Need to re-add PATH for now as well -bashcmd=$(append_cmd "$bashcmd" "export PATH=/vt/src/vitess.io/vitess/bin:/vt/src/vitess.io/vitess/dist/maven/bin:/vt/src/vitess.io/vitess/dist/chromedriver:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") +bashcmd=$(append_cmd "$bashcmd" "unset VTTOP") # Run bootstrap every time now bashcmd=$(append_cmd "$bashcmd" "./bootstrap.sh") From cfbc0924ff58e3951a5c20f8875dc8f0728a98b1 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Fri, 6 Dec 2019 13:40:00 -0700 Subject: [PATCH 20/25] Fix maven Signed-off-by: Morgan Tocker --- docker/test/run.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docker/test/run.sh b/docker/test/run.sh index 7d1b9acf61e..a198752c855 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -174,8 +174,10 @@ bashcmd=$(append_cmd "$bashcmd" "export VTROOT=/vt/src/vitess.io/vitess") bashcmd=$(append_cmd "$bashcmd" "export VTDATAROOT=/vt/vtdataroot") bashcmd=$(append_cmd "$bashcmd" "export PYTHONPATH=/vt/src/vitess.io/vitess/dist/grpc/usr/local/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/dist/py-mock-1.0.1/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/py-vtdb:/vt/src/vitess.io/vitess/dist/selenium/lib/python2.7/site-packages") -# Maven is setup in /vt/dist, need to move it. -bashcmd=$(append_cmd "$bashcmd" "mkdir -p /vt/src/vitess.io/vitess/dist; mv /vt/dist/* /vt/src/vitess.io/vitess/dist") +bashcmd=$(append_cmd "$bashcmd" "mkdir -p dist; mkdir -p bin; mkdir -p lib; mkdir -p vthook") + +# Maven was setup in /vt/dist, may need to reinstall it. +bashcmd=$(append_cmd "$bashcmd" "mvn || curl -sL --connect-timeout 10 --retry 3 http://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz | tar -xz && mv apache-maven-3.3.9 dist/maven") bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/dist; ln -s /vt/src/vitess.io/vitess/dist /vt/dist") bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/bin; ln -s /vt/src/vitess.io/vitess/bin /vt/bin") From da14904909ba1f462c42095dc8a0810a803b7dd0 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Fri, 6 Dec 2019 15:24:06 -0700 Subject: [PATCH 21/25] Found it! copying files in that should not have Signed-off-by: Morgan Tocker --- docker/test/run.sh | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/docker/test/run.sh b/docker/test/run.sh index a198752c855..31c27158583 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -165,7 +165,7 @@ bashcmd="" if [[ -z "$existing_cache_image" ]]; then # Construct "cp" command to copy the source code. - bashcmd=$(append_cmd "$bashcmd" "cp -R /tmp/src/!(vtdataroot|lib|vthook|py-vtdb) . && cp -R /tmp/src/.git .") + bashcmd=$(append_cmd "$bashcmd" "cp -R /tmp/src/!(vtdataroot|dist|bin|lib|vthook|py-vtdb) . && cp -R /tmp/src/.git .") fi @@ -175,22 +175,13 @@ bashcmd=$(append_cmd "$bashcmd" "export VTDATAROOT=/vt/vtdataroot") bashcmd=$(append_cmd "$bashcmd" "export PYTHONPATH=/vt/src/vitess.io/vitess/dist/grpc/usr/local/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/dist/py-mock-1.0.1/lib/python2.7/site-packages:/vt/src/vitess.io/vitess/py-vtdb:/vt/src/vitess.io/vitess/dist/selenium/lib/python2.7/site-packages") bashcmd=$(append_cmd "$bashcmd" "mkdir -p dist; mkdir -p bin; mkdir -p lib; mkdir -p vthook") - -# Maven was setup in /vt/dist, may need to reinstall it. -bashcmd=$(append_cmd "$bashcmd" "mvn || curl -sL --connect-timeout 10 --retry 3 http://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz | tar -xz && mv apache-maven-3.3.9 dist/maven") - bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/dist; ln -s /vt/src/vitess.io/vitess/dist /vt/dist") bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/bin; ln -s /vt/src/vitess.io/vitess/bin /vt/bin") bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/lib; ln -s /vt/src/vitess.io/vitess/lib /vt/lib") bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/vthook; ln -s /vt/src/vitess.io/vitess/vthook /vt/vthook") -# etcd is broken in the bootstrap image. -bashcmd=$(append_cmd "$bashcmd" "which etcdctl || rm -rf /vt/src/vitess.io/vitess/dist/etcd* && rm -rf /vt/src/vitess.io/vitess/bin/etcd*") - -# These were broken somehow -bashcmd=$(append_cmd "$bashcmd" "which protoc || rm -rf /vt/src/vitess.io/vitess/dist/vt-protoc-3.6.1") -bashcmd=$(append_cmd "$bashcmd" "which consul || rm -rf /vt/src/vitess.io/vitess/dist/consul") -bashcmd=$(append_cmd "$bashcmd" "unset VTTOP") +# Maven was setup in /vt/dist, may need to reinstall it. +bashcmd=$(append_cmd "$bashcmd" "command -v mvn || curl -sL --connect-timeout 10 --retry 3 http://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz | tar -xz && mv apache-maven-3.3.9 dist/maven") # Run bootstrap every time now bashcmd=$(append_cmd "$bashcmd" "./bootstrap.sh") From dd621823ea25d5cd0f746f14db01c7d96a946946 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Fri, 6 Dec 2019 15:50:55 -0700 Subject: [PATCH 22/25] Modify append_cmd to be ; instead of && Signed-off-by: Morgan Tocker --- docker/test/run.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/test/run.sh b/docker/test/run.sh index 31c27158583..8d7e6c344d9 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -48,7 +48,7 @@ function append_cmd() { local cmd="$1" local append="$2" if [[ -n "$cmd" ]]; then - cmd+=" && " + cmd+=" ; " fi cmd+="$append" echo "$cmd" @@ -181,7 +181,8 @@ bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/lib; ln -s /vt/src/vitess.io/vitess/ bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/vthook; ln -s /vt/src/vitess.io/vitess/vthook /vt/vthook") # Maven was setup in /vt/dist, may need to reinstall it. -bashcmd=$(append_cmd "$bashcmd" "command -v mvn || curl -sL --connect-timeout 10 --retry 3 http://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz | tar -xz && mv apache-maven-3.3.9 dist/maven") +bashcmd=$(append_cmd "$bashcmd" "echo 'Checking if mvn needs installing...'") +bashcmd=$(append_cmd "$bashcmd" "command -v mvn || echo 'Fetching maven...' && curl -sL --connect-timeout 10 --retry 3 http://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz | tar -xz && mv apache-maven-3.3.9 /vt/dist/maven") # Run bootstrap every time now bashcmd=$(append_cmd "$bashcmd" "./bootstrap.sh") From 22822e50650a35445ddd76f1979239f1e3fbe92e Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Fri, 6 Dec 2019 16:17:52 -0700 Subject: [PATCH 23/25] move prepared statements to 5, restore && Signed-off-by: Morgan Tocker --- Makefile | 2 +- docker/test/run.sh | 2 +- test/config.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 48ad2e27475..8eab33561d7 100644 --- a/Makefile +++ b/Makefile @@ -286,7 +286,7 @@ tools: minimaltools: echo $$(date): Installing minimal dependencies - BUILD_PYTHON=0 BULD_JAVA=0 BUILD_CONSUL=0 ./bootstrap.sh + BUILD_PYTHON=0 BUILD_JAVA=0 BUILD_CONSUL=0 ./bootstrap.sh dependency_check: ./tools/dependency_check.sh diff --git a/docker/test/run.sh b/docker/test/run.sh index 8d7e6c344d9..298dcc25981 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -48,7 +48,7 @@ function append_cmd() { local cmd="$1" local append="$2" if [[ -n "$cmd" ]]; then - cmd+=" ; " + cmd+=" && " fi cmd+="$append" echo "$cmd" diff --git a/test/config.json b/test/config.json index a8648f96063..be3ddd1b574 100644 --- a/test/config.json +++ b/test/config.json @@ -261,7 +261,7 @@ "Args": [], "Command": [], "Manual": false, - "Shard": 4, + "Shard": 5, "RetryMax": 0, "Tags": [] }, From 1ac0e522dcc7204dfebe34df7f0a708a1f152647 Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Fri, 6 Dec 2019 18:30:10 -0700 Subject: [PATCH 24/25] Fix small bug in re-bootstrap case. Signed-off-by: Morgan Tocker --- docker/test/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/run.sh b/docker/test/run.sh index 298dcc25981..828fb11162e 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -182,7 +182,7 @@ bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/vthook; ln -s /vt/src/vitess.io/vite # Maven was setup in /vt/dist, may need to reinstall it. bashcmd=$(append_cmd "$bashcmd" "echo 'Checking if mvn needs installing...'") -bashcmd=$(append_cmd "$bashcmd" "command -v mvn || echo 'Fetching maven...' && curl -sL --connect-timeout 10 --retry 3 http://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz | tar -xz && mv apache-maven-3.3.9 /vt/dist/maven") +bashcmd=$(append_cmd "$bashcmd" "command -v mvn || \$(echo 'Fetching maven...' && curl -sL --connect-timeout 10 --retry 3 http://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz | tar -xz && mv apache-maven-3.3.9 /vt/dist/maven)") # Run bootstrap every time now bashcmd=$(append_cmd "$bashcmd" "./bootstrap.sh") From 568b6147781014564f18059c983ab9f7aea61b4c Mon Sep 17 00:00:00 2001 From: Morgan Tocker Date: Fri, 6 Dec 2019 18:59:46 -0700 Subject: [PATCH 25/25] Fix check if maven needs installing Signed-off-by: Morgan Tocker --- docker/test/run.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docker/test/run.sh b/docker/test/run.sh index 828fb11162e..df0c6a20f82 100755 --- a/docker/test/run.sh +++ b/docker/test/run.sh @@ -181,8 +181,7 @@ bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/lib; ln -s /vt/src/vitess.io/vitess/ bashcmd=$(append_cmd "$bashcmd" "rm -rf /vt/vthook; ln -s /vt/src/vitess.io/vitess/vthook /vt/vthook") # Maven was setup in /vt/dist, may need to reinstall it. -bashcmd=$(append_cmd "$bashcmd" "echo 'Checking if mvn needs installing...'") -bashcmd=$(append_cmd "$bashcmd" "command -v mvn || \$(echo 'Fetching maven...' && curl -sL --connect-timeout 10 --retry 3 http://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz | tar -xz && mv apache-maven-3.3.9 /vt/dist/maven)") +bashcmd=$(append_cmd "$bashcmd" "echo 'Checking if mvn needs installing...'; if [[ ! \$(command -v mvn) ]]; then echo 'install maven'; curl -sL --connect-timeout 10 --retry 3 http://www-us.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz | tar -xz && mv apache-maven-3.3.9 /vt/dist/maven; fi; echo 'mvn check done'") # Run bootstrap every time now bashcmd=$(append_cmd "$bashcmd" "./bootstrap.sh")