Use 'python3 -m venv' instead of virtualenv

.travis.yml

@@ -17,8 +17,6 @@ env:
     # Newer versions of minikube fail on travis, see: https://github.com/kubernetes/minikube/issues/2704
     - TRAVIS_MINIKUBE_VERSION=v0.25.2
     - MACOSX_DEPLOYMENT_TARGET=10.15
-    # Migration to Python 3
-    - PYTHON_EXE=python3
 
 jobs:
   include:
@@ -203,7 +201,7 @@ jobs:
             retries: true
           update: true
           packages:
-            - python-virtualenv
+            - python3-venv
             - libpcap-dev
             - xsltproc
             - libxml2-utils
@@ -223,7 +221,7 @@ addons:
       retries: true
     update: true
     packages:
-      - python-virtualenv
+      - python3-venv
       - libpcap-dev
       - xsltproc
       - libxml2-utils
@@ -232,6 +230,7 @@ addons:
 
 before_install:
   - python --version
+  - python3 --version
   - umask 022
   - chmod -R go-w $GOPATH/src/github.com/elastic/beats
   # Docker-compose installation

Makefile

@@ -5,7 +5,8 @@ PROJECTS=libbeat $(BEATS)
 PROJECTS_ENV=libbeat filebeat metricbeat
 PYTHON_ENV?=$(BUILD_DIR)/python-env
 PYTHON_EXE?=python3
-VIRTUALENV_PARAMS?=
+PYTHON_ENV_EXE=${PYTHON_ENV}/bin/$(notdir ${PYTHON_EXE})
+VENV_PARAMS?=
 FIND=find . -type f -not -path "*/vendor/*" -not -path "*/build/*" -not -path "*/.git/*"
 GOLINT=golint
 GOLINT_REPO=golang.org/x/lint/golint
@@ -137,12 +138,12 @@ docs:
 .PHONY: notice
 notice: python-env
 	@echo "Generating NOTICE"
-	@$(PYTHON_ENV)/bin/python dev-tools/generate_notice.py .
+	@${PYTHON_ENV_EXE} dev-tools/generate_notice.py .
 
 # Sets up the virtual python environment
 .PHONY: python-env
 python-env:
-	@test -d $(PYTHON_ENV) || virtualenv -p $(PYTHON_EXE) $(VIRTUALENV_PARAMS) $(PYTHON_ENV)
+	@test -d $(PYTHON_ENV) || ${PYTHON_EXE} -m venv $(VENV_PARAMS) $(PYTHON_ENV)
 	@$(PYTHON_ENV)/bin/pip install -q --upgrade pip autopep8==1.3.5 pylint==1.9.5 future==0.18.2
 	@# Work around pip bug. See: https://github.com/pypa/pip/issues/4464
 	@find $(PYTHON_ENV) -type d -name dist-packages -exec sh -c "echo dist-packages > {}.pth" ';'

Vagrantfile

@@ -129,7 +129,7 @@ def linuxGvmProvision(arch="amd64")
   return <<SCRIPT
 mkdir -p ~/bin
 if [ ! -e "~/bin/gvm" ]; then
-  curl -sL -o ~/bin/gvm https://github.com/andrewkroh/gvm/releases/download/v0.1.0/gvm-linux-#{arch}
+  curl -sL -o ~/bin/gvm https://github.com/andrewkroh/gvm/releases/download/v0.2.1/gvm-linux-#{arch}
   chmod +x ~/bin/gvm
   ~/bin/gvm #{GO_VERSION}
   echo 'export GOPATH=$HOME/go' >> ~/.bash_profile
@@ -145,7 +145,7 @@ def linuxDebianProvision()
 #!/usr/bin/env bash
 set -eio pipefail
 apt-get update
-apt-get install -y make gcc python-pip python-virtualenv git
+apt-get install -y make gcc python3 python3-pip python3-venv git
 SCRIPT
 end
 
@@ -245,7 +245,8 @@ Vagrant.configure(2) do |config|
 
     c.vm.provision "shell", inline: $unixProvision, privileged: false
     c.vm.provision "shell", inline: linuxGvmProvision, privileged: false
-    c.vm.provision "shell", inline: "yum install -y make gcc python-pip python-virtualenv git rpm-devel"
+    c.vm.provision "shell", inline: "yum install -y make gcc git rpm-devel epel-release"
+    c.vm.provision "shell", inline: "yum install -y python34 python34-pip"
   end
 
   config.vm.define "centos7", primary: true do |c|
@@ -254,25 +255,16 @@ Vagrant.configure(2) do |config|
 
     c.vm.provision "shell", inline: $unixProvision, privileged: false
     c.vm.provision "shell", inline: linuxGvmProvision, privileged: false
-    c.vm.provision "shell", inline: "yum install -y make gcc python-pip python-virtualenv git rpm-devel"
+    c.vm.provision "shell", inline: "yum install -y make gcc python3 python3-pip git rpm-devel"
   end
 
-  config.vm.define "fedora29", primary: true do |c|
-    c.vm.box = "bento/fedora-29"
+  config.vm.define "fedora31", primary: true do |c|
+    c.vm.box = "bento/fedora-31"
     c.vm.network :forwarded_port, guest: 22, host: 2231, id: "ssh", auto_correct: true
 
     c.vm.provision "shell", inline: $unixProvision, privileged: false
     c.vm.provision "shell", inline: linuxGvmProvision, privileged: false
-    c.vm.provision "shell", inline: "dnf install -y make gcc python-pip python-virtualenv git rpm-devel"
-  end
-
-  config.vm.define "sles12", primary: true do |c|
-    c.vm.box = "elastic/sles-12-x86_64"
-    c.vm.network :forwarded_port, guest: 22, host: 2232, id: "ssh", auto_correct: true
-
-    c.vm.provision "shell", inline: $unixProvision, privileged: false
-    c.vm.provision "shell", inline: linuxGvmProvision, privileged: false
-    c.vm.provision "shell", inline: "pip install virtualenv"
+    c.vm.provision "shell", inline: "dnf install -y make gcc python3 python3-pip git rpm-devel"
   end
 
   config.vm.define "archlinux", primary: true do |c|
@@ -281,6 +273,6 @@ Vagrant.configure(2) do |config|
 
     c.vm.provision "shell", inline: $unixProvision, privileged: false
     c.vm.provision "shell", inline: linuxGvmProvision, privileged: false
-    c.vm.provision "shell", inline: "pacman -Sy && pacman -S --noconfirm make gcc python-pip python-virtualenv git"
+    c.vm.provision "shell", inline: "pacman -Sy && pacman -S --noconfirm make gcc python python-pip git"
   end
 end

auditbeat/Dockerfile

@@ -3,14 +3,14 @@ FROM golang:1.13.6
 RUN \
     apt-get update \
       && apt-get install -y --no-install-recommends \
-         python-pip \
-         virtualenv \
          python3 \
+         python3-pip \
+         python3-venv \
          librpm-dev \
       && rm -rf /var/lib/apt/lists/*
 
 ENV PYTHON_ENV=/tmp/python-env
 
-RUN pip install --upgrade pip
-RUN pip install --upgrade setuptools
-RUN pip install --upgrade docker-compose==1.23.2
+RUN pip3 install --upgrade pip
+RUN pip3 install --upgrade setuptools
+RUN pip3 install --upgrade docker-compose==1.23.2

dev-tools/mage/docs.go

@@ -75,7 +75,7 @@ func (docsBuilder) FieldDocs(fieldsYML string) error {
 		return err
 	}
 
-	python, err := LookVirtualenvPath(ve, "python")
+	python, err := LookVirtualenvPath(ve, pythonExe)
 	if err != nil {
 		return err
 	}

dev-tools/mage/kibana.go

@@ -67,7 +67,7 @@ func KibanaDashboards(moduleDirs ...string) error {
 	}
 
 	// Convert 7.x dashboards to strings.
-	err = sh.Run("python",
+	err = sh.Run(pythonExe,
 		filepath.Join(esBeatsDir, "libbeat/scripts/unpack_dashboards.py"),
 		"--glob="+filepath.Join(kibanaBuildDir, "7/dashboard/*.json"))
 	if err != nil {

dev-tools/mage/pytest.go

@@ -37,6 +37,11 @@ const (
 )
 
 var (
+	// pythonExe points to the python executable to use. It defaults to python3
+	// so this must be on the PATH. The PYTHON_EXE environment can be used to
+	// modify the executable used.
+	pythonExe = EnvOr("PYTHON_EXE", "python3")
+
 	// VirtualenvReqs specifies a list of virtualenv requirements files to be
 	// used when calling PythonVirtualenv(). It defaults to the libbeat
 	// requirements.txt file.
@@ -167,17 +172,9 @@ func PythonVirtualenv() (string, error) {
 		return pythonVirtualenvDir, nil
 	}
 
-	// If set use PYTHON_EXE env var as the python interpreter.
-	var args []string
-	if pythonExe := os.Getenv("PYTHON_EXE"); pythonExe != "" {
-		args = append(args, "-p", pythonExe)
-	}
-	args = append(args, ve)
-
-	// Execute virtualenv.
+	// Create a virtual environment only if the dir does not exist.
 	if _, err := os.Stat(ve); err != nil {
-		// Run virtualenv if the dir does not exist.
-		if err := sh.Run("virtualenv", args...); err != nil {
+		if err := sh.Run(pythonExe, "-m", "venv", ve); err != nil {
 			return "", err
 		}
 	}
@@ -188,7 +185,7 @@ func PythonVirtualenv() (string, error) {
 	}
 
 	pip := virtualenvPath(ve, "pip")
-	args = []string{"install"}
+	args := []string{"install"}
 	if !mg.Verbose() {
 		args = append(args, "--quiet")
 	}

filebeat/Dockerfile

@@ -4,15 +4,15 @@ RUN \
     apt-get update \
       && apt-get install -y --no-install-recommends \
          netcat \
-         python-pip \
          rsync \
-         virtualenv \
          python3 \
+         python3-pip \
+         python3-venv \
          libpcap-dev \
       && rm -rf /var/lib/apt/lists/*
 
 ENV PYTHON_ENV=/tmp/python-env
 
-RUN pip install --upgrade pip
-RUN pip install --upgrade setuptools
-RUN pip install --upgrade docker-compose==1.23.2
+RUN pip3 install --upgrade pip
+RUN pip3 install --upgrade setuptools
+RUN pip3 install --upgrade docker-compose==1.23.2

filebeat/tests/load/Makefile

@@ -1,5 +1,5 @@
 run:
-	python load.py
+	python3 load.py
 
 clean:
 	rm -r logs

heartbeat/Dockerfile

@@ -1,17 +1,19 @@
 FROM golang:1.13.6
-MAINTAINER Nicolas Ruflin <[email protected]>
 
-RUN set -x && \
-    apt-get update && \
-    apt-get install -y --no-install-recommends \
-         netcat python-pip virtualenv python3 && \
-    apt-get clean
+RUN \
+    apt-get update \
+      && apt-get install -y --no-install-recommends \
+         netcat \
+         python3 \
+         python3-pip \
+         python3-venv \
+      && rm -rf /var/lib/apt/lists/*
 
 ENV PYTHON_ENV=/tmp/python-env
 
-RUN pip install --upgrade pip
-RUN pip install --upgrade setuptools
-RUN pip install --upgrade docker-compose==1.23.2
+RUN pip3 install --upgrade pip
+RUN pip3 install --upgrade setuptools
+RUN pip3 install --upgrade docker-compose==1.23.2
 
 # Setup work environment
 ENV HEARTBEAT_PATH /go/src/github.com/elastic/beats/heartbeat

journalbeat/Dockerfile

@@ -1,15 +1,21 @@
 FROM golang:1.13.6
-MAINTAINER Noémi Ványi <[email protected]>
 
-RUN set -x && \
-    apt-get update && \
-    apt-get install -y --no-install-recommends \
-      python-pip virtualenv python3 libsystemd-dev libc6-dev-i386 gcc-arm-linux-gnueabi && \
-    apt-get clean
+RUN \
+    apt-get update \
+      && apt-get install -y --no-install-recommends \
+         libsystemd-dev \
+         libc6-dev-i386 \
+         gcc-arm-linux-gnueabi \
+         python3 \
+         python3-pip \
+         python3-venv \
+      && rm -rf /var/lib/apt/lists/*
 
 ENV PYTHON_ENV=/tmp/python-env
 
-RUN pip install --upgrade setuptools
+RUN pip3 install --upgrade pip
+RUN pip3 install --upgrade setuptools
+RUN pip3 install --upgrade docker-compose==1.23.2
 
 # Setup work environment
 ENV JOURNALBEAT_PATH /go/src/github.com/elastic/beats/journalbeat

libbeat/Dockerfile

@@ -1,21 +1,20 @@
-# Beats dockerfile used for testing
 FROM golang:1.13.6
-MAINTAINER Nicolas Ruflin <[email protected]>
 
-RUN set -x && \
-    apt-get update && \
-    apt-get install -y --no-install-recommends \
-         netcat python-pip virtualenv python3 libpcap-dev && \
-    apt-get clean
+RUN \
+    apt-get update \
+      && apt-get install -y --no-install-recommends \
+         netcat \
+         libpcap-dev \
+         python3 \
+         python3-pip \
+         python3-venv \
+      && rm -rf /var/lib/apt/lists/*
 
 ENV PYTHON_ENV=/tmp/python-env
 
-RUN test -d ${PYTHON_ENV} || virtualenv -p python3 ${PYTHON_ENV}
-COPY ./tests/system/requirements.txt /tmp/requirements.txt
-
-# Upgrade pip to make sure to have the most recent version
-RUN . ${PYTHON_ENV}/bin/activate && pip install -U pip
-RUN . ${PYTHON_ENV}/bin/activate && pip install -Ur /tmp/requirements.txt
+RUN pip3 install --upgrade pip
+RUN pip3 install --upgrade setuptools
+RUN pip3 install --upgrade docker-compose==1.23.2
 
 # Libbeat specific
 RUN mkdir -p /etc/pki/tls/certs