Migrate scripts to python 3

.travis.yml

@@ -166,12 +166,6 @@ jobs:
       go: $TRAVIS_GO_VERSION
       stage: test
 
-    # Docs
-    - os: linux
-      env: TARGETS="docs"
-      go: $TRAVIS_GO_VERSION
-      stage: test
-
     # Kubernetes
     - os: linux
       install: deploy/kubernetes/.travis/setup.sh
@@ -199,7 +193,6 @@ jobs:
             retries: true
           update: true
           packages:
-            - python-virtualenv
             - libpcap-dev
             - xsltproc
             - libxml2-utils
@@ -218,16 +211,22 @@ addons:
     config:
       retries: true
     update: true
+    sources:
+      - deadsnakes
     packages:
-      - python-virtualenv
       - libpcap-dev
       - xsltproc
       - libxml2-utils
       - libsystemd-journal-dev
       - librpm-dev
+      # From deadsnakes PPA
+      - python3.6
+      - python3.6-venv
 
 before_install:
+  - if [ x$TRAVIS_DIST = xtrusty ]; then sudo ln -sf python3.6 /usr/bin/python3; fi
   - python --version
+  - python3 --version
   - umask 022
   - chmod -R go-w $GOPATH/src/github.com/elastic/beats
   # Docker-compose installation

Makefile

@@ -4,7 +4,9 @@ BEATS?=auditbeat filebeat heartbeat journalbeat metricbeat packetbeat winlogbeat
 PROJECTS=libbeat $(BEATS)
 PROJECTS_ENV=libbeat filebeat metricbeat
 PYTHON_ENV?=$(BUILD_DIR)/python-env
-VIRTUALENV_PARAMS?=
+PYTHON_EXE?=python3
+PYTHON_ENV_EXE=${PYTHON_ENV}/bin/$(notdir ${PYTHON_EXE})
+VENV_PARAMS?=
 FIND=find . -type f -not -path "*/vendor/*" -not -path "*/build/*" -not -path "*/.git/*"
 GOLINT=golint
 GOLINT_REPO=golang.org/x/lint/golint
@@ -88,8 +90,8 @@ clean-vendor:
 .PHONY: check
 check: python-env
 	@$(foreach var,$(PROJECTS) dev-tools $(PROJECTS_XPACK_MAGE),$(MAKE) -C $(var) check || exit 1;)
-	@# Checks also python files which are not part of the beats
-	@$(FIND) -name *.py -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120  {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false)
+	@$(FIND) -name *.py -name *.py -not -path "*/build/*" -not -path "*/vendor/*" -exec $(PYTHON_ENV)/bin/autopep8 -d --max-line-length 120  {} \; | (! grep . -q) || (echo "Code differs from autopep8's style" && false)
+	@$(FIND) -name *.py -not -path "*/build/*" -not -path "*/vendor/*" | xargs $(PYTHON_ENV)/bin/pylint --py3k -E || (echo "Code is not compatible with Python 3" && false)
 	@# Validate that all updates were committed
 	@$(MAKE) update
 	@$(MAKE) check-headers
@@ -136,13 +138,13 @@ docs:
 .PHONY: notice
 notice: python-env
 	@echo "Generating NOTICE"
-	@$(PYTHON_ENV)/bin/python dev-tools/generate_notice.py .
+	@${PYTHON_ENV_EXE} dev-tools/generate_notice.py .
 
 # Sets up the virtual python environment
 .PHONY: python-env
 python-env:
-	@test -d $(PYTHON_ENV) || virtualenv $(VIRTUALENV_PARAMS) $(PYTHON_ENV)
-	@$(PYTHON_ENV)/bin/pip install -q --upgrade pip autopep8==1.3.5 six
+	@test -d $(PYTHON_ENV) || ${PYTHON_EXE} -m venv $(VENV_PARAMS) $(PYTHON_ENV)
+	@$(PYTHON_ENV)/bin/pip install -q --upgrade pip autopep8==1.3.5 pylint==2.4.4
 	@# Work around pip bug. See: https://github.com/pypa/pip/issues/4464
 	@find $(PYTHON_ENV) -type d -name dist-packages -exec sh -c "echo dist-packages > {}.pth" ';'
 

Vagrantfile

@@ -24,8 +24,8 @@
 # freebsd and openbsd
 # -------------------
 #   - Use gmake instead of make.
-#   - Folder syncing doesn't work well. Consider copying the files into the box or
-#     cloning the project inside the box.
+#   - Folder syncing doesn't work well. Consider copying the files into the box
+#     or cloning the project inside the box.
 ###
 
 # Read the branch's Go version from the .go-version file.
@@ -82,27 +82,14 @@ if (-Not (Get-Command "choco" -ErrorAction SilentlyContinue)) {
 choco feature disable -n=showDownloadProgress
 
 if (-Not (Get-Command "python" -ErrorAction SilentlyContinue)) {
-    echo "Installing python2"
-    choco install python2 -y -r
+    echo "Installing python 3"
+    choco install python -y -r --version 3.8.1.20200110
     refreshenv
-    $env:PATH = "$env:PATH;C:\\Python27;C:\\Python27\\Scripts"
+    $env:PATH = "$env:PATH;C:\\Python38;C:\\Python38\\Scripts"
 }
 
-if (-Not (Get-Command "pip" -ErrorAction SilentlyContinue)) {
-    echo "Installing pip"
-    Invoke-WebRequest https://bootstrap.pypa.io/get-pip.py -OutFile get-pip.py
-    python get-pip.py -U --force-reinstall 2>&1 | %{ "$_" }
-    rm get-pip.py
-    Invoke-WebRequest
-} else {
-    echo "Updating pip"
-    python -m pip install --upgrade pip 2>&1 | %{ "$_" }
-}
-
-if (-Not (Get-Command "virtualenv" -ErrorAction SilentlyContinue)) {
-    echo "Installing virtualenv"
-    python -m pip install virtualenv 2>&1 | %{ "$_" }
-}
+echo "Updating pip"
+python -m pip install --upgrade pip 2>&1 | %{ "$_" }
 
 if (-Not (Get-Command "git" -ErrorAction SilentlyContinue)) {
     echo "Installing git"
@@ -113,6 +100,9 @@ if (-Not (Get-Command "gcc" -ErrorAction SilentlyContinue)) {
     echo "Installing mingw (gcc)"
     choco install mingw -y -r
 }
+
+echo "Setting PYTHON_ENV in VM to point to C:\\beats-python-env."
+[System.Environment]::SetEnvironmentVariable("PYTHON_ENV", "C:\\beats-python-env", [System.EnvironmentVariableTarget]::Machine)
 SCRIPT
 
 # Provisioning for Unix/Linux
@@ -129,7 +119,7 @@ def linuxGvmProvision(arch="amd64")
   return <<SCRIPT
 mkdir -p ~/bin
 if [ ! -e "~/bin/gvm" ]; then
-  curl -sL -o ~/bin/gvm https://github.com/andrewkroh/gvm/releases/download/v0.1.0/gvm-linux-#{arch}
+  curl -sL -o ~/bin/gvm https://github.com/andrewkroh/gvm/releases/download/v0.2.1/gvm-linux-#{arch}
   chmod +x ~/bin/gvm
   ~/bin/gvm #{GO_VERSION}
   echo 'export GOPATH=$HOME/go' >> ~/.bash_profile
@@ -145,7 +135,7 @@ def linuxDebianProvision()
 #!/usr/bin/env bash
 set -eio pipefail
 apt-get update
-apt-get install -y make gcc python-pip python-virtualenv git
+apt-get install -y make gcc python3 python3-pip python3-venv git
 SCRIPT
 end
 
@@ -245,7 +235,8 @@ Vagrant.configure(2) do |config|
 
     c.vm.provision "shell", inline: $unixProvision, privileged: false
     c.vm.provision "shell", inline: linuxGvmProvision, privileged: false
-    c.vm.provision "shell", inline: "yum install -y make gcc python-pip python-virtualenv git rpm-devel"
+    c.vm.provision "shell", inline: "yum install -y make gcc git rpm-devel epel-release"
+    c.vm.provision "shell", inline: "yum install -y python34 python34-pip"
   end
 
   config.vm.define "centos7", primary: true do |c|
@@ -254,25 +245,16 @@ Vagrant.configure(2) do |config|
 
     c.vm.provision "shell", inline: $unixProvision, privileged: false
     c.vm.provision "shell", inline: linuxGvmProvision, privileged: false
-    c.vm.provision "shell", inline: "yum install -y make gcc python-pip python-virtualenv git rpm-devel"
+    c.vm.provision "shell", inline: "yum install -y make gcc python3 python3-pip git rpm-devel"
   end
 
-  config.vm.define "fedora29", primary: true do |c|
-    c.vm.box = "bento/fedora-29"
+  config.vm.define "fedora31", primary: true do |c|
+    c.vm.box = "bento/fedora-31"
     c.vm.network :forwarded_port, guest: 22, host: 2231, id: "ssh", auto_correct: true
 
     c.vm.provision "shell", inline: $unixProvision, privileged: false
     c.vm.provision "shell", inline: linuxGvmProvision, privileged: false
-    c.vm.provision "shell", inline: "dnf install -y make gcc python-pip python-virtualenv git rpm-devel"
-  end
-
-  config.vm.define "sles12", primary: true do |c|
-    c.vm.box = "elastic/sles-12-x86_64"
-    c.vm.network :forwarded_port, guest: 22, host: 2232, id: "ssh", auto_correct: true
-
-    c.vm.provision "shell", inline: $unixProvision, privileged: false
-    c.vm.provision "shell", inline: linuxGvmProvision, privileged: false
-    c.vm.provision "shell", inline: "pip install virtualenv"
+    c.vm.provision "shell", inline: "dnf install -y make gcc python3 python3-pip git rpm-devel"
   end
 
   config.vm.define "archlinux", primary: true do |c|
@@ -281,6 +263,6 @@ Vagrant.configure(2) do |config|
 
     c.vm.provision "shell", inline: $unixProvision, privileged: false
     c.vm.provision "shell", inline: linuxGvmProvision, privileged: false
-    c.vm.provision "shell", inline: "pacman -Sy && pacman -S --noconfirm make gcc python-pip python-virtualenv git"
+    c.vm.provision "shell", inline: "pacman -Sy && pacman -S --noconfirm make gcc python python-pip git"
   end
 end

auditbeat/Dockerfile

@@ -3,11 +3,14 @@ FROM golang:1.13.8
 RUN \
     apt-get update \
       && apt-get install -y --no-install-recommends \
-         python-pip \
-         virtualenv \
+         python3 \
+         python3-pip \
+         python3-venv \
          librpm-dev \
       && rm -rf /var/lib/apt/lists/*
 
-RUN pip install --upgrade pip
-RUN pip install --upgrade setuptools
-RUN pip install --upgrade docker-compose==1.23.2
+ENV PYTHON_ENV=/tmp/python-env
+
+RUN pip3 install --upgrade pip
+RUN pip3 install --upgrade setuptools
+RUN pip3 install --upgrade docker-compose==1.23.2

auditbeat/docs/fields.asciidoc

@@ -2949,7 +2949,7 @@ Example: `docker` and `k8s` labels.
 
 type: object
 
-example: {'application': 'foo-bar', 'env': 'production'}
+example: { "application": "foo-bar", "env": "production" }
 
 --
 

auditbeat/tests/system/auditbeat.py

@@ -24,7 +24,7 @@ def setUpClass(self):
 
     def create_file(self, path, contents):
         f = open(path, 'wb')
-        f.write(contents)
+        f.write(bytes(contents, "utf-8"))
         f.close()
 
     def check_event(self, event, expected):

auditbeat/tests/system/test_show_command.py

@@ -73,16 +73,16 @@ def test_show_auditd_rules(self):
         os.unlink(output_file)
         assert len(lines) >= len(rules)
         # get rid of automatic rule
-        if '-F key=rule' not in lines[0]:
+        if b'-F key=rule' not in lines[0]:
             del lines[0]
 
         for i in range(len(rules)):
             expected = rules[i]
             got = lines[i].strip()
-            assert expected == got, \
+            assert expected == got.decode("utf-8"), \
                 "rule {0} doesn't match. expected='{1}' got='{2}'".format(
-                    i, expected, got
-                )
+                    i, expected, got.decode("utf-8")
+            )
 
     @unittest.skipUnless(is_root(), "Requires root")
     def test_show_auditd_status(self):
@@ -107,7 +107,7 @@ def test_show_auditd_status(self):
         self.run_beat(extra_args=['show', 'auditd-status'],
                       exit_code=0,
                       output=output_file)
-        fhandle = os.fdopen(fd, 'rb')
+        fhandle = os.fdopen(fd, 'r')
         lines = fhandle.readlines()
         fhandle.close()
         os.unlink(output_file)
@@ -122,5 +122,5 @@ def test_show_auditd_status(self):
             assert n >= 0, "Field '{0}' has negative value {1}".format(k, v)
             fields[k] = True
 
-        for (k, v) in fields.iteritems():
+        for (k, v) in fields.items():
             assert v, "Field {0} not found".format(k)

dev-tools/aggregate_coverage.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 """Simple script to concatenate coverage reports.
 """

dev-tools/cherrypick_pr

@@ -1,6 +1,8 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 """Cherry pick and backport a PR"""
+from __future__ import print_function
 
+from builtins import input
 import sys
 import os
 import argparse
@@ -45,7 +47,7 @@ def main():
         formatter_class=argparse.RawDescriptionHelpFormatter,
         epilog=usage)
     parser.add_argument("to_branch",
-                        help="To branch (e.g 5.0)")
+                        help="To branch (e.g 7.x)")
     parser.add_argument("pr_number",
                         help="The PR number being merged (e.g. 2345)")
     parser.add_argument("commit_hashes", metavar="hash", nargs="+",
@@ -75,7 +77,7 @@ def main():
     tmp_branch = "backport_{}_{}".format(args.pr_number, args.to_branch)
 
     if not vars(args)["continue"]:
-        if not args.yes and raw_input("This will destroy all local changes. " +
+        if not args.yes and input("This will destroy all local changes. " +
                                       "Continue? [y/n]: ") != "y":
             return 1
         check_call("git reset --hard", shell=True)
@@ -110,15 +112,16 @@ def main():
 
     if args.diff:
         call("git diff {}".format(args.to_branch), shell=True)
-        if raw_input("Continue? [y/n]: ") != "y":
+        if input("Continue? [y/n]: ") != "y":
             print("Aborting cherry-pick.")
             return 1
 
     print("Ready to push branch.")
 
     remote = args.remote
     if not remote:
-        remote = raw_input("To which remote should I push? (your fork): ")
+        remote = input("To which remote should I push? (your fork): ")
+
     call("git push {} :{} > /dev/null".format(remote, tmp_branch),
          shell=True)
     check_call("git push --set-upstream {} {}"
@@ -138,15 +141,15 @@ def main():
         # get the github username from the remote where we pushed
         remote_url = check_output("git remote get-url {}".format(remote),
                                   shell=True)
-        remote_user = re.search("github.com[:/](.+)/beats", remote_url).group(1)
+        remote_user = re.search("github.com[:/](.+)/beats", str(remote_url)).group(1)
 
         # create PR
         request = session.post(base + "/pulls", json=dict(
-            title="Cherry-pick #{} to {}: {}".format(args.pr_number, args.to_branch, original_pr["title"].encode('utf-8')),
+            title="Cherry-pick #{} to {}: {}".format(args.pr_number, args.to_branch, original_pr["title"]),
             head=remote_user + ":" + tmp_branch,
             base=args.to_branch,
             body="Cherry-pick of PR #{} to {} branch. Original message: \n\n{}"
-            .format(args.pr_number, args.to_branch, original_pr["body"].encode('utf-8'))
+            .format(args.pr_number, args.to_branch, original_pr["body"])
         ))
         if request.status_code > 299:
             print("Creating PR failed: {}".format(request.json()))

dev-tools/deploy

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import os
 import argparse
 from subprocess import check_call

dev-tools/generate_notice.py

@@ -1,5 +1,3 @@
-from __future__ import print_function
-
 import glob
 import os
 import datetime
@@ -332,7 +330,7 @@ def detect_license_summary(content):
     # replace all white spaces with a single space
     content = re.sub(r"\s+", ' ', content)
     # replace smart quotes with less intelligent ones
-    content = content.replace(b'\xe2\x80\x9c', '"').replace(b'\xe2\x80\x9d', '"')
+    content = content.replace(bytes(b'\xe2\x80\x9c').decode(), '"').replace(bytes(b'\xe2\x80\x9d').decode(), '"')
     if any(sentence in content[0:1000] for sentence in APACHE2_LICENSE_TITLES):
         return "Apache-2.0"
     if any(sentence in content[0:1000] for sentence in MIT_LICENSES):

dev-tools/get_version

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 import os
 import re
 import argparse

dev-tools/jenkins_ci.ps1

@@ -64,6 +64,11 @@ $packages = ($packages|group|Select -ExpandProperty Name) -join ","
 exec { go test -race -c -cover -covermode=atomic -coverpkg $packages } "go test -race -cover FAILURE"
 
 if (Test-Path "tests\system") {
-    Set-Location -Path tests\system
-    exec { nosetests --with-timer --with-xunit --xunit-file=../../build/TEST-system.xml } "System test FAILURE"
+    echo "Running python tests"
+    choco install python -y -r --no-progress --version 3.8.1.20200110
+    refreshenv
+    $env:PATH = "C:\Python38;C:\Python38\Scripts;$env:PATH"
+    $env:PYTHON_ENV = "$env:TEMP\python-env"
+    python --version
+    exec { mage pythonUnitTest } "System test FAILURE"
 }