chore: standardise how unit/integration tests is run on CI, adding a build command to compress the system-tests files

Jenkinsfile

@@ -664,7 +664,7 @@ def withBeatsEnv(Map args = [:], Closure body) {
         error("Error '${err.toString()}'")
       } finally {
         if (archive) {
-          archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: upload)
+          archiveTestOutput(directory: directory, testResults: testResults, artifacts: artifacts, id: args.id, upload: upload)
         }
         tearDown()
       }
@@ -752,6 +752,8 @@ def getCommonModuleInTheChangeSet(String directory) {
 * to bypass some issues when working with big repositories.
 */
 def archiveTestOutput(Map args = [:]) {
+  def directory = args.get('directory', '')
+
   catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
     if (isUnix()) {
       fixPermissions("${WORKSPACE}")
@@ -775,13 +777,19 @@ def archiveTestOutput(Map args = [:]) {
     }
     if (args.upload) {
       catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') {
-        def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim()
-        log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball")
-        if (folder.trim()) {
-          // TODO: nodeOS() should support ARM
-          def os_suffix = isArm() ? 'linux' : nodeOS()
-          def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + os_suffix
-          tarAndUploadArtifacts(file: "${name}.tgz", location: folder)
+        withMageEnv(version: "${env.GO_VERSION}"){
+          dir(directory){
+            cmd(label: "Archive system tests files", script: 'mage packageSystemTests')
+          }
+        }
+        def fileName = 'build/system-tests-*.tar.gz' // see dev-tools/mage/pkg.go#PackageSystemTests method
+        dir("${BASE_DIR}"){
+          googleStorageUploadExt(
+            bucket: "gs://${JOB_GCS_BUCKET}/${env.JOB_NAME}-${env.BUILD_ID}",
+            credentialsId: "${JOB_GCS_EXT_CREDENTIALS}",
+            pattern: "${fileName}",
+            sharedPublicly: true
+          )
         }
       }
     }

dev-tools/mage/common.go

@@ -353,6 +353,71 @@ func unzip(sourceFile, destinationDir string) error {
 	return nil
 }
 
+// Tar compress a directory using tar + gzip algorithms
+func Tar(src string, targetFile string) error {
+	fmt.Printf(">> creating TAR file from directory: %s, target: %s\n", src, targetFile)
+	var buf bytes.Buffer
+
+	// tar > gzip > buf
+	zr := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(zr)
+
+	// walk through every file in the folder
+	filepath.Walk(src, func(file string, fi os.FileInfo, errFn error) error {
+		// if a symlink, skip file
+		if fi.Mode().Type() == os.ModeSymlink {
+			fmt.Printf(">> skipping symlink: %s\n", file)
+			return nil
+		}
+
+		// generate tar header
+		header, err := tar.FileInfoHeader(fi, file)
+		if err != nil {
+			fmt.Printf(">> error getting file info header: %s\n", err)
+			return err
+		}
+
+		// must provide real name
+		// (see https://golang.org/src/archive/tar/common.go?#L626)
+		header.Name = filepath.ToSlash(file)
+
+		// write header
+		if err := tw.WriteHeader(header); err != nil {
+			fmt.Printf(">> error writing header: %s\n", err)
+			return err
+		}
+
+		// if not a dir, write file content
+		if !fi.IsDir() {
+			data, err := os.Open(file)
+			if err != nil {
+				fmt.Printf(">> error opening file: %s\n", err)
+				return err
+			}
+			defer data.Close()
+			if _, err := io.Copy(tw, data); err != nil {
+				fmt.Printf(">> error compressing file: %s\n", err)
+				return err
+			}
+		}
+		return nil
+	})
+
+	// produce tar
+	if err := tw.Close(); err != nil {
+		fmt.Printf(">> error closing tar file: %s\n", err)
+		return err
+	}
+	// produce gzip
+	if err := zr.Close(); err != nil {
+		fmt.Printf(">> error closing gzip file: %s\n", err)
+		return err
+	}
+
+	// write the .tar.gzip
+	return ioutil.WriteFile(targetFile, buf.Bytes(), os.FileMode(0600))
+}
+
 func untar(sourceFile, destinationDir string) error {
 	file, err := os.Open(sourceFile)
 	if err != nil {

dev-tools/mage/target/systemtest/python.go

@@ -0,0 +1,55 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package systemtest
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	devtools "github.com/elastic/beats/v7/dev-tools/mage"
+)
+
+// PackageSystemTests packages the python system tests results
+func PackageSystemTests() error {
+	excludeds := []string{".ci", ".git", ".github", "vendor", "dev-tools"}
+
+	// include run as it's the directory we want to compress
+	systemTestsDir := fmt.Sprintf("build%[1]csystem-tests%[1]crun", os.PathSeparator)
+	files, err := devtools.FindFilesRecursive(func(path string, _ os.FileInfo) bool {
+		base := filepath.Base(path)
+		for _, excluded := range excludeds {
+			if strings.HasPrefix(base, excluded) {
+				return false
+			}
+		}
+
+		return strings.HasPrefix(path, systemTestsDir)
+	})
+	if err != nil {
+		return err
+	}
+
+	if len(files) == 0 {
+		fmt.Printf(">> there are no system test files under %s", systemTestsDir)
+		return nil
+	}
+
+	return devtools.Tar(systemTestsDir, devtools.MustExpand("{{ elastic_beats_dir }}/build/system-tests-"+devtools.MustExpand("{{ repo.SubDir }}")+".tar.gz"))
+}

filebeat/Jenkinsfile.yml

@@ -35,9 +35,16 @@ stages:
             branches: true     ## for all the branches
             tags: true         ## for all the tags
         stage: extended
-    build:
-        mage: "mage build test"
-        withModule: true       ## run the ITs only if the changeset affects a specific module.
+    unitTest:
+        mage: "mage build unitTest"
+        stage: mandatory
+    goIntegTest:
+        mage: "mage goIntegTest"
+        withModule: true
+        stage: mandatory
+    pythonIntegTest:
+        mage: "mage pythonIntegTest"       ## run the ITs only if the changeset affects a specific module.
+        withModule: true
         stage: mandatory
     macos:
         mage: "mage build unitTest"

filebeat/magefile.go

@@ -34,6 +34,8 @@ import (
 	// mage:import generate
 	_ "github.com/elastic/beats/v7/filebeat/scripts/mage/generate"
 	// mage:import
+	_ "github.com/elastic/beats/v7/dev-tools/mage/target/systemtest"
+	// mage:import
 	_ "github.com/elastic/beats/v7/dev-tools/mage/target/unittest"
 	// mage:import
 	"github.com/elastic/beats/v7/dev-tools/mage/target/test"

heartbeat/Jenkinsfile.yml

@@ -35,8 +35,16 @@ stages:
             branches: true     ## for all the branches
             tags: true         ## for all the tags
         stage: extended
-    build:
-        mage: "mage build test"
+    unitTest:
+        mage: "mage build unitTest"
+        stage: mandatory
+    goIntegTest:
+        mage: "mage goIntegTest"
+        withModule: true
+        stage: mandatory
+    pythonIntegTest:
+        mage: "mage pythonIntegTest"
+        withModule: true
         stage: mandatory
     macos:
         mage: "mage build unitTest"

heartbeat/magefile.go

@@ -34,6 +34,8 @@ import (
 	// mage:import
 	"github.com/elastic/beats/v7/dev-tools/mage/target/build"
 	// mage:import
+	_ "github.com/elastic/beats/v7/dev-tools/mage/target/systemtest"
+	// mage:import
 	"github.com/elastic/beats/v7/dev-tools/mage/target/unittest"
 	// mage:import
 	"github.com/elastic/beats/v7/dev-tools/mage/target/integtest"

metricbeat/Jenkinsfile.yml

@@ -68,15 +68,15 @@ stages:
         platforms:             ## override default labels in this specific stage.
             - "windows-10"
         stage: extended
-    windows-8:
+    windows-2008:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
-            - "windows-8"
+            - "windows-2008-r2"
         stage: extended
-    windows-2008:
+    windows-8:
         mage: "mage build unitTest"
         platforms:             ## override default labels in this specific stage.
-            - "windows-2008-r2"
+            - "windows-8"
         stage: extended
     windows-7:
         mage: "mage build unitTest"

metricbeat/magefile.go

@@ -45,6 +45,8 @@ import (
 	// mage:import
 	_ "github.com/elastic/beats/v7/dev-tools/mage/target/pkg"
 	// mage:import
+	_ "github.com/elastic/beats/v7/dev-tools/mage/target/systemtest"
+	// mage:import
 	"github.com/elastic/beats/v7/dev-tools/mage/target/test"
 	// mage:import
 	"github.com/elastic/beats/v7/dev-tools/mage/target/unittest"

x-pack/filebeat/Jenkinsfile.yml

@@ -35,9 +35,16 @@ stages:
             branches: true     ## for all the branches
             tags: true         ## for all the tags
         stage: extended
-    build:
-        mage: "mage build test"
-        withModule: true       ## run the ITs only if the changeset affects a specific module.
+    unitTest:
+        mage: "mage build unitTest"
+        stage: mandatory
+    goIntegTest:
+        mage: "mage goIntegTest"
+        withModule: true
+        stage: mandatory
+    pythonIntegTest:
+        mage: "mage pythonIntegTest"       ## run the ITs only if the changeset affects a specific module.
+        withModule: true
         stage: mandatory
     macos:
         mage: "mage build unitTest"

x-pack/filebeat/magefile.go

@@ -24,6 +24,8 @@ import (
 	// mage:import
 	_ "github.com/elastic/beats/v7/dev-tools/mage/target/compose"
 	// mage:import
+	_ "github.com/elastic/beats/v7/dev-tools/mage/target/systemtest"
+	// mage:import
 	_ "github.com/elastic/beats/v7/dev-tools/mage/target/unittest"
 	// mage:import
 	"github.com/elastic/beats/v7/dev-tools/mage/target/test"

x-pack/metricbeat/Jenkinsfile.yml

@@ -24,6 +24,14 @@ stages:
     unitTest:
         mage: "mage build unitTest"
         stage: mandatory
+    goIntegTest:
+        mage: "mage goIntegTest"
+        withModule: true
+        stage: mandatory
+    pythonIntegTest:
+        mage: "mage pythonIntegTest"
+        withModule: true
+        stage: mandatory
     cloud:
         cloud: "mage build test"
         withModule: true       ## run the ITs only if the changeset affects a specific module.

x-pack/metricbeat/magefile.go

@@ -26,6 +26,8 @@ import (
 	// mage:import
 	_ "github.com/elastic/beats/v7/dev-tools/mage/target/compose"
 	// mage:import
+	_ "github.com/elastic/beats/v7/dev-tools/mage/target/systemtest"
+	// mage:import
 	"github.com/elastic/beats/v7/dev-tools/mage/target/test"
 	// mage:import
 	_ "github.com/elastic/beats/v7/metricbeat/scripts/mage/target/metricset"