[CI] Pipeline 2.0 for monorepos

.ci/scripts/generate_build_table.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+import os
+import yaml
+
+if __name__ == "__main__":
+
+    print("| Beat  | Stage  | Command  | MODULE  | Platforms  | When |")
+    print("|-------|--------|----------|---------|------------|------|")
+    for root, dirs, files in os.walk("."):
+        dirs.sort()
+        for file in files:
+            if file.endswith("Jenkinsfile.yml") and root != ".":
+                with open(os.path.join(root, file), 'r') as f:
+                    doc = yaml.load(f, Loader=yaml.FullLoader)
+                module = root.replace(".{}".format(os.sep), '')
+                platforms = [doc["platform"]]
+                when = ""
+                if "branches" in doc["when"]:
+                    when = f"{when}/:palm_tree:"
+                if "changeset" in doc["when"]:
+                    when = f"{when}/:file_folder:"
+                if "comments" in doc["when"]:
+                    when = f"{when}/:speech_balloon:"
+                if "labels" in doc["when"]:
+                    when = f"{when}/:label:"
+                if "parameters" in doc["when"]:
+                    when = f"{when}/:smiley:"
+                if "tags" in doc["when"]:
+                    when = f"{when}/:taco:"
+                for stage in doc["stages"]:
+                    withModule = False
+                    if "make" in doc["stages"][stage]:
+                        command = doc["stages"][stage]["make"]
+                    if "mage" in doc["stages"][stage]:
+                        command = doc["stages"][stage]["mage"]
+                    if "platforms" in doc["stages"][stage]:
+                        platforms = doc["stages"][stage]["platforms"]
+                    if "withModule" in doc["stages"][stage]:
+                        withModule = doc["stages"][stage]["withModule"]
+                    if "when" in doc["stages"][stage]:
+                        when = f"{when}/:star:"
+                    print("| {} | {} | `{}` | {} | `{}` | {} |".format(
+                        module, stage, command, withModule, platforms, when))
+
+print("> :palm_tree: -> Git Branch based")
+print("> :label: -> GitHub Pull Request Label based")
+print("> :file_folder: -> Changeset based")
+print("> :speech_balloon: -> GitHub Pull Request comment based")
+print("> :taco: -> Git tag based")
+print("> :smiley: -> Manual UI interaction based")
+print("> :star: -> More specific cases based")

.ci/scripts/get-vendor-dependencies.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+#
+# Given the go module it will list all the dependencies that will be later on
+# used by the CI to enable/disable specific stages as long as the changeset
+# matches any of those patterns.
+#
+
+GO_VERSION=${GO_VERSION:?"GO_VERSION environment variable is not set"}
+BEATS=${1:?"parameter missing."}
+eval "$(gvm "${GO_VERSION}")"
+
+go list -deps ./"${BEATS}" \
+| grep 'elastic/beats' \
+| sort \
+| sed -e "s#github.com/elastic/beats/v7/##g" \
+| awk '{print "^" $1 "/.*"}'

.ci/scripts/install-tools.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+.ci/scripts/install-go.sh
+.ci/scripts/install-docker-compose.sh
+.ci/scripts/install-terraform.sh
+make mage

Jenkinsfile.yml

@@ -0,0 +1,47 @@
+projects:
+    - "auditbeat"
+    - "deploy/kubernetes"
+    - "filebeat"
+    - "generator"
+    - "heartbeat"
+    - "journalbeat"
+    - "libbeat"
+    - "metricbeat"
+    - "packetbeat"
+    - "winlogbeat"
+    - "x-pack/auditbeat"
+    - "x-pack/dockerlogbeat"
+    - "x-pack/elastic-agent"
+    - "x-pack/filebeat"
+    - "x-pack/functionbeat"
+    - "x-pack/libbeat"
+    - "x-pack/metricbeat"
+    - "x-pack/winlogbeat"
+    ##- "x-pack/heartbeat"    It's not yet in the 1.0 pipeline.
+    ##- "x-pack/journalbeat"  It's not yet in the 1.0 pipeline.
+    ##- "x-pack/packetbeat"   It's not yet in the 1.0 pipeline.
+
+## Changeset macros that are defined here and used in each specific 2.0 pipeline.
+changeset:
+    ci:
+        - "^Jenkinsfile"
+        - "^\\.ci/scripts/.*"
+    oss:
+        - "^go.mod"
+        - "^dev-tools/.*"
+        - "^libbeat/.*"
+        - "^testing/.*"
+    xpack:
+        - "^go.mod"
+        - "^dev-tools/.*"
+        - "^libbeat/.*"
+        - "^testing/.*"
+        - "^x-pack/libbeat/.*"
+
+## Proposal
+## TBC: This will allow to configure what to do based on the PR configuration
+disabled:
+    when:
+        labels:      ## Skip the GitHub Pull Request builds if there is a GitHub label match
+            - "skip-ci"
+        draft: true  ## Skip the GitHub Pull Request builds with Draft PRs.

README.md

@@ -89,17 +89,28 @@ For testing purposes, we generate snapshot builds that you can find [here](https
 
 ## CI
 
+### PR Comments
+
 It is possible to trigger some jobs by putting a comment on a GitHub PR.
 (This service is only available for users affiliated with Elastic and not for open-source contributors.)
 
 * [beats][]
   * `jenkins run the tests please` or `jenkins run tests` or `/test` will kick off a default build.
   * `/test macos` will kick off a default build with also the `macos` stages.
+  * `/test <beat-name>` will kick off the default build for the given PR in addition to the `<beat-name>` build itself.
+  * `/test <beat-name> for macos` will kick off a default build with also the `macos` stage for the `<beat-name>`.
 * [apm-beats-update][]
   * `/run apm-beats-update`
 * [apm-beats-packaging][]
   * `/package` or `/packaging` will kick of a build to generate the packages for beats.
 
+### PR Labels
+
+It's possible to configure the build on a GitHub PR by labelling the PR with the below labels
+
+* `<beat-name>` to force the following builds to run the stages for the `<beat-name>`
+* `macOS` to force the following builds to run the `macos` stages.
+
 [beats]: https://beats-ci.elastic.co/job/Beats/job/beats/
 [apm-beats-update]: https://beats-ci.elastic.co/job/Beats/job/apm-beats-update/
 [apm-beats-packaging]: https://beats-ci.elastic.co/job/Beats/job/packaging/

auditbeat/Jenkinsfile.yml

@@ -0,0 +1,34 @@
+when:
+    branches: true             ## for all the branches
+    changeset:                 ## when PR contains any of those entries in the changeset
+        - "^auditbeat/.*"
+        - "@ci"                ## special token regarding the changeset for the ci
+        - "@oss"               ## special token regarding the changeset for the oss
+    comments:                  ## when PR comment contains any of those entries
+        - "/test auditbeat"
+    labels:                    ## when PR labels matche any of those entries
+        - "auditbeat"
+    parameters:                ## when parameter was selected in the UI.
+        - "auditbeat"
+    tags: true                 ## for all the tags
+platform: "linux && ubuntu-18" ## default label for all the stages
+stages:
+    build:
+        mage: "mage build test"
+    crosscompile: 
+        make: "make -C auditbeat crosscompile"
+    macos:
+        mage: "mage build unitTest"
+        platforms:             ## override default label in this specific stage.
+          - "macosx"
+        when:                  ## Aggregate when with the top-level one.
+            comments:
+                - "/test auditbeat for macos"
+            labels:
+                - "macOS"
+            parameters:
+                - "macosTest"
+    windows:
+        mage: "mage build unitTest"
+        platforms:             ## override default labels in this specific stage.
+            - "windows-2019"

deploy/kubernetes/Jenkinsfile.yml

@@ -0,0 +1,15 @@
+when:
+    branches: true             ## for all the branches
+    changeset:                 ## when PR contains any of those entries in the changeset
+        - "^deploy/kubernetes/.*"
+    comments:                  ## when PR comment contains any of those entries
+        - "/test deploy/kubernetes"
+    labels:                    ## when PR labels matche any of those entries
+        - "kubernetes"
+    parameters:                ## when parameter was selected in the UI.
+        - "kubernetes"
+    tags: true                 ## for all the tags
+platform: "linux && ubuntu-18" ## default label for all the stages
+stages:
+    k8sTest:
+        k8sTest: "v1.18.2,v1.17.2,v1.16.4,v1.15.7,v1.14.10"

filebeat/Jenkinsfile.yml

@@ -0,0 +1,33 @@
+when:
+    branches: true             ## for all the branches
+    changeset:                 ## when PR contains any of those entries in the changeset
+        - "^filebeat/.*"
+        - "@ci"                ## special token regarding the changeset for the ci
+        - "@oss"               ## special token regarding the changeset for the oss
+    comments:                  ## when PR comment contains any of those entries
+        - "/test filebeat"
+    labels:                    ## when PR labels matche any of those entries
+        - "filebeat"
+    parameters:                ## when parameter was selected in the UI.
+        - "filebeat"
+    tags: true                 ## for all the tags
+platform: "linux && ubuntu-18" ## default label for all the stages
+stages:
+    build:
+        mage: "mage build test"
+        withModule: true       ## run the ITs only if the changeset affects a specific module.
+    macos:
+        mage: "mage build unitTest"
+        platforms:             ## override default label in this specific stage.
+          - "macosx"
+        when:                  ## Aggregate when with the top-level one.
+            comments:
+                - "/test filebeat for macos"
+            labels:
+                - "macOS"
+            parameters:
+                - "macosTest"
+    windows:
+        mage: "mage build unitTest"
+        platforms:             ## override default labels in this specific stage.
+            - "windows-2019"

generator/Jenkinsfile.yml

@@ -0,0 +1,43 @@
+when:
+    branches: true             ## for all the branches
+    changeset:                 ## when PR contains any of those entries in the changeset
+        - "^generator/.*"
+        - "#generator/common/beatgen" ## special token regarding the project dependency
+        - "#metricbeat/beater" ## special token regarding the project dependency
+        - "@ci"                ## special token regarding the changeset for the ci
+        - "@oss"               ## special token regarding the changeset for the oss
+    comments:                  ## when PR comment contains any of those entries
+        - "/test generator"
+    labels:                    ## when PR labels matche any of those entries
+        - "generator"
+    parameters:                ## when parameter was selected in the UI.
+        - "generator"
+    tags: true                 ## for all the tags
+platform: "linux && ubuntu-18" ## default label for all the stages
+stages:
+    metricbeat-test:
+        make: "make -C generator/_templates/metricbeat test test-package"
+    beat-test:
+        make: "make -C generator/_templates/beat test test-package"
+    macos-metricbeat:
+        make: "make -C generator/_templates/metricbeat test"
+        platforms:             ## override default label in this specific stage.
+          - "macosx"
+        when:                  ## Aggregate when with the top-level one.
+            comments:
+                - "/test generator for macos"
+            labels:
+                - "macOS"
+            parameters:
+                - "macosTest"
+    macos-beat:
+        make: "make -C generator/_templates/beat test"
+        platforms:             ## override default label in this specific stage.
+            - "macosx"
+        when:                  ## Aggregate when with the top-level one.
+            comments:
+                - "/test generator for macos"
+            labels:
+                - "macOS"
+            parameters:
+                - "macosTest"

heartbeat/Jenkinsfile.yml

@@ -0,0 +1,32 @@
+when:
+    branches: true             ## for all the branches
+    changeset:                 ## when PR contains any of those entries in the changeset
+        - "^heartbeat/.*"
+        - "@ci"                ## special token regarding the changeset for the ci
+        - "@oss"               ## special token regarding the changeset for the oss
+    comments:                  ## when PR comment contains any of those entries
+        - "/test heartbeat"
+    labels:                    ## when PR labels matche any of those entries
+        - "heartbeat"
+    parameters:                ## when parameter was selected in the UI.
+        - "heartbeat"
+    tags: true                 ## for all the tags
+platform: "linux && ubuntu-18" ## default label for all the stages
+stages:
+    build:
+        mage: "mage build test"
+    macos:
+        mage: "mage build unitTest"
+        platforms:             ## override default label in this specific stage.
+          - "macosx"
+        when:                  ## Aggregate when with the top-level one.
+            comments:
+                - "/test heartbeat for macos"
+            labels:
+                - "macOS"
+            parameters:
+                - "macosTest"
+    windows:
+        mage: "mage build unitTest"
+        platforms:             ## override default labels in this specific stage.
+            - "windows-2019"

journalbeat/Jenkinsfile.yml

@@ -0,0 +1,17 @@
+when:
+    branches: true             ## for all the branches
+    changeset:                 ## when PR contains any of those entries in the changeset
+        - "^journalbeat/.*"
+        - "@ci"                ## special token regarding the changeset for the ci
+        - "@oss"               ## special token regarding the changeset for the oss
+    comments:                  ## when PR comment contains any of those entries
+        - "/test journalbeat"
+    labels:                    ## when PR labels matche any of those entries
+        - "journalbeat"
+    parameters:                ## when parameter was selected in the UI.
+        - "journalbeat"
+    tags: true                 ## for all the tags
+platform: "linux && ubuntu-18" ## default label for all the stages
+stages:
+    unitTest:
+        mage: "mage build unitTest"

libbeat/Jenkinsfile.yml

@@ -0,0 +1,20 @@
+when:
+    branches: true             ## for all the branches
+    changeset:                 ## when PR contains any of those entries in the changeset
+        - "@ci"                ## special token regarding the changeset for the ci
+        - "@oss"               ## special token regarding the changeset for the oss
+    comments:                  ## when PR comment contains any of those entries
+        - "/test libbeat"
+    labels:                    ## when PR labels matche any of those entries
+        - "libbeat"
+    parameters:                ## when parameter was selected in the UI.
+        - "libbeat"
+    tags: true                 ## for all the tags
+platform: "linux && ubuntu-18" ## default label for all the stages
+stages:
+    build:
+        mage: "mage build test"
+    crosscompile:
+        make: "make -C libbeat crosscompile"
+    stress-tests:
+        make: "make STRESS_TEST_OPTIONS='-timeout=20m -race -v -parallel 1' -C libbeat stress-tests"

metricbeat/Jenkinsfile.yml

@@ -0,0 +1,40 @@
+when:
+    branches: true             ## for all the branches
+    changeset:                 ## when PR contains any of those entries in the changeset
+        - "^metricbeat/.*"
+        - "@ci"                ## special token regarding the changeset for the ci
+        - "@oss"               ## special token regarding the changeset for the oss
+    comments:                  ## when PR comment contains any of those entries
+        - "/test metricbeat"
+    labels:                    ## when PR labels matche any of those entries
+        - "metricbeat"
+    parameters:                ## when parameter was selected in the UI.
+        - "metricbeat"
+    tags: true                 ## for all the tags
+platform: "linux && ubuntu-18" ## default label for all the stages
+stages:
+    unitTest:
+        mage: "mage build unitTest"
+    goIntegTest:
+        mage: "mage goIntegTest"
+        withModule: true
+    pythonIntegTest:
+        mage: "mage pythonIntegTest"
+        withModule: true
+    crosscompile:
+        make: "make -C metricbeat crosscompile"
+    macos:
+        mage: "mage build unitTest"
+        platforms:             ## override default label in this specific stage.
+          - "macosx"
+        when:                  ## Aggregate when with the top-level one.
+            comments:
+                - "/test metricbeat for macos"
+            labels:
+                - "macOS"
+            parameters:
+                - "macosTest"
+    windows:
+        mage: "mage build unitTest"
+        platforms:             ## override default labels in this specific stage.
+            - "windows-2019"