Merge remote-tracking branch 'upstream/master' into feature/use-debian-9

* upstream/master: (430 commits)
  CI: increase timeout (elastic#26764)
  Heartbeat: add datastream fields to synthetics (elastic#26774)
  Osquerybeat: Change the query timeout from 3 secs to 60 secs (elastic#26775)
  Remove experimental warning for inputs with variables. (elastic#26762)
  Add latest k8s versions in testing (elastic#26729)
  change type of max_bytes to ByteType (elastic#26699)
  [Elastic Agent] Fix broken enrollment command (elastic#26749)
  Update agent managed manifest to include enrolment token variable (elastic#26756)
  Filebeat: Ensure module pipelines compatibility with previous versions of Elasticsearch (elastic#26737)
  Forward port changelog for 7.13.3 (elastic#26731) to master (elastic#26754)
  Upgrade PyYAML dependency used for tests (elastic#26746)
  Add agent fleet enrolment k8s manifest (elastic#26566)
  CI: retry the step only (elastic#26736)
  Osquerybeat: Fix the configuration poll interval setting (elastic#26739)
  [Filebeat] Replace copy_from with templated value (elastic#26631)
  Reduce the verbosity of the debug log for monitoring (elastic#26583)
  Add instructions on testing metricbeat kubernetes module (elastic#26643)
  Revert "[CI] fight the flakiness with some retry option in the CI only for the Pull Requests (elastic#26617)" (elastic#26704)
  Packaging: linux/armv7 is not supported (elastic#26706)
  Cyberarkpas: Link to official docs on how to setup TLS (elastic#26614)
  ...

.backportrc.json

@@ -1,6 +1,6 @@
 {
   "upstream": "elastic/beats",
-  "branches": [ { "name": "7.x", "checked": true }, "7.12", "7.11" ],
+  "branches": [ { "name": "7.x", "checked": true }, "7.14", "7.13", "7.12" ],
   "labels": ["backport"],
   "autoAssign": true,
   "prTitle": "Cherry-pick to {targetBranch}: {commitMessages}"

.ci/bump-go-release-version.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+#
+# Given the Golang release version this script will bump the version.
+#
+# This script is executed by the automation we are putting in place
+# and it requires the git add/commit commands.
+#
+# Parameters:
+#	$1 -> the Golang release version to be bumped. Mandatory.
+#
+set -euo pipefail
+MSG="parameter missing."
+GO_RELEASE_VERSION=${1:?$MSG}
+
+OS=$(uname -s| tr '[:upper:]' '[:lower:]')
+
+if [ "${OS}" == "darwin" ] ; then
+	SED="sed -i .bck"
+else
+	SED="sed -i"
+fi
+
+echo "Update go version ${GO_RELEASE_VERSION}"
+echo "${GO_RELEASE_VERSION}" > .go-version
+git add .go-version
+
+find . -maxdepth 3 -name Dockerfile -print0 |
+    while IFS= read -r -d '' line; do
+        ${SED} -E -e "s#(FROM golang):[0-9]+\.[0-9]+\.[0-9]+#\1:${GO_RELEASE_VERSION}#g" "$line"
+        ${SED} -E -e "s#(ARG GO_VERSION)=[0-9]+\.[0-9]+\.[0-9]+#\1=${GO_RELEASE_VERSION}#g" "$line"
+        git add "${line}"
+    done
+
+${SED} -E -e "s#(:go-version:) [0-9]+\.[0-9]+\.[0-9]+#\1 ${GO_RELEASE_VERSION}#g" libbeat/docs/version.asciidoc
+git add libbeat/docs/version.asciidoc
+
+git diff --staged --quiet || git commit -m "[Automation] Update go release version to ${GO_RELEASE_VERSION}"
+git --no-pager log -1
+
+echo "You can now push and create a Pull Request"

.ci/bump-stack-version.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+#
+# Given the stack version this script will bump the version.
+#
+# This script is executed by the automation we are putting in place
+# and it requires the git add/commit commands.
+#
+# Parameters:
+#	$1 -> the version to be bumped. Mandatory.
+#	$2 -> whether to create a branch where to commit the changes to.
+#		  this is required when reusing an existing Pull Request.
+#		  Optional. Default true.
+#
+set -euo pipefail
+MSG="parameter missing."
+VERSION=${1:?$MSG}
+CREATE_BRANCH=${2:-true}
+
+OS=$(uname -s| tr '[:upper:]' '[:lower:]')
+
+if [ "${OS}" == "darwin" ] ; then
+	SED="sed -i .bck"
+else
+	SED="sed -i"
+fi
+
+FILES="testing/environments/snapshot-oss.yml
+testing/environments/snapshot.yml
+"
+
+echo "Update stack with version ${VERSION}"
+for FILE in ${FILES} ; do
+	${SED} -E -e "s#(image: docker\.elastic\.co/.*):[0-9]+\.[0-9]+\.[0-9]+(-[a-f0-9]{8})?#\1:${VERSION}#g" $FILE
+done
+
+echo "Commit changes"
+if [ "$CREATE_BRANCH" = "true" ]; then
+	base=$(git rev-parse --abbrev-ref HEAD | sed 's#/#-#g')
+	git checkout -b "update-stack-version-$(date "+%Y%m%d%H%M%S")-${base}"
+else
+	echo "Branch creation disabled."
+fi
+for FILE in ${FILES} ; do
+	git add $FILE
+done
+git diff --staged --quiet || git commit -m "[Automation] Update elastic stack version to ${VERSION} for testing"
+git --no-pager log -1
+
+echo "You can now push and create a Pull Request"

.ci/packaging.groovy

@@ -138,7 +138,11 @@ pipeline {
                     'linux/amd64',
                     'linux/386',
                     'linux/arm64',
-                    'linux/armv7',
+                    // armv7 packaging isn't working, and we don't currently
+                    // need it for release. Do not re-enable it without
+                    // confirming it is fixed, you will break the packaging
+                    // pipeline!
+                    //'linux/armv7',
                     // The platforms above are disabled temporarly as crossbuild images are
                     // not available. See: https://github.com/elastic/golang-crossbuild/issues/71
                     //'linux/ppc64le',
@@ -425,11 +429,13 @@ def triggerE2ETests(String suite) {
 
   def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"
   def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}"
+  def beatVersion = "${env.BEAT_VERSION}-SNAPSHOT"
 
   def parameters = [
     booleanParam(name: 'forceSkipGitChecks', value: true),
     booleanParam(name: 'forceSkipPresubmit', value: true),
     booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
+    string(name: 'BEAT_VERSION', value: beatVersion),
     booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true),
     string(name: 'runTestsSuites', value: suite),
     string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),

.ci/packer_cache.sh

@@ -45,6 +45,8 @@ function dockerPullImages() {
   docker.elastic.co/kibana/kibana:${SNAPSHOT}
   docker.elastic.co/logstash/logstash:${SNAPSHOT}
   docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-arm
+  docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-armhf
+  docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-armel
   docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-base-arm-debian9
   docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-darwin
   docker.elastic.co/beats-dev/golang-crossbuild:${GO_VERSION}-main

.ci/scripts/generate_build_table.py

@@ -5,8 +5,8 @@
 
 if __name__ == "__main__":
 
-    print("| Beat  | Stage  | Command  | MODULE  | Platforms  | When |")
-    print("|-------|--------|----------|---------|------------|------|")
+    print("| Beat  | Stage  | Category | Command  | MODULE  | Platforms  | When |")
+    print("|-------|--------|----------|----------|---------|------------|------|")
     for root, dirs, files in os.walk("."):
         dirs.sort()
         for file in files:
@@ -18,6 +18,9 @@
                     withModule = False
                     platforms = [doc["platform"]]
                     when = "mandatory"
+                    category = 'default'
+                    if "stage" in doc["stages"][stage]:
+                        category = doc["stages"][stage]["stage"]
                     if "make" in doc["stages"][stage]:
                         command = doc["stages"][stage]["make"].replace("\n", " ")
                     if "mage" in doc["stages"][stage]:
@@ -33,5 +36,5 @@
                     if "when" in doc["stages"][stage]:
                         if "not_changeset_full_match" not in doc["stages"][stage]["when"]:
                             when = "optional"
-                    print("| {} | {} | `{}` | {} | `{}` | {} |".format(
-                        module, stage, command, withModule, platforms, when))
+                    print("| {} | {} | `{}` | `{}` | {} | `{}` | {} |".format(
+                        module, stage, category, command, withModule, platforms, when))

.ci/scripts/kind-setup.sh

@@ -1,5 +1,23 @@
 #!/usr/bin/env bash
 set -exuo pipefail
 
-kind create cluster --image kindest/node:${K8S_VERSION}
+kind create cluster --image kindest/node:${K8S_VERSION} --config - <<EOF
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+nodes:
+- role: control-plane
+  kubeadmConfigPatches:
+  - |
+    kind: ClusterConfiguration
+    scheduler:
+      extraArgs:
+        bind-address: "0.0.0.0"
+        port: "10251"
+        secure-port: "10259"
+    controllerManager:
+      extraArgs:
+        bind-address: "0.0.0.0"
+        port: "10252"
+        secure-port: "10257"
+EOF
 kubectl cluster-info

.gitignore

@@ -50,3 +50,6 @@ x-pack/elastic-agent/pkg/agent/operation/tests/scripts/serviceable-1.0-darwin-x8
 # Terraform
 *.terraform
 *.tfstate*
+
+# Files generated with the bump version automations
+*.bck

.go-version

@@ -1 +1 @@
-1.15.9
+1.16.5

.mergify.yml

@@ -1,9 +1,21 @@
 pull_request_rules:
+  - name: forward-port patches to master branch
+    conditions:
+      - merged
+      - label=backport-v8.0.0
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        branches:
+          - "master"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
   - name: backport patches to 7.x branch
     conditions:
       - merged
-      - base=master
-      - label=backport-v7.13.0
+      - label=backport-v7.15.0
     actions:
       backport:
         assignees:
@@ -12,32 +24,33 @@ pull_request_rules:
           - "7.x"
         labels:
           - "backport"
-  - name: backport patches to 7.12 branch
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
+  - name: backport patches to 7.13 branch
     conditions:
       - merged
-      - base=master
-      - label=backport-v7.12.0
+      - label=backport-v7.13.0
     actions:
       backport:
         assignees:
           - "{{ author }}"
         branches:
-          - "7.12"
+          - "7.13"
         labels:
           - "backport"
-  - name: backport patches to 7.11 branch
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
+  - name: backport patches to 7.12 branch
     conditions:
       - merged
-      - base=master
-      - label=backport-v7.11.0
+      - label=backport-v7.12.0
     actions:
       backport:
         assignees:
           - "{{ author }}"
         branches:
-          - "7.11"
+          - "7.12"
         labels:
           - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
   - name: ask to resolve conflict
     conditions:
       - conflict
@@ -52,3 +65,89 @@ pull_request_rules:
             git merge upstream/{{base}}
             git push upstream {{head}}
             ```
+  - name: automatic approval for mergify pull requests with bump updates
+    conditions:
+      - author=mergify[bot]
+      - check-success=beats-ci/pr-merge
+      - label=automation
+      - files~=^testing/environments/snapshot.*\.yml$
+    actions:
+      review:
+        type: APPROVE
+        message: Automatically approving mergify
+  - name: automatic squash and merge with success checks and the files matching the regex ^testing/environments/snapshot* are modified.
+    conditions:
+      - check-success=beats-ci/pr-merge
+      - label=automation
+      - files~=^testing/environments/snapshot.*\.yml$
+    actions:
+      merge:
+        method: squash
+        strict: smart+fasttrack
+  - name: delete upstream branch after merging changes on testing/environments/snapshot* or it's closed
+    conditions:
+      - or:
+        - merged
+        - closed
+      - and:
+        - label=automation
+        - head~=^update-stack-version
+        - files~=^testing/environments/snapshot.*\.yml$
+    actions:
+      delete_head_branch:
+  - name: delete upstream branch after merging changes on .go-version or it's closed
+    conditions:
+      - or:
+        - merged
+        - closed
+      - and:
+        - label=automation
+        - head~=^update-go-version
+        - files~=^\.go-version$
+    actions:
+      delete_head_branch:
+  - name: backport patches to 7.14 branch
+    conditions:
+      - merged
+      - label=backport-v7.14.0
+    actions:
+      backport:
+        assignees:
+          - "{{ author }}"
+        branches:
+          - "7.14"
+        labels:
+          - "backport"
+        title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
+  - name: automatic approval for mergify pull requests with changes in bump-rules
+    conditions:
+      - author=mergify[bot]
+      - check-success=beats-ci/pr-merge
+      - label=automation
+      - files~=^\.mergify\.yml$
+      - head~=^add-backport-next.*
+    actions:
+      review:
+        type: APPROVE
+        message: Automatically approving mergify
+  - name: automatic squash and merge with success checks and the files matching the regex ^.mergify.yml is modified.
+    conditions:
+      - check-success=beats-ci/pr-merge
+      - label=automation
+      - files~=^\.mergify\.yml$
+      - head~=^add-backport-next.*
+    actions:
+      merge:
+        method: squash
+        strict: smart+fasttrack
+  - name: delete upstream branch with changes on ^.mergify.yml that has been merged or closed
+    conditions:
+      - or:
+        - merged
+        - closed
+      - and:
+        - label=automation
+        - head~=^add-backport-next.*
+        - files~=^\.mergify\.yml$
+    actions:
+      delete_head_branch:

CHANGELOG-developer.asciidoc

@@ -208,3 +208,4 @@ The list below covers the major changes between 6.3.0 and 7.0.0-alpha2 only.
 - Allow/Merge fields.yml overrides {pull}9188[9188]
 - Filesets can now define multiple ingest pipelines, with the first one considered as the entry point pipeline. {pull}8914[8914]
 - Add `group_measurements_by_instance` option to windows perfmon metricset. {pull}8688[8688]
+- Bump ECS version to 1.10.0. {issue}25734[25734]

CHANGELOG-developer.next.asciidoc

@@ -54,6 +54,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Make implementing `Close` required for `reader.Reader` interfaces. {pull}20455[20455]
 - Remove `NumCPU` as clients should update the CPU count on the fly in case of config changes in a VM. {pull}23154[23154]
 - Remove Metricbeat EventFetcher and EventsFetcher interface. Use the reporter interface instead. {pull}25093[25093]
+- Update Darwin build image to a debian 10 base that increases the MacOS SDK and minimum supported version used in build to 10.14. {issue}24193[24193]
 
 ==== Bugfixes
 
@@ -112,3 +113,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Update Go version to 1.15.8. {pull}23955[23955]
 - Update Go version to 1.15.9. {pull}24442[24442]
 - Update Go version to 1.15.10. {pull}24606[24606]
+- Update Go version to 1.15.12. {pull}25629[25629]
+- Update Go version to 1.16.4. {issue}25346[25346] {pull}25671[25671]
+- Add sorting to array fields for generated data files (*-generated.json) {pull}25320[25320]
+- Update Go version to 1.16.5. {issue}26182[26182] {pull}26186[26186]