Merge remote-tracking branch 'upstream/master' into feature/ci-runbld…

…-refactor

* upstream/master:
  [Ingest manager] Copy Action store on upgrade (elastic#21298)
  [CI] Pipeline 2.0 for monorepos (elastic#20104)
  Stop running agent container as root by default (elastic#21213)
  Stop running auditbeat container as root by default (elastic#21202)
  Fix autodiscover flaky tests (elastic#21242)
  [Ingest Manager] Enabled dev builds (elastic#21241)
  Fix librpm installation in auditbeat build (elastic#21239)
  Fix prometheus default config (elastic#21253)
  Fix dev guide test command (elastic#21254)
  Move aws lambda metricset to GA (elastic#21255)
  [Docs] Typo in table syntax (elastic#20227)
  [ECS] Adds related.hosts to capture all hostnames and host identifiers on an event. (elastic#21160)

.ci/scripts/generate_build_table.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python3
+
+import os
+import yaml
+
+if __name__ == "__main__":
+
+    print("| Beat  | Stage  | Command  | MODULE  | Platforms  | When |")
+    print("|-------|--------|----------|---------|------------|------|")
+    for root, dirs, files in os.walk("."):
+        dirs.sort()
+        for file in files:
+            if file.endswith("Jenkinsfile.yml") and root != ".":
+                with open(os.path.join(root, file), 'r') as f:
+                    doc = yaml.load(f, Loader=yaml.FullLoader)
+                module = root.replace(".{}".format(os.sep), '')
+                platforms = [doc["platform"]]
+                when = ""
+                if "branches" in doc["when"]:
+                    when = f"{when}/:palm_tree:"
+                if "changeset" in doc["when"]:
+                    when = f"{when}/:file_folder:"
+                if "comments" in doc["when"]:
+                    when = f"{when}/:speech_balloon:"
+                if "labels" in doc["when"]:
+                    when = f"{when}/:label:"
+                if "parameters" in doc["when"]:
+                    when = f"{when}/:smiley:"
+                if "tags" in doc["when"]:
+                    when = f"{when}/:taco:"
+                for stage in doc["stages"]:
+                    withModule = False
+                    if "make" in doc["stages"][stage]:
+                        command = doc["stages"][stage]["make"]
+                    if "mage" in doc["stages"][stage]:
+                        command = doc["stages"][stage]["mage"]
+                    if "platforms" in doc["stages"][stage]:
+                        platforms = doc["stages"][stage]["platforms"]
+                    if "withModule" in doc["stages"][stage]:
+                        withModule = doc["stages"][stage]["withModule"]
+                    if "when" in doc["stages"][stage]:
+                        when = f"{when}/:star:"
+                    print("| {} | {} | `{}` | {} | `{}` | {} |".format(
+                        module, stage, command, withModule, platforms, when))
+
+print("> :palm_tree: -> Git Branch based")
+print("> :label: -> GitHub Pull Request Label based")
+print("> :file_folder: -> Changeset based")
+print("> :speech_balloon: -> GitHub Pull Request comment based")
+print("> :taco: -> Git tag based")
+print("> :smiley: -> Manual UI interaction based")
+print("> :star: -> More specific cases based")

.ci/scripts/get-vendor-dependencies.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+#
+# Given the go module it will list all the dependencies that will be later on
+# used by the CI to enable/disable specific stages as long as the changeset
+# matches any of those patterns.
+#
+
+GO_VERSION=${GO_VERSION:?"GO_VERSION environment variable is not set"}
+BEATS=${1:?"parameter missing."}
+eval "$(gvm "${GO_VERSION}")"
+
+go list -deps ./"${BEATS}" \
+| grep 'elastic/beats' \
+| sort \
+| sed -e "s#github.com/elastic/beats/v7/##g" \
+| awk '{print "^" $1 "/.*"}'

.ci/scripts/install-tools.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+.ci/scripts/install-go.sh
+.ci/scripts/install-docker-compose.sh
+.ci/scripts/install-terraform.sh
+make mage

CHANGELOG.next.asciidoc

@@ -28,6 +28,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - File integrity dataset (macOS): Replace unnecessary `file.origin.raw` (type keyword) with `file.origin.text` (type `text`). {issue}12423[12423] {pull}15630[15630]
 - Change event.kind=error to event.kind=event to comply with ECS. {issue}18870[18870] {pull}20685[20685]
 - Change network.direction values to ECS recommended values (inbound, outbound). {issue}12445[12445] {pull}20695[20695]
+- Docker container needs to be explicitly run as user root for auditing. {pull}21202[21202]
 
 *Filebeat*
 
@@ -589,6 +590,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added new module for Zoom webhooks {pull}20414[20414]
 - Add type and sub_type to panw panos fileset {pull}20912[20912]
 - Always attempt community_id processor on zeek module {pull}21155[21155]
+- Add related.hosts ecs field to all modules {pull}21160[21160]
 
 *Heartbeat*
 
@@ -713,6 +715,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Move `compute_vm_scaleset` to light metricset. {pull}21038[21038] {issue}20985[20985]
 - Sanitize `event.host`. {pull}21022[21022]
 - Add overview and platform health dashboards to Cloud Foundry module. {pull}21124[21124]
+- Release lambda metricset in aws module as GA. {issue}21251[21251] {pull}21255[21255]
 
 *Packetbeat*