Merge remote-tracking branch 'upstream/master' into feature/remove-jo…

…urnalbeat-ci * upstream/master: (49 commits) [CI]: use the downstream packaging pipeline for branches/tags (elastic#28589) fix: use declarative style for complete variant of the elastic-agent (elastic#28526) x-pack/auditbeat/tracing: fix regexp for kprobe description line (elastic#28609) docs: Update `api_key` example on elasticsearch output (elastic#28606) chore: add build scripts to CODEOWNERS (elastic#28615) Osquerybeat: Fix host_processes missing cmdline arguments (elastic#28622) Add note about changes to regexp package in Golang (elastic#28616) CI: nightly/weekly builds for 7.x targeting 7.16 instead (elastic#28612) Osquerybeat: Fix extenstion unable to start on windows (elastic#28598) Osquerybeat: Return the query result count with the action response (elastic#28576) Agent: Allow custom response properties in the action response (elastic#28575) [Heartbeat] Only setuid in elastic-agent image (elastic#28577) Fix formatting of `mapStateJSON` and `layerListJSON` in dashboard assets (elastic#28530) CI: refactor the run e2e build (elastic#28502) Use fsnotify with long windows name-safe changes (elastic#28517) Remove unneeded mergify config backport: Add 7.16 branch (elastic#28560) Add proxy_url support to threatintel module's malwarebazaar fileset (elastic#28533) Osquerybeat: Implement host_users, host_groups, host_processes tables as a part of our osquery_extension. (elastic#28434) [Heartbeat] Make run_once syntax a boolean (elastic#28548) ...
v1v · Oct 26, 2021 · fcc08d9 · fcc08d9
2 parents dd86b5d + ccc880e
commit fcc08d9
Show file tree

Hide file tree

Showing 331 changed files with 10,072 additions and 3,398 deletions.
diff --git a/.backportrc.json b/.backportrc.json
diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy
@@ -416,38 +416,14 @@ def runE2ETests(){
       };
     }
 
-    triggerE2ETests(suites)
+    runE2E(runTestsSuites: suites,
+           beatVersion: "${env.BEAT_VERSION}-SNAPSHOT",
+           gitHubCheckName: env.GITHUB_CHECK_E2E_TESTS_NAME,
+           gitHubCheckRepo: env.REPO,
+           gitHubCheckSha1: env.GIT_BASE_COMMIT)
   }
 }
 
-def triggerE2ETests(String suite) {
-  echo("Triggering E2E tests for PR-${env.CHANGE_ID}. Test suites: ${suite}.")
-
-  def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"
-  def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}"
-  def beatVersion = "${env.BEAT_VERSION}-SNAPSHOT"
-
-  def parameters = [
-    booleanParam(name: 'forceSkipGitChecks', value: true),
-    booleanParam(name: 'forceSkipPresubmit', value: true),
-    booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
-    string(name: 'BEAT_VERSION', value: beatVersion),
-    string(name: 'runTestsSuites', value: suite),
-    string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
-    string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
-    string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
-  ]
-
-  build(job: "${e2eTestsPipeline}",
-    parameters: parameters,
-    propagate: false,
-    wait: false
-  )
-
-  def notifyContext = "${env.GITHUB_CHECK_E2E_TESTS_NAME}"
-  githubNotify(context: "${notifyContext}", description: "${notifyContext} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${e2eTestsPipeline.replaceAll('/','+')}")
-}
-
 def withMacOSEnv(Closure body){
   withEnvMask( vars: [
       [var: "KEYCHAIN_PASS", password: getVaultSecret(secret: "secret/jenkins-ci/macos-codesign-keychain").data.password],

diff --git a/.ci/schedule-daily.groovy b/.ci/schedule-daily.groovy
@@ -21,8 +21,8 @@ pipeline {
     stage('Nighly beats builds') {
       steps {
         runBuild(quietPeriod: 0, job: 'Beats/beats/master')
-        runBuild(quietPeriod: 2000, job: 'Beats/beats/7.x')
-        runBuild(quietPeriod: 4000, job: 'Beats/beats/7.<minor>')
+        runBuild(quietPeriod: 2000, job: 'Beats/beats/7.16')
+        runBuild(quietPeriod: 4000, job: 'Beats/beats/7.15')
       }
     }
   }
@@ -35,9 +35,5 @@ pipeline {
 
 def runBuild(Map args = [:]) {
   def jobName = args.job
-  if (jobName.contains('7.<minor>')) {
-    def parts = stackVersions.release().split('\\.')
-    jobName = args.job.replaceAll('<minor>', parts[1])
-  }
   build(quietPeriod: args.quietPeriod, job: jobName, parameters: [booleanParam(name: 'macosTest', value: true)], wait: false, propagate: false)
 }
diff --git a/.ci/schedule-weekly.groovy b/.ci/schedule-weekly.groovy
@@ -18,12 +18,11 @@ pipeline {
     cron('H H(1-2) * * 0')
   }
   stages {
-    stage('Nighly beats builds') {
+    stage('Weekly beats builds') {
       steps {
         runBuild(quietPeriod: 0, job: 'Beats/beats/master')
-        runBuild(quietPeriod: 1000, job: 'Beats/beats/7.x')
-        // <minor> is an alias to be replaced with the latest release branch
-        runBuild(quietPeriod: 2000, job: 'Beats/beats/7.<minor>')
+        runBuild(quietPeriod: 1000, job: 'Beats/beats/7.16')
+        runBuild(quietPeriod: 2000, job: 'Beats/beats/7.15')
       }
     }
   }
@@ -36,9 +35,5 @@ pipeline {
 
 def runBuild(Map args = [:]) {
   def jobName = args.job
-  if (jobName.contains('7.<minor>')) {
-    def parts = stackVersions.release().split('\\.')
-    jobName = args.job.replaceAll('<minor>', parts[1])
-  }
   build(quietPeriod: args.quietPeriod, job: jobName, parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
 }
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -42,6 +42,11 @@
 # Winlogbeat
 /x-pack/winlogbeat/ @elastic/security-external-integrations
 
+# Beats Build Specific
+/dev-tools/mage/ @elastic/beats
+/dev-tools/make/ @elastic/beats 
+/dev-tools/packaging/ @elastic/beats 
+
 # CI Specific
 /.ci/ @elastic/observablt-robots
 /Jenkinsfile @elastic/observablt-robots
diff --git a/.mergify.yml b/.mergify.yml
@@ -26,9 +26,9 @@ pull_request_rules:
             git merge upstream/{{base}}
             git push upstream {{head}}
             ```
-  - name: automatic approval for mergify pull requests with bump updates
+  - name: automatic approval for automated pull requests with bump updates
     conditions:
-      - author=mergify[bot]
+      - author=apmmachine
       - check-success=beats-ci/pr-merge
       - label=automation
       - files~=^testing/environments/snapshot.*\.yml$
@@ -45,7 +45,7 @@ pull_request_rules:
     actions:
       merge:
         method: squash
-        strict: smart+fasttrack
+        strict: false
   - name: delete upstream branch after merging changes on testing/environments/snapshot* or it's closed
     conditions:
       - or:
@@ -89,7 +89,7 @@ pull_request_rules:
     actions:
       merge:
         method: squash
-        strict: smart+fasttrack
+        strict: false
   - name: delete upstream branch with changes on ^.mergify.yml that has been merged or closed
     conditions:
       - or:
@@ -124,42 +124,42 @@ pull_request_rules:
       label:
         remove:
           - backport-skip
-  - name: backport patches to 7.x branch
+  - name: backport patches to 7.14 branch
     conditions:
       - merged
-      - label=backport-v7.16.0
+      - label=backport-v7.14.0
     actions:
       backport:
         assignees:
           - "{{ author }}"
         branches:
-          - "7.x"
+          - "7.14"
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
-  - name: backport patches to 7.14 branch
+  - name: backport patches to 7.15 branch
     conditions:
       - merged
-      - label=backport-v7.14.0
+      - label=backport-v7.15.0
     actions:
       backport:
         assignees:
           - "{{ author }}"
         branches:
-          - "7.14"
+          - "7.15"
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
-  - name: backport patches to 7.15 branch
+  - name: backport patches to 7.16 branch
     conditions:
       - merged
-      - label=backport-v7.15.0
+      - label=backport-v7.16.0
     actions:
       backport:
         assignees:
           - "{{ author }}"
         branches:
-          - "7.15"
+          - "7.16"
         labels:
           - "backport"
         title: "[{{ destination_branch }}](backport #{{ number }}) {{ title }}"
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -0,0 +1,12 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.0.1
+    hooks:
+    -   id: check-merge-conflict
+
+-   repo: https://github.com/elastic/apm-pipeline-library.git
+    rev: current
+    hooks:
+    -   id: check-jenkins-pipelines
+        files: ^(.ci/(.*\.groovy|Jenkinsfile)|Jenkinsfile)$
+    -   id: check-jjbb
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -23,15 +23,19 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove deprecated/undocumented IncludeCreatorMetadata setting from kubernetes metadata config options {pull}28006[28006]
 - Remove deprecated fields from kubernetes module {pull}28046[28046]
 - Remove deprecated config option aws_partition. {pull}28120[28120]
+- Improve stats API {pull}27963[27963]
 - Enable IMDSv2 support for `add_cloud_metadata` processor on AWS. {issue}22101[22101] {pull}28285[28285]
 - Update kubernetes.namespace from keyword to group field and add name, labels, annotations, uuid as its fields {pull}27917[27917]
+- Previously, RE2 and thus Golang had a bug where `(|a)*` matched more characters than `(|a)+`. To stay consistent with PCRE, the bug was fixed. Configurations that rely on the old, buggy behaviour has to be adjusted. See more about Golang bug: https://github.com/golang/go/issues/46123 {pull}27543[27543]
 
 *Auditbeat*
 
 - File integrity dataset (macOS): Replace unnecessary `file.origin.raw` (type keyword) with `file.origin.text` (type `text`). {issue}12423[12423] {pull}15630[15630]
 - Change event.kind=error to event.kind=event to comply with ECS. {issue}18870[18870] {pull}20685[20685]
 - File integrity dataset: Remove non-ECS `hash.*` fields. Hashes are under `file.hash.*`. {issue}19039[19039] {pull}28378[28378]
 - Auditd dataset: Removes the authentication_success and authentication_failure event.type values for user logins. {issue}19039[19039] {pull}28378[28378]
+- Fix handling of long file names on Windows. {issue}25334[25334] {pull}28517[28517]
+- System/socket dataset: Fix uninstallation of return kprobes. {issue}28608[28608] {pull}28609[28609]
 
 *Filebeat*
 
@@ -95,6 +99,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove deprecated fields in Kafka module. {pull}27938[27938]
 - Remove deprecated fields in coredns module. {pull}28196[28196]
 - Remove old `httpjson` config implementation. {pull}28054[28054]
+- Added dataset `threatq` to the `threatintel` module to ingest indicators from ThreatQ {issue}27423[27423]
 
 *Heartbeat*
 
@@ -117,16 +122,20 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove deprecated fields in Docker module. {issue}11835[11835] {pull}27933[27933]
 - Remove deprecated fields in Kafka module. {pull}27938[27938]
 - Remove deprecated config option default_region from aws module. {pull}28120[28120]
+- Remove network and diskio metrics from ec2 metricset. {pull}28316[28316]
+- Rename read/write_io.ops_per_sec to read/write.iops in rds metricset. {pull}28350[28350]
 - Remove linux-only metrics from diskio, memory {pull}28292[28292]
 - Remove deprecated config option perfmon.counters from windows/perfmon metricset. {pull}28282[28282]
 - Remove deprecated fields in Redis module. {issue}11835[11835] {pull}28246[28246]
+- Align fields to Beats naming conventions in GCP module. {issue}27231[27231] {pull}27974[27974]
 
 *Packetbeat*
 
 - Redis: fix incorrectly handle with two-words redis command. {issue}14872[14872] {pull}14873[14873]
 - `event.category` no longer contains the value `network_traffic` because this is not a valid ECS event category value. {pull}20556[20556]
 - Added redact_headers configuration option, to allow HTTP request headers to be redacted whilst keeping the header field included in the beat. {pull}15353[15353]
 - Add dns.question.subdomain and dns.question.top_level_domain fields. {pull}14578[14578]
+- Remove deprecated TLS fields in favor of tls.server.x509 and tls.client.x509 ECS fields. {pull}28487[28487]
 
 *Winlogbeat*
 
@@ -432,6 +441,7 @@ for a few releases. Please use other tools provided by Elastic to fetch data fro
 - Change vsphere.datastore.capacity.used.pct value to betweeen 0 and 1. {pull}23148[23148]
 - `beat` module respects `basepath` config option. {pull}28162[28162]
 - Fix list_docker.go {pull}28374[28374]
+- Divide RDS metric cpu.total.pct by 100. {pull}28456[28456]
 
 *Packetbeat*
 
@@ -446,6 +456,7 @@ for a few releases. Please use other tools provided by Elastic to fetch data fro
 - Protect against accessing undefined variables in Sysmon module. {issue}22219[22219] {pull}22236[22236]
 - Protect against accessing an undefined variable in Security module. {pull}22937[22937]
 - Tolerate faults when Windows Event Log session is interrupted {issue}27947[27947] {pull}28191[28191]
+- Add ECS 1.9 new users fields {pull}26509[26509]
 
 *Functionbeat*
 
@@ -766,7 +777,10 @@ for a few releases. Please use other tools provided by Elastic to fetch data fro
 - Move processing to ingest node for AWS vpcflow fileset. {pull}28168[28168]
 - Release zoom module as GA. {pull}28106[28106]
 - Add support for secondary object attribute handling in ThreatIntel MISP module {pull}28124[28124]
+- Azure signinlogs - Add support for ManagedIdentitySignInLogs, NonInteractiveUserSignInLogs, and ServicePrincipalSignInLogs. {issue}23653[23653]
 - Add `base64Decode` and `base64DecodeNoPad` functions to `httpsjon` templates. {pull}28385[28385]
+- Add latency config option for aws-cloudwatch input. {pull}28509[28509]
+- Added proxy support to threatintel/malwarebazaar. {pull}28533[28533]
 
 *Heartbeat*
 
@@ -901,6 +915,7 @@ for a few releases. Please use other tools provided by Elastic to fetch data fro
 - Move openmetrics module to oss. {pull}26561[26561]
 - Add `gke` metricset collection to `gcp` module {pull}26824[26824]
 - Added a new beta `enterprisesearch` module for Elastic Enterprise Search {pull}27549[27549]
+- Preliminary AIX support {pull}27954[27954]
 - Register additional name for `storage` metricset in the azure module. {pull}28447[28447]
 
 *Packetbeat*

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -203,10 +203,24 @@ def runLinting() {
     }
   }
   mapParallelTasks['default'] = { cmd(label: 'make check-default', script: 'make check-default') }
-
+  mapParallelTasks['pre-commit'] = runPreCommit()
   parallel(mapParallelTasks)
 }
 
+def runPreCommit() {
+  return {
+    withNode(labels: 'ubuntu-18 && immutable', forceWorkspace: true){
+      withGithubNotify(context: 'Check pre-commit', tab: 'tests') {
+        deleteDir()
+        unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
+        dir("${BASE_DIR}"){
+          preCommit(commit: "${GIT_BASE_COMMIT}", junit: true)
+        }
+      }
+    }
+  }
+}
+
 def runBuildAndTest(Map args = [:]) {
   def filterStage = args.get('filterStage', 'mandatory')
   deleteDir()
@@ -519,10 +533,19 @@ def getBeatsName(baseDir) {
 */
 def e2e(Map args = [:]) {
   if (!args.e2e?.get('enabled', false)) { return }
+  // Skip running the tests on branches or tags if configured.
+  if (!isPR() && args.e2e?.get('when', false)) {
+    if (isBranch() && !args.e2e.when.get('branches', true)) { return }
+    if (isTag() && !args.e2e.when.get('tags', true)) { return }
+  }
   if (args.e2e.get('entrypoint', '')?.trim()) {
     e2e_with_entrypoint(args)
   } else {
-    e2e_with_job(args)
+    runE2E(testMatrixFile: args.e2e?.get('testMatrixFile', ''),
+           beatVersion: "${env.VERSION}-SNAPSHOT",
+           gitHubCheckName: "e2e-${args.context}",
+           gitHubCheckRepo: env.REPO,
+           gitHubCheckSha1: env.GIT_BASE_COMMIT)
   }
 }
 
@@ -557,35 +580,6 @@ def e2e_with_entrypoint(Map args = [:]) {
   }
 }
 
-/**
-* This method triggers the end 2 end testing job.
-*/
-def e2e_with_job(Map args = [:]) {
-  def jobName = args.e2e?.get('job')
-  def testMatrixFile = args.e2e?.get('testMatrixFile', '')
-  def notifyContext = "e2e-${args.context}"
-  def e2eTestsPipeline = "${jobName}/${isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"}"
-
-  def parameters = [
-    booleanParam(name: 'forceSkipGitChecks', value: true),
-    booleanParam(name: 'forceSkipPresubmit', value: true),
-    booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
-    string(name: 'BEAT_VERSION', value: "${env.VERSION}-SNAPSHOT"),
-    string(name: 'testMatrixFile', value: testMatrixFile),
-    string(name: 'GITHUB_CHECK_NAME', value: notifyContext),
-    string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
-    string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
-  ]
-
-  build(job: "${e2eTestsPipeline}",
-    parameters: parameters,
-    propagate: false,
-    wait: false
-  )
-
-  githubNotify(context: "${notifyContext}", description: "${notifyContext} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${e2eTestsPipeline.replaceAll('/','+')}")
-}
-
 /**
 * This method runs the given command supporting two kind of scenarios:
 *  - make -C <folder> then the dir(location) is not required, aka by disaling isMage: false

diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml
@@ -2,8 +2,7 @@ projects:
     - "auditbeat"
     - "deploy/kubernetes"
     - "filebeat"
-# temporarly disable generator tests: https://github.com/elastic/beats/issues/28361
-#    - "generator"
+    - "generator"
     - "heartbeat"
     - "libbeat"
     - "metricbeat"