Update python3 fix auditbeat and dashboards

.ci/apm-beats-update.groovy

@@ -0,0 +1,107 @@
+#!/usr/bin/env groovy
+@Library('apm@current') _
+
+pipeline {
+  agent { label 'linux && immutable' }
+  environment {
+    REPO = 'apm-server'
+    BASE_DIR = "src/github.com/elastic/${env.REPO}"
+    NOTIFY_TO = credentials('notify-to')
+    GITHUB_CHECK_ITS_NAME = 'APM Server Beats update'
+    PATH = "${env.PATH}:${env.WORKSPACE}/bin"
+    HOME = "${env.WORKSPACE}"
+    GOPATH = "${env.WORKSPACE}"
+  }
+  options {
+    timeout(time: 2, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '100', artifactNumToKeepStr: '30', daysToKeepStr: '30'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+    rateLimitBuilds(throttle: [count: 60, durationName: 'hour', userBoost: true])
+    quietPeriod(10)
+  }
+  triggers {
+    issueCommentTrigger('(?i).*/run\\s+(?:apm-beats-update\\W+)?.*')
+  }
+  stages {
+    /**
+     Checkout the code and stash it, to use it on other stages.
+    */
+    stage('Checkout') {
+      options { skipDefaultCheckout() }
+      steps {
+        deleteDir()
+        gitCheckout(basedir: "beats")
+        script {
+          dir("beats"){
+            env.GO_VERSION = readFile(".go-version").trim()
+            def regexps =[
+              "^devtools/mage.*",
+              "^libbeat/scripts/Makefile",
+            ]
+            env.BEATS_UPDATED = isGitRegionMatch(patterns: regexps)
+
+            // Skip all the stages except docs for PR's with asciidoc changes only
+            env.ONLY_DOCS = isGitRegionMatch(patterns: [ '.*\\.asciidoc' ], comparator: 'regexp', shouldMatchAll: true)
+          }
+        }
+      }
+    }
+    /**
+    updates beats updates the framework part and go parts of beats.
+    Then build and test.
+    Finally archive the results.
+    */
+    stage('Update Beats') {
+      options { skipDefaultCheckout() }
+      when {
+        beforeAgent true
+        anyOf {
+          branch 'master'
+          branch "\\d+\\.\\d+"
+          branch "v\\d?"
+          tag "v\\d+\\.\\d+\\.\\d+*"
+          allOf {
+            expression { return env.BEATS_UPDATED != "false" || isCommentTrigger() }
+            changeRequest()
+          }
+
+        }
+      }
+      steps {
+        withGithubNotify(context: 'Check Apm Server Beats Update') {
+          deleteDir()
+          gitCheckout(basedir: "${BASE_DIR}",
+            repo: "[email protected]:elastic/${REPO}.git",
+            branch: 'master',
+            credentialsId: 'credentials-id',
+            githubNotifyFirstTimeContributor: false,
+            depth: 1,
+            reference: "/var/lib/jenkins/.git-references/${REPO}.git"
+          )
+          dir("${BASE_DIR}"){
+            sh(label: 'Update Beats script', script: """
+              export BEATS_VERSION=${env.GIT_BASE_COMMIT}
+              git config --global --add remote.origin.fetch "+refs/pull/*/head:refs/remotes/origin/pr/*"
+              script/jenkins/update-beats.sh
+            """)
+          }
+        }
+      }
+      post {
+        always {
+          catchError(buildResult: 'SUCCESS', message: 'Failed to grab test results tar files', stageResult: 'SUCCESS') {
+            tar(file: "update-beats-system-tests-linux-files.tgz", archive: true, dir: "system-tests", pathPrefix: "${BASE_DIR}/build")
+          }
+        }
+      }
+    }
+  }
+  post {
+    cleanup {
+      notifyBuildResult()
+    }
+  }
+}

CHANGELOG.next.asciidoc

@@ -38,16 +38,21 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Affecting all Beats*
 
-TLS or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146]
+- TLS or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146]
+- Fix panic in the Logstash output when trying to send events to closed connection. {pull}15568[15568]
+- Fix missing output in dockerlogbeat {pull}15719[15719]
 
 *Auditbeat*
 
+- system/socket: Fixed compatibility issue with kernel 5.x. {pull}15771[15771]
 
 *Filebeat*
 
+- Add shared_credential_file to cloudtrail config {issue}15652[15652] {pull}15656[15656]
 
 *Heartbeat*
 
+- Fixed excessive memory usage introduced in 7.5 due to over-allocating memory for HTTP checks. {pull}15639[15639]
 
 *Journalbeat*
 
@@ -73,109 +78,27 @@ TLS or Beats that accept connections over TLS and validate client certificates.
 
 
 *Filebeat*
-
+- Set event.outcome field based on googlecloud audit log output. {pull}15731[15731]
 
 *Heartbeat*
 
+- Allow a list of status codes for HTTP checks. {pull}15587[15587]
+
 
 *Journalbeat*
 
 *Metricbeat*
 
-- Add AWS SQS metricset. {pull}10684[10684] {issue}10053[10053]
-- Add AWS s3_request metricset. {pull}10949[10949] {issue}10055[10055]
-- Add s3_daily_storage metricset. {pull}10940[10940] {issue}10055[10055]
-- Add `coredns` metricbeat module. {pull}10585[10585]
-- Add SSL support for Metricbeat HTTP server. {pull}11482[11482] {issue}11457[11457]
-- The `elasticsearch.index` metricset (with `xpack.enabled: true`) now collects `refresh.external_total_time_in_millis` fields from Elasticsearch. {pull}11616[11616]
-- Allow module configurations to have variants {pull}9118[9118]
-- Add `timeseries.instance` field calculation. {pull}10293[10293]
-- Added new disk states and raid level to the system/raid metricset. {pull}11613[11613]
-- Added `path_name` and `start_name` to service metricset on windows module {issue}8364[8364] {pull}11877[11877]
-- Add check on object name in the counter path if the instance name is missing {issue}6528[6528] {pull}11878[11878]
-- Add AWS cloudwatch metricset. {pull}11798[11798] {issue}11734[11734]
-- Add `regions` in aws module config to specify target regions for querying cloudwatch metrics. {issue}11932[11932] {pull}11956[11956]
-- Keep `etcd` followers members from reporting `leader` metricset events {pull}12004[12004]
-- Add overview dashboard to Consul module {pull}10665[10665]
-- New fields were added in the mysql/status metricset. {pull}12227[12227]
-- Add Kubernetes metricset `proxy`. {pull}12312[12312]
-- Add Kubernetes proxy dashboard to Kubernetes module {pull}12734[12734]
-- Always report Pod UID in the `pod` metricset. {pull}12345[12345]
-- Add Vsphere Virtual Machine operating system to `os` field in Vsphere virtualmachine module. {pull}12391[12391]
-- Add validation for elasticsearch and kibana modules' metricsets when xpack.enabled is set to true. {pull}12386[12386]
-- Add CockroachDB module. {pull}12467[12467]
-- Add support for metricbeat modules based on existing modules (a.k.a. light modules) {issue}12270[12270] {pull}12465[12465]
-- Add a system/entropy metricset {pull}12450[12450]
-- Add kubernetes metricset `controllermanager` {pull}12409[12409]
-- Add Kubernetes controller manager dashboard to Kubernetes module {pull}12744[12744]
-- Allow redis URL format in redis hosts config. {pull}12408[12408]
-- Add tags into ec2 metricset. {issue}[12263]12263 {pull}12372[12372]
-- Add metrics to kubernetes apiserver metricset. {pull}12922[12922]
-- Add kubernetes metricset `scheduler` {pull}12521[12521]
-- Add Kubernetes scheduler dashboard to Kubernetes module {pull}12749[12749]
-- Add `beat` module. {pull}12181[12181] {pull}12615[12615]
-- Collect tags for cloudwatch metricset in aws module. {issue}[12263]12263 {pull}12480[12480]
-- Add AWS RDS metricset. {pull}11620[11620] {issue}10054[10054]
-- Add Oracle Module {pull}11890[11890]
-- Add Oracle Tablespaces Dashboard {pull}12736[12736]
-- Collect client provided name for rabbitmq connection. {issue}12851[12851] {pull}12852[12852]
-- Add support to load default aws config file to get credentials. {pull}12727[12727] {issue}12708[12708]
-- Add statistic option into cloudwatch metricset. {issue}12370[12370] {pull}12840[12840]
-- Add support for kubernetes cronjobs {pull}13001[13001]
-- Add cgroup memory stats to docker/memory metricset {pull}12916[12916]
-- Add AWS elb metricset. {pull}12952[12952] {issue}11701[11701]
-- Add AWS ebs metricset. {pull}13167[13167] {issue}11699[11699]
-- Add `metricset.period` field with the configured fetching period. {pull}13242[13242] {issue}12616[12616]
-- Add rate metrics for ec2 metricset. {pull}13203[13203]
-- Add refresh list of perf counters at every fetch {issue}13091[13091]
-- Add Performance metricset to Oracle module {pull}12547[12547]
-- Add proc/vmstat data to the system/memory metricset on linux {pull}13322[13322]
-- Use DefaultMetaGeneratorConfig in MetadataEnrichers to initialize configurations {pull}13414[13414]
-- Add module for statsd. {pull}13109[13109]
-- Add support for NATS version 2. {pull}13601[13601]
-- Add `docker.cpu.*.norm.pct` metrics for `cpu` metricset of Docker Metricbeat module. {pull}13695[13695]
-- Add `instance` label by default when using Prometheus collector. {pull}13737[13737]
-- Add azure module. {pull}13196[13196] {pull}13859[13859] {pull}13988[13988]
-- Add Apache Tomcat module {pull}13491[13491]
-- Add ECS `container.id` and `container.runtime` to kubernetes `state_container` metricset. {pull}13884[13884]
-- Add `job` label by default when using Prometheus collector. {pull}13878[13878]
-- Add `state_resourcequota` metricset for Kubernetes module. {pull}13693[13693]
-- Add tags filter in ec2 metricset. {pull}13872[13872] {issue}13145[13145]
-- Add cloud.account.id and cloud.account.name into events from aws module. {issue}13551[13551] {pull}13558[13558]
-- Add `metrics_path` as known hint for autodiscovery {pull}13996[13996]
-- Leverage KUBECONFIG when creating k8s client. {pull}13916[13916]
-- Add ability to filter by tags for cloudwatch metricset. {pull}13758[13758] {issue}13145[13145]
-- Release cloudwatch, s3_daily_storage, s3_request, sqs and rds metricset as GA. {pull}14114[14114] {issue}14059[14059]
-- Add Oracle overview dashboard {pull}14021[14021]
-- Release CoreDNS module as GA. {pull}14308[14308]
-- Release CouchDB module as GA. {pull}14300[14300]
-- Add `elasticsearch/enrich` metricset. {pull}14243[14243] {issue}14221[14221]
-- Add support for Application ELB and Network ELB. {pull}14123[14123] {issue}13538[13538] {issue}13539[13539]
-- Release aws ebs metricset as GA. {pull}14312[14312] {issue}14060[14060]
-- Add `connection.state` field for RabbitMQ module. {pull}13981[13981]
-- Add more TCP states to Metricbeat system socket_summary. {pull}14347[14347]
-- Add Kafka JMX metricsets. {pull}14330[14330]
-- Add metrics to envoyproxy server metricset and support for envoy proxy 1.12. {pull}14416[14416] {issue}13642[13642]
-- Release kubernetes modules `controllermanager`, `scheduler`, `proxy`, `state_cronjob` and `state_resourcequota` as GA. {pull}14584[14584]
-- Add module for ActiveMQ. {pull}14580[14580]
-- Enable script processor. {pull}14711[14711]
-- Enable wildcard for cloudwatch metricset namespace. {pull}14971[14971] {issue}14965[14965]
-- Add `kube-state-metrics` `state_service` metrics for kubernetes module. {pull}14794[14794]
-- Add `kube-state-metrics` `state_persistentvolume` metrics for kubernetes module. {pull}14859[14859]
-- Add `kube-state-metrics` `state_persistentvolumeclaim` metrics for kubernetes module. {pull}15066[15066]
-- Add usage metricset in aws modules. {pull}14925[14925] {issue}14935[14935]
-- Add billing metricset in aws modules. {pull}14801[14801] {issue}14934[14934]
-- Add AWS SNS metricset. {pull}14946[14946]
-- Add overview dashboard for AWS SNS module {pull}14977[14977]
-- Add `index` option to all modules to specify a module-specific output index. {pull}15100[15100]
-- Add a `system/service` metricset for systemd data. {pull}14206[14206]
+- Move the windows pdh implementation from perfmon to a shared location in order for future modules/metricsets to make use of. {pull}15503[15503]
 - Add lambda metricset in aws module. {pull}15260[15260]
 - Expand data for the `system/memory` metricset {pull}15492[15492]
 - Add azure `storage` metricset in order to retrieve metric values for storage accounts. {issue}14548[14548] {pull}15342[15342]
 - Add cost warnings for the azure module. {pull}15356[15356]
 - Add DynamoDB AWS Metricbeat light module {pull}15097[15097]
 - Release elb module as GA. {pull}15485[15485]
 - Add a `system/network_summary` metricset {pull}15196[15196]
+- Add mesh metricset for Istio Metricbeat module{pull}15535[15535]
+- Make the `system/cpu` metricset collect normalized CPU metrics by default. {issue}15618[15618] {pull}15729[15729]
 
 *Packetbeat*
 

Jenkinsfile

@@ -1,11 +1,6 @@
 #!/usr/bin/env groovy
 
-library identifier: 'apm@master',
-retriever: modernSCM(
-  [$class: 'GitSCMSource',
-  credentialsId: 'f94e9298-83ae-417e-ba91-85c279771570',
-  id: '37cf2c00-2cc7-482e-8c62-7bbffef475e2',
-  remote: '[email protected]:elastic/apm-pipeline-library.git'])
+@Library('apm@current') _
 
 pipeline {
   agent { label 'ubuntu && immutable' }
@@ -36,12 +31,7 @@ pipeline {
     stage('Checkout') {
       options { skipDefaultCheckout() }
       steps {
-        //TODO we need to configure the library in Jenkins to use privileged methods.
-        //gitCheckout(basedir: "${BASE_DIR}")
-        dir("${BASE_DIR}"){
-          checkout scm
-          githubEnv()
-        }
+        gitCheckout(basedir: "${BASE_DIR}")
         stash allowEmpty: true, name: 'source', useDefaultExcludes: false
         script {
           env.GO_VERSION = readFile("${BASE_DIR}/.go-version").trim()