merge from master

.ci/packer_cache.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+#
+# this file is run daily to generate worker packer images
+#
+
+source /usr/local/bin/bash_standard_lib.sh
+
+# Docker images used on Dockerfiles 2019-07-12
+# aerospike:3.9.0
+# alpine:edge
+# apache/couchdb:1.7
+# busybox:latest
+# ceph/daemon:master-6373c6a-jewel-centos-7-x86_64
+# cockroachdb/cockroach:v19.1.1
+# consul:1.4.2
+# coredns/coredns:1.5.0
+# couchbase:4.5.1
+# debian:latest
+# debian:stretch
+# docker.elastic.co/beats-dev/fpm:1.11.0
+# docker.elastic.co/beats-dev/golang-crossbuild:1.12.4-arm
+# docker.elastic.co/beats-dev/golang-crossbuild:1.12.4-darwin
+# docker.elastic.co/beats-dev/golang-crossbuild:1.12.4-main
+# docker.elastic.co/beats-dev/golang-crossbuild:1.12.4-main-debian7
+# docker.elastic.co/beats-dev/golang-crossbuild:1.12.4-mips
+# docker.elastic.co/beats-dev/golang-crossbuild:1.12.4-ppc
+# docker.elastic.co/beats-dev/golang-crossbuild:1.12.4-s390x
+# docker.elastic.co/beats/metricbeat:6.5.4
+# docker.elastic.co/beats/metricbeat:7.2.0
+# docker.elastic.co/elasticsearch/elasticsearch:7.2.0
+# docker.elastic.co/kibana/kibana:7.2.0
+# docker.elastic.co/logstash/logstash:7.2.0
+# docker.elastic.co/observability-ci/database-instantclient:12.2.0.1
+# envoyproxy/envoy:v1.7.0
+# exekias/localkube-image
+# golang:1.10.3
+# golang:1.11.5
+# golang:1.12.1
+# golang:1.12.4
+# haproxy:1.8
+# httpd:2.4.20
+# java:8-jdk-alpine
+# jplock/zookeeper:3.4.8
+# maven:3.3-jdk-8
+# memcached:1.4.35-alpine
+# microsoft/mssql-server-linux:2017-GA
+# mongo:3.4
+# mysql:5.7.12
+# nats:1.3.0
+# nginx:1.9
+# oraclelinux:7
+# postgres:9.5.3
+# prom/prometheus:v2.6.0
+# python:3.6-alpine
+# quay.io/coreos/etcd:v3.3.10
+# rabbitmq:3.7.4-management
+# redis:3.2.12-alpine
+# redis:3.2.4-alpine
+# store/oracle/database-enterprise:12.2.0.1
+# traefik:1.6-alpine
+# tsouza/nginx-php-fpm:php-7.1
+# ubuntu:16.04
+# ubuntu:trusty
+
+DOCKER_IMAGES="docker.elastic.co/observability-ci/database-instantclient:12.2.0.1
+docker.elastic.co/observability-ci/database-enterprise:12.2.0.1"
+
+for di in ${DOCKER_IMAGES}
+do
+(retry 2 docker pull ${di}) || echo "Error pulling ${di} Docker image, we continue"
+done
+
+docker tag \
+  docker.elastic.co/observability-ci/database-instantclient:12.2.0.1 \
+  store/oracle/database-instantclient:12.2.0.1 \
+  || echo "Error setting the Oracle Instant Client tag"
+docker tag \
+  docker.elastic.co/observability-ci/database-enterprise:12.2.0.1 \
+  store/oracle/database-enterprise:12.2.0.1 \
+  || echo "Error setting the Oracle Dtabase tag"

.go-version

@@ -1 +1 @@
-1.13.7
+1.13.8

.travis.yml

@@ -148,29 +148,24 @@ jobs:
       stage: test
 
     # Generators
-    - os: linux
-      env: TARGETS="-C generator/metricbeat test test-package"
-      go: $TRAVIS_GO_VERSION
-      stage: test
-    - os: linux
-      env: TARGETS="-C generator/beat test test-package"
-      go: $TRAVIS_GO_VERSION
-      stage: test
-
-    - os: osx
-      env: TARGETS="-C generator/metricbeat test"
-      go: $TRAVIS_GO_VERSION
-      stage: test
-    - os: osx
-      env: TARGETS="-C generator/beat test"
-      go: $TRAVIS_GO_VERSION
-      stage: test
+    # Temporarily disable generator jobs
+    #- os: linux
+    #  env: TARGETS="-C generator/_templates/metricbeat test test-package"
+    #  go: $TRAVIS_GO_VERSION
+    #  stage: test
+    #- os: linux
+    #  env: TARGETS="-C generator/_templates/beat test test-package"
+    #  go: $TRAVIS_GO_VERSION
+    #  stage: test
 
-    # Docs
-    - os: linux
-      env: TARGETS="docs"
-      go: $TRAVIS_GO_VERSION
-      stage: test
+    #- os: osx
+    #  env: TARGETS="-C generator/_templates/metricbeat test"
+    #  go: $TRAVIS_GO_VERSION
+    #  stage: test
+    #- os: osx
+    #  env: TARGETS="-C generator/_templates/beat test"
+    #  go: $TRAVIS_GO_VERSION
+    #  stage: test
 
     # Kubernetes
     - os: linux
@@ -199,7 +194,6 @@ jobs:
             retries: true
           update: true
           packages:
-            - python-virtualenv
             - libpcap-dev
             - xsltproc
             - libxml2-utils
@@ -218,16 +212,22 @@ addons:
     config:
       retries: true
     update: true
+    sources:
+      - deadsnakes
     packages:
-      - python-virtualenv
       - libpcap-dev
       - xsltproc
       - libxml2-utils
       - libsystemd-journal-dev
       - librpm-dev
+      # From deadsnakes PPA
+      - python3.6
+      - python3.6-venv
 
 before_install:
+  - if [ x$TRAVIS_DIST = xtrusty ]; then sudo ln -sf python3.6 /usr/bin/python3; fi
   - python --version
+  - python3 --version
   - umask 022
   - chmod -R go-w $GOPATH/src/github.com/elastic/beats
   # Docker-compose installation

CHANGELOG-developer.asciidoc

@@ -12,6 +12,12 @@ other Beats should be migrated.
 
 Note: This changelog was only started after the 6.3 release.
 
+=== Beats version 8.0.0
+https://github.com/elastic/beats/compare/v7.x..master[Check the HEAD diff]
+
+==== Breaking changes
+ - Replace custom Pins type for a slice of string for defining the `ca_sha256` values.
+
 === Beats version 7.5.1
 https://github.com/elastic/beats/compare/v7.5.0..v7.5.1[Check the HEAD diff]
 

CHANGELOG-developer.next.asciidoc

@@ -28,6 +28,9 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Altered all remaining uses of mapval to use the renamed and enhanced version: https://github.com/elastic/go-lookslike[go-lookslike] instead, which is a separate project. The mapval tree is now gone. {pull}14165[14165]
 - Move light modules to OSS. {pull}14369[14369]
 - Deprecate test flags, `generate` and `update_expected`, in favor of `data`. {pull}15292[15292]
+- Python 3 is required now to run python tests and tools. {pull}14798[14798]
+- The type `memqueue.Broker` is no longer exported; instead of `memqueue.NewBroker`, call `memqueue.NewQueue` (which provides the same public interface). {pull}16667[16667]
+- `queue.Eventer` has been renamed to `queue.ACKListener` {pull}16691[16691]
 
 ==== Bugfixes
 

CHANGELOG.asciidoc

@@ -128,6 +128,7 @@ processing events. (CVE-2019-17596) See https://www.elastic.co/community/securit
 - Libbeat HTTP's Server can listen to a unix socket using the `unix:///tmp/hello.sock` syntax. {pull}13655[13655]
 - Libbeat HTTP's Server can listen to a Windows named pipe using the `npipe:///hello` syntax. {pull}13655[13655]
 - Adding new `Enterprise` license type to the licenser. {issue}14246[14246]
+- Add endpoint config in AWS config to support using custom endpoint accessing AWS APIs. {issue}16245[16245] {pull}16263[16263]
 
 *Auditbeat*
 
@@ -178,6 +179,7 @@ processing events. (CVE-2019-17596) See https://www.elastic.co/community/securit
 - Release cloudwatch, s3_daily_storage, s3_request, sqs and rds metricset as GA. {pull}14114[14114] {issue}14059[14059]
 - Add `elasticsearch/enrich` metricset. {pull}14243[14243] {issue}14221[14221]
 - Add new dashboards for Azure vms, vm guest metrics, vm scale sets {pull}14000[14000]
+- Add vpc metricset for aws module. {pull}16111[16111] {issue}14854[14854]
 
 *Functionbeat*
 

CHANGELOG.next.asciidoc

@@ -12,6 +12,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 - The document id fields has been renamed from @metadata.id to @metadata._id {pull}15859[15859]
 - Variable substitution from environment variables is not longer supported. {pull}15937{15937}
+- Change aws_elb autodiscover provider field name from elb_listener.* to aws.elb.*. {issue}16219[16219] {pull}16402{16402}
+- Remove `AddDockerMetadata` and `AddKubernetesMetadata` processors from the `script` processor. They can still be used as normal processors in the configuration. {issue}16349[16349] {pull}16514[16514]
 
 *Auditbeat*
 
@@ -56,6 +58,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Upgrade go-ucfg to latest v0.8.1. {pull}15937{15937}
 - Fix index names for indexing not always guaranteed to be lower case. {pull}16081[16081]
 - Add `ssl.ca_sha256` option to the supported TLS option, this allow to check that a specific certificate is used as part of the verified chain. {issue}15717[15717]
+- Fix loading processors from annotation hints. {pull}16348[16348]
+- Fix an issue that could cause redundant configuration reloads. {pull}16440[16440]
+- Fix k8s pods labels broken schema. {pull}16480[16480]
+- Fix k8s pods annotations broken schema. {pull}16554[16554]
+- Upgrade go-ucfg to latest v0.8.3. {pull}16450{16450}
 
 *Auditbeat*
 
@@ -71,6 +78,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Prevent Elasticsearch from spewing log warnings about redundant wildcards when setting up ingest pipelines for the `elasticsearch` module. {issue}15840[15840] {pull}15900[15900]
 - Fix mapping error for cloudtrail additionalEventData field {pull}16088[16088]
 - Fix a connection error in httpjson input. {pull}16123[16123]
+- Fix s3 input with cloudtrail fileset reading json file. {issue}16374[16374] {pull}16441[16441]
+- Rewrite azure filebeat dashboards, due to changes in kibana. {pull}16466[16466]
+- Adding the var definitions in azure manifest files, fix for errors when executing command setup. {issue}16270[16270] {pull}16468[16468]
+- Fix merging of fileset inputs to replace paths and append processors. {pull}16450{16450}
+- Add queue_url definition in manifest file for aws module. {pull}16640{16640}
 
 *Heartbeat*
 
@@ -89,6 +101,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fixed issue `logstash-xpack` module suddenly ceasing to monitor Logstash. {issue}15974[15974] {pull}16044[16044]
 - Fix skipping protocol scheme by light modules. {pull}16205[pull]
 - Made `logstash-xpack` module once again have parity with internally-collected Logstash monitoring data. {pull}16198[16198]
+- Change sqs metricset to use average as statistic method. {pull}16438[16438]
+- Revert changes in `docker` module: add size flag to docker.container. {pull}16600[16600]
+- Fix imports after PR was merged before rebase. {pull}16756[16756]
 
 *Packetbeat*
 
@@ -99,13 +114,20 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Functionbeat*
 
+- Fix timeout option of GCP functions. {issue}16282[16282] {pull}16287[16287]
 
 ==== Added
 
 *Affecting all Beats*
 
 - Add document_id setting to decode_json_fields processor. {pull}15859[15859]
+- Include network information by default on add_host_metadata and add_observer_metadata. {issue}15347[15347] {pull}16077[16077]
 - Add `aws_ec2` provider for autodiscover. {issue}12518[12518] {pull}14823[14823]
+- Add monitoring variable `libbeat.config.scans` to distinguish scans of the configuration directory from actual reloads of its contents. {pull}16440[16440]
+- Add support for multiple password in redis output. {issue}16058[16058] {pull}16206[16206]
+- Windows .exe files now have embedded file version info. {issue}15232[15232]t
+- Remove experimental flag from `setup.template.append_fields` {pull}16576[16576]
+- Add `add_cloudfoundry_metadata` processor to annotate events with Cloud Foundry application data. {pull}16621[16621
 
 *Auditbeat*
 
@@ -121,6 +143,18 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - move create-[module,fileset,fields] to mage and enable in x-pack/filebeat {pull}15836[15836]
 - Add ECS tls and categorization fields to apache module. {issue}16032[16032] {pull}16121[16121]
 - Add MQTT input. {issue}15602[15602] {pull}16204[16204]
+- Add ECS categorization fields to activemq module. {issue}16151[16151] {pull}16201[16201]
+- Add a TLS test and more debug output to httpjson input {pull}16315[16315]
+- Add an SSL config example in config.yml for filebeat MISP module. {pull}16320[16320]
+- Improve ECS categorization, container & process field mappings in auditd module. {issue}16153[16153] {pull}16280[16280]
+- Improve ECS field mappings in aws module. {issue}16154[16154] {pull}16307[16307]
+- Improve ECS categorization field mappings in googlecloud module. {issue}16030[16030] {pull}16500[16500]
+- Improve ECS field mappings in haproxy module. {issue}16162[16162] {pull}16529[16529]
+- Improve ECS categorization field mappings in kibana module. {issue}16168[16168] {pull}16652[16652]
+- Improve the decode_cef processor by reducing the number of memory allocations. {pull}16587[16587]
+- Add `cloudfoundry` input to send events from Cloud Foundry. {pull}16586[16586]
+- Improve ECS categorization field mappings in iis module. {issue}16165[16165] {pull}16618[16618]
+- Improve ECS categorization field mapping in kafka module. {issue}16167[16167] {pull}16645[16645]
 
 *Heartbeat*
 
@@ -153,8 +187,19 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add collecting AuroraDB metrics in rds metricset. {issue}14142[14142] {pull}16004[16004]
 - Reuse connections in SQL module. {pull}16001[16001]
 - Improve the `logstash` module (when `xpack.enabled` is set to `true`) to use the override `cluster_uuid` returned by Logstash APIs. {issue}15772[15772] {pull}15795[15795]
+- Add region parameter in googlecloud module. {issue}15780[15780] {pull}16203[16203]
 - Add kubernetes storage class support via kube-state-metrics. {pull}16145[16145]
+- Add database_account azure metricset. {issue}15758[15758]
+- Add support for NATS 2.1. {pull}16317[16317]
 - Add Load Balancing metricset to GCP {pull}15559[15559]
+- Add support for Dropwizard metrics 4.1. {pull}16332[16332]
+- Add azure container metricset in order to monitor containers. {issue}15751[15751] {pull}16421[16421]
+- Improve the `haproxy` module to support metrics exposed via HTTPS. {issue}14579[14579] {pull}16333[16333]
+- Add filtering option for prometheus collector. {pull}16420[16420]
+- Add metricsets based on Ceph Manager Daemon to the `ceph` module. {issue}7723[7723] {pull}16254[16254]
+- Release `statsd` module as GA. {pull}16447[16447] {issue}14280[14280]
+- Add `cloudfoundry` module to send events from Cloud Foundry. {pull}16671[16671]
+- Add `redisenterprise` module. {pull}16482[16482] {issue}15269[15269]
 
 *Packetbeat*