diff --git a/.circleci/config.yml b/.circleci/config.yml index 2c8713b1926da..874a28bb4c73a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4,89 +4,91 @@ defaults: working_directory: '/go/src/github.com/influxdata/telegraf' environment: GOFLAGS: -p=8 - go-1_10: &go-1_10 - docker: - - image: 'quay.io/influxdb/telegraf-ci:1.10.8' - go-1_11: &go-1_11 - docker: - - image: 'quay.io/influxdb/telegraf-ci:1.11.13' go-1_12: &go-1_12 docker: - - image: 'quay.io/influxdb/telegraf-ci:1.12.9' + - image: 'quay.io/influxdb/telegraf-ci:1.12.14' + environment: + GO111MODULE: 'on' + go-1_13: &go-1_13 + docker: + - image: 'quay.io/influxdb/telegraf-ci:1.13.5' version: 2 jobs: deps: - <<: [ *defaults, *go-1_12 ] + <<: [ *defaults, *go-1_13 ] steps: - checkout - restore_cache: - key: vendor-{{ checksum "Gopkg.lock" }} + key: go-mod-v1-{{ checksum "go.sum" }} - run: 'make deps' - - run: 'dep check' + - run: 'make tidy' - save_cache: - name: 'vendored deps' - key: vendor-{{ checksum "Gopkg.lock" }} + name: 'go module cache' + key: go-mod-v1-{{ checksum "go.sum" }} paths: - - './vendor' + - '/go/pkg/mod' - persist_to_workspace: - root: '/go/src' + root: '/go' paths: - '*' - test-go-1.10: - <<: [ *defaults, *go-1_10 ] - steps: - - attach_workspace: - at: '/go/src' - # disabled due to gofmt differences (1.10 vs 1.11). - #- run: 'make check' - - run: 'make test' - test-go-1.11: - <<: [ *defaults, *go-1_11 ] + test-go-1.12: + <<: [ *defaults, *go-1_12 ] steps: - attach_workspace: - at: '/go/src' + at: '/go' + - run: 'make' - run: 'make check' - run: 'make test' - test-go-1.12: + test-go-1.12-386: <<: [ *defaults, *go-1_12 ] steps: - attach_workspace: - at: '/go/src' + at: '/go' + - run: 'GOARCH=386 make' - run: 'GOARCH=386 make check' - run: 'GOARCH=386 make test' - test-go-1.12-386: - <<: [ *defaults, *go-1_12 ] + test-go-1.13: + <<: [ *defaults, *go-1_13 ] + steps: + - attach_workspace: + at: '/go' + - run: 'make' + - run: 'make check' + - run: 'make test' + test-go-1.13-386: + <<: [ *defaults, *go-1_13 ] steps: - attach_workspace: - at: '/go/src' + at: '/go' + - run: 'GOARCH=386 make' - run: 'GOARCH=386 make check' - run: 'GOARCH=386 make test' package: - <<: [ *defaults, *go-1_12 ] + <<: [ *defaults, *go-1_13 ] steps: - attach_workspace: - at: '/go/src' + at: '/go' - run: 'make package' - store_artifacts: path: './build' destination: 'build' release: - <<: [ *defaults, *go-1_12 ] + <<: [ *defaults, *go-1_13 ] steps: - attach_workspace: - at: '/go/src' + at: '/go' - run: 'make package-release' - store_artifacts: path: './build' destination: 'build' nightly: - <<: [ *defaults, *go-1_12 ] + <<: [ *defaults, *go-1_13 ] steps: - attach_workspace: - at: '/go/src' + at: '/go' - run: 'make package-nightly' - store_artifacts: path: './build' @@ -100,25 +102,25 @@ workflows: filters: tags: only: /.*/ - - 'test-go-1.10': + - 'test-go-1.12': requires: - 'deps' filters: tags: only: /.*/ - - 'test-go-1.11': + - 'test-go-1.12-386': requires: - 'deps' filters: tags: only: /.*/ - - 'test-go-1.12': + - 'test-go-1.13': requires: - 'deps' filters: tags: only: /.*/ - - 'test-go-1.12-386': + - 'test-go-1.13-386': requires: - 'deps' filters: @@ -126,16 +128,16 @@ workflows: only: /.*/ - 'package': requires: - - 'test-go-1.10' - - 'test-go-1.11' - 'test-go-1.12' - 'test-go-1.12-386' + - 'test-go-1.13' + - 'test-go-1.13-386' - 'release': requires: - - 'test-go-1.10' - - 'test-go-1.11' - 'test-go-1.12' - 'test-go-1.12-386' + - 'test-go-1.13' + - 'test-go-1.13-386' filters: tags: only: /.*/ @@ -144,24 +146,24 @@ workflows: nightly: jobs: - 'deps' - - 'test-go-1.10': + - 'test-go-1.12': requires: - 'deps' - - 'test-go-1.11': + - 'test-go-1.12-386': requires: - 'deps' - - 'test-go-1.12': + - 'test-go-1.13': requires: - 'deps' - - 'test-go-1.12-386': + - 'test-go-1.13-386': requires: - 'deps' - 'nightly': requires: - - 'test-go-1.10' - - 'test-go-1.11' - 'test-go-1.12' - 'test-go-1.12-386' + - 'test-go-1.13' + - 'test-go-1.13-386' triggers: - schedule: cron: "0 7 * * *" diff --git a/.github/ISSUE_TEMPLATE/Bug_report.md b/.github/ISSUE_TEMPLATE/Bug_report.md index 188df248e5e7c..ee9a35d4fa38e 100644 --- a/.github/ISSUE_TEMPLATE/Bug_report.md +++ b/.github/ISSUE_TEMPLATE/Bug_report.md @@ -3,26 +3,39 @@ name: Bug report about: Create a report to help us improve --- + ### Relevant telegraf.conf: - + ```toml ``` ### System info: - + ### Steps to reproduce: + + 1. ... 2. ... ### Expected behavior: + + ### Actual behavior: + + ### Additional info: - + diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a6e7fffbea51..64f233db43723 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,9 +1,96 @@ -## v1.13 [unreleased] +## v1.14 [unreleased] + +#### Release Notes + +- In the `sqlserver` input, the `sqlserver_azurestats` measurement has been + renamed to `sqlserver_azure_db_resource_stats` due to an issue where numeric + metrics were previously being reported incorrectly as strings. + +- The `date` processor now uses the UTC timezone when creating its tag. In + previous versions the local time was used. + +#### New Inputs + +- [infiniband](/plugins/inputs/infiniband/README.md) - Contributed by @willfurnell + +#### New Outputs + +- [warp10](/plugins/outputs/warp10/README.md) - Contributed by @aurrelhebert + +#### Features + +- [#6730](https://github.com/influxdata/telegraf/pull/6730): Add page_faults for mongodb wired tiger. +- [#6798](https://github.com/influxdata/telegraf/pull/6798): Add use_sudo option to ipmi_sensor input. +- [#6764](https://github.com/influxdata/telegraf/pull/6764): Add ability to collect pod labels to kubernetes input. +- [#6770](https://github.com/influxdata/telegraf/pull/6770): Expose unbound-control config file option. +- [#6508](https://github.com/influxdata/telegraf/pull/6508): Add support for new nginx plus api endpoints. +- [#6342](https://github.com/influxdata/telegraf/pull/6342): Add kafka SASL version control to support Azure Event Hub. +- [#6869](https://github.com/influxdata/telegraf/pull/6869): Add RBPEX IO statistics to DatabaseIO query in sqlserver input. +- [#6869](https://github.com/influxdata/telegraf/pull/6869): Add space on disk for each file to DatabaseIO query in the sqlserver input. +- [#6869](https://github.com/influxdata/telegraf/pull/6869): Calculate DB Name instead of GUID in physical_db_name in the sqlserver input. +- [#6733](https://github.com/influxdata/telegraf/pull/6733): Add latency stats to mongo input. +- [#6844](https://github.com/influxdata/telegraf/pull/6844): Add source and port tags to jenkins_job metrics. +- [#6886](https://github.com/influxdata/telegraf/pull/6886): Add date offset and timezone options to date processor. +- [#6859](https://github.com/influxdata/telegraf/pull/6859): Exclude resources by inventory path in vsphere input. + +#### Bugfixes + +- [#6397](https://github.com/influxdata/telegraf/issues/6397): Fix conversion to floats in AzureDBResourceStats query in the sqlserver input. +- [#6867](https://github.com/influxdata/telegraf/issues/6867): Fix case sensitive collation in sqlserver input. + +## v1.13.2 [unreleased] + +#### Bugfixes + +- [#2652](https://github.com/influxdata/telegraf/issues/2652): Warn without error when processes input is started on Windows. +- [#6890](https://github.com/influxdata/telegraf/issues/6890): Only parse certificate blocks in x509_cert input. +- [#6883](https://github.com/influxdata/telegraf/issues/6883): Add custom attributes for all resource types in vsphere input. +- [#6899](https://github.com/influxdata/telegraf/pull/6899): Fix URL agent address form with udp in snmp input. +- [#6619](https://github.com/influxdata/telegraf/issues/6619): Change logic to allow recording of device fields when attributes is false. +- [#6903](https://github.com/influxdata/telegraf/issues/6903): Do not add invalid timestamps to kafka messages. + +## v1.13.1 [2020-01-08] + +#### Bugfixes + +- [#6788](https://github.com/influxdata/telegraf/issues/6788): Fix ServerProperty query stops working on Azure after failover. +- [#6803](https://github.com/influxdata/telegraf/pull/6803): Add leading period to OID in SNMP v1 generic traps. +- [#6823](https://github.com/influxdata/telegraf/pull/6823): Fix missing config fields in prometheus serializer. +- [#6694](https://github.com/influxdata/telegraf/issues/6694): Fix panic on connection loss with undelivered messages in mqtt_consumer. +- [#6679](https://github.com/influxdata/telegraf/issues/6679): Encode query hash fields as hex strings in sqlserver input. +- [#6345](https://github.com/influxdata/telegraf/issues/6345): Invalidate diskio cache if the metadata mtime has changed. +- [#6800](https://github.com/influxdata/telegraf/issues/6800): Show platform not supported warning only on plugin creation. +- [#6814](https://github.com/influxdata/telegraf/issues/6814): Fix rabbitmq cannot complete gather after request error. +- [#6846](https://github.com/influxdata/telegraf/issues/6846): Fix /sbin/init --version executed on Telegraf startup. +- [#6847](https://github.com/influxdata/telegraf/issues/6847): Use last path element as field key if path fully specified in cisco_telemetry_gnmi input. + +## v1.13 [2019-12-12] + +#### Release Notes + +- Official packages built with Go 1.13.5. +- The `prometheus` input and `prometheus_client` output have a new mapping to + and from Telegraf metrics, which can be enabled by setting `metric_version = 2`. + The original mapping is deprecated. When both plugins have the same setting, + passthrough metrics will be unchanged. Refer to the `prometheus` input for + details about the mapping. #### New Inputs - [azure_storage_queue](/plugins/inputs/azure_storage_queue/README.md) - Contributed by @mjiderhamn +- [ethtool](/plugins/inputs/ethtool/README.md) - Contributed by @philippreston +- [snmp_trap](/plugins/inputs/snmp_trap/README.md) - Contributed by @influxdata - [suricata](/plugins/inputs/suricata/README.md) - Contributed by @satta +- [synproxy](/plugins/inputs/synproxy/README.md) - Contributed by @rfrenayworldstream +- [systemd_units](/plugins/inputs/systemd_units/README.md) - Contributed by @benschweizer + +#### New Processors + +- [clone](/plugins/processors/clone/README.md) - Contributed by @adrianlzt + +#### New Aggregators + +- [merge](/plugins/aggregators/merge/README.md) - Contributed by @influxdata #### Features @@ -13,8 +100,103 @@ - [#6177](https://github.com/influxdata/telegraf/pull/6177): Support NX-OS telemetry extensions in cisco_telemetry_mdt. - [#6415](https://github.com/influxdata/telegraf/pull/6415): Allow graphite parser to create Inf and NaN values. - [#6434](https://github.com/influxdata/telegraf/pull/6434): Use prefix base detection for ints in grok parser. +- [#6465](https://github.com/influxdata/telegraf/pull/6465): Add more performance counter metrics to sqlserver input. +- [#6476](https://github.com/influxdata/telegraf/pull/6476): Add millisecond unix time support to grok parser. +- [#6473](https://github.com/influxdata/telegraf/pull/6473): Add container id as optional source tag to docker and docker_log input. +- [#6504](https://github.com/influxdata/telegraf/pull/6504): Add lang parameter to OpenWeathermap input plugin. +- [#6540](https://github.com/influxdata/telegraf/pull/6540): Log file open errors at debug level in tail input. +- [#6553](https://github.com/influxdata/telegraf/pull/6553): Add timeout option to cloudwatch input. +- [#6549](https://github.com/influxdata/telegraf/pull/6549): Support custom success codes in http input. +- [#6530](https://github.com/influxdata/telegraf/pull/6530): Improve ipvs input error strings and logging. +- [#6532](https://github.com/influxdata/telegraf/pull/6532): Add strict mode to JSON parser that can be disable to ignore invalid items. +- [#6543](https://github.com/influxdata/telegraf/pull/6543): Add support for Kubernetes 1.16 and remove deprecated API usage. +- [#6283](https://github.com/influxdata/telegraf/pull/6283): Add gathering of RabbitMQ federation link metrics. +- [#6356](https://github.com/influxdata/telegraf/pull/6356): Add bearer token defaults for Kubernetes plugins. +- [#5870](https://github.com/influxdata/telegraf/pull/5870): Add support for SNMP over TCP. +- [#6603](https://github.com/influxdata/telegraf/pull/6603): Add support for per output flush jitter. +- [#6650](https://github.com/influxdata/telegraf/pull/6650): Add a nameable file tag to file input plugin. +- [#6640](https://github.com/influxdata/telegraf/pull/6640): Add Splunk MultiMetric support. +- [#6680](https://github.com/influxdata/telegraf/pull/6668): Add support for sending HTTP Basic Auth in influxdb input +- [#5767](https://github.com/influxdata/telegraf/pull/5767): Add ability to configure the url tag in the prometheus input. +- [#5767](https://github.com/influxdata/telegraf/pull/5767): Add prometheus metric_version=2 mapping to internal metrics/line protocol. +- [#6703](https://github.com/influxdata/telegraf/pull/6703): Add prometheus metric_version=2 support to prometheus_client output. +- [#6660](https://github.com/influxdata/telegraf/pull/6660): Add content_encoding compression support to socket_listener. +- [#6689](https://github.com/influxdata/telegraf/pull/6689): Add high resolution metrics support to CloudWatch output. +- [#6716](https://github.com/influxdata/telegraf/pull/6716): Add SReclaimable and SUnreclaim to mem input. +- [#6695](https://github.com/influxdata/telegraf/pull/6695): Allow multiple certificates per file in x509_cert input. +- [#6686](https://github.com/influxdata/telegraf/pull/6686): Add additional tags to the x509 input. +- [#6703](https://github.com/influxdata/telegraf/pull/6703): Add batch data format support to file output. +- [#6688](https://github.com/influxdata/telegraf/pull/6688): Support partition assignement strategy configuration in kafka_consumer. +- [#6731](https://github.com/influxdata/telegraf/pull/6731): Add node type tag to mongodb input. +- [#6669](https://github.com/influxdata/telegraf/pull/6669): Add uptime_ns field to mongodb input. +- [#6735](https://github.com/influxdata/telegraf/pull/6735): Support resolution of symlinks in filecount input. +- [#6746](https://github.com/influxdata/telegraf/pull/6746): Set message timestamp to the metric time in kafka output. +- [#6740](https://github.com/influxdata/telegraf/pull/6740): Add base64decode operation to string processor. +- [#6790](https://github.com/influxdata/telegraf/pull/6790): Add option to control collecting global variables to mysql input. + +#### Bugfixes + +- [#6484](https://github.com/influxdata/telegraf/issues/6484): Show correct default settings in mysql sample config. +- [#6583](https://github.com/influxdata/telegraf/issues/6583): Use 1h or 3h rain values as appropriate in openweathermap input. +- [#6573](https://github.com/influxdata/telegraf/issues/6573): Fix not a valid field error in Windows with nvidia input. +- [#6614](https://github.com/influxdata/telegraf/issues/6614): Fix influxdb output serialization on connection closed. +- [#6690](https://github.com/influxdata/telegraf/issues/6690): Fix ping skips remaining hosts after dns lookup error. +- [#6684](https://github.com/influxdata/telegraf/issues/6684): Log mongodb oplog auth errors at debug level. +- [#6705](https://github.com/influxdata/telegraf/issues/6705): Remove trailing underscore trimming from json flattener. +- [#6421](https://github.com/influxdata/telegraf/issues/6421): Revert change causing cpu usage to be capped at 100 percent. +- [#6523](https://github.com/influxdata/telegraf/issues/6523): Accept any media type in the prometheus input. +- [#6769](https://github.com/influxdata/telegraf/issues/6769): Fix unix socket dial arguments in uwsgi input. +- [#6757](https://github.com/influxdata/telegraf/issues/6757): Replace colon chars in prometheus output labels with metric_version=1. +- [#6773](https://github.com/influxdata/telegraf/issues/6773): Set TrimLeadingSpace when TrimSpace is on in csv parser. + +## v1.12.6 [2019-11-19] + +#### Bugfixes + +- [#6666](https://github.com/influxdata/telegraf/issues/6666): Fix many plugin errors are logged at debug logging level. +- [#6652](https://github.com/influxdata/telegraf/issues/6652): Use nanosecond precision in docker_log input. +- [#6642](https://github.com/influxdata/telegraf/issues/6642): Fix interface option with method = native in ping input. +- [#6680](https://github.com/influxdata/telegraf/pull/6680): Fix panic in mongodb input if shard connection pool stats are unreadable. + +## v1.12.5 [2019-11-12] + +#### Bugfixes + +- [#6576](https://github.com/influxdata/telegraf/issues/6576): Fix incorrect results in ping input plugin. +- [#6610](https://github.com/influxdata/telegraf/pull/6610): Add missing character replacement to sql_instance tag. +- [#6337](https://github.com/influxdata/telegraf/issues/6337): Change no metric error message to debug level in cloudwatch input. +- [#6602](https://github.com/influxdata/telegraf/issues/6602): Add missing ServerProperties query to sqlserver input docs. +- [#6643](https://github.com/influxdata/telegraf/pull/6643): Fix mongodb connections_total_created field loading. +- [#6627](https://github.com/influxdata/telegraf/issues/6578): Fix metric creation when node is offline in jenkins input. +- [#6649](https://github.com/influxdata/telegraf/issues/6615): Fix docker uptime_ns calculation when container has been restarted. +- [#6647](https://github.com/influxdata/telegraf/issues/6646): Fix mysql field type conflict in conversion of gtid_mode to an integer. +- [#5529](https://github.com/influxdata/telegraf/issues/5529): Fix mysql field type conflict with ssl_verify_depth and ssl_ctx_verify_depth. + +## v1.12.4 [2019-10-23] + +#### Release Notes + +- Official packages built with Go 1.12.12. + +#### Bugfixes + +- [#6521](https://github.com/influxdata/telegraf/issues/6521): Fix metric generation with ping input native method. +- [#6541](https://github.com/influxdata/telegraf/issues/6541): Exclude alias tag if unset from plugin internal stats. +- [#6564](https://github.com/influxdata/telegraf/issues/6564): Fix socket_mode option in powerdns_recursor input. + +## v1.12.3 [2019-10-07] + +#### Bugfixes + +- [#6445](https://github.com/influxdata/telegraf/issues/6445): Use batch serialization format in exec output. +- [#6455](https://github.com/influxdata/telegraf/issues/6455): Build official packages with Go 1.12.10. +- [#6464](https://github.com/influxdata/telegraf/pull/6464): Use case insensitive serial numer match in smart input. +- [#6469](https://github.com/influxdata/telegraf/pull/6469): Add auth header only when env var is set. +- [#6468](https://github.com/influxdata/telegraf/pull/6468): Fix running multiple mysql and sqlserver plugin instances. +- [#6471](https://github.com/influxdata/telegraf/issues/6471): Fix database routing on retry with exclude_database_tag. +- [#6488](https://github.com/influxdata/telegraf/issues/6488): Fix logging panic in exec input with nagios data format. -## v1.12.2 [unreleased] +## v1.12.2 [2019-09-24] #### Bugfixes @@ -22,6 +204,7 @@ - [#6394](https://github.com/influxdata/telegraf/issues/6394): Fix parsing of BATTDATE in apcupsd input. - [#6398](https://github.com/influxdata/telegraf/issues/6398): Keep boolean values listed in json_string_fields. - [#6393](https://github.com/influxdata/telegraf/issues/6393): Disable Go plugin support in official builds. +- [#6391](https://github.com/influxdata/telegraf/issues/6391): Fix path handling issues in cisco_telemetry_gnmi. ## v1.12.1 [2019-09-10] diff --git a/Gopkg.lock b/Gopkg.lock index 22520af3aa52d..477aff14aa4a0 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -64,21 +64,31 @@ version = "0.2.0" [[projects]] - digest = "1:5923e22a060ab818a015593422f9e8a35b9d881d4cfcfed0669a82959b11c7ee" + digest = "1:e4a02906493a47ee87ef61aeea130ce6624da07349a6dc62494a4e72b550ca8e" name = "github.com/Azure/go-autorest" packages = [ "autorest", "autorest/adal", "autorest/azure", "autorest/azure/auth", + "autorest/azure/cli", "autorest/date", + "logger", + "tracing", ] pruneopts = "" - revision = "1f7cd6cfe0adea687ad44a512dfe76140f804318" - version = "v10.12.0" + revision = "3492b2aff5036c67228ab3c7dba3577c871db200" + version = "v13.3.0" [[projects]] branch = "master" + digest = "1:005d83d9daaea4e3fc7b2eedf28f68ebf87df7d331a874e5d7d14f643467e7d9" + name = "github.com/Mellanox/rdmamap" + packages = ["."] + pruneopts = "" + revision = "7c3c4763a6ee6a4d624fe133135dc3a7c483111c" + +[[projects]] digest = "1:298712a3ee36b59c3ca91f4183bd75d174d5eaa8b4aed5072831f126e2e752f6" name = "github.com/Microsoft/ApplicationInsights-Go" packages = [ @@ -87,6 +97,7 @@ ] pruneopts = "" revision = "d2df5d440eda5372f24fcac03839a64d6cb5f7e5" + version = "v0.4.2" [[projects]] digest = "1:45ec6eb579713a01991ad07f538fed3b576ee55f5ce9f248320152a9270d9258" @@ -97,12 +108,12 @@ version = "v0.4.9" [[projects]] - digest = "1:322bf7f4bb312294fc551f6e2c82d02f2ab8f94920f4163b3deeb07a8141ac79" + digest = "1:33f56caa9ab45fedc63d3d1d3e342d9f9d00726071f22c67d06b0cd26d49a55e" name = "github.com/Shopify/sarama" packages = ["."] pruneopts = "" - revision = "b12709e6ca29240128c89fe0b30b6a76be42b457" - source = "https://github.com/influxdata/sarama.git" + revision = "" + version = "v1.24.1" [[projects]] digest = "1:f82b8ac36058904227087141017bb82f4b0fc58272990a4cdae3e2d6d222644e" @@ -301,12 +312,12 @@ version = "v3.2.0" [[projects]] - branch = "master" - digest = "1:654ac9799e7a8a586d8690bb2229a4f3408bbfe2c5494bf4dfe043053eeb5496" + digest = "1:459dfcae44c32c1a6831fb99c75b40e7139aa800a04f55f6e47fedb33ee4407d" name = "github.com/dimchansky/utfbom" packages = ["."] pruneopts = "" - revision = "6c6132ff69f0f6c088739067407b5d32c52e1d0f" + revision = "d2133a1ce379ef6fa992b0514a77146c60db9d1c" + version = "v1.1.0" [[projects]] digest = "1:522eff2a1f014a64fb403db60fc0110653e4dc5b59779894d208e697b0708ddc" @@ -413,8 +424,7 @@ packages = [ ".", "apis/apiextensions/v1beta1", - "apis/apps/v1beta1", - "apis/apps/v1beta2", + "apis/apps/v1", "apis/core/v1", "apis/extensions/v1beta1", "apis/meta/v1", @@ -438,12 +448,12 @@ revision = "25d852aebe32c875e9c044af3eef9c7dc6bc777f" [[projects]] - digest = "1:c6f371f2b02c751a83be83139a12a5467e55393feda16d4f8dfa95adfc4efede" + digest = "1:7a9dc29b3fbc9a6440d98fcff422a2ce1a613975697ea560e3610084234f91ec" name = "github.com/glinton/ping" packages = ["."] pruneopts = "" - revision = "1983bc2fd5de3ea00aa5457bbc8774300e889db9" - version = "v0.1.1" + revision = "d3c0ecf4df108179eccdff2176f4ff569c3aab37" + version = "v0.1.3" [[projects]] digest = "1:df89444601379b2e1ee82bf8e6b72af9901cbeed4b469fa380a519c89c339310" @@ -506,6 +516,14 @@ revision = "5ccd90ef52e1e632236f7326478d4faa74f99438" version = "v0.2.3" +[[projects]] + digest = "1:181fe10dcb708edd7c68c5781928b6657612771f81dd1773287386b6982c94e2" + name = "github.com/gofrs/uuid" + packages = ["."] + pruneopts = "" + revision = "3a54a6416087bae7aa0ac32dd79fe1bf87bc99e4" + version = "v2.1.0" + [[projects]] digest = "1:6e73003ecd35f4487a5e88270d3ca0a81bc80dc88053ac7e4dcfec5fba30d918" name = "github.com/gogo/protobuf" @@ -514,6 +532,14 @@ revision = "636bf0302bc95575d69441b25a2603156ffdddf1" version = "v1.1.1" +[[projects]] + digest = "1:68c64bb61d55dcd17c82ca0b871ddddb5ae18b30cfe26f6bfd4b6df6287dc2e0" + name = "github.com/golang/mock" + packages = ["gomock"] + pruneopts = "" + revision = "9fa652df1129bef0e734c9cf9bf6dbae9ef3b9fa" + version = "1.3.1" + [[projects]] digest = "1:f958a1c137db276e52f0b50efee41a1a389dcdded59a69711f3e872757dab34b" name = "github.com/golang/protobuf" @@ -708,7 +734,7 @@ revision = "7c63b0a71ef8300adc255344d275e10e5c3a71ec" [[projects]] - digest = "1:a7998e19ebb78fdd341cdaf3825fded9030ae27af9c70d298c05d88744e16a0b" + digest = "1:e248df365cb87001738e8c9368a6a27c504328047b196d89687c1ca918279a82" name = "github.com/jackc/pgx" packages = [ ".", @@ -720,8 +746,8 @@ "stdlib", ] pruneopts = "" - revision = "8faa4453fc7051d1076053f8854077753ab912f2" - version = "v3.4.0" + revision = "c73e7d75061bb42b0282945710f344cfe1113d10" + version = "v3.6.0" [[projects]] digest = "1:d45477e90c25c8c6d7d4237281167aa56079382fc042db4b44a8328071649bfa" @@ -742,28 +768,20 @@ revision = "c2b33e84" [[projects]] - branch = "master" - digest = "1:2c5ad58492804c40bdaf5d92039b0cde8b5becd2b7feeb37d7d1cc36a8aa8dbe" - name = "github.com/kardianos/osext" - packages = ["."] - pruneopts = "" - revision = "ae77be60afb1dcacde03767a8c37337fad28ac14" - -[[projects]] - branch = "master" - digest = "1:fed90fa725d3b1bac0a760de64426834dfef4546474cf182f2ec94285afa74a8" + digest = "1:b498ceccf0d2efa0af877b1dda20d3742ef9ff7475123e8e922016f0b737069b" name = "github.com/kardianos/service" packages = ["."] pruneopts = "" - revision = "615a14ed75099c9eaac6949e22ac2341bf9d3197" + revision = "56787a3ea05e9b262708192e7ce3b500aba73561" + version = "v1.0.0" [[projects]] - digest = "1:a12b6f20a7e5eb7412d2e5cd15e1262a021f735fa958d664d9e7ba2160eefd0a" + digest = "1:3e160bec100719bb664ce5192b42e82e66b290397da4c0845aed5ce3cfce60cb" name = "github.com/karrick/godirwalk" packages = ["."] pruneopts = "" - revision = "2de2192f9e35ce981c152a873ed943b93b79ced4" - version = "v1.7.5" + revision = "532e518bccc921708e14b29e16503b1bf5c898cc" + version = "v1.12.0" [[projects]] branch = "master" @@ -773,6 +791,20 @@ pruneopts = "" revision = "95032a82bc518f77982ea72343cc1ade730072f0" +[[projects]] + digest = "1:4ceab6231efd01210f2b8b6ab360d480d49c0f44df63841ca0465920a387495d" + name = "github.com/klauspost/compress" + packages = [ + "fse", + "huff0", + "snappy", + "zstd", + "zstd/internal/xxhash", + ] + pruneopts = "" + revision = "4e96aec082898e4dad17d8aca1a7e2d01362ff6c" + version = "v1.9.2" + [[projects]] branch = "master" digest = "1:1ed9eeebdf24aadfbca57eb50e6455bd1d2474525e0f0d4454de8c8e9bc7ee9a" @@ -829,20 +861,20 @@ revision = "eb3dd99a75fe58389e357b732691320dcf706b5f" [[projects]] - digest = "1:4c8d8358c45ba11ab7bb15df749d4df8664ff1582daead28bae58cf8cbe49890" + digest = "1:1eef80a63549d929a5d922dc3d9ad0d489ed490f52b90887ad577b65a16d071c" name = "github.com/miekg/dns" packages = ["."] pruneopts = "" - revision = "5a2b9fab83ff0f8bfc99684bd5f43a37abe560f1" - version = "v1.0.8" + revision = "f4db2ca6edc3af0ee51bf332099cc480bcf3ef9d" + version = "v1.0.10" [[projects]] - branch = "master" - digest = "1:99651e95333755cbe5c9768c1b80031300acca64a80870b40309202b32585a5a" + digest = "1:6dbb0eb72090871f2e58d1e37973fe3cb8c0f45f49459398d3fc740cb30e13bd" name = "github.com/mitchellh/go-homedir" packages = ["."] pruneopts = "" - revision = "3864e76763d94a6df2f9960b16a20a33da9f9a66" + revision = "af06845cf3004701891bf4fdb884bfe4920b3727" + version = "v1.1.0" [[projects]] branch = "master" @@ -1055,6 +1087,13 @@ pruneopts = "" revision = "e2704e165165ec55d062f5919b4b29494e9fa790" +[[projects]] + digest = "1:a18bd4e530f3f36fe91a5d1fd57d492f25287546e613f892d21c2b76b848517d" + name = "github.com/safchain/ethtool" + packages = ["."] + pruneopts = "" + revision = "42ed695e3de80b9d695f280295fd7994639f209d" + [[projects]] branch = "master" digest = "1:7fc2f428767a2521abc63f1a663d981f61610524275d6c0ea645defadd4e916f" @@ -1064,15 +1103,14 @@ revision = "c4fab1ac1bec58281ad0667dc3f0907a9476ac47" [[projects]] - digest = "1:7f569d906bdd20d906b606415b7d794f798f91a62fcfb6a4daa6d50690fb7a3f" + digest = "1:47081c00d00c1dfc9a530c2556e78be391a5c24db1043efe6d406af882a169a1" name = "github.com/satori/go.uuid" packages = ["."] pruneopts = "" - revision = "f58768cc1a7a7e77a3bd49e98cdd21419399b6a3" - version = "v1.2.0" + revision = "b2ce2384e17bbe0c6d34077efa39dbab3e09123b" [[projects]] - digest = "1:55dcddb2ba6ab25098ee6b96f176f39305f1fde7ea3d138e7e10bb64a5bf45be" + digest = "1:9024df427b3c8a80a0c4b34e535e5e1ae922c7174e3242b6c7f30ffb3b9f715e" name = "github.com/shirou/gopsutil" packages = [ "cpu", @@ -1085,8 +1123,8 @@ "process", ] pruneopts = "" - revision = "e4ec7b275ada47ca32799106c2dba142d96aaf93" - version = "v2.19.8" + revision = "fc7e5e7af6052e36e83e5539148015ed2c09d8f9" + version = "v2.19.11" [[projects]] branch = "master" @@ -1105,12 +1143,12 @@ version = "v1.0.5" [[projects]] - branch = "master" - digest = "1:4b0cabe65ca903a7b2a3e6272c5304eb788ce196d35ecb901c6563e5e7582443" + digest = "1:a1cb5e999ad98b9838147e11ed1bdb000e750ee8872e2e21c74d9464cc9110c0" name = "github.com/soniah/gosnmp" packages = ["."] pruneopts = "" - revision = "96b86229e9b3ffb4b954144cdc7f98fe3ee1003f" + revision = "40eae407a1f8cbbe3f3f14c57bde0b16db1cfe85" + version = "v1.22.0" [[projects]] branch = "master" @@ -1167,7 +1205,10 @@ [[projects]] digest = "1:026b6ceaabbacaa147e94a63579efc3d3c73e00c73b67fa5c43ab46191ed04eb" name = "github.com/vishvananda/netlink" - packages = ["nl"] + packages = [ + ".", + "nl", + ] pruneopts = "" revision = "b2de5d10e38ecce8607e6b438b6d174f389a004e" @@ -1283,7 +1324,7 @@ [[projects]] branch = "master" - digest = "1:0773b5c3be42874166670a20aa177872edb450cd9fc70b1df97303d977702a50" + digest = "1:d709f6b44dffe11337b3730ebf5ae6bb1bc9273a1c204266921205158a5a523f" name = "golang.org/x/crypto" packages = [ "bcrypt", @@ -1297,7 +1338,7 @@ "ssh/terminal", ] pruneopts = "" - revision = "a2144134853fc9a27a7b1e3eb4f19f1a76df13c9" + revision = "87dc89f01550277dc22b74ffcf4cd89fa2f40f4c" source = "https://github.com/golang/crypto.git" [[projects]] @@ -1622,12 +1663,12 @@ version = "v1.1.0" [[projects]] - digest = "1:367baf06b7dbd0ef0bbdd785f6a79f929c96b0c18e9d3b29c0eed1ac3f5db133" - name = "gopkg.in/ldap.v2" + digest = "1:cff622452aa789a1b2212d401f6b618ca1751a02229d26e002eb645ec22818f2" + name = "gopkg.in/ldap.v3" packages = ["."] pruneopts = "" - revision = "bb7a9ca6e4fbc2129e3db588a34bc970ffe811a9" - version = "v2.5.1" + revision = "caa044a2bfa324b735baee1722e8e2e372f76864" + version = "v3.1.0" [[projects]] branch = "v2" @@ -1682,6 +1723,7 @@ "github.com/Azure/azure-storage-queue-go/azqueue", "github.com/Azure/go-autorest/autorest", "github.com/Azure/go-autorest/autorest/azure/auth", + "github.com/Mellanox/rdmamap", "github.com/Microsoft/ApplicationInsights-Go/appinsights", "github.com/Shopify/sarama", "github.com/StackExchange/wmi", @@ -1712,8 +1754,7 @@ "github.com/docker/libnetwork/ipvs", "github.com/eclipse/paho.mqtt.golang", "github.com/ericchiang/k8s", - "github.com/ericchiang/k8s/apis/apps/v1beta1", - "github.com/ericchiang/k8s/apis/apps/v1beta2", + "github.com/ericchiang/k8s/apis/apps/v1", "github.com/ericchiang/k8s/apis/core/v1", "github.com/ericchiang/k8s/apis/extensions/v1beta1", "github.com/ericchiang/k8s/apis/meta/v1", @@ -1725,6 +1766,8 @@ "github.com/go-redis/redis", "github.com/go-sql-driver/mysql", "github.com/gobwas/glob", + "github.com/gofrs/uuid", + "github.com/gogo/protobuf/proto", "github.com/golang/protobuf/proto", "github.com/golang/protobuf/ptypes/duration", "github.com/golang/protobuf/ptypes/empty", @@ -1766,7 +1809,7 @@ "github.com/prometheus/client_golang/prometheus/promhttp", "github.com/prometheus/client_model/go", "github.com/prometheus/common/expfmt", - "github.com/satori/go.uuid", + "github.com/safchain/ethtool", "github.com/shirou/gopsutil/cpu", "github.com/shirou/gopsutil/disk", "github.com/shirou/gopsutil/host", @@ -1774,6 +1817,7 @@ "github.com/shirou/gopsutil/mem", "github.com/shirou/gopsutil/net", "github.com/shirou/gopsutil/process", + "github.com/sirupsen/logrus", "github.com/soniah/gosnmp", "github.com/streadway/amqp", "github.com/stretchr/testify/assert", @@ -1819,7 +1863,7 @@ "google.golang.org/grpc/peer", "google.golang.org/grpc/status", "gopkg.in/gorethink/gorethink.v3", - "gopkg.in/ldap.v2", + "gopkg.in/ldap.v3", "gopkg.in/mgo.v2", "gopkg.in/mgo.v2/bson", "gopkg.in/olivere/elastic.v5", diff --git a/Gopkg.toml b/Gopkg.toml index 2d545e224bd61..b4304c61c2d9b 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -88,7 +88,7 @@ [[constraint]] name = "github.com/kardianos/service" - branch = "master" + version = "1.0.0" [[constraint]] name = "github.com/kballard/go-shellquote" @@ -100,11 +100,11 @@ [[constraint]] name = "github.com/Microsoft/ApplicationInsights-Go" - branch = "master" + version = "0.4.2" [[constraint]] name = "github.com/miekg/dns" - version = "1.0.8" + version = "1.0.10" [[constraint]] name = "github.com/multiplay/go-ts3" @@ -139,21 +139,20 @@ branch = "master" [[constraint]] - name = "github.com/satori/go.uuid" - version = "1.2.0" + name = "github.com/gofrs/uuid" + version = "2.0.0" [[constraint]] name = "github.com/shirou/gopsutil" - version = "2.18.12" + version = "2.19.7" [[constraint]] name = "github.com/Shopify/sarama" - revision = "b12709e6ca29240128c89fe0b30b6a76be42b457" - source = "https://github.com/influxdata/sarama.git" + version = "1.24.0" [[constraint]] name = "github.com/soniah/gosnmp" - branch = "master" + version = "1.22.0" [[constraint]] name = "github.com/StackExchange/wmi" @@ -201,10 +200,6 @@ name = "gopkg.in/gorethink/gorethink.v3" version = "3.0.5" -[[constraint]] - name = "gopkg.in/ldap.v2" - version = "2.5.1" - [[constraint]] name = "gopkg.in/mgo.v2" branch = "v2" @@ -235,7 +230,7 @@ [[constraint]] name = "github.com/Azure/go-autorest" - version = "10.12.0" + version = "^13.0.0" [[constraint]] name = "github.com/Azure/azure-storage-queue-go" @@ -260,7 +255,7 @@ [[constraint]] name = "github.com/karrick/godirwalk" - version = "1.7.5" + version = "1.10" [[override]] name = "github.com/harlow/kinesis-consumer" @@ -301,3 +296,19 @@ [[constraint]] branch = "master" name = "github.com/cisco-ie/nx-telemetry-proto" + +[[constraint]] + branch = "master" + name = "github.com/Mellanox/rdmamap" + +[[constraint]] + name = "gopkg.in/ldap.v3" + version = "3.1.0" + +[[constraint]] + name = "github.com/safchain/ethtool" + revision = "42ed695e3de80b9d695f280295fd7994639f209d" + +[[override]] + name = "github.com/satori/go.uuid" + revision = "b2ce2384e17bbe0c6d34077efa39dbab3e09123b" diff --git a/Makefile b/Makefile index 3c0fb3952be6d..27aefdeb7bc5e 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ all: .PHONY: deps deps: - dep ensure -vendor-only + go mod download .PHONY: telegraf telegraf: @@ -83,8 +83,18 @@ vet: exit 1; \ fi +.PHONY: tidy +tidy: + go mod verify + go mod tidy + @if ! git diff --quiet go.mod go.sum; then \ + echo "please run go mod tidy and check in changes"; \ + exit 1; \ + fi + .PHONY: check check: fmtcheck vet + @$(MAKE) --no-print-directory tidy .PHONY: test-all test-all: fmtcheck vet @@ -129,17 +139,12 @@ plugin-%: @echo "Starting dev environment for $${$(@)} input plugin..." @docker-compose -f plugins/inputs/$${$(@)}/dev/docker-compose.yml up +.PHONY: ci-1.13 +ci-1.13: + docker build -t quay.io/influxdb/telegraf-ci:1.13.5 - < scripts/ci-1.13.docker + docker push quay.io/influxdb/telegraf-ci:1.13.5 + .PHONY: ci-1.12 ci-1.12: - docker build -t quay.io/influxdb/telegraf-ci:1.12.9 - < scripts/ci-1.12.docker - docker push quay.io/influxdb/telegraf-ci:1.12.9 - -.PHONY: ci-1.11 -ci-1.11: - docker build -t quay.io/influxdb/telegraf-ci:1.11.13 - < scripts/ci-1.11.docker - docker push quay.io/influxdb/telegraf-ci:1.11.13 - -.PHONY: ci-1.10 -ci-1.10: - docker build -t quay.io/influxdb/telegraf-ci:1.10.8 - < scripts/ci-1.10.docker - docker push quay.io/influxdb/telegraf-ci:1.10.8 + docker build -t quay.io/influxdb/telegraf-ci:1.12.14 - < scripts/ci-1.12.docker + docker push quay.io/influxdb/telegraf-ci:1.12.14 diff --git a/README.md b/README.md index 5f34ebf2ad0ef..81990320d8693 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,16 @@ There are many ways to contribute: - Answer questions and discuss here on github and on the [Community Site](https://community.influxdata.com/) - [Contribute plugins](CONTRIBUTING.md) +## Minimum Requirements + +Telegraf shares the same [minimum requirements][] as Go: +- Linux kernel version 2.6.23 or later +- Windows 7 or later +- FreeBSD 11.2 or later +- MacOS 10.11 El Capitan or later + +[minimum requirements]: https://github.com/golang/go/wiki/MinimumRequirements#minimum-requirements + ## Installation: You can download the binaries directly from the [downloads](https://www.influxdata.com/downloads) page @@ -40,17 +50,17 @@ Ansible role: https://github.com/rossmcdonald/telegraf ### From Source: -Telegraf requires golang version 1.10 or newer, the Makefile requires GNU make. +Telegraf requires Go version 1.12 or newer, the Makefile requires GNU make. -1. [Install Go](https://golang.org/doc/install) >=1.10 (1.12 recommended) -2. [Install dep](https://golang.github.io/dep/docs/installation.html) ==v0.5.0 -3. Download Telegraf source: +1. [Install Go](https://golang.org/doc/install) >=1.12 (1.13 recommended) +2. Clone the Telegraf repository: ``` - go get -d github.com/influxdata/telegraf + cd ~/src + git clone https://github.com/influxdata/telegraf.git ``` -4. Run make from the source directory +3. Run `make` from the source directory ``` - cd "$HOME/go/src/github.com/influxdata/telegraf" + cd ~/src/telegraf make ``` @@ -142,7 +152,7 @@ For documentation on the latest development code see the [documentation index][d * [apache](./plugins/inputs/apache) * [apcupsd](./plugins/inputs/apcupsd) * [aurora](./plugins/inputs/aurora) -* [aws cloudwatch](./plugins/inputs/cloudwatch) +* [aws cloudwatch](./plugins/inputs/cloudwatch) (Amazon Cloudwatch) * [azure_storage_queue](./plugins/inputs/azure_storage_queue) * [bcache](./plugins/inputs/bcache) * [beanstalkd](./plugins/inputs/beanstalkd) @@ -171,8 +181,9 @@ For documentation on the latest development code see the [documentation index][d * [docker](./plugins/inputs/docker) * [docker_log](./plugins/inputs/docker_log) * [dovecot](./plugins/inputs/dovecot) -* [ecs](./plugins/inputs/ecs) (Amazon Elastic Container Service, Fargate) +* [aws ecs](./plugins/inputs/ecs) (Amazon Elastic Container Service, Fargate) * [elasticsearch](./plugins/inputs/elasticsearch) +* [ethtool](./plugins/inputs/ethtool) * [exec](./plugins/inputs/exec) (generic executable plugin, support JSON, influx, graphite and nagios) * [fail2ban](./plugins/inputs/fail2ban) * [fibaro](./plugins/inputs/fibaro) @@ -191,6 +202,7 @@ For documentation on the latest development code see the [documentation index][d * [http](./plugins/inputs/http) (generic HTTP plugin, supports using input data formats) * [http_response](./plugins/inputs/http_response) * [icinga2](./plugins/inputs/icinga2) +* [infiniband](./plugins/inputs/infiniband) * [influxdb](./plugins/inputs/influxdb) * [influxdb_listener](./plugins/inputs/influxdb_listener) * [internal](./plugins/inputs/internal) @@ -205,7 +217,7 @@ For documentation on the latest development code see the [documentation index][d * [jti_openconfig_telemetry](./plugins/inputs/jti_openconfig_telemetry) * [kafka_consumer](./plugins/inputs/kafka_consumer) * [kapacitor](./plugins/inputs/kapacitor) -* [kinesis](./plugins/inputs/kinesis_consumer) +* [aws kinesis](./plugins/inputs/kinesis_consumer) (Amazon Kinesis) * [kernel](./plugins/inputs/kernel) * [kernel_vmstat](./plugins/inputs/kernel_vmstat) * [kibana](./plugins/inputs/kibana) @@ -271,6 +283,7 @@ For documentation on the latest development code see the [documentation index][d * [smart](./plugins/inputs/smart) * [snmp_legacy](./plugins/inputs/snmp_legacy) * [snmp](./plugins/inputs/snmp) +* [snmp_trap](./plugins/inputs/snmp_trap) * [socket_listener](./plugins/inputs/socket_listener) * [solr](./plugins/inputs/solr) * [sql server](./plugins/inputs/sqlserver) (microsoft) @@ -278,8 +291,10 @@ For documentation on the latest development code see the [documentation index][d * [statsd](./plugins/inputs/statsd) * [suricata](./plugins/inputs/suricata) * [swap](./plugins/inputs/swap) +* [synproxy](./plugins/inputs/synproxy) * [syslog](./plugins/inputs/syslog) * [sysstat](./plugins/inputs/sysstat) +* [systemd_units](./plugins/inputs/systemd_units) * [system](./plugins/inputs/system) * [tail](./plugins/inputs/tail) * [temp](./plugins/inputs/temp) @@ -290,7 +305,7 @@ For documentation on the latest development code see the [documentation index][d * [twemproxy](./plugins/inputs/twemproxy) * [udp_listener](./plugins/inputs/socket_listener) * [unbound](./plugins/inputs/unbound) -* [uswgi](./plugins/inputs/uswgi) +* [uwsgi](./plugins/inputs/uwsgi) * [varnish](./plugins/inputs/varnish) * [vsphere](./plugins/inputs/vsphere) VMware vSphere * [webhooks](./plugins/inputs/webhooks) @@ -335,6 +350,7 @@ For documentation on the latest development code see the [documentation index][d ## Processor Plugins +* [clone](./plugins/processors/clone) * [converter](./plugins/processors/converter) * [date](./plugins/processors/date) * [enum](./plugins/processors/enum) @@ -354,6 +370,7 @@ For documentation on the latest development code see the [documentation index][d * [basicstats](./plugins/aggregators/basicstats) * [final](./plugins/aggregators/final) * [histogram](./plugins/aggregators/histogram) +* [merge](./plugins/aggregators/merge) * [minmax](./plugins/aggregators/minmax) * [valuecounter](./plugins/aggregators/valuecounter) @@ -393,4 +410,5 @@ For documentation on the latest development code see the [documentation index][d * [syslog](./plugins/outputs/syslog) * [tcp](./plugins/outputs/socket_writer) * [udp](./plugins/outputs/socket_writer) +* [warp10](./plugins/outputs/warp10) * [wavefront](./plugins/outputs/wavefront) diff --git a/agent/agent.go b/agent/agent.go index e2ef79b840f4f..aa8d07e67833d 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -503,6 +503,12 @@ func (a *Agent) runOutputs( interval = output.Config.FlushInterval } + jitter := jitter + // Overwrite agent flush_jitter if this plugin has its own. + if output.Config.FlushJitter != nil { + jitter = *output.Config.FlushJitter + } + wg.Add(1) go func(output *models.RunningOutput) { defer wg.Done() diff --git a/appveyor.yml b/appveyor.yml index c2349dd322020..559647e352f1a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,40 +1,33 @@ version: "{build}" cache: - - C:\Cache - - C:\gopath\pkg\dep\sources -> Gopkg.lock + - C:\gopath\pkg\mod -> go.sum + - C:\ProgramData\chocolatey\bin -> appveyor.yml + - C:\ProgramData\chocolatey\lib -> appveyor.yml clone_folder: C:\gopath\src\github.com\influxdata\telegraf environment: + GOVERSION: 1.13.5 GOPATH: C:\gopath platform: x64 install: - - IF NOT EXIST "C:\Cache" mkdir C:\Cache - - IF NOT EXIST "C:\Cache\go1.12.9.msi" curl -o "C:\Cache\go1.12.9.msi" https://storage.googleapis.com/golang/go1.12.9.windows-amd64.msi - - IF NOT EXIST "C:\Cache\gnuwin32-bin.zip" curl -o "C:\Cache\gnuwin32-bin.zip" https://dl.influxdata.com/telegraf/ci/make-3.81-bin.zip - - IF NOT EXIST "C:\Cache\gnuwin32-dep.zip" curl -o "C:\Cache\gnuwin32-dep.zip" https://dl.influxdata.com/telegraf/ci/make-3.81-dep.zip - - IF EXIST "C:\Go" rmdir /S /Q C:\Go - - msiexec.exe /i "C:\Cache\go1.12.9.msi" /quiet - - 7z x "C:\Cache\gnuwin32-bin.zip" -oC:\GnuWin32 -y - - 7z x "C:\Cache\gnuwin32-dep.zip" -oC:\GnuWin32 -y - - go get -d github.com/golang/dep - - cd "%GOPATH%\src\github.com\golang\dep" - - git checkout -q v0.5.0 - - go install -ldflags="-X main.version=v0.5.0" ./cmd/dep + - choco install golang --version "%GOVERSION%" + - choco install make - cd "%GOPATH%\src\github.com\influxdata\telegraf" - git config --system core.longpaths true - go version - go env build_script: - - cmd: C:\GnuWin32\bin\make + - make deps + - make telegraf test_script: - - cmd: C:\GnuWin32\bin\make check - - cmd: C:\GnuWin32\bin\make test-windows + - make check + - make test-windows artifacts: - path: telegraf.exe diff --git a/cmd/telegraf/telegraf.go b/cmd/telegraf/telegraf.go index f865cee5149c6..7b013cc6cc72f 100644 --- a/cmd/telegraf/telegraf.go +++ b/cmd/telegraf/telegraf.go @@ -11,6 +11,7 @@ import ( "os" "os/signal" "runtime" + "sort" "strings" "syscall" "time" @@ -327,14 +328,24 @@ func main() { // switch for flags which just do something and exit immediately switch { case *fOutputList: - fmt.Println("Available Output Plugins:") + fmt.Println("Available Output Plugins: ") + names := make([]string, 0, len(outputs.Outputs)) for k := range outputs.Outputs { + names = append(names, k) + } + sort.Strings(names) + for _, k := range names { fmt.Printf(" %s\n", k) } return case *fInputList: fmt.Println("Available Input Plugins:") + names := make([]string, 0, len(inputs.Inputs)) for k := range inputs.Inputs { + names = append(names, k) + } + sort.Strings(names) + for _, k := range names { fmt.Printf(" %s\n", k) } return diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index 36feac79148fc..428ffeab44aa2 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -124,12 +124,14 @@ The agent table configures Telegraf and the defaults used across all plugins. - **flush_interval**: Default flushing [interval][] for all outputs. Maximum flush_interval will be - flush_interval + flush_jitter + flush_interval + flush_jitter. - **flush_jitter**: - Jitter the flush [interval][] by a random amount. This is primarily to avoid - large write spikes for users running a large number of telegraf instances. - ie, a jitter of 5s and interval 10s means flushes will happen every 10-15s + Default flush jitter for all outputs. This jitters the flush [interval][] + by a random amount. This is primarily to avoid large write spikes for users + running a large number of telegraf instances. ie, a jitter of 5s and interval + 10s means flushes will happen every 10-15s. + - **precision**: Collected metrics are rounded to the precision specified as an [interval][]. @@ -144,11 +146,14 @@ The agent table configures Telegraf and the defaults used across all plugins. Log only error level messages. - **logtarget**: - Log target - `file`, `stderr` or `eventlog` (Windows only). - The empty string means to log to stderr. + Log target controls the destination for logs and can be one of "file", + "stderr" or, on Windows, "eventlog". When set to "file", the output file is + determined by the "logfile" setting. - **logfile**: - Log file name. + Name of the file to be logged to when using the "file" logtarget. If set to + the empty string then logs are written to stderr. + - **logfile_rotation_interval**: The logfile will be rotated after the time interval specified. When set to @@ -188,6 +193,7 @@ driven operation. Parameters that can be used with any input plugin: +- **alias**: Name an instance of a plugin. - **interval**: How often to gather this metric. Normal plugins use a single global interval, but if one particular input should be run less or more often, you can configure that here. @@ -253,8 +259,11 @@ databases, network services, and messaging systems. Parameters that can be used with any output plugin: +- **alias**: Name an instance of a plugin. - **flush_interval**: The maximum time between flushes. Use this setting to override the agent `flush_interval` on a per plugin basis. +- **flush_jitter**: The amount of time to jitter the flush interval. Use this + setting to override the agent `flush_jitter` on a per plugin basis. - **metric_batch_size**: The maximum number of metrics to send at once. Use this setting to override the agent `metric_batch_size` on a per plugin basis. - **metric_buffer_limit**: The maximum number of unsent metrics to buffer. @@ -270,6 +279,7 @@ Override flush parameters for a single output: ```toml [agent] flush_interval = "10s" + flush_jitter = "5s" metric_batch_size = 1000 [[outputs.influxdb]] @@ -279,6 +289,7 @@ Override flush parameters for a single output: [[outputs.file]] files = [ "stdout" ] flush_interval = "1s" + flush_jitter = "1s" metric_batch_size = 10 ``` @@ -290,6 +301,7 @@ input plugins and before any aggregator plugins. Parameters that can be used with any processor plugin: +- **alias**: Name an instance of a plugin. - **order**: The order in which the processor(s) are executed. If this is not specified then processor execution order will be random. @@ -324,6 +336,7 @@ processors have been applied. Parameters that can be used with any aggregator plugin: +- **alias**: Name an instance of a plugin. - **period**: The period on which to flush & clear each aggregator. All metrics that are sent with timestamps outside of this period will be ignored by the aggregator. diff --git a/docs/DATA_FORMATS_OUTPUT.md b/docs/DATA_FORMATS_OUTPUT.md index f3ac028b980d6..a8650b250f3fd 100644 --- a/docs/DATA_FORMATS_OUTPUT.md +++ b/docs/DATA_FORMATS_OUTPUT.md @@ -5,10 +5,11 @@ standard data formats that may be selected from when configuring many output plugins. 1. [InfluxDB Line Protocol](/plugins/serializers/influx) -1. [JSON](/plugins/serializers/json) +1. [Carbon2](/plugins/serializers/carbon2) 1. [Graphite](/plugins/serializers/graphite) +1. [JSON](/plugins/serializers/json) +1. [Prometheus](/plugins/serializers/prometheus) 1. [SplunkMetric](/plugins/serializers/splunkmetric) -1. [Carbon2](/plugins/serializers/carbon2) 1. [Wavefront](/plugins/serializers/wavefront) You will be able to identify the plugins with support by the presence of a diff --git a/docs/FAQ.md b/docs/FAQ.md index 1d1c490aa2ffc..8819ee65786a3 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -10,10 +10,13 @@ docker run --name telegraf -v /etc:/hostfs/etc:ro -v /proc:/hostfs/proc:ro -v /sys:/hostfs/sys:ro - -v /var/run/utmp:/var/run/utmp:ro + -v /var:/hostfs/var:ro + -v /run:/hostfs/run:ro -e HOST_ETC=/hostfs/etc -e HOST_PROC=/hostfs/proc -e HOST_SYS=/hostfs/sys + -e HOST_VAR=/hostfs/var + -e HOST_RUN=/hostfs/run -e HOST_MOUNT_PREFIX=/hostfs telegraf ``` @@ -40,6 +43,33 @@ If running as a service add the environment variable to `/etc/default/telegraf`: GODEBUG=netdns=cgo ``` +### Q: How can I manage series cardinality? + +High [series cardinality][], when not properly managed, can cause high load on +your database. Telegraf attempts to avoid creating series with high +cardinality, but some monitoring workloads such as tracking containers are are +inherently high cardinality. These workloads can still be monitored, but care +must be taken to manage cardinality growth. + +You can use the following techniques to avoid cardinality issues: + +- Use [metric filtering][] options to exclude unneeded measurements and tags. +- Write to a database with an appropriate [retention policy][]. +- Limit series cardinality in your database using the + [max-series-per-database][] and [max-values-per-tag][] settings. +- Consider using the [Time Series Index][tsi]. +- Monitor your databases using the [show cardinality][] commands. +- Consult the [InfluxDB documentation][influx docs] for the most up-to-date techniques. + +[series cardinality]: https://docs.influxdata.com/influxdb/v1.7/concepts/glossary/#series-cardinality +[metric filtering]: https://github.com/influxdata/telegraf/blob/master/docs/CONFIGURATION.md#metric-filtering +[retention policy]: https://docs.influxdata.com/influxdb/latest/guides/downsampling_and_retention/ +[max-series-per-database]: https://docs.influxdata.com/influxdb/latest/administration/config/#max-series-per-database-1000000 +[max-values-per-tag]: https://docs.influxdata.com/influxdb/latest/administration/config/#max-values-per-tag-100000 +[tsi]: https://docs.influxdata.com/influxdb/latest/concepts/time-series-index/ +[show cardinality]: https://docs.influxdata.com/influxdb/latest/query_language/spec/#show-cardinality +[influx docs]: https://docs.influxdata.com/influxdb/latest/ + ### Q: When will the next version be released? The latest release date estimate can be viewed on the diff --git a/docs/LICENSE_OF_DEPENDENCIES.md b/docs/LICENSE_OF_DEPENDENCIES.md index e0332196bf71e..71636a0b86622 100644 --- a/docs/LICENSE_OF_DEPENDENCIES.md +++ b/docs/LICENSE_OF_DEPENDENCIES.md @@ -41,7 +41,9 @@ following works: - github.com/go-redis/redis [BSD 2-Clause "Simplified" License](https://github.com/go-redis/redis/blob/master/LICENSE) - github.com/go-sql-driver/mysql [Mozilla Public License 2.0](https://github.com/go-sql-driver/mysql/blob/master/LICENSE) - github.com/gobwas/glob [MIT License](https://github.com/gobwas/glob/blob/master/LICENSE) +- github.com/gofrs/uuid [MIT License](https://github.com/gofrs/uuid/blob/master/LICENSE) - github.com/gogo/protobuf [BSD 3-Clause Clear License](https://github.com/gogo/protobuf/blob/master/LICENSE) +- github.com/golang/mock [Apache License 2.0](https://github.com/golang/mock/blob/master/LICENSE) - github.com/golang/protobuf [BSD 3-Clause "New" or "Revised" License](https://github.com/golang/protobuf/blob/master/LICENSE) - github.com/golang/snappy [BSD 3-Clause "New" or "Revised" License](https://github.com/golang/snappy/blob/master/LICENSE) - github.com/google/go-cmp [BSD 3-Clause "New" or "Revised" License](https://github.com/google/go-cmp/blob/master/LICENSE) @@ -66,12 +68,14 @@ following works: - github.com/kardianos/osext [BSD 3-Clause "New" or "Revised" License](https://github.com/kardianos/osext/blob/master/LICENSE) - github.com/kardianos/service [zlib License](https://github.com/kardianos/service/blob/master/LICENSE) - github.com/kballard/go-shellquote [MIT License](https://github.com/kballard/go-shellquote/blob/master/LICENSE) +- github.com/klauspost/compress [BSD 3-Clause Clear License](https://github.com/klauspost/compress/blob/master/LICENSE) - github.com/kr/logfmt [MIT License](https://github.com/kr/logfmt/blob/master/Readme) - github.com/kubernetes/apimachinery [Apache License 2.0](https://github.com/kubernetes/apimachinery/blob/master/LICENSE) - github.com/leodido/ragel-machinery [MIT License](https://github.com/leodido/ragel-machinery/blob/develop/LICENSE) - github.com/mailru/easyjson [MIT License](https://github.com/mailru/easyjson/blob/master/LICENSE) - github.com/matttproud/golang_protobuf_extensions [Apache License 2.0](https://github.com/matttproud/golang_protobuf_extensions/blob/master/LICENSE) - github.com/mdlayher/apcupsd [MIT License](https://github.com/mdlayher/apcupsd/blob/master/LICENSE.md) +- github.com/Mellanox/rdmamap [Apache License 2.0](https://github.com/Mellanox/rdmamap/blob/master/LICENSE) - github.com/Microsoft/ApplicationInsights-Go [MIT License](https://github.com/Microsoft/ApplicationInsights-Go/blob/master/LICENSE) - github.com/Microsoft/go-winio [MIT License](https://github.com/Microsoft/go-winio/blob/master/LICENSE) - github.com/miekg/dns [BSD 3-Clause Clear License](https://github.com/miekg/dns/blob/master/LICENSE) @@ -98,7 +102,6 @@ following works: - github.com/prometheus/procfs [Apache License 2.0](https://github.com/prometheus/procfs/blob/master/LICENSE) - github.com/rcrowley/go-metrics [MIT License](https://github.com/rcrowley/go-metrics/blob/master/LICENSE) - github.com/samuel/go-zookeeper [BSD 3-Clause Clear License](https://github.com/samuel/go-zookeeper/blob/master/LICENSE) -- github.com/satori/go.uuid [MIT License](https://github.com/satori/go.uuid/blob/master/LICENSE) - github.com/shirou/gopsutil [BSD 3-Clause Clear License](https://github.com/shirou/gopsutil/blob/master/LICENSE) - github.com/shirou/w32 [BSD 3-Clause Clear License](https://github.com/shirou/w32/blob/master/LICENSE) - github.com/Shopify/sarama [MIT License](https://github.com/Shopify/sarama/blob/master/LICENSE) diff --git a/docs/WINDOWS_SERVICE.md b/docs/WINDOWS_SERVICE.md index 5b630076c5265..b0b6ee5adf358 100644 --- a/docs/WINDOWS_SERVICE.md +++ b/docs/WINDOWS_SERVICE.md @@ -48,18 +48,21 @@ Telegraf can manage its own service through the --service flag: ## Install multiple services -You can install multiple telegraf instances with --service-name flag: +Running multiple instances of Telegraf is seldom needed, as you can run +multiple instances of each plugin and route metric flow using the metric +filtering options. However, if you do need to run multiple telegraf instances +on a single system, you can install the service with the `--service-name` and +`--service-display-name` flags to give the services unique names: ``` - > C:\"Program Files"\Telegraf\telegraf.exe --service install --service-name telegraf-1 - > C:\"Program Files"\Telegraf\telegraf.exe --service install --service-name telegraf-2 - > C:\"Program Files"\Telegraf\telegraf.exe --service uninstall --service-name telegraf-1 +> C:\"Program Files"\Telegraf\telegraf.exe --service install --service-name telegraf-1 --service-display-name "Telegraf 1" +> C:\"Program Files"\Telegraf\telegraf.exe --service install --service-name telegraf-2 --service-display-name "Telegraf 2" ``` ## Troubleshooting When Telegraf runs as a Windows service, Telegraf logs messages to Windows events log before configuration file with logging settings is loaded. -Check event log for an error reported by `telegraf` service in case of Telegraf service reports failure on its start: Event Viewer->Windows Logs->Application +Check event log for an error reported by `telegraf` service in case of Telegraf service reports failure on its start: Event Viewer->Windows Logs->Application **Troubleshooting common error #1067** diff --git a/etc/telegraf.conf b/etc/telegraf.conf index 49edc842fe107..28edd51924e34 100644 --- a/etc/telegraf.conf +++ b/etc/telegraf.conf @@ -35,7 +35,9 @@ ## This controls the size of writes that Telegraf sends to output plugins. metric_batch_size = 1000 - ## Maximum number of unwritten metrics per output. + ## Maximum number of unwritten metrics per output. Increasing this value + ## allows for longer periods of output downtime without dropping metrics at the + ## cost of higher maximum memory usage. metric_buffer_limit = 10000 ## Collection jitter is used to jitter the collection by a random amount. @@ -66,7 +68,13 @@ ## Log only error level messages. # quiet = false - ## Log file name, the empty string means to log to stderr. + ## Log target controls the destination for logs and can be one of "file", + ## "stderr" or, on Windows, "eventlog". When set to "file", the output file + ## is determined by the "logfile" setting. + # logtarget = "file" + + ## Name of the file to be logged to when using the "file" logtarget. If set to + ## the empty string then logs are written to stderr. # logfile = "" ## The logfile will be rotated after the time interval specified. When set @@ -412,6 +420,9 @@ # ## You could use basicstats aggregator to calculate those fields. If not all statistic # ## fields are available, all fields would still be sent as raw metrics. # # write_statistics = false +# +# ## Enable high resolution metrics of 1 second (if not enabled, standard resolution are of 60 seconds precision) +# # high_resolution_metrics = false # # Configuration for CrateDB to send metrics to. @@ -516,6 +527,11 @@ # ## Files to write to, "stdout" is a specially handled file. # files = ["stdout", "/tmp/metrics.out"] # +# ## Use batch serialization format instead of line based delimiting. The +# ## batch format allows for the production of non line based output formats and +# ## may more effiently encode metric groups. +# # use_batch_format = false +# # ## The file will be rotated after the time interval specified. When set # ## to 0 no time based rotation is performed. # # rotation_interval = "0d" @@ -657,6 +673,7 @@ # ## # ## Multiple URLs can be specified for a single cluster, only ONE of the # ## urls will be written to each interval. +# ## ex: urls = ["https://us-west-2-1.aws.cloud2.influxdata.com"] # urls = ["http://127.0.0.1:9999"] # # ## Token for authentication. @@ -805,6 +822,7 @@ # # max_message_bytes = 1000000 # # ## Optional TLS Config +# # enable_tls = true # # tls_ca = "/etc/telegraf/ca.pem" # # tls_cert = "/etc/telegraf/cert.pem" # # tls_key = "/etc/telegraf/key.pem" @@ -815,6 +833,9 @@ # # sasl_username = "kafka" # # sasl_password = "secret" # +# ## SASL protocol version. When connecting to Azure EventHub set to 0. +# # sasl_version = 1 +# # ## Data format to output. # ## Each data format has its own unique set of configuration options, read # ## more about them here: @@ -1029,6 +1050,14 @@ # ## Address to listen on # listen = ":9273" # +# ## Metric version controls the mapping from Telegraf metrics into +# ## Prometheus format. When using the prometheus input, use the same value in +# ## both plugins to ensure metrics are round-tripped without modification. +# ## +# ## example: metric_version = 1; deprecated in 1.13 +# ## metric_version = 2; recommended version +# # metric_version = 1 +# # ## Use HTTP Basic Authentication. # # basic_username = "Foo" # # basic_password = "Bar" @@ -1292,6 +1321,18 @@ ############################################################################### +# # Clone metrics and apply modifications. +# [[processors.clone]] +# ## All modifications on inputs and aggregators can be overridden: +# # name_override = "new_name" +# # name_prefix = "new_name_prefix" +# # name_suffix = "new_name_suffix" +# +# ## Tags to be added (all values must be strings) +# # [processors.clone.tags] +# # additional_tag = "tag_value" + + # # Convert values to another metric value type # [[processors.converter]] # ## Tags to convert @@ -1479,6 +1520,10 @@ # # [[processors.strings.left]] # # field = "message" # # width = 10 +# +# ## Decode a base64 encoded utf-8 string +# # [[processors.strings.base64decode]] +# # field = "message" # # Restricts the number of tags that can pass through this filter and chooses which tags to preserve when over the limit. @@ -1557,6 +1602,7 @@ # [[aggregators.basicstats]] # ## The period on which to flush & clear the aggregator. # period = "30s" +# # ## If true, the original metric will be dropped by the # ## aggregator and will not get sent to the output plugins. # drop_original = false @@ -1607,6 +1653,13 @@ # # fields = ["io_time", "read_time", "write_time"] +# # Merge metrics into multifield metrics by series key +# [[aggregators.merge]] +# ## If true, the original metric will be dropped by the +# ## aggregator and will not get sent to the output plugins. +# drop_original = true + + # # Keep the aggregate min/max of each metric passing through. # [[aggregators.minmax]] # ## General Aggregator Arguments: @@ -1816,6 +1869,18 @@ # # insecure_skip_verify = false +# # Gather Azure Storage Queue metrics +# [[inputs.azure_storage_queue]] +# ## Required Azure Storage Account name +# account_name = "mystorageaccount" +# +# ## Required Azure Storage Account access key +# account_key = "storageaccountaccesskey" +# +# ## Set to false to disable peeking age of oldest message (executes faster) +# # peek_oldest_message_age = true + + # # Read metrics of bcache from stats_total and dirty_data # [[inputs.bcache]] # ## Bcache sets path @@ -2013,6 +2078,9 @@ # ## See http://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_limits.html # # ratelimit = 25 # +# ## Timeout for http requests made by the cloudwatch client. +# # timeout = "5s" +# # ## Namespace-wide statistic filters. These allow fewer queries to be made to # ## cloudwatch. # # statistic_include = [ "average", "sum", "minimum", "maximum", sample_count" ] @@ -2202,6 +2270,9 @@ # ## Only collect metrics for these containers, collect all if empty # container_names = [] # +# ## Set the source tag for the metrics to the container ID hostname, eg first 12 chars +# source_tag = false +# # ## Containers to include and exclude. Globs accepted. # ## Note that an empty array for both will include all containers # container_name_include = [] @@ -2220,8 +2291,10 @@ # ## Whether to report for each container per-device blkio (8:0, 8:1...) and # ## network (eth0, eth1, ...) stats or not # perdevice = true +# # ## Whether to report for each container total blkio and network stats or not # total = false +# # ## Which environment variables should we use as a tag # ##tag_env = ["JAVA_HOME", "HEAP_SIZE"] # @@ -2246,8 +2319,10 @@ # ## # ## If no servers are specified, then localhost is used as the host. # servers = ["localhost:24242"] +# # ## Type is one of "user", "domain", "ip", or "global" # type = "global" +# # ## Wildcard matches like "*.com". An empty string "" is same as "*" # ## If type = "ip" filters should be # filters = [""] @@ -2332,6 +2407,15 @@ # # insecure_skip_verify = false +# # Returns ethtool statistics for given interfaces +# [[inputs.ethtool]] +# ## List of interfaces to pull metrics for +# # interface_include = ["eth0"] +# +# ## List of interfaces to ignore when pulling metrics. +# # interface_exclude = ["eth1"] + + # # Read metrics from one or more commands that can output to stdout # [[inputs.exec]] # ## Commands array @@ -2389,6 +2473,10 @@ # ## more about them here: # ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md # data_format = "influx" +# +# ## Name a tag containing the name of the file the data was parsed from. Leave empty +# ## to disable. +# # file_tag = "" # # Count files in a directory @@ -2414,6 +2502,9 @@ # ## Only count regular files. Defaults to true. # regular_only = true # +# ## Follow all symlinks while walking the directory tree. Defaults to false. +# follow_symlinks = false +# # ## Only count files that are at least this size. If size is # ## a negative number, only count files that are smaller than the # ## absolute value of size. Acceptable units are B, KiB, MiB, KB, ... @@ -2438,6 +2529,7 @@ # ## See https://github.com/gobwas/glob for more examples # ## # files = ["/var/log/**.log"] +# # ## If true, read the entire file and calculate an md5 checksum. # md5 = false @@ -2604,6 +2696,9 @@ # ## Amount of time allowed to complete the HTTP request # # timeout = "5s" # +# ## List of success status codes +# # success_status_codes = [200] +# # ## Data format to consume. # ## Each data format has its own unique set of configuration options, read # ## more about them here: @@ -2707,10 +2802,10 @@ # # Gather Icinga2 status # [[inputs.icinga2]] -# ## Required Icinga2 server address (default: "https://localhost:5665") +# ## Required Icinga2 server address # # server = "https://localhost:5665" # -# ## Required Icinga2 object type ("services" or "hosts, default "services") +# ## Required Icinga2 object type ("services" or "hosts") # # object_type = "services" # # ## Credentials for basic HTTP authentication @@ -2740,6 +2835,10 @@ # "http://localhost:8086/debug/vars" # ] # +# ## Username and password to send using HTTP Basic Authentication. +# # username = "" +# # password = "" +# # ## Optional TLS Config # # tls_ca = "/etc/telegraf/ca.pem" # # tls_cert = "/etc/telegraf/cert.pem" @@ -2777,6 +2876,11 @@ # ## optionally specify the path to the ipmitool executable # # path = "/usr/bin/ipmitool" # ## +# ## Setting 'use_sudo' to true will make use of sudo to run ipmitool. +# ## Sudo must be configured to allow the telegraf user to run ipmitool +# ## without a password. +# # use_sudo = false +# ## # ## optionally force session privilege level. Can be CALLBACK, USER, OPERATOR, ADMINISTRATOR # # privilege = "ADMINISTRATOR" # ## @@ -2838,7 +2942,7 @@ # # Read jobs and cluster metrics from Jenkins instances # [[inputs.jenkins]] -# ## The Jenkins URL +# ## The Jenkins URL in the format "schema://host:port" # url = "http://my-jenkins-instance:8080" # # username = "admin" # # password = "admin" @@ -3059,6 +3163,8 @@ # # namespace = "default" # # ## Use bearer token for authorization. ('bearer_token' takes priority) +# ## If both of these are empty, we'll use the default serviceaccount: +# ## at: /run/secrets/kubernetes.io/serviceaccount/token # # bearer_token = "/path/to/bearer/token" # ## OR # # bearer_token_string = "abc_123" @@ -3090,10 +3196,17 @@ # url = "http://127.0.0.1:10255" # # ## Use bearer token for authorization. ('bearer_token' takes priority) +# ## If both of these are empty, we'll use the default serviceaccount: +# ## at: /run/secrets/kubernetes.io/serviceaccount/token # # bearer_token = "/path/to/bearer/token" # ## OR # # bearer_token_string = "abc_123" # +# ## Pod labels to be added as tags. An empty array for both include and +# ## exclude will include all labels. +# # label_include = [] +# # label_exclude = ["*"] +# # ## Set response_timeout (default 5 seconds) # # response_timeout = "5s" # @@ -3220,8 +3333,10 @@ # [[inputs.mesos]] # ## Timeout, in ms. # timeout = 100 +# # ## A list of Mesos masters. # masters = ["http://localhost:5050"] +# # ## Master metrics groups to be collected, by default, all enabled. # master_collections = [ # "resources", @@ -3236,8 +3351,10 @@ # "registrar", # "allocator", # ] +# # ## A list of Mesos slaves, default is [] # # slaves = [] +# # ## Slave metrics groups to be collected, by default, all enabled. # # slave_collections = [ # # "resources", @@ -3282,8 +3399,10 @@ # # ## When true, collect per database stats # # gather_perdb_stats = false +# # ## When true, collect per collection stats # # gather_col_stats = false +# # ## List of db where collections stats are collected # ## If empty, all db are concerned # # col_stats_dbs = ["local"] @@ -3346,55 +3465,59 @@ # ## <1.6: metric_version = 1 (or unset) # metric_version = 2 # -# ## the limits for metrics form perf_events_statements -# perf_events_statements_digest_text_limit = 120 -# perf_events_statements_limit = 250 -# perf_events_statements_time_limit = 86400 -# # # ## if the list is empty, then metrics are gathered from all databasee tables -# table_schema_databases = [] -# # +# # table_schema_databases = [] +# # ## gather metrics from INFORMATION_SCHEMA.TABLES for databases provided above list -# gather_table_schema = false -# # +# # gather_table_schema = false +# # ## gather thread state counts from INFORMATION_SCHEMA.PROCESSLIST -# gather_process_list = true -# # +# # gather_process_list = false +# # ## gather user statistics from INFORMATION_SCHEMA.USER_STATISTICS -# gather_user_statistics = true -# # +# # gather_user_statistics = false +# # ## gather auto_increment columns and max values from information schema -# gather_info_schema_auto_inc = true -# # +# # gather_info_schema_auto_inc = false +# # ## gather metrics from INFORMATION_SCHEMA.INNODB_METRICS -# gather_innodb_metrics = true -# # +# # gather_innodb_metrics = false +# # ## gather metrics from SHOW SLAVE STATUS command output -# gather_slave_status = true -# # +# # gather_slave_status = false +# # ## gather metrics from SHOW BINARY LOGS command output -# gather_binary_logs = false -# # +# # gather_binary_logs = false +# +# ## gather metrics from PERFORMANCE_SCHEMA.GLOBAL_VARIABLES +# # gather_global_variables = true +# # ## gather metrics from PERFORMANCE_SCHEMA.TABLE_IO_WAITS_SUMMARY_BY_TABLE -# gather_table_io_waits = false -# # +# # gather_table_io_waits = false +# # ## gather metrics from PERFORMANCE_SCHEMA.TABLE_LOCK_WAITS -# gather_table_lock_waits = false -# # +# # gather_table_lock_waits = false +# # ## gather metrics from PERFORMANCE_SCHEMA.TABLE_IO_WAITS_SUMMARY_BY_INDEX_USAGE -# gather_index_io_waits = false -# # +# # gather_index_io_waits = false +# # ## gather metrics from PERFORMANCE_SCHEMA.EVENT_WAITS -# gather_event_waits = false -# # +# # gather_event_waits = false +# # ## gather metrics from PERFORMANCE_SCHEMA.FILE_SUMMARY_BY_EVENT_NAME -# gather_file_events_stats = false -# # +# # gather_file_events_stats = false +# # ## gather metrics from PERFORMANCE_SCHEMA.EVENTS_STATEMENTS_SUMMARY_BY_DIGEST -# gather_perf_events_statements = false -# # +# # gather_perf_events_statements = false +# +# ## the limits for metrics form perf_events_statements +# # perf_events_statements_digest_text_limit = 120 +# # perf_events_statements_limit = 250 +# # perf_events_statements_time_limit = 86400 +# # ## Some queries we may want to run less often (such as SHOW GLOBAL VARIABLES) -# interval_slow = "30m" +# ## example: interval_slow = "30m" +# # interval_slow = "" # # ## Optional TLS Config (will be used if tls=custom parameter specified in server uri) # # tls_ca = "/etc/telegraf/ca.pem" @@ -3669,6 +3792,12 @@ # ## City ID's to collect weather data from. # city_id = ["5391959"] # +# ## Language of the description field. Can be one of "ar", "bg", +# ## "ca", "cz", "de", "el", "en", "fa", "fi", "fr", "gl", "hr", "hu", +# ## "it", "ja", "kr", "la", "lt", "mk", "nl", "pl", "pt", "ro", "ru", +# ## "se", "sk", "sl", "es", "tr", "ua", "vi", "zh_cn", "zh_tw" +# # lang = "en" +# # ## APIs to fetch; can contain "weather" or "forecast". # fetch = ["weather", "forecast"] # @@ -3745,35 +3874,47 @@ # # Ping given url(s) and return statistics # [[inputs.ping]] -# ## List of urls to ping +# ## Hosts to send ping packets to. # urls = ["example.org"] # -# ## Number of pings to send per collection (ping -c ) +# ## Method used for sending pings, can be either "exec" or "native". When set +# ## to "exec" the systems ping command will be executed. When set to "native" +# ## the plugin will send pings directly. +# ## +# ## While the default is "exec" for backwards compatibility, new deployments +# ## are encouraged to use the "native" method for improved compatibility and +# ## performance. +# # method = "exec" +# +# ## Number of ping packets to send per interval. Corresponds to the "-c" +# ## option of the ping command. # # count = 1 # -# ## Interval, in s, at which to ping. 0 == default (ping -i ) +# ## Time to wait between sending ping packets in seconds. Operates like the +# ## "-i" option of the ping command. # # ping_interval = 1.0 # -# ## Per-ping timeout, in s. 0 == no timeout (ping -W ) +# ## If set, the time to wait for a ping response in seconds. Operates like +# ## the "-W" option of the ping command. # # timeout = 1.0 # -# ## Total-ping deadline, in s. 0 == no deadline (ping -w ) +# ## If set, the total ping deadline, in seconds. Operates like the -w option +# ## of the ping command. # # deadline = 10 # -# ## Interface or source address to send ping from (ping -I[-S] ) +# ## Interface or source address to send ping from. Operates like the -I or -S +# ## option of the ping command. # # interface = "" # -# ## How to ping. "native" doesn't have external dependencies, while "exec" depends on 'ping'. -# # method = "exec" -# -# ## Specify the ping executable binary, default is "ping" -# # binary = "ping" +# ## Specify the ping executable binary. +# # binary = "ping" # -# ## Arguments for ping command. When arguments is not empty, system binary will be used and -# ## other options (ping_interval, timeout, etc) will be ignored. +# ## Arguments for ping command. When arguments is not empty, the command from +# ## the binary option will be used and other options (ping_interval, timeout, +# ## etc) will be ignored. # # arguments = ["-c", "3"] # -# ## Use only ipv6 addresses when resolving hostnames. +# ## Use only IPv6 addresses when resolving a hostname. # # ipv6 = false @@ -3892,6 +4033,15 @@ # ## Note that an empty array for both will include all queues # queue_name_include = [] # queue_name_exclude = [] +# +# ## Federation upstreams include and exclude when gathering the rabbitmq_federation measurement. +# ## If neither are specified, metrics for all federation upstreams are gathered. +# ## Federation link metrics will only be gathered for queues and exchanges +# ## whose non-federation metrics will be collected (e.g a queue excluded +# ## by the 'queue_name_exclude' option will also be excluded from federation). +# ## Globs accepted. +# # federation_upstream_include = ["dataCentre-*"] +# # federation_upstream_exclude = [] # # Read raindrops stats (raindrops - real-time stats for preforking Rack servers) @@ -4016,61 +4166,46 @@ # # Retrieves SNMP values from remote agents # [[inputs.snmp]] -# agents = [ "127.0.0.1:161" ] -# ## Timeout for each SNMP query. -# timeout = "5s" -# ## Number of retries to attempt within timeout. -# retries = 3 -# ## SNMP version, values can be 1, 2, or 3 -# version = 2 +# ## Agent addresses to retrieve values from. +# ## example: agents = ["udp://127.0.0.1:161"] +# ## agents = ["tcp://127.0.0.1:161"] +# agents = ["udp://127.0.0.1:161"] +# +# ## Timeout for each request. +# # timeout = "5s" +# +# ## SNMP version; can be 1, 2, or 3. +# # version = 2 # # ## SNMP community string. -# community = "public" -# -# ## The GETBULK max-repetitions parameter -# max_repetitions = 10 -# -# ## SNMPv3 auth parameters -# #sec_name = "myuser" -# #auth_protocol = "md5" # Values: "MD5", "SHA", "" -# #auth_password = "pass" -# #sec_level = "authNoPriv" # Values: "noAuthNoPriv", "authNoPriv", "authPriv" -# #context_name = "" -# #priv_protocol = "" # Values: "DES", "AES", "" -# #priv_password = "" -# -# ## measurement name -# name = "system" -# [[inputs.snmp.field]] -# name = "hostname" -# oid = ".1.0.0.1.1" -# [[inputs.snmp.field]] -# name = "uptime" -# oid = ".1.0.0.1.2" -# [[inputs.snmp.field]] -# name = "load" -# oid = ".1.0.0.1.3" -# [[inputs.snmp.field]] -# oid = "HOST-RESOURCES-MIB::hrMemorySize" +# # community = "public" # -# [[inputs.snmp.table]] -# ## measurement name -# name = "remote_servers" -# inherit_tags = [ "hostname" ] -# [[inputs.snmp.table.field]] -# name = "server" -# oid = ".1.0.0.0.1.0" -# is_tag = true -# [[inputs.snmp.table.field]] -# name = "connections" -# oid = ".1.0.0.0.1.1" -# [[inputs.snmp.table.field]] -# name = "latency" -# oid = ".1.0.0.0.1.2" +# ## Number of retries to attempt. +# # retries = 3 # -# [[inputs.snmp.table]] -# ## auto populate table's fields using the MIB -# oid = "HOST-RESOURCES-MIB::hrNetworkTable" +# ## The GETBULK max-repetitions parameter. +# # max_repetitions = 10 +# +# ## SNMPv3 authentication and encryption options. +# ## +# ## Security Name. +# # sec_name = "myuser" +# ## Authentication protocol; one of "MD5", "SHA", or "". +# # auth_protocol = "MD5" +# ## Authentication password. +# # auth_password = "pass" +# ## Security Level; one of "noAuthNoPriv", "authNoPriv", or "authPriv". +# # sec_level = "authNoPriv" +# ## Context Name. +# # context_name = "" +# ## Privacy protocol used for encrypted messages; one of "DES", "AES" or "". +# # priv_protocol = "" +# ## Privacy password used for encrypted messages. +# # priv_password = "" +# +# ## Add fields and tables defining the variables you wish to collect. This +# ## example collects the system uptime and interface variables. Reference the +# ## full plugin documentation for configuration details. # # DEPRECATED! PLEASE USE inputs.snmp INSTEAD. @@ -4197,7 +4332,8 @@ # ## By default, the host is localhost, listening on default port, TCP 1433. # ## for Windows, the user is the currently running AD user (SSO). # ## See https://github.com/denisenkom/go-mssqldb for detailed connection -# ## parameters. +# ## parameters, in particular, tls connections can be created like so: +# ## "encrypt=true;certificate=;hostNameInCertificate=" # # servers = [ # # "Server=192.168.1.10;Port=1433;User Id=;Password=;app name=telegraf;log=1;", # # ] @@ -4226,6 +4362,7 @@ # ## - AzureDBResourceStats # ## - AzureDBResourceGovernance # ## - SqlRequests +# ## - ServerProperties # exclude_query = [ 'Schedulers' ] @@ -4309,6 +4446,11 @@ # # value = 'one_of("sda", "sdb")' +# # Get synproxy counter statistics from procfs +# [[inputs.synproxy]] +# # no configuration + + # # Sysstat metrics collector # [[inputs.sysstat]] # ## Path to the sadc command. @@ -4318,18 +4460,15 @@ # ## Arch: /usr/lib/sa/sadc # ## RHEL/CentOS: /usr/lib64/sa/sadc # sadc_path = "/usr/lib/sa/sadc" # required -# # -# # +# # ## Path to the sadf command, if it is not in PATH # # sadf_path = "/usr/bin/sadf" -# # -# # +# # ## Activities is a list of activities, that are passed as argument to the # ## sadc collector utility (e.g: DISK, SNMP etc...) # ## The more activities that are added, the more data is collected. # # activities = ["DISK"] -# # -# # +# # ## Group metrics to measurements. # ## # ## If group is false each metric will be prefixed with a description @@ -4337,8 +4476,7 @@ # ## # ## If Group is true, corresponding metrics are grouped to a single measurement. # # group = true -# # -# # +# # ## Options for the sadf command. The values on the left represent the sadf # ## options and the values on the right their description (which are used for # ## grouping and prefixing metrics). @@ -4362,8 +4500,7 @@ # -w = "task" # # -H = "hugepages" # only available for newer linux distributions # # "-I ALL" = "interrupts" # requires INT activity -# # -# # +# # ## Device tags can be used to add additional tags for devices. # ## For example the configuration below adds a tag vg with value rootvg for # ## all metrics with sda devices. @@ -4371,6 +4508,17 @@ # # vg = "rootvg" +# # Gather systemd units state +# [[inputs.systemd_units]] +# ## Set timeout for systemctl execution +# # timeout = "1s" +# # +# ## Filter for a specific unit type, default is "service", other possible +# ## values are "socket", "target", "device", "mount", "automount", "swap", +# ## "timer", "path", "slice" and "scope ": +# # unittype = "service" + + # # Reads metrics from a Teamspeak 3 Server via ServerQuery # [[inputs.teamspeak]] # ## Server address for Teamspeak 3 ServerQuery @@ -4450,6 +4598,9 @@ # ## The default location of the unbound-control binary can be overridden with: # # binary = "/usr/sbin/unbound-control" # +# ## The default location of the unbound config file can be overridden with: +# # config_file = "/etc/unbound/unbound.conf" +# # ## The default timeout of 1s can be overriden with: # # timeout = "1s" # @@ -4736,6 +4887,9 @@ # ## transport only. # # tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"] # +# ## Define (for certain nested telemetry measurements with embedded tags) which fields are tags +# # embedded_tags = ["Cisco-IOS-XR-qos-ma-oper:qos/interface-table/interface/input/service-policy-names/service-policy-instance/statistics/class-stats/class-name"] +# # ## Define aliases to map telemetry encoding paths to simple measurement names # [inputs.cisco_telemetry_mdt.aliases] # ifstats = "ietf-interfaces:interfaces-state/interface/statistics" @@ -4896,6 +5050,9 @@ # # docker_label_include = [] # # docker_label_exclude = [] # +# ## Set the source tag for the metrics to the container ID hostname, eg first 12 chars +# source_tag = false +# # ## Optional TLS Config # # tls_ca = "/etc/telegraf/ca.pem" # # tls_cert = "/etc/telegraf/cert.pem" @@ -5097,22 +5254,30 @@ # # version = "" # # ## Optional TLS Config +# # enable_tls = true # # tls_ca = "/etc/telegraf/ca.pem" # # tls_cert = "/etc/telegraf/cert.pem" # # tls_key = "/etc/telegraf/key.pem" # ## Use TLS but skip chain & host verification # # insecure_skip_verify = false # -# ## Optional SASL Config +# ## SASL authentication credentials. These settings should typically be used +# ## with TLS encryption enabled using the "enable_tls" option. # # sasl_username = "kafka" # # sasl_password = "secret" # +# ## SASL protocol version. When connecting to Azure EventHub set to 0. +# # sasl_version = 1 +# # ## Name of the consumer group. # # consumer_group = "telegraf_metrics_consumers" # # ## Initial offset position; one of "oldest" or "newest". # # offset = "oldest" # +# ## Consumer group partition assignment strategy; one of "range", "roundrobin" or "sticky". +# # balance_strategy = "range" +# # ## Maximum length of a message to consume, in bytes (default 0/unlimited); # ## larger messages are dropped # max_message_len = 1000000 @@ -5138,12 +5303,16 @@ # [[inputs.kafka_consumer_legacy]] # ## topic(s) to consume # topics = ["telegraf"] +# # ## an array of Zookeeper connection strings # zookeeper_peers = ["localhost:2181"] +# # ## Zookeeper Chroot # zookeeper_chroot = "" +# # ## the name of the consumer group # consumer_group = "telegraf_metrics_consumers" +# # ## Offset (must be either "oldest" or "newest") # offset = "oldest" # @@ -5308,7 +5477,7 @@ # # max_undelivered_messages = 1000 # # ## Persistent session disables clearing of the client session on connection. -# ## In order for this option to work you must also set client_id to identity +# ## In order for this option to work you must also set client_id to identify # ## the client. To receive messages that arrived while the client is offline, # ## also set the qos option to 1 or 2 and don't forget to also set the QoS when # ## publishing. @@ -5342,6 +5511,7 @@ # # ## subject(s) to consume # subjects = ["telegraf"] +# # ## name a queue group # queue_group = "telegraf_consumers" # @@ -5385,8 +5555,10 @@ # [[inputs.nsq_consumer]] # ## Server option still works but is deprecated, we just prepend it to the nsqd array. # # server = "localhost:4150" +# # ## An array representing the NSQD TCP HTTP Endpoints # nsqd = ["localhost:4150"] +# # ## An array representing the NSQLookupd HTTP Endpoints # nsqlookupd = ["localhost:4161"] # topic = "telegraf" @@ -5501,7 +5673,10 @@ # ## field is used to define custom tags (separated by commas) # ## The optional "measurement" value can be used to override the default # ## output measurement name ("postgresql"). -# # +# ## +# ## The script option can be used to specify the .sql file path. +# ## If script and sqlquery options specified at same time, sqlquery will be used +# ## # ## Structure : # ## [[inputs.postgresql_extensible.query]] # ## sqlquery string @@ -5527,6 +5702,18 @@ # ## An array of urls to scrape metrics from. # urls = ["http://localhost:9100/metrics"] # +# ## Metric version controls the mapping from Prometheus metrics into +# ## Telegraf metrics. When using the prometheus_client output, use the same +# ## value in both plugins to ensure metrics are round-tripped without +# ## modification. +# ## +# ## example: metric_version = 1; deprecated in 1.13 +# ## metric_version = 2; recommended version +# # metric_version = 1 +# +# ## Url tag name (tag containing scrapped url. optional, default is "url") +# # url_tag = "scrapeUrl" +# # ## An array of Kubernetes services to scrape metrics from. # # kubernetes_services = ["http://my-service-dns.my-namespace:9100/metrics"] # @@ -5554,7 +5741,7 @@ # # username = "" # # password = "" # -# ## Specify timeout duration for slower prometheus clients (default is 3s) +# ## Specify timeout duration for slower prometheus clients (default is 3s) # # response_timeout = "3s" # # ## Optional TLS Config @@ -5565,6 +5752,20 @@ # # insecure_skip_verify = false +# # Receive SNMP traps +# [[inputs.snmp_trap]] +# ## Transport, local address, and port to listen on. Transport must +# ## be "udp://". Omit local address to listen on all interfaces. +# ## example: "udp://127.0.0.1:1234" +# ## +# ## Special permissions may be required to listen on a port less than +# ## 1024. See README.md for details +# ## +# # service_address = "udp://:162" +# ## Timeout running snmptranslate command +# # timeout = "5s" + + # # Generic socket listener capable of handling multiple socket types. # [[inputs.socket_listener]] # ## URL to listen on @@ -5620,6 +5821,10 @@ # ## more about them here: # ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md # # data_format = "influx" +# +# ## Content encoding for message payloads, can be set to "gzip" to or +# ## "identity" to apply no encoding. +# # content_encoding = "identity" # # Statsd UDP/TCP Server @@ -5682,6 +5887,18 @@ # percentile_limit = 1000 +# # Suricata stats plugin +# [[inputs.suricata]] +# ## Data sink for Suricata stats log +# # This is expected to be a filename of a +# # unix socket to be created for listening. +# source = "/var/run/suricata-stats.sock" +# +# # Delimiter for flattening field keys, e.g. subitem "alert" of "detect" +# # becomes "detect_alert" when delimiter is "_". +# delimiter = "_" + + # # Accepts syslog messages following RFC5424 format with transports as per RFC5426, RFC5425, or RFC6587 # [[inputs.syslog]] # ## Specify an ip or hostname with port - eg., tcp://localhost:6514, tcp://10.0.0.1:6514 diff --git a/etc/telegraf_windows.conf b/etc/telegraf_windows.conf index 0d72e79e8c210..c3586cafd835a 100644 --- a/etc/telegraf_windows.conf +++ b/etc/telegraf_windows.conf @@ -9,9 +9,9 @@ # Use 'telegraf -config telegraf.conf -test' to see what metrics a config # file would generate. # -# Environment variables can be used anywhere in this config file, simply prepend -# them with $. For strings the variable must be within quotes (ie, "$STR_VAR"), -# for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR) +# Environment variables can be used anywhere in this config file, simply surround +# them with ${}. For strings the variable must be within quotes (ie, "${STR_VAR}"), +# for numbers and booleans they should be plain (ie, ${INT_VAR}, ${BOOL_VAR}) # Global tags can be specified here in key="value" format. @@ -35,7 +35,9 @@ ## This controls the size of writes that Telegraf sends to output plugins. metric_batch_size = 1000 - ## Maximum number of unwritten metrics per output. + ## Maximum number of unwritten metrics per output. Increasing this value + ## allows for longer periods of output downtime without dropping metrics at the + ## cost of higher maximum memory usage. metric_buffer_limit = 10000 ## Collection jitter is used to jitter the collection by a random amount. @@ -66,7 +68,13 @@ ## Log only error level messages. # quiet = false - ## Log file name, the empty string means to log to stderr. + ## Log target controls the destination for logs and can be one of "file", + ## "stderr" or, on Windows, "eventlog". When set to "file", the output file + ## is determined by the "logfile" setting. + # logtarget = "file" + + ## Name of the file to be logged to when using the "file" logtarget. If set to + ## the empty string then logs are written to stderr. # logfile = "" ## The logfile will be rotated after the time interval specified. When set @@ -89,9 +97,10 @@ ############################################################################### -# OUTPUTS # +# OUTPUT PLUGINS # ############################################################################### + # Configuration for sending metrics to InfluxDB [[outputs.influxdb]] ## The full HTTP or UDP URL for your InfluxDB instance. @@ -103,8 +112,16 @@ # urls = ["http://127.0.0.1:8086"] ## The target database for metrics; will be created as needed. + ## For UDP url endpoint database needs to be configured on server side. # database = "telegraf" + ## The value of this tag will be used to determine the database. If this + ## tag is not set the 'database' option is used as the default. + # database_tag = "" + + ## If true, the database tag will not be added to the metric. + # exclude_database_tag = false + ## If true, no CREATE DATABASE queries will be sent. Set to true when using ## Telegraf with a user without permissions to create databases or when the ## database already exists. @@ -161,6 +178,7 @@ # ## # ## Multiple URLs can be specified for a single cluster, only ONE of the # ## urls will be written to each interval. +# ## ex: urls = ["https://us-west-2-1.aws.cloud2.influxdata.com"] # urls = ["http://127.0.0.1:9999"] # # ## Token for authentication. @@ -206,10 +224,12 @@ # ## Use TLS but skip chain & host verification # # insecure_skip_verify = false + ############################################################################### -# INPUTS # +# INPUT PLUGINS # ############################################################################### + # Windows Performance Counters plugin. # These are the recommended method of monitoring system metrics on windows, # as the regular system plugins (inputs.cpu, inputs.mem, etc.) rely on WMI, diff --git a/go.mod b/go.mod new file mode 100644 index 0000000000000..cea932902ea53 --- /dev/null +++ b/go.mod @@ -0,0 +1,139 @@ +module github.com/influxdata/telegraf + +go 1.12 + +require ( + cloud.google.com/go v0.37.4 + code.cloudfoundry.org/clock v1.0.0 // indirect + collectd.org v0.3.0 + github.com/Azure/azure-event-hubs-go/v3 v3.1.0 + github.com/Azure/azure-storage-queue-go v0.0.0-20181215014128-6ed74e755687 + github.com/Azure/go-autorest/autorest v0.9.3 + github.com/Azure/go-autorest/autorest/azure/auth v0.4.2 + github.com/Mellanox/rdmamap v0.0.0-20191106181932-7c3c4763a6ee + github.com/Microsoft/ApplicationInsights-Go v0.4.2 + github.com/Microsoft/go-winio v0.4.9 // indirect + github.com/Shopify/sarama v1.24.1 + github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 + github.com/aerospike/aerospike-client-go v1.27.0 + github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf + github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 + github.com/apache/thrift v0.12.0 + github.com/armon/go-metrics v0.3.0 // indirect + github.com/aws/aws-sdk-go v1.19.41 + github.com/bitly/go-hostpool v0.1.0 // indirect + github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect + github.com/caio/go-tdigest v2.3.0+incompatible // indirect + github.com/cenkalti/backoff v2.0.0+incompatible // indirect + github.com/cisco-ie/nx-telemetry-proto v0.0.0-20190531143454-82441e232cf6 + github.com/cockroachdb/apd v1.1.0 // indirect + github.com/couchbase/go-couchbase v0.0.0-20180501122049-16db1f1fe037 + github.com/couchbase/gomemcached v0.0.0-20180502221210-0da75df14530 // indirect + github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a // indirect + github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4 + github.com/dgrijalva/jwt-go v3.2.0+incompatible + github.com/docker/distribution v2.6.0-rc.1.0.20170726174610-edc3ab29cdff+incompatible // indirect + github.com/docker/docker v1.4.2-0.20180327123150-ed7b6428c133 + github.com/docker/go-connections v0.3.0 // indirect + github.com/docker/go-units v0.3.3 // indirect + github.com/docker/libnetwork v0.8.0-dev.2.0.20181012153825-d7b61745d166 + github.com/eclipse/paho.mqtt.golang v1.2.0 + github.com/ericchiang/k8s v1.2.0 + github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 + github.com/glinton/ping v0.1.3 + github.com/go-logfmt/logfmt v0.4.0 + github.com/go-ole/go-ole v1.2.1 // indirect + github.com/go-redis/redis v6.12.0+incompatible + github.com/go-sql-driver/mysql v1.4.1 + github.com/gobwas/glob v0.2.3 + github.com/gofrs/uuid v2.1.0+incompatible + github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d + github.com/golang/mock v1.3.1-0.20190508161146-9fa652df1129 // indirect + github.com/golang/protobuf v1.3.2 + github.com/google/go-cmp v0.3.1 + github.com/google/go-github v17.0.0+incompatible + github.com/google/go-querystring v1.0.0 // indirect + github.com/gorilla/mux v1.6.2 + github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect + github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect + github.com/harlow/kinesis-consumer v0.3.1-0.20181230152818-2f58b136fee0 + github.com/hashicorp/consul v1.2.1 + github.com/hashicorp/go-msgpack v0.5.5 // indirect + github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90 // indirect + github.com/hashicorp/memberlist v0.1.5 // indirect + github.com/hashicorp/serf v0.8.1 // indirect + github.com/influxdata/go-syslog/v2 v2.0.1 + github.com/influxdata/tail v1.0.1-0.20180327235535-c43482518d41 + github.com/influxdata/toml v0.0.0-20190415235208-270119a8ce65 + github.com/influxdata/wlog v0.0.0-20160411224016-7c63b0a71ef8 + github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 // indirect + github.com/jackc/pgx v3.6.0+incompatible + github.com/jcmturner/gofork v1.0.0 // indirect + github.com/kardianos/service v1.0.0 + github.com/karrick/godirwalk v1.12.0 + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 + github.com/klauspost/compress v1.9.2 // indirect + github.com/kubernetes/apimachinery v0.0.0-20190119020841-d41becfba9ee + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353 // indirect + github.com/lib/pq v1.3.0 // indirect + github.com/mailru/easyjson v0.0.0-20180717111219-efc7eb8984d6 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 + github.com/mdlayher/apcupsd v0.0.0-20190314144147-eb3dd99a75fe + github.com/miekg/dns v1.0.14 + github.com/mitchellh/go-testing-interface v1.0.0 // indirect + github.com/multiplay/go-ts3 v1.0.0 + github.com/naoina/go-stringutil v0.1.0 // indirect + github.com/nats-io/gnatsd v1.2.0 + github.com/nats-io/go-nats v1.5.0 + github.com/nats-io/nuid v1.0.0 // indirect + github.com/nsqio/go-nsq v1.0.7 + github.com/openconfig/gnmi v0.0.0-20180912164834-33a1865c3029 + github.com/opencontainers/go-digest v1.0.0-rc1 // indirect + github.com/opencontainers/image-spec v1.0.1 // indirect + github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 // indirect + github.com/opentracing/opentracing-go v1.0.2 // indirect + github.com/openzipkin/zipkin-go-opentracing v0.3.4 + github.com/pkg/errors v0.8.1 + github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 + github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f + github.com/prometheus/common v0.2.0 + github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8 + github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec // indirect + github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b // indirect + github.com/shirou/gopsutil v2.19.11+incompatible + github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect + github.com/shopspring/decimal v0.0.0-20200105231215-408a2507e114 // indirect + github.com/sirupsen/logrus v1.2.0 + github.com/soniah/gosnmp v1.22.0 + github.com/streadway/amqp v0.0.0-20180528204448-e5adc2ada8b8 + github.com/stretchr/testify v1.4.0 + github.com/tedsuo/ifrit v0.0.0-20191009134036-9a97d0632f00 // indirect + github.com/tidwall/gjson v1.3.0 + github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e // indirect + github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc // indirect + github.com/vjeantet/grok v1.0.0 + github.com/vmware/govmomi v0.19.0 + github.com/wavefronthq/wavefront-sdk-go v0.9.2 + github.com/wvanbergen/kafka v0.0.0-20171203153745-e2edea948ddf + github.com/wvanbergen/kazoo-go v0.0.0-20180202103751-f72d8611297a // indirect + github.com/yuin/gopher-lua v0.0.0-20180630135845-46796da1b0b4 // indirect + golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 + golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 + golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 + gonum.org/v1/gonum v0.6.2 // indirect + google.golang.org/api v0.5.0 + google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69 + google.golang.org/grpc v1.21.0 + gopkg.in/fatih/pool.v2 v2.0.0 // indirect + gopkg.in/gorethink/gorethink.v3 v3.0.5 + gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect + gopkg.in/ldap.v3 v3.1.0 + gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce + gopkg.in/olivere/elastic.v5 v5.0.70 + gopkg.in/yaml.v2 v2.2.4 + gotest.tools v2.2.0+incompatible // indirect + k8s.io/apimachinery v0.17.1 // indirect +) + +replace github.com/devigned/tab => github.com/R290/tab v0.1.2 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000000000..0d25c69953ab0 --- /dev/null +++ b/go.sum @@ -0,0 +1,598 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.37.4 h1:glPeL3BQJsbF6aIIYfZizMwc5LTYz250bDMjttbBGAU= +cloud.google.com/go v0.37.4/go.mod h1:NHPJ89PdicEuT9hdPXMROBD91xc5uRDxsMtSB16k7hw= +code.cloudfoundry.org/clock v1.0.0 h1:kFXWQM4bxYvdBw2X8BbBeXwQNgfoWv1vqAk2ZZyBN2o= +code.cloudfoundry.org/clock v1.0.0/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= +collectd.org v0.3.0 h1:iNBHGw1VvPJxH2B6RiFWFZ+vsjo1lCdRszBeOuwGi00= +collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= +github.com/Azure/azure-amqp-common-go/v3 v3.0.0 h1:j9tjcwhypb/jek3raNrwlCIl7iKQYOug7CLpSyBBodc= +github.com/Azure/azure-amqp-common-go/v3 v3.0.0/go.mod h1:SY08giD/XbhTz07tJdpw1SoxQXHPN30+DI3Z04SYqyg= +github.com/Azure/azure-event-hubs-go/v3 v3.1.0 h1:j+/WXzke3PTRu5gAgSpWgWJVfpwIyaedIqqgdgkjAe0= +github.com/Azure/azure-event-hubs-go/v3 v3.1.0/go.mod h1:hR40byNJjKkS74+3RhloPQ8sJ8zFQeJ920Uk3oYY0+k= +github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= +github.com/Azure/azure-pipeline-go v0.1.9 h1:u7JFb9fFTE6Y/j8ae2VK33ePrRqJqoCM/IWkQdAZ+rg= +github.com/Azure/azure-pipeline-go v0.1.9/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= +github.com/Azure/azure-sdk-for-go v37.1.0+incompatible h1:aFlw3lP7ZHQi4m1kWCpcwYtczhDkGhDoRaMTaxcOf68= +github.com/Azure/azure-sdk-for-go v37.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-storage-blob-go v0.6.0/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= +github.com/Azure/azure-storage-queue-go v0.0.0-20181215014128-6ed74e755687 h1:7MiZ6Th+YTmwUdrKmFg5OMsGYz7IdQwjqL0RPxkhhOQ= +github.com/Azure/azure-storage-queue-go v0.0.0-20181215014128-6ed74e755687/go.mod h1:K6am8mT+5iFXgingS9LUc7TmbsW6XBw3nxaRyaMyWc8= +github.com/Azure/go-amqp v0.12.6 h1:34yItuwhA/nusvq2sPSNPQxZLCf/CtaogYH8n578mnY= +github.com/Azure/go-amqp v0.12.6/go.mod h1:qApuH6OFTSKZFmCOxccvAv5rLizBQf4v8pRmG138DPo= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest v0.9.3 h1:OZEIaBbMdUE/Js+BQKlpO81XlISgipr6yDJ+PSwsgi4= +github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S6HY6e6wcHn37qQMBQlvY3lc= +github.com/Azure/go-autorest/autorest/adal v0.8.1 h1:pZdL8o72rK+avFWl+p9nE8RWi1JInZrWJYlnpfXJwHk= +github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest/azure/auth v0.4.2 h1:iM6UAvjR97ZIeR93qTcwpKNMpV+/FTWjwEbuPD495Tk= +github.com/Azure/go-autorest/autorest/azure/auth v0.4.2/go.mod h1:90gmfKdlmKgfjUpnCEpOJzsUEjrWDSLwHIG73tSXddM= +github.com/Azure/go-autorest/autorest/azure/cli v0.3.1 h1:LXl088ZQlP0SBppGFsRZonW6hSvwgL5gRByMbvUbx8U= +github.com/Azure/go-autorest/autorest/azure/cli v0.3.1/go.mod h1:ZG5p860J94/0kI9mNJVoIoLgXcirM2gF5i2kWloofxw= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0 h1:yW+Zlqf26583pE43KhfnhFcdmSWlm5Ew6bxipnr/tbM= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0 h1:qJumjCaCudz+OcqE9/XtEPfvtOjOmKaui4EOpFI6zZc= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/autorest/to v0.3.0 h1:zebkZaadz7+wIQYgC7GXaz3Wb28yKYfVkkBKwc38VF8= +github.com/Azure/go-autorest/autorest/to v0.3.0/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= +github.com/Azure/go-autorest/autorest/validation v0.2.0 h1:15vMO4y76dehZSq7pAaOLQxC6dZYsSrj2GQpflyM/L4= +github.com/Azure/go-autorest/autorest/validation v0.2.0/go.mod h1:3EEqHnBxQGHXRYq3HT1WyXAvT7LLY3tl70hw6tQIbjI= +github.com/Azure/go-autorest/logger v0.1.0 h1:ruG4BSDXONFRrZZJ2GUXDiUyVpayPmb1GnWeHDdaNKY= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.5.0 h1:TRn4WjSnkcSy5AEG3pnbtFSwNtwzjr4VYyQflFE619k= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/Mellanox/rdmamap v0.0.0-20191106181932-7c3c4763a6ee h1:atI/FFjXh6hIVlPE1Jup9m8N4B9q/OSbMUe2EBahs+w= +github.com/Mellanox/rdmamap v0.0.0-20191106181932-7c3c4763a6ee/go.mod h1:jDA6v0TUYrFEIAE5uGJ29LQOeONIgMdP4Rkqb8HUnPM= +github.com/Microsoft/ApplicationInsights-Go v0.4.2 h1:HIZoGXMiKNwAtMAgCSSX35j9mP+DjGF9ezfBvxMDLLg= +github.com/Microsoft/ApplicationInsights-Go v0.4.2/go.mod h1:CukZ/G66zxXtI+h/VcVn3eVVDGDHfXM2zVILF7bMmsg= +github.com/Microsoft/go-winio v0.4.9 h1:3RbgqgGVqmcpbOiwrjbVtDHLlJBGF6aE+yHmNtBNsFQ= +github.com/Microsoft/go-winio v0.4.9/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/R290/tab v0.1.2 h1:BTz2qU0BgzWpn42PNbHpyOQWhWCTYBUOmaSmbZnb/BE= +github.com/R290/tab v0.1.2/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= +github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= +github.com/Shopify/sarama v1.24.1 h1:svn9vfN3R1Hz21WR2Gj0VW9ehaDGkiOS+VqlIcZOkMI= +github.com/Shopify/sarama v1.24.1/go.mod h1:fGP8eQ6PugKEI0iUETYYtnP6d1pH/bdDMTel1X5ajsU= +github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= +github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= +github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/aerospike/aerospike-client-go v1.27.0 h1:VC6/Wqqm3Qlp4/utM7Zts3cv4A2HPn8rVFp/XZKTWgE= +github.com/aerospike/aerospike-client-go v1.27.0/go.mod h1:zj8LBEnWBDOVEIJt8LvaRvDG5ARAoa5dBeHaB472NRc= +github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9 h1:FXrPTd8Rdlc94dKccl7KPmdmIbVh/OjelJ8/vgMRzcQ= +github.com/amir/raidman v0.0.0-20170415203553-1ccc43bfb9c9/go.mod h1:eliMa/PW+RDr2QLWRmLH1R1ZA4RInpmvOzDDXtaIZkc= +github.com/apache/thrift v0.12.0 h1:pODnxUFNcjP9UTLZGTdeh+j16A8lJbRvD3rOtrk/7bs= +github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.3.0 h1:B7AQgHi8QSEi4uHu7Sbsga+IJDU+CENgjxoo81vDUqU= +github.com/armon/go-metrics v0.3.0/go.mod h1:zXjbSimjXTd7vOpY8B0/2LpvNvDoXBuplAD+gJD3GYs= +github.com/aws/aws-sdk-go v1.19.41 h1:veutzvQP/lOmYmtX26S9mTFJLO6sp7/UsxFcCjglu4A= +github.com/aws/aws-sdk-go v1.19.41/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/bitly/go-hostpool v0.1.0 h1:XKmsF6k5el6xHG3WPJ8U0Ku/ye7njX7W81Ng7O2ioR0= +github.com/bitly/go-hostpool v0.1.0/go.mod h1:4gOCgp6+NZnVqlKyZ/iBZFTAJKembaVENUpMkpg42fw= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/caio/go-tdigest v2.3.0+incompatible h1:zP6nR0nTSUzlSqqr7F/LhslPlSZX/fZeGmgmwj2cxxY= +github.com/caio/go-tdigest v2.3.0+incompatible/go.mod h1:sHQM/ubZStBUmF1WbB8FAm8q9GjDajLC5T7ydxE3JHI= +github.com/cenkalti/backoff v2.0.0+incompatible h1:5IIPUHhlnUZbcHQsQou5k1Tn58nJkeJL9U+ig5CHJbY= +github.com/cenkalti/backoff v2.0.0+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/cisco-ie/nx-telemetry-proto v0.0.0-20190531143454-82441e232cf6 h1:57RI0wFkG/smvVTcz7F43+R0k+Hvci3jAVQF9lyMoOo= +github.com/cisco-ie/nx-telemetry-proto v0.0.0-20190531143454-82441e232cf6/go.mod h1:ugEfq4B8T8ciw/h5mCkgdiDRFS4CkqqhH2dymDB4knc= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= +github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= +github.com/couchbase/go-couchbase v0.0.0-20180501122049-16db1f1fe037 h1:Dbz60fpCq04vRxVVVJLbQuL0G7pRt0Gyo2BkozFc4SQ= +github.com/couchbase/go-couchbase v0.0.0-20180501122049-16db1f1fe037/go.mod h1:TWI8EKQMs5u5jLKW/tsb9VwauIrMIxQG1r5fMsswK5U= +github.com/couchbase/gomemcached v0.0.0-20180502221210-0da75df14530 h1:F8nmbiuX+gCz9xvWMi6Ak8HQntB4ATFXP46gaxifbp4= +github.com/couchbase/gomemcached v0.0.0-20180502221210-0da75df14530/go.mod h1:srVSlQLB8iXBVXHgnqemxUXqN6FCvClgCMPCsjBDR7c= +github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a h1:Y5XsLCEhtEI8qbD9RP3Qlv5FXdTDHxZM9UPUnMRgBp8= +github.com/couchbase/goutils v0.0.0-20180530154633-e865a1461c8a/go.mod h1:BQwMFlJzDjFDG3DJUdU0KORxn88UlsOULuxLExMh3Hs= +github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4 h1:YcpmyvADGYw5LqMnHqSkyIELsHCGF6PkrmM31V8rF7o= +github.com/denisenkom/go-mssqldb v0.0.0-20190707035753-2be1aa521ff4/go.mod h1:zAg7JM8CkOJ43xKXIj7eRO9kmWm/TW578qo+oDO6tuM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= +github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/docker/distribution v2.6.0-rc.1.0.20170726174610-edc3ab29cdff+incompatible h1:357nGVUC8gSpeSc2Axup8HfrfTLLUfWfCsCUhiQSKIg= +github.com/docker/distribution v2.6.0-rc.1.0.20170726174610-edc3ab29cdff+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v1.4.2-0.20180327123150-ed7b6428c133 h1:Kus8nU6ctI/u/l86ljUJl6GpUtmO7gtD/krn4u5dr0M= +github.com/docker/docker v1.4.2-0.20180327123150-ed7b6428c133/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/go-connections v0.3.0 h1:3lOnM9cSzgGwx8VfK/NGOW5fLQ0GjIlCkaktF+n1M6o= +github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= +github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/libnetwork v0.8.0-dev.2.0.20181012153825-d7b61745d166 h1:KgEcrKF0NWi9GT/OvDp9ioXZIrHRbP8S5o+sot9gznQ= +github.com/docker/libnetwork v0.8.0-dev.2.0.20181012153825-d7b61745d166/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/eapache/go-resiliency v1.1.0 h1:1NtRmCAqadE2FN4ZcN6g90TP3uk8cg9rn9eNK2197aU= +github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= +github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/eclipse/paho.mqtt.golang v1.2.0 h1:1F8mhG9+aO5/xpdtFkW4SxOJB67ukuDC3t2y2qayIX0= +github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= +github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/ericchiang/k8s v1.2.0 h1:vxrMwEzY43oxu8aZyD/7b1s8tsBM+xoUoxjWECWFbPI= +github.com/ericchiang/k8s v1.2.0/go.mod h1:/OmBgSq2cd9IANnsGHGlEz27nwMZV2YxlpXuQtU3Bz4= +github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/frankban/quicktest v1.4.1 h1:Wv2VwvNn73pAdFIVUQRXYDFp31lXKbqblIXo/Q5GPSg= +github.com/frankban/quicktest v1.4.1/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 h1:Mn26/9ZMNWSw9C9ERFA1PUxfmGpolnw2v0bKOREu5ew= +github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32/go.mod h1:GIjDIg/heH5DOkXY3YJ/wNhfHsQHoXGjl8G8amsYQ1I= +github.com/glinton/ping v0.1.3 h1:8/9mj+hCgfba0X25E0Xs7cy+Zg9jGQVyulMVlUBrDDA= +github.com/glinton/ping v0.1.3/go.mod h1:uY+1eqFUyotrQxF1wYFNtMeHp/swbYRsoGzfcPZ8x3o= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= +github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-redis/redis v6.12.0+incompatible h1:s+64XI+z/RXqGHz2fQSgRJOEwqqSXeX3dliF7iVkMbE= +github.com/go-redis/redis v6.12.0+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= +github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/gofrs/uuid v2.1.0+incompatible h1:8oEj3gioPmmDAOLQUZdnW+h4FZu9aSE/SQIas1E9pzA= +github.com/gofrs/uuid v2.1.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d h1:3PaI8p3seN09VjbTYC/QWlUZdZ1qS1zGjy7LH2Wt07I= +github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1-0.20190508161146-9fa652df1129 h1:tT8iWCYw4uOem71yYA3htfH+LNopJvcqZQshm56G5L4= +github.com/golang/mock v1.3.1-0.20190508161146-9fa652df1129/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c h1:964Od4U6p2jUkFxvCydnIczKteheJEzHRToSGK3Bnlw= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= +github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4 h1:hU4mGcQI4DaAYW+IbTun+2qEZVFxK0ySjQLTbS0VQKc= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= +github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= +github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gotestyourself/gotestyourself v2.2.0+incompatible h1:AQwinXlbQR2HvPjQZOmDhRqsv5mZf+Jb1RnSLxcqZcI= +github.com/gotestyourself/gotestyourself v2.2.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= +github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= +github.com/harlow/kinesis-consumer v0.3.1-0.20181230152818-2f58b136fee0 h1:U0KvGD9CJIl1nbgu9yLsfWxMT6WqL8fG0IBB7RvOZZQ= +github.com/harlow/kinesis-consumer v0.3.1-0.20181230152818-2f58b136fee0/go.mod h1:dk23l2BruuUzRP8wbybQbPn3J7sZga2QHICCeaEy5rQ= +github.com/hashicorp/consul v1.2.1 h1:66MuuTfV4aOXTQM7cjAIKUWFOITSk4XZlMhE09ymVbg= +github.com/hashicorp/consul v1.2.1/go.mod h1:mFrjN1mfidgJfYP1xrJCF+AfRhr6Eaqhb2+sfyn/OOI= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90 h1:VBj0QYQ0u2MCJzBfeYXGexnAl17GsH1yidnoxCqqD9E= +github.com/hashicorp/go-rootcerts v0.0.0-20160503143440-6bb64b370b90/go.mod h1:o4zcYY1e0GEZI6eSEr+43QDYmuGglw1qSO6qdHUHCgg= +github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/memberlist v0.1.5 h1:AYBsgJOW9gab/toO5tEB8lWetVgDKZycqkebJ8xxpqM= +github.com/hashicorp/memberlist v0.1.5/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/hashicorp/serf v0.8.1 h1:mYs6SMzu72+90OcPa5wr3nfznA4Dw9UyR791ZFNOIf4= +github.com/hashicorp/serf v0.8.1/go.mod h1:h/Ru6tmZazX7WO/GDmwdpS975F019L4t5ng5IgwbNrE= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/influxdata/go-syslog/v2 v2.0.1 h1:l44S4l4Q8MhGQcoOxJpbo+QQYxJqp0vdgIVHh4+DO0s= +github.com/influxdata/go-syslog/v2 v2.0.1/go.mod h1:hjvie1UTaD5E1fTnDmxaCw8RRDrT4Ve+XHr5O2dKSCo= +github.com/influxdata/tail v1.0.1-0.20180327235535-c43482518d41 h1:HxQo1NpNXQDpvEBzthbQLmePvTLFTa5GzSFUjL03aEs= +github.com/influxdata/tail v1.0.1-0.20180327235535-c43482518d41/go.mod h1:xTFF2SILpIYc5N+Srb0d5qpx7d+f733nBrbasb13DtQ= +github.com/influxdata/toml v0.0.0-20190415235208-270119a8ce65 h1:vvyMtD5LTJc1W9sQKjDkAWdcg0478CszSdzlHtiAXCY= +github.com/influxdata/toml v0.0.0-20190415235208-270119a8ce65/go.mod h1:zApaNFpP/bTpQItGZNNUMISDMDAnTXu9UqJ4yT3ocz8= +github.com/influxdata/wlog v0.0.0-20160411224016-7c63b0a71ef8 h1:W2IgzRCb0L9VzMujq/QuTaZUKcH8096jWwP519mHN6Q= +github.com/influxdata/wlog v0.0.0-20160411224016-7c63b0a71ef8/go.mod h1:/2NMgWB1DHM1ti/gqhOlg+LJeBVk6FqR5aVGYY0hlwI= +github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 h1:vr3AYkKovP8uR8AvSGGUK1IDqRa5lAAvEkZG1LKaCRc= +github.com/jackc/fake v0.0.0-20150926172116-812a484cc733/go.mod h1:WrMFNQdiFJ80sQsxDoMokWK1W5TQtxBFNpzWTD84ibQ= +github.com/jackc/pgx v3.6.0+incompatible h1:bJeo4JdVbDAW8KB2m8XkFeo8CPipREoG37BwEoKGz+Q= +github.com/jackc/pgx v3.6.0+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= +github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8= +github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7 h1:K//n/AqR5HjG3qxbrBCL4vJPW0MVFSs9CPK1OOJdRME= +github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= +github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= +github.com/kardianos/service v1.0.0 h1:HgQS3mFfOlyntWX8Oke98JcJLqt1DBcHR4kxShpYef0= +github.com/kardianos/service v1.0.0/go.mod h1:8CzDhVuCuugtsHyZoTvsOBuvonN/UDBvl0kH+BUxvbo= +github.com/karrick/godirwalk v1.12.0 h1:nkS4xxsjiZMvVlazd0mFyiwD4BR9f3m6LXGhM2TUx3Y= +github.com/karrick/godirwalk v1.12.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/klauspost/compress v1.9.2 h1:LfVyl+ZlLlLDeQ/d2AqfGIIH4qEDu0Ed2S5GyhCWIWY= +github.com/klauspost/compress v1.9.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515 h1:T+h1c/A9Gawja4Y9mFVWj2vyii2bbUNDw3kt9VxK2EY= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kubernetes/apimachinery v0.0.0-20190119020841-d41becfba9ee h1:MB75LRhfeLER2RF7neSVpYuX/lL8aPi3yPtv5vdOJmk= +github.com/kubernetes/apimachinery v0.0.0-20190119020841-d41becfba9ee/go.mod h1:Pe/YBTPc3vqoMkbuIWPH8CF9ehINdvNyS0dP3J6HC0s= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353 h1:X/79QL0b4YJVO5+OsPH9rF2u428CIrGL/jLmPsoOQQ4= +github.com/leesper/go_rng v0.0.0-20190531154944-a612b043e353/go.mod h1:N0SVk0uhy+E1PZ3C9ctsPRlvOPAFPkCNlcPBDkt0N3U= +github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165 h1:bCiVCRCs1Heq84lurVinUPy19keqGEe4jh5vtK37jcg= +github.com/leodido/ragel-machinery v0.0.0-20181214104525-299bdde78165/go.mod h1:WZxr2/6a/Ar9bMDc2rN/LJrE/hF6bXE4LPyDSIxwAfg= +github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= +github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20180717111219-efc7eb8984d6 h1:8/+Y8SKf0xCZ8cCTfnrMdY7HNzlEjPAt3bPjalNb6CA= +github.com/mailru/easyjson v0.0.0-20180717111219-efc7eb8984d6/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mdlayher/apcupsd v0.0.0-20190314144147-eb3dd99a75fe h1:yMrL+YorbzaBpj/h3BbLMP+qeslPZYMbzcpHFBNy1Yk= +github.com/mdlayher/apcupsd v0.0.0-20190314144147-eb3dd99a75fe/go.mod h1:y3mw3VG+t0m20OMqpG8RQqw8cDXvShVb+L8Z8FEnebw= +github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/multiplay/go-ts3 v1.0.0 h1:loxtEFqvYtpoGh1jOqEt6aDzctYuQsi3vb3dMpvWiWw= +github.com/multiplay/go-ts3 v1.0.0/go.mod h1:14S6cS3fLNT3xOytrA/DkRyAFNuQLMLEqOYAsf87IbQ= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= +github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= +github.com/nats-io/gnatsd v1.2.0 h1:WKLzmB8LyP4CiVJuAoZMxdYBurENVX4piS358tjcBhw= +github.com/nats-io/gnatsd v1.2.0/go.mod h1:nqco77VO78hLCJpIcVfygDP2rPGfsEHkGTUk94uh5DQ= +github.com/nats-io/go-nats v1.5.0 h1:OrEQSvQQrP+A+9EBBxY86Z4Es6uaUdObZ5UhWHn9b08= +github.com/nats-io/go-nats v1.5.0/go.mod h1:+t7RHT5ApZebkrQdnn6AhQJmhJJiKAvJUio1PiiCtj0= +github.com/nats-io/nuid v1.0.0 h1:44QGdhbiANq8ZCbUkdn6W5bqtg+mHuDE4wOUuxxndFs= +github.com/nats-io/nuid v1.0.0/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nsqio/go-nsq v1.0.7 h1:O0pIZJYTf+x7cZBA0UMY8WxFG79lYTURmWzAAh48ljY= +github.com/nsqio/go-nsq v1.0.7/go.mod h1:XP5zaUs3pqf+Q71EqUJs3HYfBIqfK6G83WQMdNN+Ito= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1 h1:q/mM8GF/n0shIN8SaAZ0V+jnLPzen6WIVZdiwrRlMlo= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/openconfig/gnmi v0.0.0-20180912164834-33a1865c3029 h1:lXQqyLroROhwR2Yq/kXbLzVecgmVeZh2TFLg6OxCd+w= +github.com/openconfig/gnmi v0.0.0-20180912164834-33a1865c3029/go.mod h1:t+O9It+LKzfOAhKTT5O0ehDix+MTqbtT0T9t+7zzOvc= +github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= +github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 h1:lM6RxxfUMrYL/f8bWEUqdXrANWtrL7Nndbm9iFN0DlU= +github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= +github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= +github.com/openzipkin/zipkin-go-opentracing v0.3.4 h1:x/pBv/5VJNWkcHF1G9xqhug8Iw7X1y1zOMzDmyuvP2g= +github.com/openzipkin/zipkin-go-opentracing v0.3.4/go.mod h1:js2AbwmHW0YD9DwIw2JhQWmbfFi/UnWyYwdVhqbCDOE= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pierrec/lz4 v2.2.6+incompatible h1:6aCX4/YZ9v8q69hTyiR7dNLnTA3fgtKHVVW5BCd5Znw= +github.com/pierrec/lz4 v2.2.6+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829 h1:D+CiwcpGTW6pL6bv6KI3KbyEyCKyS+1JWS2h8PNDnGA= +github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f h1:BVwpUVJDADN2ufcGik7W992pyps0wZ888b/y9GXcLTU= +github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.2.0 h1:kUZDBDTdBVBYBj5Tmh2NZLlF60mfjA27rM34b+cVwNU= +github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1 h1:/K3IL0Z1quvmJ7X0A1AwNEK7CRkVK3YwfOU/QAL4WGg= +github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8 h1:2c1EFnZHIPCW8qKWgHMH/fX2PkSabFc5mrVzfUNdg5U= +github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= +github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec h1:6ncX5ko6B9LntYM0YBRXkiSaZMmLYeZ/NWcmeB43mMY= +github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= +github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b h1:gQZ0qzfKHQIybLANtM3mBXNUtOfsCFXeTsnBqCsx1KM= +github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shirou/gopsutil v2.19.11+incompatible h1:lJHR0foqAjI4exXqWsU3DbH7bX1xvdhGdnXTIARA9W4= +github.com/shirou/gopsutil v2.19.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= +github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= +github.com/shopspring/decimal v0.0.0-20200105231215-408a2507e114 h1:Pm6R878vxWWWR+Sa3ppsLce/Zq+JNTs6aVvRu13jv9A= +github.com/shopspring/decimal v0.0.0-20200105231215-408a2507e114/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/soniah/gosnmp v1.22.0 h1:jVJi8+OGvR+JHIaZKMmnyNP0akJd2vEgNatybwhZvxg= +github.com/soniah/gosnmp v1.22.0/go.mod h1:DuEpAS0az51+DyVBQwITDsoq4++e3LTNckp2GoasF2I= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/streadway/amqp v0.0.0-20180528204448-e5adc2ada8b8 h1:l6epF6yBwuejBfhGkM5m8VSNM/QAm7ApGyH35ehA7eQ= +github.com/streadway/amqp v0.0.0-20180528204448-e5adc2ada8b8/go.mod h1:1WNBiOZtZQLpVAyu0iTduoJL9hEsMloAK5XWrtW0xdY= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/tedsuo/ifrit v0.0.0-20191009134036-9a97d0632f00 h1:mujcChM89zOHwgZBBNr5WZ77mBXP1yR+gLThGCYZgAg= +github.com/tedsuo/ifrit v0.0.0-20191009134036-9a97d0632f00/go.mod h1:eyZnKCc955uh98WQvzOm0dgAeLnf2O0Rz0LPoC5ze+0= +github.com/tidwall/gjson v1.3.0 h1:kfpsw1W3trbg4Xm6doUtqSl9+LhLB6qJ9PkltVAQZYs= +github.com/tidwall/gjson v1.3.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= +github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc= +github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= +github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e h1:f1yevOHP+Suqk0rVc13fIkzcLULJbyQcXDba2klljD0= +github.com/vishvananda/netlink v0.0.0-20171020171820-b2de5d10e38e/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= +github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc h1:R83G5ikgLMxrBvLh22JhdfI8K6YXEPHx5P03Uu3DRs4= +github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= +github.com/vjeantet/grok v1.0.0 h1:uxMqatJP6MOFXsj6C1tZBnqqAThQEeqnizUZ48gSJQQ= +github.com/vjeantet/grok v1.0.0/go.mod h1:/FWYEVYekkm+2VjcFmO9PufDU5FgXHUz9oy2EGqmQBo= +github.com/vmware/govmomi v0.19.0 h1:CR6tEByWCPOnRoRyhLzuHaU+6o2ybF3qufNRWS/MGrY= +github.com/vmware/govmomi v0.19.0/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59bHWk6aFU= +github.com/wavefronthq/wavefront-sdk-go v0.9.2 h1:/LvWgZYNjHFUg+ZUX+qv+7e+M8sEMi0lM15zPp681Gk= +github.com/wavefronthq/wavefront-sdk-go v0.9.2/go.mod h1:hQI6y8M9OtTCtc0xdwh+dCER4osxXdEAeCpacjpDZEU= +github.com/wvanbergen/kafka v0.0.0-20171203153745-e2edea948ddf h1:TOV5PC6fIWwFOFra9xJfRXZcL2pLhMI8oNuDugNxg9Q= +github.com/wvanbergen/kafka v0.0.0-20171203153745-e2edea948ddf/go.mod h1:nxx7XRXbR9ykhnC8lXqQyJS0rfvJGxKyKw/sT1YOttg= +github.com/wvanbergen/kazoo-go v0.0.0-20180202103751-f72d8611297a h1:ILoU84rj4AQ3q6cjQvtb9jBjx4xzR/Riq/zYhmDQiOk= +github.com/wvanbergen/kazoo-go v0.0.0-20180202103751-f72d8611297a/go.mod h1:vQQATAGxVK20DC1rRubTJbZDDhhpA4QfU02pMdPxGO4= +github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/yuin/gopher-lua v0.0.0-20180630135845-46796da1b0b4 h1:f6CCNiTjQZ0uWK4jPwhwYB8QIGGfn0ssD9kVzRUUUpk= +github.com/yuin/gopher-lua v0.0.0-20180630135845-46796da1b0b4/go.mod h1:aEV29XrmTYFr3CiRxZeGHpkvbwq+prZduBqMaascyCU= +go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= +go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190404164418-38d8ce5564a5/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413 h1:ULYEB3JvPRE/IfO+9uO7vKV/xzVTO7XPAwm8xbf4w2g= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2 h1:y102fOLFqhV41b+4GPiJoa0k/x+pJcEi2/HB1Y5T6fU= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= +gonum.org/v1/gonum v0.6.2 h1:4r+yNT0+8SWcOkXP+63H2zQbN+USnC73cjGUxnDF94Q= +gonum.org/v1/gonum v0.6.2/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0 h1:OE9mWmgKkjJyEmDAAtGMPjXu+YNeGvK9VTSHY6+Qihc= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= +google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= +google.golang.org/api v0.5.0 h1:lj9SyhMzyoa38fgFF0oO2T6pjs5IzkLPKfVtxpyCRMM= +google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69 h1:4rNOqY4ULrKzS6twXa619uQgI7h9PaVd4ZhjFQ7C5zs= +google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.21.0 h1:G+97AoqBnmZIT91cLG/EkCoK9NSelj64P8bOHHNmGn0= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d h1:TxyelI5cVkbREznMhfzycHdkp5cLA7DpE+GKjSslYhM= +gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fatih/pool.v2 v2.0.0 h1:xIFeWtxifuQJGk/IEPKsTduEKcKvPmhoiVDGpC40nKg= +gopkg.in/fatih/pool.v2 v2.0.0/go.mod h1:8xVGeu1/2jr2wm5V9SPuMht2H5AEmf5aFMGSQixtjTY= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gorethink/gorethink.v3 v3.0.5 h1:e2Uc/Xe+hpcVQFsj6MuHlYog3r0JYpnTzwDj/y2O4MU= +gopkg.in/gorethink/gorethink.v3 v3.0.5/go.mod h1:+3yIIHJUGMBK+wyPH+iN5TP+88ikFDfZdqTlK3Y9q8I= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw= +gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM= +gopkg.in/jcmturner/dnsutils.v1 v1.0.1/go.mod h1:m3v+5svpVOhtFAP/wSz+yzh4Mc0Fg7eRhxkJMWSIz9Q= +gopkg.in/jcmturner/goidentity.v3 v3.0.0 h1:1duIyWiTaYvVx3YX2CYtpJbUFd7/UuPYCfgXtQ3VTbI= +gopkg.in/jcmturner/goidentity.v3 v3.0.0/go.mod h1:oG2kH0IvSYNIu80dVAyu/yoefjq1mNfM5bm88whjWx4= +gopkg.in/jcmturner/gokrb5.v7 v7.2.3/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/gokrb5.v7 v7.3.0 h1:0709Jtq/6QXEuWRfAm260XqlpcwL1vxtO1tUE2qK8Z4= +gopkg.in/jcmturner/gokrb5.v7 v7.3.0/go.mod h1:l8VISx+WGYp+Fp7KRbsiUuXTTOnxIc3Tuvyavf11/WM= +gopkg.in/jcmturner/rpc.v1 v1.1.0 h1:QHIUxTX1ISuAv9dD2wJ9HWQVuWDX/Zc0PfeC2tjc4rU= +gopkg.in/jcmturner/rpc.v1 v1.1.0/go.mod h1:YIdkC4XfD6GXbzje11McwsDuOlZQSb9W4vfLvuNnlv8= +gopkg.in/ldap.v3 v3.1.0 h1:DIDWEjI7vQWREh0S8X5/NFPCZ3MCVd55LmXKPW4XLGE= +gopkg.in/ldap.v3 v3.1.0/go.mod h1:dQjCc0R0kfyFjIlWNMH1DORwUASZyDxo2Ry1B51dXaQ= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/olivere/elastic.v5 v5.0.70 h1:DqFG2Odzs74JCz6SssgJjd6qpGnsOAzNc7+l5EnvsnE= +gopkg.in/olivere/elastic.v5 v5.0.70/go.mod h1:FylZT6jQWtfHsicejzOm3jIMVPOAksa80i3o+6qtQRk= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +k8s.io/apimachinery v0.17.1 h1:zUjS3szTxoUjTDYNvdFkYt2uMEXLcthcbp+7uZvWhYM= +k8s.io/apimachinery v0.17.1/go.mod h1:b9qmWdKlLuU9EBh+06BtLcSf/Mu89rWL33naRxs1uZg= +k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= +k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= +k8s.io/kube-openapi v0.0.0-20191107075043-30be4d16710a/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= +rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/internal/config/config.go b/internal/config/config.go index d7fe11427a588..586acce719706 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -75,6 +75,7 @@ func NewConfig() *Config { Interval: internal.Duration{Duration: 10 * time.Second}, RoundInterval: true, FlushInterval: internal.Duration{Duration: 10 * time.Second}, + LogTarget: "file", LogfileRotationMaxArchives: 5, }, @@ -146,10 +147,13 @@ type AgentConfig struct { // Quiet is the option for running in quiet mode Quiet bool `toml:"quiet"` - // Log target - file, stderr, eventlog (Windows only). The empty string means to log to stderr. + // Log target controls the destination for logs and can be one of "file", + // "stderr" or, on Windows, "eventlog". When set to "file", the output file + // is determined by the "logfile" setting. LogTarget string `toml:"logtarget"` - // Log file name . + // Name of the file to be logged to when using the "file" logtarget. If set to + // the empty string then logs are written to stderr. Logfile string `toml:"logfile"` // The file will be rotated after the time interval specified. When set @@ -290,7 +294,13 @@ var agentConfig = ` ## Log only error level messages. # quiet = false - ## Log file name, the empty string means to log to stderr. + ## Log target controls the destination for logs and can be one of "file", + ## "stderr" or, on Windows, "eventlog". When set to "file", the output file + ## is determined by the "logfile" setting. + # logtarget = "file" + + ## Name of the file to be logged to when using the "file" logtarget. If set to + ## the empty string then logs are written to stderr. # logfile = "" ## The logfile will be rotated after the time interval specified. When set @@ -841,13 +851,14 @@ func loadConfig(config string) ([]byte, error) { } func fetchConfig(u *url.URL) ([]byte, error) { - v := os.Getenv("INFLUX_TOKEN") - req, err := http.NewRequest("GET", u.String(), nil) if err != nil { return nil, err } - req.Header.Add("Authorization", "Token "+v) + + if v, exists := os.LookupEnv("INFLUX_TOKEN"); exists { + req.Header.Add("Authorization", "Token "+v) + } req.Header.Add("Accept", "application/toml") resp, err := http.DefaultClient.Do(req) if err != nil { @@ -1941,6 +1952,18 @@ func buildSerializer(name string, tbl *ast.Table) (serializers.Serializer, error } } + if node, ok := tbl.Fields["splunkmetric_multimetric"]; ok { + if kv, ok := node.(*ast.KeyValue); ok { + if b, ok := kv.Value.(*ast.Boolean); ok { + var err error + c.SplunkmetricMultiMetric, err = b.Boolean() + if err != nil { + return nil, err + } + } + } + } + if node, ok := tbl.Fields["wavefront_source_override"]; ok { if kv, ok := node.(*ast.KeyValue); ok { if ary, ok := kv.Value.(*ast.Array); ok { @@ -1965,6 +1988,42 @@ func buildSerializer(name string, tbl *ast.Table) (serializers.Serializer, error } } + if node, ok := tbl.Fields["prometheus_export_timestamp"]; ok { + if kv, ok := node.(*ast.KeyValue); ok { + if b, ok := kv.Value.(*ast.Boolean); ok { + var err error + c.PrometheusExportTimestamp, err = b.Boolean() + if err != nil { + return nil, err + } + } + } + } + + if node, ok := tbl.Fields["prometheus_sort_metrics"]; ok { + if kv, ok := node.(*ast.KeyValue); ok { + if b, ok := kv.Value.(*ast.Boolean); ok { + var err error + c.PrometheusSortMetrics, err = b.Boolean() + if err != nil { + return nil, err + } + } + } + } + + if node, ok := tbl.Fields["prometheus_string_as_label"]; ok { + if kv, ok := node.(*ast.KeyValue); ok { + if b, ok := kv.Value.(*ast.Boolean); ok { + var err error + c.PrometheusStringAsLabel, err = b.Boolean() + if err != nil { + return nil, err + } + } + } + } + delete(tbl.Fields, "influx_max_line_bytes") delete(tbl.Fields, "influx_sort_fields") delete(tbl.Fields, "influx_uint_support") @@ -1974,8 +2033,12 @@ func buildSerializer(name string, tbl *ast.Table) (serializers.Serializer, error delete(tbl.Fields, "template") delete(tbl.Fields, "json_timestamp_units") delete(tbl.Fields, "splunkmetric_hec_routing") + delete(tbl.Fields, "splunkmetric_multimetric") delete(tbl.Fields, "wavefront_source_override") delete(tbl.Fields, "wavefront_use_strict") + delete(tbl.Fields, "prometheus_export_timestamp") + delete(tbl.Fields, "prometheus_sort_metrics") + delete(tbl.Fields, "prometheus_string_as_label") return serializers.NewSerializer(c) } @@ -2015,6 +2078,19 @@ func buildOutput(name string, tbl *ast.Table) (*models.OutputConfig, error) { } } + if node, ok := tbl.Fields["flush_jitter"]; ok { + if kv, ok := node.(*ast.KeyValue); ok { + if str, ok := kv.Value.(*ast.String); ok { + dur, err := time.ParseDuration(str.Value) + if err != nil { + return nil, err + } + oc.FlushJitter = new(time.Duration) + *oc.FlushJitter = dur + } + } + } + if node, ok := tbl.Fields["metric_buffer_limit"]; ok { if kv, ok := node.(*ast.KeyValue); ok { if integer, ok := kv.Value.(*ast.Integer); ok { @@ -2048,6 +2124,7 @@ func buildOutput(name string, tbl *ast.Table) (*models.OutputConfig, error) { } delete(tbl.Fields, "flush_interval") + delete(tbl.Fields, "flush_jitter") delete(tbl.Fields, "metric_buffer_limit") delete(tbl.Fields, "metric_batch_size") delete(tbl.Fields, "alias") diff --git a/internal/config/config_test.go b/internal/config/config_test.go index f05419eef770c..7559bf9fea12b 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -158,6 +158,11 @@ func TestConfig_LoadDirectory(t *testing.T) { MeasurementSuffix: "_myothercollector", } eConfig.Tags = make(map[string]string) + + exec := c.Inputs[1].Input.(*exec.Exec) + require.NotNil(t, exec.Log) + exec.Log = nil + assert.Equal(t, ex, c.Inputs[1].Input, "Merged Testdata did not produce a correct exec struct.") assert.Equal(t, eConfig, c.Inputs[1].Config, diff --git a/internal/http.go b/internal/http.go index 230fdf2b722df..7ffd9bf2b8d8b 100644 --- a/internal/http.go +++ b/internal/http.go @@ -2,6 +2,7 @@ package internal import ( "crypto/subtle" + "net" "net/http" ) @@ -43,3 +44,49 @@ func (h *basicAuthHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) h.next.ServeHTTP(rw, req) } + +// IPRangeHandler returns a http handler that requires the remote address to be +// in the specified network. +func IPRangeHandler(network []*net.IPNet, onError ErrorFunc) func(h http.Handler) http.Handler { + return func(h http.Handler) http.Handler { + return &ipRangeHandler{ + network: network, + onError: onError, + next: h, + } + } +} + +type ipRangeHandler struct { + network []*net.IPNet + onError ErrorFunc + next http.Handler +} + +func (h *ipRangeHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + if len(h.network) == 0 { + h.next.ServeHTTP(rw, req) + return + } + + remoteIPString, _, err := net.SplitHostPort(req.RemoteAddr) + if err != nil { + h.onError(rw, http.StatusForbidden) + return + } + + remoteIP := net.ParseIP(remoteIPString) + if remoteIP == nil { + h.onError(rw, http.StatusForbidden) + return + } + + for _, net := range h.network { + if net.Contains(remoteIP) { + h.next.ServeHTTP(rw, req) + return + } + } + + h.onError(rw, http.StatusForbidden) +} diff --git a/internal/internal.go b/internal/internal.go index af36460e3e3d1..12e4b3af2d490 100644 --- a/internal/internal.go +++ b/internal/internal.go @@ -16,6 +16,7 @@ import ( "runtime" "strconv" "strings" + "sync" "syscall" "time" "unicode" @@ -50,6 +51,11 @@ type Number struct { Value float64 } +type ReadWaitCloser struct { + pipeReader *io.PipeReader + wg sync.WaitGroup +} + // SetVersion sets the telegraf agent version func SetVersion(v string) error { if version != "" { @@ -281,14 +287,25 @@ func ExitStatus(err error) (int, bool) { return 0, false } +func (r *ReadWaitCloser) Close() error { + err := r.pipeReader.Close() + r.wg.Wait() // wait for the gzip goroutine finish + return err +} + // CompressWithGzip takes an io.Reader as input and pipes // it through a gzip.Writer returning an io.Reader containing // the gzipped data. // An error is returned if passing data to the gzip.Writer fails -func CompressWithGzip(data io.Reader) (io.Reader, error) { +func CompressWithGzip(data io.Reader) (io.ReadCloser, error) { pipeReader, pipeWriter := io.Pipe() gzipWriter := gzip.NewWriter(pipeWriter) + rc := &ReadWaitCloser{ + pipeReader: pipeReader, + } + + rc.wg.Add(1) var err error go func() { _, err = io.Copy(gzipWriter, data) @@ -296,6 +313,7 @@ func CompressWithGzip(data io.Reader) (io.Reader, error) { // subsequent reads from the read half of the pipe will // return no bytes and the error err, or EOF if err is nil. pipeWriter.CloseWithError(err) + rc.wg.Done() }() return pipeReader, err diff --git a/internal/internal_test.go b/internal/internal_test.go index f4627ee74c329..bb186f5fcf9b0 100644 --- a/internal/internal_test.go +++ b/internal/internal_test.go @@ -3,6 +3,8 @@ package internal import ( "bytes" "compress/gzip" + "crypto/rand" + "io" "io/ioutil" "log" "os/exec" @@ -232,6 +234,38 @@ func TestCompressWithGzip(t *testing.T) { assert.Equal(t, testData, string(output)) } +type mockReader struct { + readN uint64 // record the number of calls to Read +} + +func (r *mockReader) Read(p []byte) (n int, err error) { + r.readN++ + return rand.Read(p) +} + +func TestCompressWithGzipEarlyClose(t *testing.T) { + mr := &mockReader{} + + rc, err := CompressWithGzip(mr) + assert.NoError(t, err) + + n, err := io.CopyN(ioutil.Discard, rc, 10000) + assert.NoError(t, err) + assert.Equal(t, int64(10000), n) + + r1 := mr.readN + err = rc.Close() + assert.NoError(t, err) + + n, err = io.CopyN(ioutil.Discard, rc, 10000) + assert.Error(t, io.EOF, err) + assert.Equal(t, int64(0), n) + + r2 := mr.readN + // no more read to the source after closing + assert.Equal(t, r1, r2) +} + func TestVersionAlreadySet(t *testing.T) { err := SetVersion("foo") assert.Nil(t, err) diff --git a/internal/models/running_aggregator.go b/internal/models/running_aggregator.go index ee46e5b13f557..b8957e30abf36 100644 --- a/internal/models/running_aggregator.go +++ b/internal/models/running_aggregator.go @@ -24,10 +24,14 @@ type RunningAggregator struct { } func NewRunningAggregator(aggregator telegraf.Aggregator, config *AggregatorConfig) *RunningAggregator { + tags := map[string]string{"aggregator": config.Name} + if config.Alias != "" { + tags["alias"] = config.Alias + } + logger := &Logger{ Name: logName("aggregators", config.Name, config.Alias), - Errs: selfstat.Register("aggregate", "errors", - map[string]string{"input": config.Name, "alias": config.Alias}), + Errs: selfstat.Register("aggregate", "errors", tags), } setLogIfExist(aggregator, logger) @@ -38,22 +42,22 @@ func NewRunningAggregator(aggregator telegraf.Aggregator, config *AggregatorConf MetricsPushed: selfstat.Register( "aggregate", "metrics_pushed", - map[string]string{"aggregator": config.Name, "alias": config.Alias}, + tags, ), MetricsFiltered: selfstat.Register( "aggregate", "metrics_filtered", - map[string]string{"aggregator": config.Name, "alias": config.Alias}, + tags, ), MetricsDropped: selfstat.Register( "aggregate", "metrics_dropped", - map[string]string{"aggregator": config.Name, "alias": config.Alias}, + tags, ), PushTime: selfstat.Register( "aggregate", "push_time_ns", - map[string]string{"aggregator": config.Name, "alias": config.Alias}, + tags, ), log: logger, } @@ -144,7 +148,7 @@ func (r *RunningAggregator) Add(m telegraf.Metric) bool { defer r.Unlock() if m.Time().Before(r.periodStart.Add(-r.Config.Grace)) || m.Time().After(r.periodEnd.Add(r.Config.Delay)) { - r.log.Debugf("metric is outside aggregation window; discarding. %s: m: %s e: %s g: %s", + r.log.Debugf("Metric is outside aggregation window; discarding. %s: m: %s e: %s g: %s", m.Time(), r.periodStart, r.periodEnd, r.Config.Grace) r.MetricsDropped.Incr(1) return r.Config.DropOriginal diff --git a/internal/models/running_input.go b/internal/models/running_input.go index 85f0afb81869c..c09fb1409f226 100644 --- a/internal/models/running_input.go +++ b/internal/models/running_input.go @@ -21,12 +21,15 @@ type RunningInput struct { } func NewRunningInput(input telegraf.Input, config *InputConfig) *RunningInput { + tags := map[string]string{"input": config.Name} + if config.Alias != "" { + tags["alias"] = config.Alias + } + logger := &Logger{ Name: logName("inputs", config.Name, config.Alias), - Errs: selfstat.Register("gather", "errors", - map[string]string{"input": config.Name, "alias": config.Alias}), + Errs: selfstat.Register("gather", "errors", tags), } - setLogIfExist(input, logger) return &RunningInput{ @@ -35,12 +38,12 @@ func NewRunningInput(input telegraf.Input, config *InputConfig) *RunningInput { MetricsGathered: selfstat.Register( "gather", "metrics_gathered", - map[string]string{"input": config.Name, "alias": config.Alias}, + tags, ), GatherTime: selfstat.RegisterTiming( "gather", "gather_time_ns", - map[string]string{"input": config.Name, "alias": config.Alias}, + tags, ), log: logger, } diff --git a/internal/models/running_output.go b/internal/models/running_output.go index 282c2d23b17f1..752cf34ef127d 100644 --- a/internal/models/running_output.go +++ b/internal/models/running_output.go @@ -24,6 +24,7 @@ type OutputConfig struct { Filter Filter FlushInterval time.Duration + FlushJitter *time.Duration MetricBufferLimit int MetricBatchSize int } @@ -57,12 +58,15 @@ func NewRunningOutput( batchSize int, bufferLimit int, ) *RunningOutput { + tags := map[string]string{"output": config.Name} + if config.Alias != "" { + tags["alias"] = config.Alias + } + logger := &Logger{ Name: logName("outputs", config.Name, config.Alias), - Errs: selfstat.Register("write", "errors", - map[string]string{"output": config.Name, "alias": config.Alias}), + Errs: selfstat.Register("write", "errors", tags), } - setLogIfExist(output, logger) if config.MetricBufferLimit > 0 { @@ -88,12 +92,12 @@ func NewRunningOutput( MetricsFiltered: selfstat.Register( "write", "metrics_filtered", - map[string]string{"output": config.Name, "alias": config.Alias}, + tags, ), WriteTime: selfstat.RegisterTiming( "write", "write_time_ns", - map[string]string{"output": config.Name, "alias": config.Alias}, + tags, ), log: logger, } diff --git a/internal/models/running_processor.go b/internal/models/running_processor.go index 5a12716e5ebeb..22a7d01987769 100644 --- a/internal/models/running_processor.go +++ b/internal/models/running_processor.go @@ -29,12 +29,15 @@ type ProcessorConfig struct { } func NewRunningProcessor(processor telegraf.Processor, config *ProcessorConfig) *RunningProcessor { + tags := map[string]string{"processor": config.Name} + if config.Alias != "" { + tags["alias"] = config.Alias + } + logger := &Logger{ Name: logName("processors", config.Name, config.Alias), - Errs: selfstat.Register("process", "errors", - map[string]string{"input": config.Name, "alias": config.Alias}), + Errs: selfstat.Register("process", "errors", tags), } - setLogIfExist(processor, logger) return &RunningProcessor{ diff --git a/logger/logger.go b/logger/logger.go index a52e709a89f62..a276d2e807c6c 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -2,7 +2,6 @@ package logger import ( "errors" - "io" "log" "os" @@ -112,7 +111,6 @@ func (t *telegrafLogCreator) CreateLogger(config LogConfig) (io.Writer, error) { writer = defaultWriter } } else { - log.Print("E! Empty logfile, using stderr") writer = defaultWriter } case LogTargetStderr, "": diff --git a/plugins/aggregators/all/all.go b/plugins/aggregators/all/all.go index ec04c0aaf68ff..eabfaa4bf8460 100644 --- a/plugins/aggregators/all/all.go +++ b/plugins/aggregators/all/all.go @@ -4,6 +4,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/aggregators/basicstats" _ "github.com/influxdata/telegraf/plugins/aggregators/final" _ "github.com/influxdata/telegraf/plugins/aggregators/histogram" + _ "github.com/influxdata/telegraf/plugins/aggregators/merge" _ "github.com/influxdata/telegraf/plugins/aggregators/minmax" _ "github.com/influxdata/telegraf/plugins/aggregators/valuecounter" ) diff --git a/plugins/aggregators/basicstats/README.md b/plugins/aggregators/basicstats/README.md index e6ae3b31a7c7d..8fef0c6f4886a 100644 --- a/plugins/aggregators/basicstats/README.md +++ b/plugins/aggregators/basicstats/README.md @@ -10,6 +10,7 @@ emitting the aggregate every `period` seconds. [[aggregators.basicstats]] ## The period on which to flush & clear the aggregator. period = "30s" + ## If true, the original metric will be dropped by the ## aggregator and will not get sent to the output plugins. drop_original = false diff --git a/plugins/aggregators/basicstats/basicstats.go b/plugins/aggregators/basicstats/basicstats.go index 28d1f2741c8d1..4e62ee31123a4 100644 --- a/plugins/aggregators/basicstats/basicstats.go +++ b/plugins/aggregators/basicstats/basicstats.go @@ -1,7 +1,6 @@ package basicstats import ( - "log" "math" "github.com/influxdata/telegraf" @@ -10,6 +9,7 @@ import ( type BasicStats struct { Stats []string `toml:"stats"` + Log telegraf.Logger cache map[uint64]aggregate statsConfig *configuredStats @@ -28,9 +28,9 @@ type configuredStats struct { } func NewBasicStats() *BasicStats { - mm := &BasicStats{} - mm.Reset() - return mm + return &BasicStats{ + cache: make(map[uint64]aggregate), + } } type aggregate struct { @@ -53,6 +53,7 @@ type basicstats struct { var sampleConfig = ` ## The period on which to flush & clear the aggregator. period = "30s" + ## If true, the original metric will be dropped by the ## aggregator and will not get sent to the output plugins. drop_original = false @@ -61,17 +62,17 @@ var sampleConfig = ` # stats = ["count", "min", "max", "mean", "stdev", "s2", "sum"] ` -func (m *BasicStats) SampleConfig() string { +func (*BasicStats) SampleConfig() string { return sampleConfig } -func (m *BasicStats) Description() string { +func (*BasicStats) Description() string { return "Keep the aggregate basicstats of each metric passing through." } -func (m *BasicStats) Add(in telegraf.Metric) { +func (b *BasicStats) Add(in telegraf.Metric) { id := in.HashID() - if _, ok := m.cache[id]; !ok { + if _, ok := b.cache[id]; !ok { // hit an uncached metric, create caches for first time: a := aggregate{ name: in.Name(), @@ -92,13 +93,13 @@ func (m *BasicStats) Add(in telegraf.Metric) { } } } - m.cache[id] = a + b.cache[id] = a } else { for _, field := range in.FieldList() { if fv, ok := convert(field.Value); ok { - if _, ok := m.cache[id].fields[field.Key]; !ok { + if _, ok := b.cache[id].fields[field.Key]; !ok { // hit an uncached field of a cached metric - m.cache[id].fields[field.Key] = basicstats{ + b.cache[id].fields[field.Key] = basicstats{ count: 1, min: fv, max: fv, @@ -111,7 +112,7 @@ func (m *BasicStats) Add(in telegraf.Metric) { continue } - tmp := m.cache[id].fields[field.Key] + tmp := b.cache[id].fields[field.Key] //https://en.m.wikipedia.org/wiki/Algorithms_for_calculating_variance //variable initialization x := fv @@ -138,32 +139,30 @@ func (m *BasicStats) Add(in telegraf.Metric) { //diff compute tmp.diff = fv - tmp.LAST //store final data - m.cache[id].fields[field.Key] = tmp + b.cache[id].fields[field.Key] = tmp } } } } -func (m *BasicStats) Push(acc telegraf.Accumulator) { - config := getConfiguredStats(m) - - for _, aggregate := range m.cache { +func (b *BasicStats) Push(acc telegraf.Accumulator) { + for _, aggregate := range b.cache { fields := map[string]interface{}{} for k, v := range aggregate.fields { - if config.count { + if b.statsConfig.count { fields[k+"_count"] = v.count } - if config.min { + if b.statsConfig.min { fields[k+"_min"] = v.min } - if config.max { + if b.statsConfig.max { fields[k+"_max"] = v.max } - if config.mean { + if b.statsConfig.mean { fields[k+"_mean"] = v.mean } - if config.sum { + if b.statsConfig.sum { fields[k+"_sum"] = v.sum } @@ -171,16 +170,16 @@ func (m *BasicStats) Push(acc telegraf.Accumulator) { if v.count > 1 { variance := v.M2 / (v.count - 1) - if config.variance { + if b.statsConfig.variance { fields[k+"_s2"] = variance } - if config.stdev { + if b.statsConfig.stdev { fields[k+"_stdev"] = math.Sqrt(variance) } - if config.diff { + if b.statsConfig.diff { fields[k+"_diff"] = v.diff } - if config.non_negative_diff && v.diff >= 0 { + if b.statsConfig.non_negative_diff && v.diff >= 0 { fields[k+"_non_negative_diff"] = v.diff } @@ -194,14 +193,12 @@ func (m *BasicStats) Push(acc telegraf.Accumulator) { } } -func parseStats(names []string) *configuredStats { - +// member function for logging. +func (b *BasicStats) parseStats() *configuredStats { parsed := &configuredStats{} - for _, name := range names { - + for _, name := range b.Stats { switch name { - case "count": parsed.count = true case "min": @@ -222,45 +219,32 @@ func parseStats(names []string) *configuredStats { parsed.non_negative_diff = true default: - log.Printf("W! Unrecognized basic stat '%s', ignoring", name) + b.Log.Warnf("Unrecognized basic stat %q, ignoring", name) } } return parsed } -func defaultStats() *configuredStats { - - defaults := &configuredStats{} - - defaults.count = true - defaults.min = true - defaults.max = true - defaults.mean = true - defaults.variance = true - defaults.stdev = true - defaults.sum = false - defaults.non_negative_diff = false - - return defaults -} - -func getConfiguredStats(m *BasicStats) *configuredStats { - - if m.statsConfig == nil { - - if m.Stats == nil { - m.statsConfig = defaultStats() - } else { - m.statsConfig = parseStats(m.Stats) +func (b *BasicStats) getConfiguredStats() { + if b.Stats == nil { + b.statsConfig = &configuredStats{ + count: true, + min: true, + max: true, + mean: true, + variance: true, + stdev: true, + sum: false, + non_negative_diff: false, } + } else { + b.statsConfig = b.parseStats() } - - return m.statsConfig } -func (m *BasicStats) Reset() { - m.cache = make(map[uint64]aggregate) +func (b *BasicStats) Reset() { + b.cache = make(map[uint64]aggregate) } func convert(in interface{}) (float64, bool) { @@ -276,6 +260,12 @@ func convert(in interface{}) (float64, bool) { } } +func (b *BasicStats) Init() error { + b.getConfiguredStats() + + return nil +} + func init() { aggregators.Add("basicstats", func() telegraf.Aggregator { return NewBasicStats() diff --git a/plugins/aggregators/basicstats/basicstats_test.go b/plugins/aggregators/basicstats/basicstats_test.go index 9965c5caa3536..c5a093840abc7 100644 --- a/plugins/aggregators/basicstats/basicstats_test.go +++ b/plugins/aggregators/basicstats/basicstats_test.go @@ -39,6 +39,8 @@ var m2, _ = metric.New("m1", func BenchmarkApply(b *testing.B) { minmax := NewBasicStats() + minmax.Log = testutil.Logger{} + minmax.getConfiguredStats() for n := 0; n < b.N; n++ { minmax.Add(m1) @@ -50,6 +52,8 @@ func BenchmarkApply(b *testing.B) { func TestBasicStatsWithPeriod(t *testing.T) { acc := testutil.Accumulator{} minmax := NewBasicStats() + minmax.Log = testutil.Logger{} + minmax.getConfiguredStats() minmax.Add(m1) minmax.Add(m2) @@ -106,6 +110,8 @@ func TestBasicStatsWithPeriod(t *testing.T) { func TestBasicStatsDifferentPeriods(t *testing.T) { acc := testutil.Accumulator{} minmax := NewBasicStats() + minmax.Log = testutil.Logger{} + minmax.getConfiguredStats() minmax.Add(m1) minmax.Push(&acc) @@ -181,6 +187,8 @@ func TestBasicStatsWithOnlyCount(t *testing.T) { aggregator := NewBasicStats() aggregator.Stats = []string{"count"} + aggregator.Log = testutil.Logger{} + aggregator.getConfiguredStats() aggregator.Add(m1) aggregator.Add(m2) @@ -208,6 +216,8 @@ func TestBasicStatsWithOnlyMin(t *testing.T) { aggregator := NewBasicStats() aggregator.Stats = []string{"min"} + aggregator.Log = testutil.Logger{} + aggregator.getConfiguredStats() aggregator.Add(m1) aggregator.Add(m2) @@ -235,6 +245,8 @@ func TestBasicStatsWithOnlyMax(t *testing.T) { aggregator := NewBasicStats() aggregator.Stats = []string{"max"} + aggregator.Log = testutil.Logger{} + aggregator.getConfiguredStats() aggregator.Add(m1) aggregator.Add(m2) @@ -262,6 +274,8 @@ func TestBasicStatsWithOnlyMean(t *testing.T) { aggregator := NewBasicStats() aggregator.Stats = []string{"mean"} + aggregator.Log = testutil.Logger{} + aggregator.getConfiguredStats() aggregator.Add(m1) aggregator.Add(m2) @@ -289,6 +303,8 @@ func TestBasicStatsWithOnlySum(t *testing.T) { aggregator := NewBasicStats() aggregator.Stats = []string{"sum"} + aggregator.Log = testutil.Logger{} + aggregator.getConfiguredStats() aggregator.Add(m1) aggregator.Add(m2) @@ -347,6 +363,8 @@ func TestBasicStatsWithOnlySumFloatingPointErrata(t *testing.T) { aggregator := NewBasicStats() aggregator.Stats = []string{"sum"} + aggregator.Log = testutil.Logger{} + aggregator.getConfiguredStats() aggregator.Add(sum1) aggregator.Add(sum2) @@ -368,6 +386,8 @@ func TestBasicStatsWithOnlyVariance(t *testing.T) { aggregator := NewBasicStats() aggregator.Stats = []string{"s2"} + aggregator.Log = testutil.Logger{} + aggregator.getConfiguredStats() aggregator.Add(m1) aggregator.Add(m2) @@ -393,6 +413,8 @@ func TestBasicStatsWithOnlyStandardDeviation(t *testing.T) { aggregator := NewBasicStats() aggregator.Stats = []string{"stdev"} + aggregator.Log = testutil.Logger{} + aggregator.getConfiguredStats() aggregator.Add(m1) aggregator.Add(m2) @@ -418,6 +440,8 @@ func TestBasicStatsWithMinAndMax(t *testing.T) { aggregator := NewBasicStats() aggregator.Stats = []string{"min", "max"} + aggregator.Log = testutil.Logger{} + aggregator.getConfiguredStats() aggregator.Add(m1) aggregator.Add(m2) @@ -452,6 +476,8 @@ func TestBasicStatsWithDiff(t *testing.T) { aggregator := NewBasicStats() aggregator.Stats = []string{"diff"} + aggregator.Log = testutil.Logger{} + aggregator.getConfiguredStats() aggregator.Add(m1) aggregator.Add(m2) @@ -477,6 +503,8 @@ func TestBasicStatsWithNonNegativeDiff(t *testing.T) { aggregator := NewBasicStats() aggregator.Stats = []string{"non_negative_diff"} + aggregator.Log = testutil.Logger{} + aggregator.getConfiguredStats() aggregator.Add(m1) aggregator.Add(m2) @@ -500,7 +528,9 @@ func TestBasicStatsWithNonNegativeDiff(t *testing.T) { func TestBasicStatsWithAllStats(t *testing.T) { acc := testutil.Accumulator{} minmax := NewBasicStats() + minmax.Log = testutil.Logger{} minmax.Stats = []string{"count", "min", "max", "mean", "stdev", "s2", "sum"} + minmax.getConfiguredStats() minmax.Add(m1) minmax.Add(m2) @@ -564,6 +594,8 @@ func TestBasicStatsWithNoStats(t *testing.T) { aggregator := NewBasicStats() aggregator.Stats = []string{} + aggregator.Log = testutil.Logger{} + aggregator.getConfiguredStats() aggregator.Add(m1) aggregator.Add(m2) @@ -579,6 +611,8 @@ func TestBasicStatsWithUnknownStat(t *testing.T) { aggregator := NewBasicStats() aggregator.Stats = []string{"crazy"} + aggregator.Log = testutil.Logger{} + aggregator.getConfiguredStats() aggregator.Add(m1) aggregator.Add(m2) @@ -596,6 +630,8 @@ func TestBasicStatsWithUnknownStat(t *testing.T) { func TestBasicStatsWithDefaultStats(t *testing.T) { aggregator := NewBasicStats() + aggregator.Log = testutil.Logger{} + aggregator.getConfiguredStats() aggregator.Add(m1) aggregator.Add(m2) diff --git a/plugins/aggregators/merge/README.md b/plugins/aggregators/merge/README.md new file mode 100644 index 0000000000000..89f7f0983c692 --- /dev/null +++ b/plugins/aggregators/merge/README.md @@ -0,0 +1,25 @@ +# Merge Aggregator + +Merge metrics together into a metric with multiple fields into the most memory +and network transfer efficient form. + +Use this plugin when fields are split over multiple metrics, with the same +measurement, tag set and timestamp. By merging into a single metric they can +be handled more efficiently by the output. + +### Configuration + +```toml +[[aggregators.merge]] + ## If true, the original metric will be dropped by the + ## aggregator and will not get sent to the output plugins. + drop_original = true +``` + +### Example + +```diff +- cpu,host=localhost usage_time=42 1567562620000000000 +- cpu,host=localhost idle_time=42 1567562620000000000 ++ cpu,host=localhost idle_time=42,usage_time=42 1567562620000000000 +``` diff --git a/plugins/aggregators/merge/merge.go b/plugins/aggregators/merge/merge.go new file mode 100644 index 0000000000000..083c8fd3e6b0a --- /dev/null +++ b/plugins/aggregators/merge/merge.go @@ -0,0 +1,66 @@ +package seriesgrouper + +import ( + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/metric" + "github.com/influxdata/telegraf/plugins/aggregators" +) + +const ( + description = "Merge metrics into multifield metrics by series key" + sampleConfig = ` + ## If true, the original metric will be dropped by the + ## aggregator and will not get sent to the output plugins. + drop_original = true +` +) + +type Merge struct { + grouper *metric.SeriesGrouper + log telegraf.Logger +} + +func (a *Merge) Init() error { + a.grouper = metric.NewSeriesGrouper() + return nil +} + +func (a *Merge) Description() string { + return description +} + +func (a *Merge) SampleConfig() string { + return sampleConfig +} + +func (a *Merge) Add(m telegraf.Metric) { + tags := m.Tags() + for _, field := range m.FieldList() { + err := a.grouper.Add(m.Name(), tags, m.Time(), field.Key, field.Value) + if err != nil { + a.log.Errorf("Error adding metric: %v", err) + } + } +} + +func (a *Merge) Push(acc telegraf.Accumulator) { + // Always use nanosecond precision to avoid rounding metrics that were + // produced at a precision higher than the agent default. + acc.SetPrecision(time.Nanosecond) + + for _, m := range a.grouper.Metrics() { + acc.AddMetric(m) + } +} + +func (a *Merge) Reset() { + a.grouper = metric.NewSeriesGrouper() +} + +func init() { + aggregators.Add("merge", func() telegraf.Aggregator { + return &Merge{} + }) +} diff --git a/plugins/aggregators/merge/merge_test.go b/plugins/aggregators/merge/merge_test.go new file mode 100644 index 0000000000000..2f2703c8f4b7c --- /dev/null +++ b/plugins/aggregators/merge/merge_test.go @@ -0,0 +1,186 @@ +package seriesgrouper + +import ( + "testing" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/require" +) + +func TestSimple(t *testing.T) { + plugin := &Merge{} + + err := plugin.Init() + require.NoError(t, err) + + plugin.Add( + testutil.MustMetric( + "cpu", + map[string]string{ + "cpu": "cpu0", + }, + map[string]interface{}{ + "time_idle": 42, + }, + time.Unix(0, 0), + ), + ) + require.NoError(t, err) + + plugin.Add( + testutil.MustMetric( + "cpu", + map[string]string{ + "cpu": "cpu0", + }, + map[string]interface{}{ + "time_guest": 42, + }, + time.Unix(0, 0), + ), + ) + require.NoError(t, err) + + var acc testutil.Accumulator + plugin.Push(&acc) + + expected := []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{ + "cpu": "cpu0", + }, + map[string]interface{}{ + "time_idle": 42, + "time_guest": 42, + }, + time.Unix(0, 0), + ), + } + + testutil.RequireMetricsEqual(t, expected, acc.GetTelegrafMetrics()) +} + +func TestNanosecondPrecision(t *testing.T) { + plugin := &Merge{} + + err := plugin.Init() + require.NoError(t, err) + + plugin.Add( + testutil.MustMetric( + "cpu", + map[string]string{ + "cpu": "cpu0", + }, + map[string]interface{}{ + "time_idle": 42, + }, + time.Unix(0, 1), + ), + ) + require.NoError(t, err) + + plugin.Add( + testutil.MustMetric( + "cpu", + map[string]string{ + "cpu": "cpu0", + }, + map[string]interface{}{ + "time_guest": 42, + }, + time.Unix(0, 1), + ), + ) + require.NoError(t, err) + + var acc testutil.Accumulator + acc.SetPrecision(time.Second) + plugin.Push(&acc) + + expected := []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{ + "cpu": "cpu0", + }, + map[string]interface{}{ + "time_idle": 42, + "time_guest": 42, + }, + time.Unix(0, 1), + ), + } + + testutil.RequireMetricsEqual(t, expected, acc.GetTelegrafMetrics()) +} + +func TestReset(t *testing.T) { + plugin := &Merge{} + + err := plugin.Init() + require.NoError(t, err) + + plugin.Add( + testutil.MustMetric( + "cpu", + map[string]string{ + "cpu": "cpu0", + }, + map[string]interface{}{ + "time_idle": 42, + }, + time.Unix(0, 0), + ), + ) + require.NoError(t, err) + + var acc testutil.Accumulator + plugin.Push(&acc) + + plugin.Reset() + + plugin.Add( + testutil.MustMetric( + "cpu", + map[string]string{ + "cpu": "cpu0", + }, + map[string]interface{}{ + "time_guest": 42, + }, + time.Unix(0, 0), + ), + ) + require.NoError(t, err) + + plugin.Push(&acc) + + expected := []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{ + "cpu": "cpu0", + }, + map[string]interface{}{ + "time_idle": 42, + }, + time.Unix(0, 0), + ), + testutil.MustMetric( + "cpu", + map[string]string{ + "cpu": "cpu0", + }, + map[string]interface{}{ + "time_guest": 42, + }, + time.Unix(0, 0), + ), + } + + testutil.RequireMetricsEqual(t, expected, acc.GetTelegrafMetrics()) +} diff --git a/plugins/common/kafka/sasl.go b/plugins/common/kafka/sasl.go new file mode 100644 index 0000000000000..cd3358b3833ec --- /dev/null +++ b/plugins/common/kafka/sasl.go @@ -0,0 +1,25 @@ +package kafka + +import ( + "errors" + + "github.com/Shopify/sarama" +) + +func SASLVersion(kafkaVersion sarama.KafkaVersion, saslVersion *int) (int16, error) { + if saslVersion == nil { + if kafkaVersion.IsAtLeast(sarama.V1_0_0_0) { + return sarama.SASLHandshakeV1, nil + } + return sarama.SASLHandshakeV0, nil + } + + switch *saslVersion { + case 0: + return sarama.SASLHandshakeV0, nil + case 1: + return sarama.SASLHandshakeV1, nil + default: + return 0, errors.New("invalid SASL version") + } +} diff --git a/plugins/common/logrus/hook.go b/plugins/common/logrus/hook.go new file mode 100644 index 0000000000000..a7f99023be1ba --- /dev/null +++ b/plugins/common/logrus/hook.go @@ -0,0 +1,35 @@ +package logrus + +import ( + "io/ioutil" + "log" + "strings" + "sync" + + "github.com/sirupsen/logrus" +) + +var once sync.Once + +type LogHook struct { +} + +// Install a logging hook into the logrus standard logger, diverting all logs +// through the Telegraf logger at debug level. This is useful for libraries +// that directly log to the logrus system without providing an override method. +func InstallHook() { + once.Do(func() { + logrus.SetOutput(ioutil.Discard) + logrus.AddHook(&LogHook{}) + }) +} + +func (h *LogHook) Fire(entry *logrus.Entry) error { + msg := strings.ReplaceAll(entry.Message, "\n", " ") + log.Print("D! [logrus] ", msg) + return nil +} + +func (h *LogHook) Levels() []logrus.Level { + return logrus.AllLevels +} diff --git a/plugins/inputs/all/all.go b/plugins/inputs/all/all.go index 6934266429b2d..76c04f6427b9a 100644 --- a/plugins/inputs/all/all.go +++ b/plugins/inputs/all/all.go @@ -38,6 +38,8 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/dovecot" _ "github.com/influxdata/telegraf/plugins/inputs/ecs" _ "github.com/influxdata/telegraf/plugins/inputs/elasticsearch" + _ "github.com/influxdata/telegraf/plugins/inputs/ethtool" + _ "github.com/influxdata/telegraf/plugins/inputs/eventhub" _ "github.com/influxdata/telegraf/plugins/inputs/exec" _ "github.com/influxdata/telegraf/plugins/inputs/fail2ban" _ "github.com/influxdata/telegraf/plugins/inputs/fibaro" @@ -55,6 +57,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/http_response" _ "github.com/influxdata/telegraf/plugins/inputs/httpjson" _ "github.com/influxdata/telegraf/plugins/inputs/icinga2" + _ "github.com/influxdata/telegraf/plugins/inputs/infiniband" _ "github.com/influxdata/telegraf/plugins/inputs/influxdb" _ "github.com/influxdata/telegraf/plugins/inputs/influxdb_listener" _ "github.com/influxdata/telegraf/plugins/inputs/internal" @@ -135,6 +138,7 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/smart" _ "github.com/influxdata/telegraf/plugins/inputs/snmp" _ "github.com/influxdata/telegraf/plugins/inputs/snmp_legacy" + _ "github.com/influxdata/telegraf/plugins/inputs/snmp_trap" _ "github.com/influxdata/telegraf/plugins/inputs/socket_listener" _ "github.com/influxdata/telegraf/plugins/inputs/solr" _ "github.com/influxdata/telegraf/plugins/inputs/sqlserver" @@ -142,9 +146,11 @@ import ( _ "github.com/influxdata/telegraf/plugins/inputs/statsd" _ "github.com/influxdata/telegraf/plugins/inputs/suricata" _ "github.com/influxdata/telegraf/plugins/inputs/swap" + _ "github.com/influxdata/telegraf/plugins/inputs/synproxy" _ "github.com/influxdata/telegraf/plugins/inputs/syslog" _ "github.com/influxdata/telegraf/plugins/inputs/sysstat" _ "github.com/influxdata/telegraf/plugins/inputs/system" + _ "github.com/influxdata/telegraf/plugins/inputs/systemd_units" _ "github.com/influxdata/telegraf/plugins/inputs/tail" _ "github.com/influxdata/telegraf/plugins/inputs/tcp_listener" _ "github.com/influxdata/telegraf/plugins/inputs/teamspeak" diff --git a/plugins/inputs/amqp_consumer/amqp_consumer.go b/plugins/inputs/amqp_consumer/amqp_consumer.go index 6cf6004f54596..cee425f60e29c 100644 --- a/plugins/inputs/amqp_consumer/amqp_consumer.go +++ b/plugins/inputs/amqp_consumer/amqp_consumer.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "log" "math/rand" "strings" "sync" @@ -55,6 +54,7 @@ type AMQPConsumer struct { tls.ClientConfig ContentEncoding string `toml:"content_encoding"` + Log telegraf.Logger deliveries map[telegraf.TrackingID]amqp.Delivery @@ -241,11 +241,11 @@ func (a *AMQPConsumer) Start(acc telegraf.Accumulator) error { break } - log.Printf("I! [inputs.amqp_consumer] connection closed: %s; trying to reconnect", err) + a.Log.Infof("Connection closed: %s; trying to reconnect", err) for { msgs, err := a.connect(amqpConf) if err != nil { - log.Printf("E! AMQP connection failed: %s", err) + a.Log.Errorf("AMQP connection failed: %s", err) time.Sleep(10 * time.Second) continue } @@ -272,14 +272,14 @@ func (a *AMQPConsumer) connect(amqpConf *amqp.Config) (<-chan amqp.Delivery, err p := rand.Perm(len(brokers)) for _, n := range p { broker := brokers[n] - log.Printf("D! [inputs.amqp_consumer] connecting to %q", broker) + a.Log.Debugf("Connecting to %q", broker) conn, err := amqp.DialConfig(broker, *amqpConf) if err == nil { a.conn = conn - log.Printf("D! [inputs.amqp_consumer] connected to %q", broker) + a.Log.Debugf("Connected to %q", broker) break } - log.Printf("D! [inputs.amqp_consumer] error connecting to %q", broker) + a.Log.Debugf("Error connecting to %q", broker) } if a.conn == nil { @@ -288,7 +288,7 @@ func (a *AMQPConsumer) connect(amqpConf *amqp.Config) (<-chan amqp.Delivery, err ch, err := a.conn.Channel() if err != nil { - return nil, fmt.Errorf("Failed to open a channel: %s", err) + return nil, fmt.Errorf("Failed to open a channel: %s", err.Error()) } if a.Exchange != "" { @@ -395,7 +395,7 @@ func declareExchange( ) } if err != nil { - return fmt.Errorf("error declaring exchange: %v", err) + return fmt.Errorf("Error declaring exchange: %v", err) } return nil } @@ -437,7 +437,7 @@ func declareQueue( ) } if err != nil { - return nil, fmt.Errorf("error declaring queue: %v", err) + return nil, fmt.Errorf("Error declaring queue: %v", err) } return &queue, nil } @@ -486,8 +486,7 @@ func (a *AMQPConsumer) onMessage(acc telegraf.TrackingAccumulator, d amqp.Delive // this message. rejErr := d.Ack(false) if rejErr != nil { - log.Printf("E! [inputs.amqp_consumer] Unable to reject message: %d: %v", - d.DeliveryTag, rejErr) + a.Log.Errorf("Unable to reject message: %d: %v", d.DeliveryTag, rejErr) a.conn.Close() } } @@ -519,15 +518,13 @@ func (a *AMQPConsumer) onDelivery(track telegraf.DeliveryInfo) bool { if track.Delivered() { err := delivery.Ack(false) if err != nil { - log.Printf("E! [inputs.amqp_consumer] Unable to ack written delivery: %d: %v", - delivery.DeliveryTag, err) + a.Log.Errorf("Unable to ack written delivery: %d: %v", delivery.DeliveryTag, err) a.conn.Close() } } else { err := delivery.Reject(false) if err != nil { - log.Printf("E! [inputs.amqp_consumer] Unable to reject failed delivery: %d: %v", - delivery.DeliveryTag, err) + a.Log.Errorf("Unable to reject failed delivery: %d: %v", delivery.DeliveryTag, err) a.conn.Close() } } @@ -541,7 +538,7 @@ func (a *AMQPConsumer) Stop() { a.wg.Wait() err := a.conn.Close() if err != nil && err != amqp.ErrClosed { - log.Printf("E! [inputs.amqp_consumer] Error closing AMQP connection: %s", err) + a.Log.Errorf("Error closing AMQP connection: %s", err) return } } diff --git a/plugins/inputs/azure_storage_queue/azure_storage_queue.go b/plugins/inputs/azure_storage_queue/azure_storage_queue.go index 0fa7b0fd65936..6d132a5ef0171 100644 --- a/plugins/inputs/azure_storage_queue/azure_storage_queue.go +++ b/plugins/inputs/azure_storage_queue/azure_storage_queue.go @@ -1,9 +1,8 @@ -package activemq +package azure_storage_queue import ( "context" "errors" - "log" "net/url" "strings" "time" @@ -17,6 +16,7 @@ type AzureStorageQueue struct { StorageAccountName string `toml:"account_name"` StorageAccountKey string `toml:"account_key"` PeekOldestMessageAge bool `toml:"peek_oldest_message_age"` + Log telegraf.Logger serviceURL *azqueue.ServiceURL } @@ -92,7 +92,7 @@ func (a *AzureStorageQueue) Gather(acc telegraf.Accumulator) error { ctx := context.TODO() for marker := (azqueue.Marker{}); marker.NotDone(); { - log.Printf("D! [inputs.azure_storage_queue] Listing queues of storage account '%s'", a.StorageAccountName) + a.Log.Debugf("Listing queues of storage account '%s'", a.StorageAccountName) queuesSegment, err := serviceURL.ListQueuesSegment(ctx, marker, azqueue.ListQueuesSegmentOptions{ Detail: azqueue.ListQueuesSegmentDetails{Metadata: false}, @@ -103,11 +103,11 @@ func (a *AzureStorageQueue) Gather(acc telegraf.Accumulator) error { marker = queuesSegment.NextMarker for _, queueItem := range queuesSegment.QueueItems { - log.Printf("D! [inputs.azure_storage_queue] Processing queue '%s' of storage account '%s'", queueItem.Name, a.StorageAccountName) + a.Log.Debugf("Processing queue '%s' of storage account '%s'", queueItem.Name, a.StorageAccountName) queueURL := serviceURL.NewQueueURL(queueItem.Name) properties, err := queueURL.GetProperties(ctx) if err != nil { - log.Printf("E! [inputs.azure_storage_queue] Error getting properties for queue %s: %s", queueItem.Name, err.Error()) + a.Log.Errorf("Error getting properties for queue %s: %s", queueItem.Name, err.Error()) continue } var peekedMessage *azqueue.PeekedMessage @@ -115,7 +115,7 @@ func (a *AzureStorageQueue) Gather(acc telegraf.Accumulator) error { messagesURL := queueURL.NewMessagesURL() messagesResponse, err := messagesURL.Peek(ctx, 1) if err != nil { - log.Printf("E! [inputs.azure_storage_queue] Error peeking queue %s: %s", queueItem.Name, err.Error()) + a.Log.Errorf("Error peeking queue %s: %s", queueItem.Name, err.Error()) } else if messagesResponse.NumMessages() > 0 { peekedMessage = messagesResponse.Message(0) } diff --git a/plugins/inputs/ceph/ceph.go b/plugins/inputs/ceph/ceph.go index e28f977d23777..9a2fc47a315c8 100644 --- a/plugins/inputs/ceph/ceph.go +++ b/plugins/inputs/ceph/ceph.go @@ -101,12 +101,12 @@ func (c *Ceph) gatherAdminSocketStats(acc telegraf.Accumulator) error { for _, s := range sockets { dump, err := perfDump(c.CephBinary, s) if err != nil { - acc.AddError(fmt.Errorf("E! error reading from socket '%s': %v", s.socket, err)) + acc.AddError(fmt.Errorf("error reading from socket '%s': %v", s.socket, err)) continue } data, err := parseDump(dump) if err != nil { - acc.AddError(fmt.Errorf("E! error parsing dump from socket '%s': %v", s.socket, err)) + acc.AddError(fmt.Errorf("error parsing dump from socket '%s': %v", s.socket, err)) continue } for tag, metrics := range data { @@ -287,7 +287,7 @@ func flatten(data interface{}) []*metric { } } default: - log.Printf("I! Ignoring unexpected type '%T' for value %v", val, val) + log.Printf("I! [inputs.ceph] ignoring unexpected type '%T' for value %v", val, val) } return metrics diff --git a/plugins/inputs/cisco_telemetry_gnmi/cisco_telemetry_gnmi.go b/plugins/inputs/cisco_telemetry_gnmi/cisco_telemetry_gnmi.go index 9ab920bedaf63..c8c50e3686376 100644 --- a/plugins/inputs/cisco_telemetry_gnmi/cisco_telemetry_gnmi.go +++ b/plugins/inputs/cisco_telemetry_gnmi/cisco_telemetry_gnmi.go @@ -7,8 +7,8 @@ import ( "encoding/json" "fmt" "io" - "log" "net" + "path" "strings" "sync" "time" @@ -54,6 +54,8 @@ type CiscoTelemetryGNMI struct { acc telegraf.Accumulator cancel context.CancelFunc wg sync.WaitGroup + + Log telegraf.Logger } // Subscription for a GNMI client @@ -101,21 +103,27 @@ func (c *CiscoTelemetryGNMI) Start(acc telegraf.Accumulator) error { // Invert explicit alias list and prefill subscription names c.aliases = make(map[string]string, len(c.Subscriptions)+len(c.Aliases)) for _, subscription := range c.Subscriptions { + var gnmiLongPath, gnmiShortPath *gnmi.Path + // Build the subscription path without keys - gnmiPath, err := parsePath(subscription.Origin, subscription.Path, "") - if err != nil { + if gnmiLongPath, err = parsePath(subscription.Origin, subscription.Path, ""); err != nil { + return err + } + if gnmiShortPath, err = parsePath("", subscription.Path, ""); err != nil { return err } - path, _ := c.handlePath(gnmiPath, nil, "") + longPath, _ := c.handlePath(gnmiLongPath, nil, "") + shortPath, _ := c.handlePath(gnmiShortPath, nil, "") name := subscription.Name // If the user didn't provide a measurement name, use last path element if len(name) == 0 { - name = path[strings.LastIndexByte(path, '/')+1:] + name = path.Base(shortPath) } if len(name) > 0 { - c.aliases[path] = name + c.aliases[longPath] = name + c.aliases[shortPath] = name } } for alias, path := range c.Aliases { @@ -211,8 +219,8 @@ func (c *CiscoTelemetryGNMI) subscribeGNMI(ctx context.Context, address string, return fmt.Errorf("failed to send subscription request: %v", err) } - log.Printf("D! [inputs.cisco_telemetry_gnmi]: Connection to GNMI device %s established", address) - defer log.Printf("D! [inputs.cisco_telemetry_gnmi]: Connection to GNMI device %s closed", address) + c.Log.Debugf("Connection to GNMI device %s established", address) + defer c.Log.Debugf("Connection to GNMI device %s closed", address) for ctx.Err() == nil { var reply *gnmi.SubscribeResponse if reply, err = subscribeClient.Recv(); err != nil { @@ -267,16 +275,31 @@ func (c *CiscoTelemetryGNMI) handleSubscribeResponse(address string, reply *gnmi if alias, ok := c.aliases[aliasPath]; ok { name = alias } else { - log.Printf("D! [inputs.cisco_telemetry_gnmi]: No measurement alias for GNMI path: %s", name) + c.Log.Debugf("No measurement alias for GNMI path: %s", name) } } // Group metrics - for key, val := range fields { - if len(aliasPath) > 0 { + for k, v := range fields { + key := k + if len(aliasPath) < len(key) { + // This may not be an exact prefix, due to naming style + // conversion on the key. key = key[len(aliasPath)+1:] + } else { + // Otherwise use the last path element as the field key. + key = path.Base(key) + + // If there are no elements skip the item; this would be an + // invalid message. + key = strings.TrimLeft(key, "/.") + if key == "" { + c.Log.Errorf("invalid empty path: %q", k) + continue + } } - grouper.Add(name, tags, timestamp, key, val) + + grouper.Add(name, tags, timestamp, key, v) } lastAliasPath = aliasPath @@ -295,6 +318,12 @@ func (c *CiscoTelemetryGNMI) handleTelemetryField(update *gnmi.Update, tags map[ var value interface{} var jsondata []byte + // Make sure a value is actually set + if update.Val == nil || update.Val.Value == nil { + c.Log.Infof("Discarded empty or legacy type value with path: %q", path) + return aliasPath, nil + } + switch val := update.Val.Value.(type) { case *gnmi.TypedValue_AsciiVal: value = val.AsciiVal @@ -346,8 +375,10 @@ func (c *CiscoTelemetryGNMI) handlePath(path *gnmi.Path, tags map[string]string, // Parse generic keys from prefix for _, elem := range path.Elem { - builder.WriteRune('/') - builder.WriteString(elem.Name) + if len(elem.Name) > 0 { + builder.WriteRune('/') + builder.WriteString(elem.Name) + } name := builder.String() if _, exists := c.aliases[name]; exists { diff --git a/plugins/inputs/cisco_telemetry_gnmi/cisco_telemetry_gnmi_test.go b/plugins/inputs/cisco_telemetry_gnmi/cisco_telemetry_gnmi_test.go index 32ad714fd2b26..1b12886b9efcf 100644 --- a/plugins/inputs/cisco_telemetry_gnmi/cisco_telemetry_gnmi_test.go +++ b/plugins/inputs/cisco_telemetry_gnmi/cisco_telemetry_gnmi_test.go @@ -5,9 +5,11 @@ import ( "errors" "fmt" "net" + "sync" "testing" "time" + "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/testutil" "github.com/openconfig/gnmi/proto/gnmi" @@ -37,87 +39,124 @@ func TestParsePath(t *testing.T) { assert.Equal(t, errors.New("Invalid GNMI path: /foo[[/"), err) } -type mockGNMIServer struct { - t *testing.T - acc *testutil.Accumulator - server *grpc.Server - scenario int +type MockServer struct { + SubscribeF func(gnmi.GNMI_SubscribeServer) error + GRPCServer *grpc.Server } -func (m *mockGNMIServer) Capabilities(context.Context, *gnmi.CapabilityRequest) (*gnmi.CapabilityResponse, error) { +func (s *MockServer) Capabilities(context.Context, *gnmi.CapabilityRequest) (*gnmi.CapabilityResponse, error) { return nil, nil } -func (m *mockGNMIServer) Get(context.Context, *gnmi.GetRequest) (*gnmi.GetResponse, error) { +func (s *MockServer) Get(context.Context, *gnmi.GetRequest) (*gnmi.GetResponse, error) { return nil, nil } -func (m *mockGNMIServer) Set(context.Context, *gnmi.SetRequest) (*gnmi.SetResponse, error) { +func (s *MockServer) Set(context.Context, *gnmi.SetRequest) (*gnmi.SetResponse, error) { return nil, nil } -func (m *mockGNMIServer) Subscribe(server gnmi.GNMI_SubscribeServer) error { - metadata, ok := metadata.FromIncomingContext(server.Context()) - require.Equal(m.t, ok, true) - require.Equal(m.t, metadata.Get("username"), []string{"theuser"}) - require.Equal(m.t, metadata.Get("password"), []string{"thepassword"}) - - // Must read request before sending a response; even though we don't check - // the request itself currently. - _, err := server.Recv() - if err != nil { - panic(err) +func (s *MockServer) Subscribe(server gnmi.GNMI_SubscribeServer) error { + return s.SubscribeF(server) +} + +func TestWaitError(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + grpcServer := grpc.NewServer() + gnmiServer := &MockServer{ + SubscribeF: func(server gnmi.GNMI_SubscribeServer) error { + return fmt.Errorf("testerror") + }, + GRPCServer: grpcServer, } + gnmi.RegisterGNMIServer(grpcServer, gnmiServer) - switch m.scenario { - case 0: - return fmt.Errorf("testerror") - case 1: - notification := mockGNMINotification() - server.Send(&gnmi.SubscribeResponse{Response: &gnmi.SubscribeResponse_Update{Update: notification}}) - server.Send(&gnmi.SubscribeResponse{Response: &gnmi.SubscribeResponse_SyncResponse{SyncResponse: true}}) - notification.Prefix.Elem[0].Key["foo"] = "bar2" - notification.Update[0].Path.Elem[1].Key["name"] = "str2" - notification.Update[0].Val = &gnmi.TypedValue{Value: &gnmi.TypedValue_JsonVal{JsonVal: []byte{'"', '1', '2', '3', '"'}}} - server.Send(&gnmi.SubscribeResponse{Response: &gnmi.SubscribeResponse_Update{Update: notification}}) - return nil - case 2: - notification := mockGNMINotification() - server.Send(&gnmi.SubscribeResponse{Response: &gnmi.SubscribeResponse_Update{Update: notification}}) - return nil - case 3: - notification := mockGNMINotification() - notification.Prefix.Elem[0].Key["foo"] = "bar2" - notification.Update[0].Path.Elem[1].Key["name"] = "str2" - notification.Update[0].Val = &gnmi.TypedValue{Value: &gnmi.TypedValue_BoolVal{BoolVal: false}} - server.Send(&gnmi.SubscribeResponse{Response: &gnmi.SubscribeResponse_Update{Update: notification}}) - return nil - default: - return fmt.Errorf("test not implemented ;)") + plugin := &CiscoTelemetryGNMI{ + Log: testutil.Logger{}, + Addresses: []string{listener.Addr().String()}, + Encoding: "proto", + Redial: internal.Duration{Duration: 1 * time.Second}, } + + var acc testutil.Accumulator + err = plugin.Start(&acc) + require.NoError(t, err) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err := grpcServer.Serve(listener) + require.NoError(t, err) + }() + + acc.WaitError(1) + plugin.Stop() + grpcServer.Stop() + wg.Wait() + + require.Contains(t, acc.Errors, + errors.New("aborted GNMI subscription: rpc error: code = Unknown desc = testerror")) } -func TestGNMIError(t *testing.T) { +func TestUsernamePassword(t *testing.T) { listener, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) - server := grpc.NewServer() - acc := &testutil.Accumulator{} - gnmi.RegisterGNMIServer(server, &mockGNMIServer{t: t, scenario: 0, server: server, acc: acc}) - c := &CiscoTelemetryGNMI{Addresses: []string{listener.Addr().String()}, - Username: "theuser", Password: "thepassword", Encoding: "proto", - Redial: internal.Duration{Duration: 1 * time.Second}} + grpcServer := grpc.NewServer() + gnmiServer := &MockServer{ + SubscribeF: func(server gnmi.GNMI_SubscribeServer) error { + metadata, ok := metadata.FromIncomingContext(server.Context()) + if !ok { + return errors.New("failed to get metadata") + } + + username := metadata.Get("username") + if len(username) != 1 || username[0] != "theusername" { + return errors.New("wrong username") + } + + password := metadata.Get("password") + if len(password) != 1 || password[0] != "thepassword" { + return errors.New("wrong password") + } + + return errors.New("success") + }, + GRPCServer: grpcServer, + } + gnmi.RegisterGNMIServer(grpcServer, gnmiServer) + + plugin := &CiscoTelemetryGNMI{ + Log: testutil.Logger{}, + Addresses: []string{listener.Addr().String()}, + Username: "theusername", + Password: "thepassword", + Encoding: "proto", + Redial: internal.Duration{Duration: 1 * time.Second}, + } - require.NoError(t, c.Start(acc)) + var acc testutil.Accumulator + err = plugin.Start(&acc) + require.NoError(t, err) + + var wg sync.WaitGroup + wg.Add(1) go func() { - err := server.Serve(listener) + defer wg.Done() + err := grpcServer.Serve(listener) require.NoError(t, err) }() + acc.WaitError(1) - c.Stop() - server.Stop() + plugin.Stop() + grpcServer.Stop() + wg.Wait() - require.Contains(t, acc.Errors, errors.New("aborted GNMI subscription: rpc error: code = Unknown desc = testerror")) + require.Contains(t, acc.Errors, + errors.New("aborted GNMI subscription: rpc error: code = Unknown desc = success")) } func mockGNMINotification() *gnmi.Notification { @@ -167,93 +206,268 @@ func mockGNMINotification() *gnmi.Notification { } } -func TestGNMIMultiple(t *testing.T) { - listener, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - server := grpc.NewServer() - acc := &testutil.Accumulator{} - gnmi.RegisterGNMIServer(server, &mockGNMIServer{t: t, scenario: 1, server: server, acc: acc}) - - c := &CiscoTelemetryGNMI{Addresses: []string{listener.Addr().String()}, - Username: "theuser", Password: "thepassword", Encoding: "proto", - Redial: internal.Duration{Duration: 1 * time.Second}, - Subscriptions: []Subscription{{Name: "alias", Origin: "type", Path: "/model", SubscriptionMode: "sample"}}, +func TestNotification(t *testing.T) { + tests := []struct { + name string + plugin *CiscoTelemetryGNMI + server *MockServer + expected []telegraf.Metric + }{ + { + name: "multiple metrics", + plugin: &CiscoTelemetryGNMI{ + Log: testutil.Logger{}, + Encoding: "proto", + Redial: internal.Duration{Duration: 1 * time.Second}, + Subscriptions: []Subscription{ + { + Name: "alias", + Origin: "type", + Path: "/model", + SubscriptionMode: "sample", + }, + }, + }, + server: &MockServer{ + SubscribeF: func(server gnmi.GNMI_SubscribeServer) error { + notification := mockGNMINotification() + server.Send(&gnmi.SubscribeResponse{Response: &gnmi.SubscribeResponse_Update{Update: notification}}) + server.Send(&gnmi.SubscribeResponse{Response: &gnmi.SubscribeResponse_SyncResponse{SyncResponse: true}}) + notification.Prefix.Elem[0].Key["foo"] = "bar2" + notification.Update[0].Path.Elem[1].Key["name"] = "str2" + notification.Update[0].Val = &gnmi.TypedValue{Value: &gnmi.TypedValue_JsonVal{JsonVal: []byte{'"', '1', '2', '3', '"'}}} + server.Send(&gnmi.SubscribeResponse{Response: &gnmi.SubscribeResponse_Update{Update: notification}}) + return nil + }, + }, + expected: []telegraf.Metric{ + testutil.MustMetric( + "alias", + map[string]string{ + "path": "type:/model", + "source": "127.0.0.1", + "foo": "bar", + "name": "str", + "uint64": "1234", + }, + map[string]interface{}{ + "some/path": int64(5678), + }, + time.Unix(0, 0), + ), + testutil.MustMetric( + "alias", + map[string]string{ + "path": "type:/model", + "source": "127.0.0.1", + "foo": "bar", + }, + map[string]interface{}{ + "other/path": "foobar", + "other/this": "that", + }, + time.Unix(0, 0), + ), + testutil.MustMetric( + "alias", + map[string]string{ + "path": "type:/model", + "foo": "bar2", + "source": "127.0.0.1", + "name": "str2", + "uint64": "1234", + }, + map[string]interface{}{ + "some/path": "123", + }, + time.Unix(0, 0), + ), + testutil.MustMetric( + "alias", + map[string]string{ + "path": "type:/model", + "source": "127.0.0.1", + "foo": "bar2", + }, + map[string]interface{}{ + "other/path": "foobar", + "other/this": "that", + }, + time.Unix(0, 0), + ), + }, + }, + { + name: "full path field key", + plugin: &CiscoTelemetryGNMI{ + Log: testutil.Logger{}, + Encoding: "proto", + Redial: internal.Duration{Duration: 1 * time.Second}, + Subscriptions: []Subscription{ + { + Name: "PHY_COUNTERS", + Origin: "type", + Path: "/state/port[port-id=*]/ethernet/oper-speed", + SubscriptionMode: "sample", + }, + }, + }, + server: &MockServer{ + SubscribeF: func(server gnmi.GNMI_SubscribeServer) error { + response := &gnmi.SubscribeResponse{ + Response: &gnmi.SubscribeResponse_Update{ + Update: &gnmi.Notification{ + Timestamp: 1543236572000000000, + Prefix: &gnmi.Path{ + Origin: "type", + Elem: []*gnmi.PathElem{ + { + Name: "state", + }, + { + Name: "port", + Key: map[string]string{"port-id": "1"}, + }, + { + Name: "ethernet", + }, + { + Name: "oper-speed", + }, + }, + Target: "subscription", + }, + Update: []*gnmi.Update{ + { + Path: &gnmi.Path{}, + Val: &gnmi.TypedValue{ + Value: &gnmi.TypedValue_IntVal{IntVal: 42}, + }, + }, + }, + }, + }, + } + server.Send(response) + return nil + }, + }, + expected: []telegraf.Metric{ + testutil.MustMetric( + "PHY_COUNTERS", + map[string]string{ + "path": "type:/state/port/ethernet/oper-speed", + "source": "127.0.0.1", + "port_id": "1", + }, + map[string]interface{}{ + "oper_speed": 42, + }, + time.Unix(0, 0), + ), + }, + }, } - require.NoError(t, c.Start(acc)) - go func() { - err := server.Serve(listener) - require.NoError(t, err) - }() - acc.Wait(4) - c.Stop() - server.Stop() - - require.Empty(t, acc.Errors) - - tags := map[string]string{"path": "type:/model", "source": "127.0.0.1", "foo": "bar", "name": "str", "uint64": "1234"} - fields := map[string]interface{}{"some/path": int64(5678)} - acc.AssertContainsTaggedFields(t, "alias", fields, tags) - - tags = map[string]string{"path": "type:/model", "source": "127.0.0.1", "foo": "bar"} - fields = map[string]interface{}{"other/path": "foobar", "other/this": "that"} - acc.AssertContainsTaggedFields(t, "alias", fields, tags) - - tags = map[string]string{"path": "type:/model", "foo": "bar2", "source": "127.0.0.1", "name": "str2", "uint64": "1234"} - fields = map[string]interface{}{"some/path": "123"} - acc.AssertContainsTaggedFields(t, "alias", fields, tags) - - tags = map[string]string{"path": "type:/model", "source": "127.0.0.1", "foo": "bar2"} - fields = map[string]interface{}{"other/path": "foobar", "other/this": "that"} - acc.AssertContainsTaggedFields(t, "alias", fields, tags) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + tt.plugin.Addresses = []string{listener.Addr().String()} + + grpcServer := grpc.NewServer() + tt.server.GRPCServer = grpcServer + gnmi.RegisterGNMIServer(grpcServer, tt.server) + + var acc testutil.Accumulator + err = tt.plugin.Start(&acc) + require.NoError(t, err) + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + err := grpcServer.Serve(listener) + require.NoError(t, err) + }() + + acc.Wait(len(tt.expected)) + tt.plugin.Stop() + grpcServer.Stop() + wg.Wait() + + testutil.RequireMetricsEqual(t, tt.expected, acc.GetTelegrafMetrics(), + testutil.IgnoreTime()) + }) + } } -func TestGNMIMultipleRedial(t *testing.T) { +func TestRedial(t *testing.T) { listener, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) - server := grpc.NewServer() - acc := &testutil.Accumulator{} - gnmi.RegisterGNMIServer(server, &mockGNMIServer{t: t, scenario: 2, server: server, acc: acc}) - - c := &CiscoTelemetryGNMI{Addresses: []string{listener.Addr().String()}, - Username: "theuser", Password: "thepassword", Encoding: "proto", - Redial: internal.Duration{Duration: 10 * time.Millisecond}, - Subscriptions: []Subscription{{Name: "alias", Origin: "type", Path: "/model", SubscriptionMode: "sample"}}, + + plugin := &CiscoTelemetryGNMI{ + Log: testutil.Logger{}, + Addresses: []string{listener.Addr().String()}, + Encoding: "proto", + Redial: internal.Duration{Duration: 10 * time.Millisecond}, } - require.NoError(t, c.Start(acc)) + grpcServer := grpc.NewServer() + gnmiServer := &MockServer{ + SubscribeF: func(server gnmi.GNMI_SubscribeServer) error { + notification := mockGNMINotification() + server.Send(&gnmi.SubscribeResponse{Response: &gnmi.SubscribeResponse_Update{Update: notification}}) + return nil + }, + GRPCServer: grpcServer, + } + gnmi.RegisterGNMIServer(grpcServer, gnmiServer) + + var wg sync.WaitGroup + wg.Add(1) go func() { - err := server.Serve(listener) + defer wg.Done() + err := grpcServer.Serve(listener) require.NoError(t, err) }() + + var acc testutil.Accumulator + err = plugin.Start(&acc) + require.NoError(t, err) + acc.Wait(2) - server.Stop() + grpcServer.Stop() + wg.Wait() - listener, _ = net.Listen("tcp", listener.Addr().String()) - server = grpc.NewServer() - gnmi.RegisterGNMIServer(server, &mockGNMIServer{t: t, scenario: 3, server: server, acc: acc}) + // Restart GNMI server at the same address + listener, err = net.Listen("tcp", listener.Addr().String()) + require.NoError(t, err) + grpcServer = grpc.NewServer() + gnmiServer = &MockServer{ + SubscribeF: func(server gnmi.GNMI_SubscribeServer) error { + notification := mockGNMINotification() + notification.Prefix.Elem[0].Key["foo"] = "bar2" + notification.Update[0].Path.Elem[1].Key["name"] = "str2" + notification.Update[0].Val = &gnmi.TypedValue{Value: &gnmi.TypedValue_BoolVal{BoolVal: false}} + server.Send(&gnmi.SubscribeResponse{Response: &gnmi.SubscribeResponse_Update{Update: notification}}) + return nil + }, + GRPCServer: grpcServer, + } + gnmi.RegisterGNMIServer(grpcServer, gnmiServer) + + wg.Add(1) go func() { - err := server.Serve(listener) + defer wg.Done() + err := grpcServer.Serve(listener) require.NoError(t, err) }() - acc.Wait(4) - c.Stop() - server.Stop() - tags := map[string]string{"path": "type:/model", "source": "127.0.0.1", "foo": "bar", "name": "str", "uint64": "1234"} - fields := map[string]interface{}{"some/path": int64(5678)} - acc.AssertContainsTaggedFields(t, "alias", fields, tags) - - tags = map[string]string{"path": "type:/model", "source": "127.0.0.1", "foo": "bar"} - fields = map[string]interface{}{"other/path": "foobar", "other/this": "that"} - acc.AssertContainsTaggedFields(t, "alias", fields, tags) - - tags = map[string]string{"path": "type:/model", "foo": "bar2", "source": "127.0.0.1", "name": "str2", "uint64": "1234"} - fields = map[string]interface{}{"some/path": false} - acc.AssertContainsTaggedFields(t, "alias", fields, tags) - - tags = map[string]string{"path": "type:/model", "source": "127.0.0.1", "foo": "bar2"} - fields = map[string]interface{}{"other/path": "foobar", "other/this": "that"} - acc.AssertContainsTaggedFields(t, "alias", fields, tags) + acc.Wait(4) + plugin.Stop() + grpcServer.Stop() + wg.Wait() } diff --git a/plugins/inputs/cisco_telemetry_mdt/cisco_telemetry_mdt.go b/plugins/inputs/cisco_telemetry_mdt/cisco_telemetry_mdt.go index ddca8247d75ff..37ccff92617a0 100644 --- a/plugins/inputs/cisco_telemetry_mdt/cisco_telemetry_mdt.go +++ b/plugins/inputs/cisco_telemetry_mdt/cisco_telemetry_mdt.go @@ -5,7 +5,6 @@ import ( "encoding/binary" "fmt" "io" - "log" "net" "path" "strconv" @@ -43,6 +42,8 @@ type CiscoTelemetryMDT struct { Aliases map[string]string `toml:"aliases"` EmbeddedTags []string `toml:"embedded_tags"` + Log telegraf.Logger + // GRPC TLS settings internaltls.ServerConfig @@ -146,11 +147,11 @@ func (c *CiscoTelemetryMDT) acceptTCPClients() { // Individual client connection routine c.wg.Add(1) go func() { - log.Printf("D! [inputs.cisco_telemetry_mdt]: Accepted Cisco MDT TCP dialout connection from %s", conn.RemoteAddr()) + c.Log.Debugf("Accepted Cisco MDT TCP dialout connection from %s", conn.RemoteAddr()) if err := c.handleTCPClient(conn); err != nil { c.acc.AddError(err) } - log.Printf("D! [inputs.cisco_telemetry_mdt]: Closed Cisco MDT TCP dialout connection from %s", conn.RemoteAddr()) + c.Log.Debugf("Closed Cisco MDT TCP dialout connection from %s", conn.RemoteAddr()) mutex.Lock() delete(clients, conn) @@ -165,7 +166,7 @@ func (c *CiscoTelemetryMDT) acceptTCPClients() { mutex.Lock() for client := range clients { if err := client.Close(); err != nil { - log.Printf("E! [inputs.cisco_telemetry_mdt]: Failed to close TCP dialout client: %v", err) + c.Log.Errorf("Failed to close TCP dialout client: %v", err) } } mutex.Unlock() @@ -218,7 +219,7 @@ func (c *CiscoTelemetryMDT) handleTCPClient(conn net.Conn) error { func (c *CiscoTelemetryMDT) MdtDialout(stream dialout.GRPCMdtDialout_MdtDialoutServer) error { peer, peerOK := peer.FromContext(stream.Context()) if peerOK { - log.Printf("D! [inputs.cisco_telemetry_mdt]: Accepted Cisco MDT GRPC dialout connection from %s", peer.Addr) + c.Log.Debugf("Accepted Cisco MDT GRPC dialout connection from %s", peer.Addr) } var chunkBuffer bytes.Buffer @@ -252,7 +253,7 @@ func (c *CiscoTelemetryMDT) MdtDialout(stream dialout.GRPCMdtDialout_MdtDialoutS } if peerOK { - log.Printf("D! [inputs.cisco_telemetry_mdt]: Closed Cisco MDT GRPC dialout connection from %s", peer.Addr) + c.Log.Debugf("Closed Cisco MDT GRPC dialout connection from %s", peer.Addr) } return nil @@ -291,7 +292,7 @@ func (c *CiscoTelemetryMDT) handleTelemetry(data []byte) { } if keys == nil || content == nil { - log.Printf("I! [inputs.cisco_telemetry_mdt]: Message from %s missing keys or content", msg.GetNodeIdStr()) + c.Log.Infof("Message from %s missing keys or content", msg.GetNodeIdStr()) continue } @@ -412,7 +413,7 @@ func (c *CiscoTelemetryMDT) parseContentField(grouper *metric.SeriesGrouper, fie } else { c.mutex.Lock() if _, haveWarned := c.warned[path]; !haveWarned { - log.Printf("D! [inputs.cisco_telemetry_mdt]: No measurement alias for encoding path: %s", path) + c.Log.Debugf("No measurement alias for encoding path: %s", path) c.warned[path] = struct{}{} } c.mutex.Unlock() diff --git a/plugins/inputs/cisco_telemetry_mdt/cisco_telemetry_mdt_test.go b/plugins/inputs/cisco_telemetry_mdt/cisco_telemetry_mdt_test.go index 3736a85318e86..5261bd399c2f6 100644 --- a/plugins/inputs/cisco_telemetry_mdt/cisco_telemetry_mdt_test.go +++ b/plugins/inputs/cisco_telemetry_mdt/cisco_telemetry_mdt_test.go @@ -18,7 +18,7 @@ import ( ) func TestHandleTelemetryTwoSimple(t *testing.T) { - c := &CiscoTelemetryMDT{Transport: "dummy", Aliases: map[string]string{"alias": "type:model/some/path"}} + c := &CiscoTelemetryMDT{Log: testutil.Logger{}, Transport: "dummy", Aliases: map[string]string{"alias": "type:model/some/path"}} acc := &testutil.Accumulator{} c.Start(acc) @@ -93,7 +93,7 @@ func TestHandleTelemetryTwoSimple(t *testing.T) { } func TestHandleTelemetrySingleNested(t *testing.T) { - c := &CiscoTelemetryMDT{Transport: "dummy", Aliases: map[string]string{"nested": "type:model/nested/path"}} + c := &CiscoTelemetryMDT{Log: testutil.Logger{}, Transport: "dummy", Aliases: map[string]string{"nested": "type:model/nested/path"}} acc := &testutil.Accumulator{} c.Start(acc) @@ -385,7 +385,7 @@ func TestHandleNXDME(t *testing.T) { } func TestTCPDialoutOverflow(t *testing.T) { - c := &CiscoTelemetryMDT{Transport: "tcp", ServiceAddress: "127.0.0.1:57000"} + c := &CiscoTelemetryMDT{Log: testutil.Logger{}, Transport: "tcp", ServiceAddress: "127.0.0.1:57000"} acc := &testutil.Accumulator{} assert.Nil(t, c.Start(acc)) @@ -441,7 +441,7 @@ func mockTelemetryMessage() *telemetry.Telemetry { } func TestTCPDialoutMultiple(t *testing.T) { - c := &CiscoTelemetryMDT{Transport: "tcp", ServiceAddress: "127.0.0.1:57000", Aliases: map[string]string{ + c := &CiscoTelemetryMDT{Log: testutil.Logger{}, Transport: "tcp", ServiceAddress: "127.0.0.1:57000", Aliases: map[string]string{ "some": "type:model/some/path", "parallel": "type:model/parallel/path", "other": "type:model/other/path"}} acc := &testutil.Accumulator{} assert.Nil(t, c.Start(acc)) @@ -500,7 +500,7 @@ func TestTCPDialoutMultiple(t *testing.T) { } func TestGRPCDialoutError(t *testing.T) { - c := &CiscoTelemetryMDT{Transport: "grpc", ServiceAddress: "127.0.0.1:57001"} + c := &CiscoTelemetryMDT{Log: testutil.Logger{}, Transport: "grpc", ServiceAddress: "127.0.0.1:57001"} acc := &testutil.Accumulator{} assert.Nil(t, c.Start(acc)) @@ -519,7 +519,7 @@ func TestGRPCDialoutError(t *testing.T) { } func TestGRPCDialoutMultiple(t *testing.T) { - c := &CiscoTelemetryMDT{Transport: "grpc", ServiceAddress: "127.0.0.1:57001", Aliases: map[string]string{ + c := &CiscoTelemetryMDT{Log: testutil.Logger{}, Transport: "grpc", ServiceAddress: "127.0.0.1:57001", Aliases: map[string]string{ "some": "type:model/some/path", "parallel": "type:model/parallel/path", "other": "type:model/other/path"}} acc := &testutil.Accumulator{} assert.Nil(t, c.Start(acc)) diff --git a/plugins/inputs/cloud_pubsub/pubsub.go b/plugins/inputs/cloud_pubsub/pubsub.go index 845711e7d7d44..b418274f3b34a 100644 --- a/plugins/inputs/cloud_pubsub/pubsub.go +++ b/plugins/inputs/cloud_pubsub/pubsub.go @@ -5,16 +5,16 @@ import ( "fmt" "sync" - "cloud.google.com/go/pubsub" "encoding/base64" + "time" + + "cloud.google.com/go/pubsub" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/plugins/inputs" "github.com/influxdata/telegraf/plugins/parsers" "golang.org/x/oauth2/google" "google.golang.org/api/option" - "log" - "time" ) type empty struct{} @@ -43,6 +43,8 @@ type PubSub struct { Base64Data bool `toml:"base64_data"` + Log telegraf.Logger + sub subscription stubSub func() subscription @@ -134,14 +136,14 @@ func (ps *PubSub) receiveWithRetry(parentCtx context.Context) { err := ps.startReceiver(parentCtx) for err != nil && parentCtx.Err() == nil { - log.Printf("E! [inputs.cloud_pubsub] Receiver for subscription %s exited with error: %v", ps.sub.ID(), err) + ps.Log.Errorf("Receiver for subscription %s exited with error: %v", ps.sub.ID(), err) delay := defaultRetryDelaySeconds if ps.RetryReceiveDelaySeconds > 0 { delay = ps.RetryReceiveDelaySeconds } - log.Printf("I! [inputs.cloud_pubsub] Waiting %d seconds before attempting to restart receiver...", delay) + ps.Log.Infof("Waiting %d seconds before attempting to restart receiver...", delay) time.Sleep(time.Duration(delay) * time.Second) err = ps.startReceiver(parentCtx) @@ -149,7 +151,7 @@ func (ps *PubSub) receiveWithRetry(parentCtx context.Context) { } func (ps *PubSub) startReceiver(parentCtx context.Context) error { - log.Printf("I! [inputs.cloud_pubsub] Starting receiver for subscription %s...", ps.sub.ID()) + ps.Log.Infof("Starting receiver for subscription %s...", ps.sub.ID()) cctx, ccancel := context.WithCancel(parentCtx) err := ps.sub.Receive(cctx, func(ctx context.Context, msg message) { if err := ps.onMessage(ctx, msg); err != nil { @@ -159,7 +161,7 @@ func (ps *PubSub) startReceiver(parentCtx context.Context) error { if err != nil { ps.acc.AddError(fmt.Errorf("receiver for subscription %s exited: %v", ps.sub.ID(), err)) } else { - log.Printf("I! [inputs.cloud_pubsub] subscription pull ended (no error, most likely stopped)") + ps.Log.Info("Subscription pull ended (no error, most likely stopped)") } ccancel() return err diff --git a/plugins/inputs/cloud_pubsub/pubsub_test.go b/plugins/inputs/cloud_pubsub/pubsub_test.go index 6233546aa80ee..2045cf4ccbc89 100644 --- a/plugins/inputs/cloud_pubsub/pubsub_test.go +++ b/plugins/inputs/cloud_pubsub/pubsub_test.go @@ -3,10 +3,11 @@ package cloud_pubsub import ( "encoding/base64" "errors" + "testing" + "github.com/influxdata/telegraf/plugins/parsers" "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/assert" - "testing" ) const ( @@ -26,6 +27,7 @@ func TestRunParse(t *testing.T) { sub.receiver = testMessagesReceive(sub) ps := &PubSub{ + Log: testutil.Logger{}, parser: testParser, stubSub: func() subscription { return sub }, Project: "projectIDontMatterForTests", @@ -69,6 +71,7 @@ func TestRunBase64(t *testing.T) { sub.receiver = testMessagesReceive(sub) ps := &PubSub{ + Log: testutil.Logger{}, parser: testParser, stubSub: func() subscription { return sub }, Project: "projectIDontMatterForTests", @@ -112,6 +115,7 @@ func TestRunInvalidMessages(t *testing.T) { sub.receiver = testMessagesReceive(sub) ps := &PubSub{ + Log: testutil.Logger{}, parser: testParser, stubSub: func() subscription { return sub }, Project: "projectIDontMatterForTests", @@ -158,6 +162,7 @@ func TestRunOverlongMessages(t *testing.T) { sub.receiver = testMessagesReceive(sub) ps := &PubSub{ + Log: testutil.Logger{}, parser: testParser, stubSub: func() subscription { return sub }, Project: "projectIDontMatterForTests", @@ -205,6 +210,7 @@ func TestRunErrorInSubscriber(t *testing.T) { sub.receiver = testMessagesError(sub, errors.New("a fake error")) ps := &PubSub{ + Log: testutil.Logger{}, parser: testParser, stubSub: func() subscription { return sub }, Project: "projectIDontMatterForTests", diff --git a/plugins/inputs/cloud_pubsub_push/pubsub_push.go b/plugins/inputs/cloud_pubsub_push/pubsub_push.go index 8b83a440df462..d1c521349fbe3 100644 --- a/plugins/inputs/cloud_pubsub_push/pubsub_push.go +++ b/plugins/inputs/cloud_pubsub_push/pubsub_push.go @@ -6,7 +6,6 @@ import ( "encoding/base64" "encoding/json" "io/ioutil" - "log" "net" "net/http" "sync" @@ -33,6 +32,7 @@ type PubSubPush struct { WriteTimeout internal.Duration MaxBodySize internal.Size AddMeta bool + Log telegraf.Logger MaxUndeliveredMessages int `toml:"max_undelivered_messages"` @@ -227,21 +227,21 @@ func (p *PubSubPush) serveWrite(res http.ResponseWriter, req *http.Request) { var payload Payload if err = json.Unmarshal(bytes, &payload); err != nil { - log.Printf("E! [inputs.cloud_pubsub_push] Error decoding payload %s", err.Error()) + p.Log.Errorf("Error decoding payload %s", err.Error()) res.WriteHeader(http.StatusBadRequest) return } sDec, err := base64.StdEncoding.DecodeString(payload.Msg.Data) if err != nil { - log.Printf("E! [inputs.cloud_pubsub_push] Base64-Decode Failed %s", err.Error()) + p.Log.Errorf("Base64-decode failed %s", err.Error()) res.WriteHeader(http.StatusBadRequest) return } metrics, err := p.Parse(sDec) if err != nil { - log.Println("D! [inputs.cloud_pubsub_push] " + err.Error()) + p.Log.Debug(err.Error()) res.WriteHeader(http.StatusBadRequest) return } @@ -295,7 +295,7 @@ func (p *PubSubPush) receiveDelivered() { ch <- true } else { ch <- false - log.Println("D! [inputs.cloud_pubsub_push] Metric group failed to process") + p.Log.Debug("Metric group failed to process") } } } diff --git a/plugins/inputs/cloud_pubsub_push/pubsub_push_test.go b/plugins/inputs/cloud_pubsub_push/pubsub_push_test.go index 45a304e602e24..a0d71da94d7ff 100644 --- a/plugins/inputs/cloud_pubsub_push/pubsub_push_test.go +++ b/plugins/inputs/cloud_pubsub_push/pubsub_push_test.go @@ -18,6 +18,7 @@ import ( "github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/internal/models" "github.com/influxdata/telegraf/plugins/parsers" + "github.com/influxdata/telegraf/testutil" ) func TestServeHTTP(t *testing.T) { @@ -118,6 +119,7 @@ func TestServeHTTP(t *testing.T) { rr := httptest.NewRecorder() pubPush := &PubSubPush{ + Log: testutil.Logger{}, Path: "/", MaxBodySize: internal.Size{ Size: test.maxsize, diff --git a/plugins/inputs/cloudwatch/README.md b/plugins/inputs/cloudwatch/README.md index 369eadbc16290..3cd098f4706f5 100644 --- a/plugins/inputs/cloudwatch/README.md +++ b/plugins/inputs/cloudwatch/README.md @@ -70,6 +70,9 @@ API endpoint. In the following order the plugin will attempt to authenticate. ## See http://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_limits.html # ratelimit = 25 + ## Timeout for http requests made by the cloudwatch client. + # timeout = "5s" + ## Namespace-wide statistic filters. These allow fewer queries to be made to ## cloudwatch. # statistic_include = [ "average", "sum", "minimum", "maximum", sample_count" ] diff --git a/plugins/inputs/cloudwatch/cloudwatch.go b/plugins/inputs/cloudwatch/cloudwatch.go index 7aad67f5b5c3c..be4ae3700ca44 100644 --- a/plugins/inputs/cloudwatch/cloudwatch.go +++ b/plugins/inputs/cloudwatch/cloudwatch.go @@ -1,8 +1,9 @@ package cloudwatch import ( - "errors" "fmt" + "net" + "net/http" "strconv" "strings" "sync" @@ -10,7 +11,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/cloudwatch" - "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/filter" "github.com/influxdata/telegraf/internal" @@ -23,16 +23,17 @@ import ( type ( // CloudWatch contains the configuration and cache for the cloudwatch plugin. CloudWatch struct { - Region string `toml:"region"` - AccessKey string `toml:"access_key"` - SecretKey string `toml:"secret_key"` - RoleARN string `toml:"role_arn"` - Profile string `toml:"profile"` - CredentialPath string `toml:"shared_credential_file"` - Token string `toml:"token"` - EndpointURL string `toml:"endpoint_url"` - StatisticExclude []string `toml:"statistic_exclude"` - StatisticInclude []string `toml:"statistic_include"` + Region string `toml:"region"` + AccessKey string `toml:"access_key"` + SecretKey string `toml:"secret_key"` + RoleARN string `toml:"role_arn"` + Profile string `toml:"profile"` + CredentialPath string `toml:"shared_credential_file"` + Token string `toml:"token"` + EndpointURL string `toml:"endpoint_url"` + StatisticExclude []string `toml:"statistic_exclude"` + StatisticInclude []string `toml:"statistic_include"` + Timeout internal.Duration `toml:"timeout"` Period internal.Duration `toml:"period"` Delay internal.Duration `toml:"delay"` @@ -41,6 +42,8 @@ type ( CacheTTL internal.Duration `toml:"cache_ttl"` RateLimit int `toml:"ratelimit"` + Log telegraf.Logger `toml:"-"` + client cloudwatchClient statFilter filter.Filter metricCache *metricCache @@ -133,6 +136,9 @@ func (c *CloudWatch) SampleConfig() string { ## See http://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/cloudwatch_limits.html # ratelimit = 25 + ## Timeout for http requests made by the cloudwatch client. + # timeout = "5s" + ## Namespace-wide statistic filters. These allow fewer queries to be made to ## cloudwatch. # statistic_include = [ "average", "sum", "minimum", "maximum", sample_count" ] @@ -183,10 +189,7 @@ func (c *CloudWatch) Gather(acc telegraf.Accumulator) error { return err } - err = c.updateWindow(time.Now()) - if err != nil { - return err - } + c.updateWindow(time.Now()) // Get all of the possible queries so we can send groups of 100. queries, err := c.getDataQueries(filteredMetrics) @@ -194,6 +197,10 @@ func (c *CloudWatch) Gather(acc telegraf.Accumulator) error { return err } + if len(queries) == 0 { + return nil + } + // Limit concurrency or we can easily exhaust user connection limit. // See cloudwatch API request limits: // http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/cloudwatch_limits.html @@ -235,7 +242,7 @@ func (c *CloudWatch) Gather(acc telegraf.Accumulator) error { return c.aggregateMetrics(acc, results) } -func (c *CloudWatch) initializeCloudWatch() error { +func (c *CloudWatch) initializeCloudWatch() { credentialConfig := &internalaws.CredentialConfig{ Region: c.Region, AccessKey: c.AccessKey, @@ -248,10 +255,27 @@ func (c *CloudWatch) initializeCloudWatch() error { } configProvider := credentialConfig.Credentials() - cfg := &aws.Config{} + cfg := &aws.Config{ + HTTPClient: &http.Client{ + // use values from DefaultTransport + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + DualStack: true, + }).DialContext, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 10 * time.Second, + ExpectContinueTimeout: 1 * time.Second, + }, + Timeout: c.Timeout.Duration, + }, + } + loglevel := aws.LogOff c.client = cloudwatch.New(configProvider, cfg.WithLogLevel(loglevel)) - return nil } type filteredMetric struct { @@ -370,7 +394,7 @@ func (c *CloudWatch) fetchNamespaceMetrics() ([]*cloudwatch.Metric, error) { return metrics, nil } -func (c *CloudWatch) updateWindow(relativeTo time.Time) error { +func (c *CloudWatch) updateWindow(relativeTo time.Time) { windowEnd := relativeTo.Add(-c.Delay.Duration) if c.windowEnd.IsZero() { @@ -382,8 +406,6 @@ func (c *CloudWatch) updateWindow(relativeTo time.Time) error { } c.windowEnd = windowEnd - - return nil } // getDataQueries gets all of the possible queries so we can maximize the request payload. @@ -463,7 +485,8 @@ func (c *CloudWatch) getDataQueries(filteredMetrics []filteredMetric) ([]*cloudw } if len(dataQueries) == 0 { - return nil, errors.New("no metrics found to collect") + c.Log.Debug("no metrics found to collect") + return nil, nil } if c.metricCache == nil { @@ -535,6 +558,7 @@ func init() { return &CloudWatch{ CacheTTL: internal.Duration{Duration: time.Hour}, RateLimit: 25, + Timeout: internal.Duration{Duration: time.Second * 5}, } }) } diff --git a/plugins/inputs/consul/README.md b/plugins/inputs/consul/README.md index 2b2368388e39c..72bdeb23116e8 100644 --- a/plugins/inputs/consul/README.md +++ b/plugins/inputs/consul/README.md @@ -12,7 +12,7 @@ report those stats already using StatsD protocol if needed. # Gather health check statuses from services registered in Consul [[inputs.consul]] ## Consul server address - # address = "localhost" + # address = "localhost:8500" ## URI scheme for the Consul server, one of "http", "https" # scheme = "http" diff --git a/plugins/inputs/consul/consul.go b/plugins/inputs/consul/consul.go index 4b5ee4b1cae11..964eb9394a29a 100644 --- a/plugins/inputs/consul/consul.go +++ b/plugins/inputs/consul/consul.go @@ -5,7 +5,6 @@ import ( "strings" "github.com/hashicorp/consul/api" - "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal/tls" "github.com/influxdata/telegraf/plugins/inputs" @@ -28,7 +27,7 @@ type Consul struct { var sampleConfig = ` ## Consul server address - # address = "localhost" + # address = "localhost:8500" ## URI scheme for the Consul server, one of "http", "https" # scheme = "http" diff --git a/plugins/inputs/diskio/diskio.go b/plugins/inputs/diskio/diskio.go index 053765b4e5173..875ec9582485b 100644 --- a/plugins/inputs/diskio/diskio.go +++ b/plugins/inputs/diskio/diskio.go @@ -2,7 +2,6 @@ package diskio import ( "fmt" - "log" "regexp" "strings" @@ -24,6 +23,8 @@ type DiskIO struct { NameTemplates []string SkipSerialNumber bool + Log telegraf.Logger + infoCache map[string]diskInfoCache deviceFilter filter.Filter initialized bool @@ -75,7 +76,7 @@ func (s *DiskIO) init() error { if hasMeta(device) { filter, err := filter.Compile(s.Devices) if err != nil { - return fmt.Errorf("error compiling device pattern: %v", err) + return fmt.Errorf("error compiling device pattern: %s", err.Error()) } s.deviceFilter = filter } @@ -99,7 +100,7 @@ func (s *DiskIO) Gather(acc telegraf.Accumulator) error { diskio, err := s.ps.DiskIO(devices) if err != nil { - return fmt.Errorf("error getting disk io info: %s", err) + return fmt.Errorf("error getting disk io info: %s", err.Error()) } for _, io := range diskio { @@ -166,7 +167,7 @@ func (s *DiskIO) diskName(devName string) (string, []string) { } if err != nil { - log.Printf("W! Error gathering disk info: %s", err) + s.Log.Warnf("Error gathering disk info: %s", err) return devName, devLinks } @@ -199,7 +200,7 @@ func (s *DiskIO) diskTags(devName string) map[string]string { di, err := s.diskInfo(devName) if err != nil { - log.Printf("W! Error gathering disk info: %s", err) + s.Log.Warnf("Error gathering disk info: %s", err) return nil } diff --git a/plugins/inputs/diskio/diskio_linux.go b/plugins/inputs/diskio/diskio_linux.go index c727f485b1410..f2499ca17c1c2 100644 --- a/plugins/inputs/diskio/diskio_linux.go +++ b/plugins/inputs/diskio/diskio_linux.go @@ -11,6 +11,7 @@ import ( ) type diskInfoCache struct { + modifiedAt int64 // Unix Nano timestamp of the last modification of the device. This value is used to invalidate the cache udevDataPath string values map[string]string } @@ -31,7 +32,8 @@ func (s *DiskIO) diskInfo(devName string) (map[string]string, error) { s.infoCache = map[string]diskInfoCache{} } ic, ok := s.infoCache[devName] - if ok { + + if ok && stat.Mtim.Nano() == ic.modifiedAt { return ic.values, nil } @@ -42,6 +44,7 @@ func (s *DiskIO) diskInfo(devName string) (map[string]string, error) { di := map[string]string{} s.infoCache[devName] = diskInfoCache{ + modifiedAt: stat.Mtim.Nano(), udevDataPath: udevDataPath, values: di, } diff --git a/plugins/inputs/diskio/diskio_test.go b/plugins/inputs/diskio/diskio_test.go index 41c4b53e25614..b013e30bab225 100644 --- a/plugins/inputs/diskio/diskio_test.go +++ b/plugins/inputs/diskio/diskio_test.go @@ -103,6 +103,7 @@ func TestDiskIO(t *testing.T) { var acc testutil.Accumulator diskio := &DiskIO{ + Log: testutil.Logger{}, ps: &mps, Devices: tt.devices, } diff --git a/plugins/inputs/docker/README.md b/plugins/inputs/docker/README.md index 1816107ea974a..6ec95b64f52c7 100644 --- a/plugins/inputs/docker/README.md +++ b/plugins/inputs/docker/README.md @@ -26,6 +26,9 @@ to gather stats from the [Engine API](https://docs.docker.com/engine/api/v1.24/) ## Deprecated (1.4.0), use container_name_include container_names = [] + ## Set the source tag for the metrics to the container ID hostname, eg first 12 chars + source_tag = false + ## Containers to include and exclude. Collect all if empty. Globs accepted. container_name_include = [] container_name_exclude = [] @@ -93,6 +96,17 @@ volumes: - /var/run/docker.sock:/var/run/docker.sock ``` +#### source tag + +Selecting the containers measurements can be tricky if you have many containers with the same name. +To alleviate this issue you can set the below value to `true` + +```toml +source_tag = true +``` + +This will cause all measurements to have the `source` tag be set to the first 12 characters of the container id. The first 12 characters is the common hostname for containers that have no explicit hostname set, as defined by docker. + #### Kubernetes Labels Kubernetes may add many labels to your containers, if they are not needed you diff --git a/plugins/inputs/docker/docker.go b/plugins/inputs/docker/docker.go index 3c92ca278c829..915d3a3e3bc90 100644 --- a/plugins/inputs/docker/docker.go +++ b/plugins/inputs/docker/docker.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "io" - "log" "net/http" "regexp" "strconv" @@ -45,6 +44,10 @@ type Docker struct { ContainerStateInclude []string `toml:"container_state_include"` ContainerStateExclude []string `toml:"container_state_exclude"` + IncludeSourceTag bool `toml:"source_tag"` + + Log telegraf.Logger + tlsint.ClientConfig newEnvClient func() (Client, error) @@ -89,6 +92,9 @@ var sampleConfig = ` ## Only collect metrics for these containers, collect all if empty container_names = [] + ## Set the source tag for the metrics to the container ID hostname, eg first 12 chars + source_tag = false + ## Containers to include and exclude. Globs accepted. ## Note that an empty array for both will include all containers container_name_include = [] @@ -107,8 +113,10 @@ var sampleConfig = ` ## Whether to report for each container per-device blkio (8:0, 8:1...) and ## network (eth0, eth1, ...) stats or not perdevice = true + ## Whether to report for each container total blkio and network stats or not total = false + ## Which environment variables should we use as a tag ##tag_env = ["JAVA_HOME", "HEAP_SIZE"] @@ -274,7 +282,7 @@ func (d *Docker) gatherSwarmInfo(acc telegraf.Accumulator) error { fields["tasks_running"] = running[service.ID] fields["tasks_desired"] = tasksNoShutdown[service.ID] } else { - log.Printf("E! Unknow Replicas Mode") + d.Log.Error("Unknown replica mode") } // Add metrics acc.AddFields("docker_swarm", @@ -409,6 +417,13 @@ func (d *Docker) gatherInfo(acc telegraf.Accumulator) error { return nil } +func hostnameFromID(id string) string { + if len(id) > 12 { + return id[0:12] + } + return id +} + func (d *Docker) gatherContainer( container types.Container, acc telegraf.Accumulator, @@ -440,6 +455,10 @@ func (d *Docker) gatherContainer( "container_version": imageVersion, } + if d.IncludeSourceTag { + tags["source"] = hostnameFromID(container.ID) + } + ctx, cancel := context.WithTimeout(context.Background(), d.Timeout.Duration) defer cancel() @@ -527,17 +546,22 @@ func (d *Docker) gatherContainerInspect( started, err := time.Parse(time.RFC3339, info.State.StartedAt) if err == nil && !started.IsZero() { statefields["started_at"] = started.UnixNano() - statefields["uptime_ns"] = finished.Sub(started).Nanoseconds() + + uptime := finished.Sub(started) + if finished.Before(started) { + uptime = now().Sub(started) + } + statefields["uptime_ns"] = uptime.Nanoseconds() } - acc.AddFields("docker_container_status", statefields, tags, time.Now()) + acc.AddFields("docker_container_status", statefields, tags, now()) if info.State.Health != nil { healthfields := map[string]interface{}{ "health_status": info.State.Health.Status, "failing_streak": info.ContainerJSONBase.State.Health.FailingStreak, } - acc.AddFields("docker_container_health", healthfields, tags, time.Now()) + acc.AddFields("docker_container_health", healthfields, tags, now()) } } diff --git a/plugins/inputs/docker/docker_test.go b/plugins/inputs/docker/docker_test.go index 77228b00cad47..a331479d10ea1 100644 --- a/plugins/inputs/docker/docker_test.go +++ b/plugins/inputs/docker/docker_test.go @@ -11,6 +11,7 @@ import ( "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/swarm" + "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/require" ) @@ -252,6 +253,7 @@ func TestDocker_WindowsMemoryContainerStats(t *testing.T) { var acc testutil.Accumulator d := Docker{ + Log: testutil.Logger{}, newClient: func(string, *tls.Config) (Client, error) { return &MockClient{ InfoF: func(ctx context.Context) (types.Info, error) { @@ -390,6 +392,7 @@ func TestContainerLabels(t *testing.T) { } d := Docker{ + Log: testutil.Logger{}, newClient: newClientFunc, LabelInclude: tt.include, LabelExclude: tt.exclude, @@ -511,6 +514,7 @@ func TestContainerNames(t *testing.T) { } d := Docker{ + Log: testutil.Logger{}, newClient: newClientFunc, ContainerInclude: tt.include, ContainerExclude: tt.exclude, @@ -538,25 +542,22 @@ func TestContainerNames(t *testing.T) { } } -func TestContainerStatus(t *testing.T) { - type expectation struct { - // tags - Status string - // fields - ContainerID string - OOMKilled bool - Pid int - ExitCode int - StartedAt time.Time - FinishedAt time.Time - UptimeNs int64 +func FilterMetrics(metrics []telegraf.Metric, f func(telegraf.Metric) bool) []telegraf.Metric { + results := []telegraf.Metric{} + for _, m := range metrics { + if f(m) { + results = append(results, m) + } } + return results +} +func TestContainerStatus(t *testing.T) { var tests = []struct { - name string - now func() time.Time - inspect types.ContainerJSON - expect expectation + name string + now func() time.Time + inspect types.ContainerJSON + expected []telegraf.Metric }{ { name: "finished_at is zero value", @@ -564,49 +565,141 @@ func TestContainerStatus(t *testing.T) { return time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC) }, inspect: containerInspect(), - expect: expectation{ - ContainerID: "e2173b9478a6ae55e237d4d74f8bbb753f0817192b5081334dc78476296b7dfb", - Status: "running", - OOMKilled: false, - Pid: 1234, - ExitCode: 0, - StartedAt: time.Date(2018, 6, 14, 5, 48, 53, 266176036, time.UTC), - UptimeNs: int64(3 * time.Minute), + expected: []telegraf.Metric{ + testutil.MustMetric( + "docker_container_status", + map[string]string{ + "container_name": "etcd", + "container_image": "quay.io/coreos/etcd", + "container_version": "v2.2.2", + "engine_host": "absol", + "label1": "test_value_1", + "label2": "test_value_2", + "server_version": "17.09.0-ce", + "container_status": "running", + "source": "e2173b9478a6", + }, + map[string]interface{}{ + "oomkilled": false, + "pid": 1234, + "exitcode": 0, + "container_id": "e2173b9478a6ae55e237d4d74f8bbb753f0817192b5081334dc78476296b7dfb", + "started_at": time.Date(2018, 6, 14, 5, 48, 53, 266176036, time.UTC).UnixNano(), + "uptime_ns": int64(3 * time.Minute), + }, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), }, }, { name: "finished_at is non-zero value", + now: func() time.Time { + return time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC) + }, inspect: func() types.ContainerJSON { i := containerInspect() i.ContainerJSONBase.State.FinishedAt = "2018-06-14T05:53:53.266176036Z" return i }(), - expect: expectation{ - ContainerID: "e2173b9478a6ae55e237d4d74f8bbb753f0817192b5081334dc78476296b7dfb", - Status: "running", - OOMKilled: false, - Pid: 1234, - ExitCode: 0, - StartedAt: time.Date(2018, 6, 14, 5, 48, 53, 266176036, time.UTC), - FinishedAt: time.Date(2018, 6, 14, 5, 53, 53, 266176036, time.UTC), - UptimeNs: int64(5 * time.Minute), + expected: []telegraf.Metric{ + testutil.MustMetric( + "docker_container_status", + map[string]string{ + "container_name": "etcd", + "container_image": "quay.io/coreos/etcd", + "container_version": "v2.2.2", + "engine_host": "absol", + "label1": "test_value_1", + "label2": "test_value_2", + "server_version": "17.09.0-ce", + "container_status": "running", + "source": "e2173b9478a6", + }, + map[string]interface{}{ + "oomkilled": false, + "pid": 1234, + "exitcode": 0, + "container_id": "e2173b9478a6ae55e237d4d74f8bbb753f0817192b5081334dc78476296b7dfb", + "started_at": time.Date(2018, 6, 14, 5, 48, 53, 266176036, time.UTC).UnixNano(), + "finished_at": time.Date(2018, 6, 14, 5, 53, 53, 266176036, time.UTC).UnixNano(), + "uptime_ns": int64(5 * time.Minute), + }, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), }, }, { name: "started_at is zero value", + now: func() time.Time { + return time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC) + }, inspect: func() types.ContainerJSON { i := containerInspect() i.ContainerJSONBase.State.StartedAt = "" i.ContainerJSONBase.State.FinishedAt = "2018-06-14T05:53:53.266176036Z" return i }(), - expect: expectation{ - ContainerID: "e2173b9478a6ae55e237d4d74f8bbb753f0817192b5081334dc78476296b7dfb", - Status: "running", - OOMKilled: false, - Pid: 1234, - ExitCode: 0, - FinishedAt: time.Date(2018, 6, 14, 5, 53, 53, 266176036, time.UTC), + expected: []telegraf.Metric{ + testutil.MustMetric( + "docker_container_status", + map[string]string{ + "container_name": "etcd", + "container_image": "quay.io/coreos/etcd", + "container_version": "v2.2.2", + "engine_host": "absol", + "label1": "test_value_1", + "label2": "test_value_2", + "server_version": "17.09.0-ce", + "container_status": "running", + "source": "e2173b9478a6", + }, + map[string]interface{}{ + "oomkilled": false, + "pid": 1234, + "exitcode": 0, + "container_id": "e2173b9478a6ae55e237d4d74f8bbb753f0817192b5081334dc78476296b7dfb", + "finished_at": time.Date(2018, 6, 14, 5, 53, 53, 266176036, time.UTC).UnixNano(), + }, + time.Date(2018, 6, 14, 5, 51, 53, 266176036, time.UTC), + ), + }, + }, + { + name: "container has been restarted", + now: func() time.Time { + return time.Date(2019, 1, 1, 0, 0, 3, 0, time.UTC) + }, + inspect: func() types.ContainerJSON { + i := containerInspect() + i.ContainerJSONBase.State.StartedAt = "2019-01-01T00:00:02Z" + i.ContainerJSONBase.State.FinishedAt = "2019-01-01T00:00:01Z" + return i + }(), + expected: []telegraf.Metric{ + testutil.MustMetric( + "docker_container_status", + map[string]string{ + "container_name": "etcd", + "container_image": "quay.io/coreos/etcd", + "container_version": "v2.2.2", + "engine_host": "absol", + "label1": "test_value_1", + "label2": "test_value_2", + "server_version": "17.09.0-ce", + "container_status": "running", + "source": "e2173b9478a6", + }, + map[string]interface{}{ + "oomkilled": false, + "pid": 1234, + "exitcode": 0, + "container_id": "e2173b9478a6ae55e237d4d74f8bbb753f0817192b5081334dc78476296b7dfb", + "started_at": time.Date(2019, 1, 1, 0, 0, 2, 0, time.UTC).UnixNano(), + "finished_at": time.Date(2019, 1, 1, 0, 0, 1, 0, time.UTC).UnixNano(), + "uptime_ns": int64(1 * time.Second), + }, + time.Date(2019, 1, 1, 0, 0, 3, 0, time.UTC), + ), }, }, } @@ -625,7 +718,11 @@ func TestContainerStatus(t *testing.T) { return &client, nil } - d = Docker{newClient: newClientFunc} + d = Docker{ + Log: testutil.Logger{}, + newClient: newClientFunc, + IncludeSourceTag: true, + } ) // mock time @@ -636,38 +733,13 @@ func TestContainerStatus(t *testing.T) { now = time.Now }() - err := acc.GatherError(d.Gather) + err := d.Gather(&acc) require.NoError(t, err) - fields := map[string]interface{}{ - "oomkilled": tt.expect.OOMKilled, - "pid": tt.expect.Pid, - "exitcode": tt.expect.ExitCode, - "container_id": tt.expect.ContainerID, - } - - if started := tt.expect.StartedAt; !started.IsZero() { - fields["started_at"] = started.UnixNano() - fields["uptime_ns"] = tt.expect.UptimeNs - } - - if finished := tt.expect.FinishedAt; !finished.IsZero() { - fields["finished_at"] = finished.UnixNano() - } - - acc.AssertContainsTaggedFields(t, - "docker_container_status", - fields, - map[string]string{ - "container_name": "etcd", - "container_image": "quay.io/coreos/etcd", - "container_version": "v2.2.2", - "engine_host": "absol", - "label1": "test_value_1", - "label2": "test_value_2", - "server_version": "17.09.0-ce", - "container_status": tt.expect.Status, - }) + actual := FilterMetrics(acc.GetTelegrafMetrics(), func(m telegraf.Metric) bool { + return m.Name() == "docker_container_status" + }) + testutil.RequireMetricsEqual(t, tt.expected, actual) }) } } @@ -675,6 +747,7 @@ func TestContainerStatus(t *testing.T) { func TestDockerGatherInfo(t *testing.T) { var acc testutil.Accumulator d := Docker{ + Log: testutil.Logger{}, newClient: newClient, TagEnvironment: []string{"ENVVAR1", "ENVVAR2", "ENVVAR3", "ENVVAR5", "ENVVAR6", "ENVVAR7", "ENVVAR8", "ENVVAR9"}, @@ -824,6 +897,7 @@ func TestDockerGatherInfo(t *testing.T) { func TestDockerGatherSwarmInfo(t *testing.T) { var acc testutil.Accumulator d := Docker{ + Log: testutil.Logger{}, newClient: newClient, } @@ -931,6 +1005,7 @@ func TestContainerStateFilter(t *testing.T) { } d := Docker{ + Log: testutil.Logger{}, newClient: newClientFunc, ContainerStateInclude: tt.include, ContainerStateExclude: tt.exclude, @@ -992,6 +1067,7 @@ func TestContainerName(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { d := Docker{ + Log: testutil.Logger{}, newClient: tt.clientFunc, } var acc testutil.Accumulator @@ -1007,3 +1083,37 @@ func TestContainerName(t *testing.T) { }) } } + +func TestHostnameFromID(t *testing.T) { + tests := []struct { + name string + id string + expect string + }{ + { + name: "Real ID", + id: "565e3a55f5843cfdd4aa5659a1a75e4e78d47f73c3c483f782fe4a26fc8caa07", + expect: "565e3a55f584", + }, + { + name: "Short ID", + id: "shortid123", + expect: "shortid123", + }, + { + name: "No ID", + id: "", + expect: "shortid123", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + output := hostnameFromID(test.id) + if test.expect != output { + t.Logf("Container ID for hostname is wrong. Want: %s, Got: %s", output, test.expect) + } + }) + } + +} diff --git a/plugins/inputs/docker_log/README.md b/plugins/inputs/docker_log/README.md index 02f44e14c6960..d2f0dc6144ff9 100644 --- a/plugins/inputs/docker_log/README.md +++ b/plugins/inputs/docker_log/README.md @@ -43,6 +43,9 @@ The docker plugin uses the [Official Docker Client][] to gather logs from the # docker_label_include = [] # docker_label_exclude = [] + ## Set the source tag for the metrics to the container ID hostname, eg first 12 chars + source_tag = false + ## Optional TLS Config # tls_ca = "/etc/telegraf/ca.pem" # tls_cert = "/etc/telegraf/cert.pem" @@ -58,6 +61,17 @@ When using the `"ENV"` endpoint, the connection is configured using the [env]: https://godoc.org/github.com/moby/moby/client#NewEnvClient +### source tag + +Selecting the containers can be tricky if you have many containers with the same name. +To alleviate this issue you can set the below value to `true` + +```toml +source_tag = true +``` + +This will cause all data points to have the `source` tag be set to the first 12 characters of the container id. The first 12 characters is the common hostname for containers that have no explicit hostname set, as defined by docker. + ### Metrics - docker_log @@ -66,6 +80,7 @@ When using the `"ENV"` endpoint, the connection is configured using the - container_version - container_name - stream (stdout, stderr, or tty) + - source - fields: - container_id - message diff --git a/plugins/inputs/docker_log/docker_log.go b/plugins/inputs/docker_log/docker_log.go index 6a675219fc859..7cb2d94bed988 100644 --- a/plugins/inputs/docker_log/docker_log.go +++ b/plugins/inputs/docker_log/docker_log.go @@ -49,6 +49,9 @@ var sampleConfig = ` # docker_label_include = [] # docker_label_exclude = [] + ## Set the source tag for the metrics to the container ID hostname, eg first 12 chars + source_tag = false + ## Optional TLS Config # tls_ca = "/etc/telegraf/ca.pem" # tls_cert = "/etc/telegraf/cert.pem" @@ -82,6 +85,7 @@ type DockerLogs struct { ContainerExclude []string `toml:"container_name_exclude"` ContainerStateInclude []string `toml:"container_state_include"` ContainerStateExclude []string `toml:"container_state_exclude"` + IncludeSourceTag bool `toml:"source_tag"` tlsint.ClientConfig @@ -199,6 +203,7 @@ func (d *DockerLogs) matchedContainerName(names []string) string { func (d *DockerLogs) Gather(acc telegraf.Accumulator) error { ctx := context.Background() + acc.SetPrecision(time.Nanosecond) ctx, cancel := context.WithTimeout(ctx, d.Timeout.Duration) defer cancel() @@ -258,6 +263,10 @@ func (d *DockerLogs) tailContainerLogs( "container_version": imageVersion, } + if d.IncludeSourceTag { + tags["source"] = hostnameFromID(container.ID) + } + // Add matching container labels as tags for k, label := range container.Labels { if d.labelFilter.Match(k) { @@ -435,3 +444,10 @@ func init() { } }) } + +func hostnameFromID(id string) string { + if len(id) > 12 { + return id[0:12] + } + return id +} diff --git a/plugins/inputs/docker_log/docker_log_test.go b/plugins/inputs/docker_log/docker_log_test.go index ce61f6135a9fc..11cf0befd290e 100644 --- a/plugins/inputs/docker_log/docker_log_test.go +++ b/plugins/inputs/docker_log/docker_log_test.go @@ -98,6 +98,7 @@ func Test(t *testing.T) { "container_image": "influxdata/telegraf", "container_version": "1.11.0", "stream": "tty", + "source": "deadbeef", }, map[string]interface{}{ "container_id": "deadbeef", @@ -141,6 +142,7 @@ func Test(t *testing.T) { "container_image": "influxdata/telegraf", "container_version": "1.11.0", "stream": "stdout", + "source": "deadbeef", }, map[string]interface{}{ "container_id": "deadbeef", @@ -155,9 +157,10 @@ func Test(t *testing.T) { t.Run(tt.name, func(t *testing.T) { var acc testutil.Accumulator plugin := &DockerLogs{ - Timeout: internal.Duration{Duration: time.Second * 5}, - newClient: func(string, *tls.Config) (Client, error) { return tt.client, nil }, - containerList: make(map[string]context.CancelFunc), + Timeout: internal.Duration{Duration: time.Second * 5}, + newClient: func(string, *tls.Config) (Client, error) { return tt.client, nil }, + containerList: make(map[string]context.CancelFunc), + IncludeSourceTag: true, } err := plugin.Init() diff --git a/plugins/inputs/dovecot/README.md b/plugins/inputs/dovecot/README.md index c853832b69b46..d28ae3dd9b9a3 100644 --- a/plugins/inputs/dovecot/README.md +++ b/plugins/inputs/dovecot/README.md @@ -17,8 +17,10 @@ the [upgrading steps][upgrading]. ## ## If no servers are specified, then localhost is used as the host. servers = ["localhost:24242"] + ## Type is one of "user", "domain", "ip", or "global" type = "global" + ## Wildcard matches like "*.com". An empty string "" is same as "*" ## If type = "ip" filters should be filters = [""] diff --git a/plugins/inputs/dovecot/dovecot.go b/plugins/inputs/dovecot/dovecot.go index a621252e5dd4b..66282c43423b2 100644 --- a/plugins/inputs/dovecot/dovecot.go +++ b/plugins/inputs/dovecot/dovecot.go @@ -4,7 +4,6 @@ import ( "bytes" "fmt" "io" - // "log" "net" "strconv" "strings" @@ -32,8 +31,10 @@ var sampleConfig = ` ## ## If no servers are specified, then localhost is used as the host. servers = ["localhost:24242"] + ## Type is one of "user", "domain", "ip", or "global" type = "global" + ## Wildcard matches like "*.com". An empty string "" is same as "*" ## If type = "ip" filters should be filters = [""] @@ -82,12 +83,12 @@ func (d *Dovecot) Gather(acc telegraf.Accumulator) error { func (d *Dovecot) gatherServer(addr string, acc telegraf.Accumulator, qtype string, filter string) error { _, _, err := net.SplitHostPort(addr) if err != nil { - return fmt.Errorf("Error: %s on url %s\n", err, addr) + return fmt.Errorf("%q on url %s", err.Error(), addr) } c, err := net.DialTimeout("tcp", addr, defaultTimeout) if err != nil { - return fmt.Errorf("Unable to connect to dovecot server '%s': %s", addr, err) + return fmt.Errorf("enable to connect to dovecot server '%s': %s", addr, err) } defer c.Close() diff --git a/plugins/inputs/ecs/README.md b/plugins/inputs/ecs/README.md index f391a6b9c3da6..f23eb8bab04bf 100644 --- a/plugins/inputs/ecs/README.md +++ b/plugins/inputs/ecs/README.md @@ -1,6 +1,6 @@ -# ECS Input Plugin +# Amazon ECS Input Plugin -ECS, Fargate compatible, input plugin which uses the [ECS v2 metadata and +Amazon ECS, Fargate compatible, input plugin which uses the [Amazon ECS v2 metadata and stats API][task-metadata-endpoint-v2] endpoints to gather stats on running containers in a Task. diff --git a/plugins/inputs/ethtool/README.md b/plugins/inputs/ethtool/README.md new file mode 100644 index 0000000000000..3f397cdfbe36f --- /dev/null +++ b/plugins/inputs/ethtool/README.md @@ -0,0 +1,33 @@ +# Ethtool Input Plugin + +The ethtool input plugin pulls ethernet device stats. Fields pulled will depend on the network device and driver + +### Configuration: + +```toml +# Returns ethtool statistics for given interfaces +[[inputs.ethtool]] + ## List of interfaces to pull metrics for + # interface_include = ["eth0"] + + ## List of interfaces to ignore when pulling metrics. + # interface_exclude = ["eth1"] +``` + +Interfaces can be included or ignored using + +- `interface_include` +- `interface_exclude` + +Note that loopback interfaces will be automatically ignored + +### Metrics: + +Metrics are dependant on the network device and driver + +### Example Output: + +``` +ethtool,driver=igb,host=test01,interface=mgmt0 tx_queue_1_packets=280782i,rx_queue_5_csum_err=0i,tx_queue_4_restart=0i,tx_multicast=7i,tx_queue_1_bytes=39674885i,rx_queue_2_alloc_failed=0i,tx_queue_5_packets=173970i,tx_single_coll_ok=0i,rx_queue_1_drops=0i,tx_queue_2_restart=0i,tx_aborted_errors=0i,rx_queue_6_csum_err=0i,tx_queue_5_restart=0i,tx_queue_4_bytes=64810835i,tx_abort_late_coll=0i,tx_queue_4_packets=109102i,os2bmc_tx_by_bmc=0i,tx_bytes=427527435i,tx_queue_7_packets=66665i,dropped_smbus=0i,rx_queue_0_csum_err=0i,tx_flow_control_xoff=0i,rx_packets=25926536i,rx_queue_7_csum_err=0i,rx_queue_3_bytes=84326060i,rx_multicast=83771i,rx_queue_4_alloc_failed=0i,rx_queue_3_drops=0i,rx_queue_3_csum_err=0i,rx_errors=0i,tx_errors=0i,tx_queue_6_packets=183236i,rx_broadcast=24378893i,rx_queue_7_packets=88680i,tx_dropped=0i,rx_frame_errors=0i,tx_queue_3_packets=161045i,tx_packets=1257017i,rx_queue_1_csum_err=0i,tx_window_errors=0i,tx_dma_out_of_sync=0i,rx_length_errors=0i,rx_queue_5_drops=0i,tx_timeout_count=0i,rx_queue_4_csum_err=0i,rx_flow_control_xon=0i,tx_heartbeat_errors=0i,tx_flow_control_xon=0i,collisions=0i,tx_queue_0_bytes=29465801i,rx_queue_6_drops=0i,rx_queue_0_alloc_failed=0i,tx_queue_1_restart=0i,rx_queue_0_drops=0i,tx_broadcast=9i,tx_carrier_errors=0i,tx_queue_7_bytes=13777515i,tx_queue_7_restart=0i,rx_queue_5_bytes=50732006i,rx_queue_7_bytes=35744457i,tx_deferred_ok=0i,tx_multi_coll_ok=0i,rx_crc_errors=0i,rx_fifo_errors=0i,rx_queue_6_alloc_failed=0i,tx_queue_2_packets=175206i,tx_queue_0_packets=107011i,rx_queue_4_bytes=201364548i,rx_queue_6_packets=372573i,os2bmc_rx_by_host=0i,multicast=83771i,rx_queue_4_drops=0i,rx_queue_5_packets=130535i,rx_queue_6_bytes=139488035i,tx_fifo_errors=0i,tx_queue_5_bytes=84899130i,rx_queue_0_packets=24529563i,rx_queue_3_alloc_failed=0i,rx_queue_7_drops=0i,tx_queue_6_bytes=96288614i,tx_queue_2_bytes=22132949i,tx_tcp_seg_failed=0i,rx_queue_1_bytes=246703840i,rx_queue_0_bytes=1506870738i,tx_queue_0_restart=0i,rx_queue_2_bytes=111344804i,tx_tcp_seg_good=0i,tx_queue_3_restart=0i,rx_no_buffer_count=0i,rx_smbus=0i,rx_queue_1_packets=273865i,rx_over_errors=0i,os2bmc_tx_by_host=0i,rx_queue_1_alloc_failed=0i,rx_queue_7_alloc_failed=0i,rx_short_length_errors=0i,tx_hwtstamp_timeouts=0i,tx_queue_6_restart=0i,rx_queue_2_packets=207136i,tx_queue_3_bytes=70391970i,rx_queue_3_packets=112007i,rx_queue_4_packets=212177i,tx_smbus=0i,rx_long_byte_count=2480280632i,rx_queue_2_csum_err=0i,rx_missed_errors=0i,rx_bytes=2480280632i,rx_queue_5_alloc_failed=0i,rx_queue_2_drops=0i,os2bmc_rx_by_bmc=0i,rx_align_errors=0i,rx_long_length_errors=0i,rx_hwtstamp_cleared=0i,rx_flow_control_xoff=0i 1564658080000000000 +ethtool,driver=igb,host=test02,interface=mgmt0 rx_queue_2_bytes=111344804i,tx_queue_3_bytes=70439858i,multicast=83771i,rx_broadcast=24378975i,tx_queue_0_packets=107011i,rx_queue_6_alloc_failed=0i,rx_queue_6_drops=0i,rx_hwtstamp_cleared=0i,tx_window_errors=0i,tx_tcp_seg_good=0i,rx_queue_1_drops=0i,tx_queue_1_restart=0i,rx_queue_7_csum_err=0i,rx_no_buffer_count=0i,tx_queue_1_bytes=39675245i,tx_queue_5_bytes=84899130i,tx_broadcast=9i,rx_queue_1_csum_err=0i,tx_flow_control_xoff=0i,rx_queue_6_csum_err=0i,tx_timeout_count=0i,os2bmc_tx_by_bmc=0i,rx_queue_6_packets=372577i,rx_queue_0_alloc_failed=0i,tx_flow_control_xon=0i,rx_queue_2_drops=0i,tx_queue_2_packets=175206i,rx_queue_3_csum_err=0i,tx_abort_late_coll=0i,tx_queue_5_restart=0i,tx_dropped=0i,rx_queue_2_alloc_failed=0i,tx_multi_coll_ok=0i,rx_queue_1_packets=273865i,rx_flow_control_xon=0i,tx_single_coll_ok=0i,rx_length_errors=0i,rx_queue_7_bytes=35744457i,rx_queue_4_alloc_failed=0i,rx_queue_6_bytes=139488395i,rx_queue_2_csum_err=0i,rx_long_byte_count=2480288216i,rx_queue_1_alloc_failed=0i,tx_queue_0_restart=0i,rx_queue_0_csum_err=0i,tx_queue_2_bytes=22132949i,rx_queue_5_drops=0i,tx_dma_out_of_sync=0i,rx_queue_3_drops=0i,rx_queue_4_packets=212177i,tx_queue_6_restart=0i,rx_packets=25926650i,rx_queue_7_packets=88680i,rx_frame_errors=0i,rx_queue_3_bytes=84326060i,rx_short_length_errors=0i,tx_queue_7_bytes=13777515i,rx_queue_3_alloc_failed=0i,tx_queue_6_packets=183236i,rx_queue_0_drops=0i,rx_multicast=83771i,rx_queue_2_packets=207136i,rx_queue_5_csum_err=0i,rx_queue_5_packets=130535i,rx_queue_7_alloc_failed=0i,tx_smbus=0i,tx_queue_3_packets=161081i,rx_queue_7_drops=0i,tx_queue_2_restart=0i,tx_multicast=7i,tx_fifo_errors=0i,tx_queue_3_restart=0i,rx_long_length_errors=0i,tx_queue_6_bytes=96288614i,tx_queue_1_packets=280786i,tx_tcp_seg_failed=0i,rx_align_errors=0i,tx_errors=0i,rx_crc_errors=0i,rx_queue_0_packets=24529673i,rx_flow_control_xoff=0i,tx_queue_0_bytes=29465801i,rx_over_errors=0i,rx_queue_4_drops=0i,os2bmc_rx_by_bmc=0i,rx_smbus=0i,dropped_smbus=0i,tx_hwtstamp_timeouts=0i,rx_errors=0i,tx_queue_4_packets=109102i,tx_carrier_errors=0i,tx_queue_4_bytes=64810835i,tx_queue_4_restart=0i,rx_queue_4_csum_err=0i,tx_queue_7_packets=66665i,tx_aborted_errors=0i,rx_missed_errors=0i,tx_bytes=427575843i,collisions=0i,rx_queue_1_bytes=246703840i,rx_queue_5_bytes=50732006i,rx_bytes=2480288216i,os2bmc_rx_by_host=0i,rx_queue_5_alloc_failed=0i,rx_queue_3_packets=112007i,tx_deferred_ok=0i,os2bmc_tx_by_host=0i,tx_heartbeat_errors=0i,rx_queue_0_bytes=1506877506i,tx_queue_7_restart=0i,tx_packets=1257057i,rx_queue_4_bytes=201364548i,rx_fifo_errors=0i,tx_queue_5_packets=173970i 1564658090000000000 +``` diff --git a/plugins/inputs/ethtool/ethtool.go b/plugins/inputs/ethtool/ethtool.go new file mode 100644 index 0000000000000..3f8f8e15618a2 --- /dev/null +++ b/plugins/inputs/ethtool/ethtool.go @@ -0,0 +1,50 @@ +package ethtool + +import ( + "net" + + "github.com/influxdata/telegraf" +) + +type Command interface { + Init() error + DriverName(intf string) (string, error) + Interfaces() ([]net.Interface, error) + Stats(intf string) (map[string]uint64, error) +} + +type Ethtool struct { + // This is the list of interface names to include + InterfaceInclude []string `toml:"interface_include"` + + // This is the list of interface names to ignore + InterfaceExclude []string `toml:"interface_exclude"` + + Log telegraf.Logger `toml:"-"` + + // the ethtool command + command Command +} + +const ( + pluginName = "ethtool" + tagInterface = "interface" + tagDriverName = "driver" + + sampleConfig = ` + ## List of interfaces to pull metrics for + # interface_include = ["eth0"] + + ## List of interfaces to ignore when pulling metrics. + # interface_exclude = ["eth1"] +` +) + +func (e *Ethtool) SampleConfig() string { + return sampleConfig +} + +// Description returns a one-sentence description on the Input +func (e *Ethtool) Description() string { + return "Returns ethtool statistics for given interfaces" +} diff --git a/plugins/inputs/ethtool/ethtool_linux.go b/plugins/inputs/ethtool/ethtool_linux.go new file mode 100644 index 0000000000000..b8c9312cbe309 --- /dev/null +++ b/plugins/inputs/ethtool/ethtool_linux.go @@ -0,0 +1,136 @@ +// +build linux + +package ethtool + +import ( + "net" + "sync" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/filter" + "github.com/influxdata/telegraf/plugins/inputs" + "github.com/pkg/errors" + "github.com/safchain/ethtool" +) + +type CommandEthtool struct { + ethtool *ethtool.Ethtool +} + +func (e *Ethtool) Gather(acc telegraf.Accumulator) error { + + // Get the list of interfaces + interfaces, err := e.command.Interfaces() + if err != nil { + acc.AddError(err) + return nil + } + + interfaceFilter, err := filter.NewIncludeExcludeFilter(e.InterfaceInclude, e.InterfaceExclude) + if err != nil { + return err + } + + // parallelize the ethtool call in event of many interfaces + var wg sync.WaitGroup + + for _, iface := range interfaces { + + // Check this isn't a loop back and that its matched by the filter + if (iface.Flags&net.FlagLoopback == 0) && interfaceFilter.Match(iface.Name) { + wg.Add(1) + + go func(i net.Interface) { + e.gatherEthtoolStats(i, acc) + wg.Done() + }(iface) + } + } + + // Waiting for all the interfaces + wg.Wait() + return nil +} + +// Initialise the Command Tool +func (e *Ethtool) Init() error { + return e.command.Init() +} + +// Gather the stats for the interface. +func (e *Ethtool) gatherEthtoolStats(iface net.Interface, acc telegraf.Accumulator) { + + tags := make(map[string]string) + tags[tagInterface] = iface.Name + + driverName, err := e.command.DriverName(iface.Name) + if err != nil { + driverErr := errors.Wrapf(err, "%s driver", iface.Name) + acc.AddError(driverErr) + return + } + + tags[tagDriverName] = driverName + + fields := make(map[string]interface{}) + stats, err := e.command.Stats(iface.Name) + if err != nil { + statsErr := errors.Wrapf(err, "%s stats", iface.Name) + acc.AddError(statsErr) + return + } + + for k, v := range stats { + fields[k] = v + } + + acc.AddFields(pluginName, fields, tags) +} + +func NewCommandEthtool() *CommandEthtool { + return &CommandEthtool{} +} + +func (c *CommandEthtool) Init() error { + + if c.ethtool != nil { + return nil + } + + e, err := ethtool.NewEthtool() + if err == nil { + c.ethtool = e + } + + return err +} + +func (c *CommandEthtool) DriverName(intf string) (string, error) { + return c.ethtool.DriverName(intf) +} + +func (c *CommandEthtool) Stats(intf string) (map[string]uint64, error) { + return c.ethtool.Stats(intf) +} + +func (c *CommandEthtool) Interfaces() ([]net.Interface, error) { + + // Get the list of interfaces + interfaces, err := net.Interfaces() + if err != nil { + return nil, err + } + + return interfaces, nil +} + +func init() { + + inputs.Add(pluginName, func() telegraf.Input { + return &Ethtool{ + InterfaceInclude: []string{}, + InterfaceExclude: []string{}, + command: NewCommandEthtool(), + } + }) +} diff --git a/plugins/inputs/ethtool/ethtool_notlinux.go b/plugins/inputs/ethtool/ethtool_notlinux.go new file mode 100644 index 0000000000000..b022e0a46bb72 --- /dev/null +++ b/plugins/inputs/ethtool/ethtool_notlinux.go @@ -0,0 +1,23 @@ +// +build !linux + +package ethtool + +import ( + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" +) + +func (e *Ethtool) Init() error { + e.Log.Warn("Current platform is not supported") + return nil +} + +func (e *Ethtool) Gather(acc telegraf.Accumulator) error { + return nil +} + +func init() { + inputs.Add(pluginName, func() telegraf.Input { + return &Ethtool{} + }) +} diff --git a/plugins/inputs/ethtool/ethtool_test.go b/plugins/inputs/ethtool/ethtool_test.go new file mode 100644 index 0000000000000..c151c9caea018 --- /dev/null +++ b/plugins/inputs/ethtool/ethtool_test.go @@ -0,0 +1,379 @@ +package ethtool + +import ( + "net" + "testing" + + "github.com/influxdata/telegraf/testutil" + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" +) + +var command *Ethtool +var interfaceMap map[string]*InterfaceMock + +type InterfaceMock struct { + Name string + DriverName string + Stat map[string]uint64 + LoopBack bool +} + +type CommandEthtoolMock struct { + InterfaceMap map[string]*InterfaceMock +} + +func (c *CommandEthtoolMock) Init() error { + // Not required for test mock + return nil +} + +func (c *CommandEthtoolMock) DriverName(intf string) (driverName string, err error) { + i := c.InterfaceMap[intf] + if i != nil { + driverName = i.DriverName + return + } + return driverName, errors.New("interface not found") +} + +func (c *CommandEthtoolMock) Interfaces() ([]net.Interface, error) { + interfaceNames := make([]net.Interface, 0) + for k, v := range c.InterfaceMap { + + // Whether to set the flag to loopback + flag := net.FlagUp + if v.LoopBack { + flag = net.FlagLoopback + } + + // Create a dummy interface + iface := net.Interface{ + Index: 0, + MTU: 1500, + Name: k, + HardwareAddr: nil, + Flags: flag, + } + interfaceNames = append(interfaceNames, iface) + } + return interfaceNames, nil +} + +func (c *CommandEthtoolMock) Stats(intf string) (stat map[string]uint64, err error) { + i := c.InterfaceMap[intf] + if i != nil { + stat = i.Stat + return + } + return stat, errors.New("interface not found") +} + +func setup() { + + interfaceMap = make(map[string]*InterfaceMock) + + eth1Stat := map[string]uint64{ + "port_rx_1024_to_15xx": 25167245, + "port_rx_128_to_255": 1573526387, + "port_rx_15xx_to_jumbo": 137819058, + "port_rx_256_to_511": 772038107, + "port_rx_512_to_1023": 78294457, + "port_rx_64": 8798065, + "port_rx_65_to_127": 450348015, + "port_rx_bad": 0, + "port_rx_bad_bytes": 0, + "port_rx_bad_gtjumbo": 0, + "port_rx_broadcast": 6428250, + "port_rx_bytes": 893460472634, + "port_rx_control": 0, + "port_rx_dp_di_dropped_packets": 2772680304, + "port_rx_dp_hlb_fetch": 0, + "port_rx_dp_hlb_wait": 0, + "port_rx_dp_q_disabled_packets": 0, + "port_rx_dp_streaming_packets": 0, + "port_rx_good": 3045991334, + "port_rx_good_bytes": 893460472927, + "port_rx_gtjumbo": 0, + "port_rx_lt64": 0, + "port_rx_multicast": 1639566045, + "port_rx_nodesc_drops": 0, + "port_rx_overflow": 0, + "port_rx_packets": 3045991334, + "port_rx_pause": 0, + "port_rx_pm_discard_bb_overflow": 0, + "port_rx_pm_discard_mapping": 0, + "port_rx_pm_discard_qbb": 0, + "port_rx_pm_discard_vfifo_full": 0, + "port_rx_pm_trunc_bb_overflow": 0, + "port_rx_pm_trunc_qbb": 0, + "port_rx_pm_trunc_vfifo_full": 0, + "port_rx_unicast": 1399997040, + "port_tx_1024_to_15xx": 236, + "port_tx_128_to_255": 275090219, + "port_tx_15xx_to_jumbo": 926, + "port_tx_256_to_511": 48567221, + "port_tx_512_to_1023": 5142016, + "port_tx_64": 113903973, + "port_tx_65_to_127": 161935699, + "port_tx_broadcast": 8, + "port_tx_bytes": 94357131016, + "port_tx_control": 0, + "port_tx_lt64": 0, + "port_tx_multicast": 325891647, + "port_tx_packets": 604640290, + "port_tx_pause": 0, + "port_tx_unicast": 278748635, + "ptp_bad_syncs": 1, + "ptp_fast_syncs": 1, + "ptp_filter_matches": 0, + "ptp_good_syncs": 136151, + "ptp_invalid_sync_windows": 0, + "ptp_no_time_syncs": 1, + "ptp_non_filter_matches": 0, + "ptp_oversize_sync_windows": 53, + "ptp_rx_no_timestamp": 0, + "ptp_rx_timestamp_packets": 0, + "ptp_sync_timeouts": 1, + "ptp_timestamp_packets": 0, + "ptp_tx_timestamp_packets": 0, + "ptp_undersize_sync_windows": 3, + "rx-0.rx_packets": 55659234, + "rx-1.rx_packets": 87880538, + "rx-2.rx_packets": 26746234, + "rx-3.rx_packets": 103026471, + "rx-4.rx_packets": 0, + "rx_eth_crc_err": 0, + "rx_frm_trunc": 0, + "rx_inner_ip_hdr_chksum_err": 0, + "rx_inner_tcp_udp_chksum_err": 0, + "rx_ip_hdr_chksum_err": 0, + "rx_mcast_mismatch": 0, + "rx_merge_events": 0, + "rx_merge_packets": 0, + "rx_nodesc_trunc": 0, + "rx_noskb_drops": 0, + "rx_outer_ip_hdr_chksum_err": 0, + "rx_outer_tcp_udp_chksum_err": 0, + "rx_reset": 0, + "rx_tcp_udp_chksum_err": 0, + "rx_tobe_disc": 0, + "tx-0.tx_packets": 85843565, + "tx-1.tx_packets": 108642725, + "tx-2.tx_packets": 202596078, + "tx-3.tx_packets": 207561010, + "tx-4.tx_packets": 0, + "tx_cb_packets": 4, + "tx_merge_events": 11025, + "tx_pio_packets": 531928114, + "tx_pushes": 604643378, + "tx_tso_bursts": 0, + "tx_tso_fallbacks": 0, + "tx_tso_long_headers": 0, + } + eth1 := &InterfaceMock{"eth1", "driver1", eth1Stat, false} + interfaceMap[eth1.Name] = eth1 + + eth2Stat := map[string]uint64{ + "port_rx_1024_to_15xx": 11529312, + "port_rx_128_to_255": 1868952037, + "port_rx_15xx_to_jumbo": 130339387, + "port_rx_256_to_511": 843846270, + "port_rx_512_to_1023": 173194372, + "port_rx_64": 9190374, + "port_rx_65_to_127": 507806115, + "port_rx_bad": 0, + "port_rx_bad_bytes": 0, + "port_rx_bad_gtjumbo": 0, + "port_rx_broadcast": 6648019, + "port_rx_bytes": 1007358162202, + "port_rx_control": 0, + "port_rx_dp_di_dropped_packets": 3164124639, + "port_rx_dp_hlb_fetch": 0, + "port_rx_dp_hlb_wait": 0, + "port_rx_dp_q_disabled_packets": 0, + "port_rx_dp_streaming_packets": 0, + "port_rx_good": 3544857867, + "port_rx_good_bytes": 1007358162202, + "port_rx_gtjumbo": 0, + "port_rx_lt64": 0, + "port_rx_multicast": 2231999743, + "port_rx_nodesc_drops": 0, + "port_rx_overflow": 0, + "port_rx_packets": 3544857867, + "port_rx_pause": 0, + "port_rx_pm_discard_bb_overflow": 0, + "port_rx_pm_discard_mapping": 0, + "port_rx_pm_discard_qbb": 0, + "port_rx_pm_discard_vfifo_full": 0, + "port_rx_pm_trunc_bb_overflow": 0, + "port_rx_pm_trunc_qbb": 0, + "port_rx_pm_trunc_vfifo_full": 0, + "port_rx_unicast": 1306210105, + "port_tx_1024_to_15xx": 379, + "port_tx_128_to_255": 202767251, + "port_tx_15xx_to_jumbo": 558, + "port_tx_256_to_511": 31454719, + "port_tx_512_to_1023": 6865731, + "port_tx_64": 17268276, + "port_tx_65_to_127": 272816313, + "port_tx_broadcast": 6, + "port_tx_bytes": 78071946593, + "port_tx_control": 0, + "port_tx_lt64": 0, + "port_tx_multicast": 239510586, + "port_tx_packets": 531173227, + "port_tx_pause": 0, + "port_tx_unicast": 291662635, + "ptp_bad_syncs": 0, + "ptp_fast_syncs": 0, + "ptp_filter_matches": 0, + "ptp_good_syncs": 0, + "ptp_invalid_sync_windows": 0, + "ptp_no_time_syncs": 0, + "ptp_non_filter_matches": 0, + "ptp_oversize_sync_windows": 0, + "ptp_rx_no_timestamp": 0, + "ptp_rx_timestamp_packets": 0, + "ptp_sync_timeouts": 0, + "ptp_timestamp_packets": 0, + "ptp_tx_timestamp_packets": 0, + "ptp_undersize_sync_windows": 0, + "rx-0.rx_packets": 84587075, + "rx-1.rx_packets": 74029305, + "rx-2.rx_packets": 134586471, + "rx-3.rx_packets": 87531322, + "rx-4.rx_packets": 0, + "rx_eth_crc_err": 0, + "rx_frm_trunc": 0, + "rx_inner_ip_hdr_chksum_err": 0, + "rx_inner_tcp_udp_chksum_err": 0, + "rx_ip_hdr_chksum_err": 0, + "rx_mcast_mismatch": 0, + "rx_merge_events": 0, + "rx_merge_packets": 0, + "rx_nodesc_trunc": 0, + "rx_noskb_drops": 0, + "rx_outer_ip_hdr_chksum_err": 0, + "rx_outer_tcp_udp_chksum_err": 0, + "rx_reset": 0, + "rx_tcp_udp_chksum_err": 0, + "rx_tobe_disc": 0, + "tx-0.tx_packets": 232521451, + "tx-1.tx_packets": 97876137, + "tx-2.tx_packets": 106822111, + "tx-3.tx_packets": 93955050, + "tx-4.tx_packets": 0, + "tx_cb_packets": 1, + "tx_merge_events": 8402, + "tx_pio_packets": 481040054, + "tx_pushes": 531174491, + "tx_tso_bursts": 128, + "tx_tso_fallbacks": 0, + "tx_tso_long_headers": 0, + } + eth2 := &InterfaceMock{"eth2", "driver1", eth2Stat, false} + interfaceMap[eth2.Name] = eth2 + + // dummy loopback including dummy stat to ensure that the ignore feature is working + lo0Stat := map[string]uint64{ + "dummy": 0, + } + lo0 := &InterfaceMock{"lo0", "", lo0Stat, true} + interfaceMap[lo0.Name] = lo0 + + c := &CommandEthtoolMock{interfaceMap} + command = &Ethtool{ + InterfaceInclude: []string{}, + InterfaceExclude: []string{}, + command: c, + } +} + +func toStringMapInterface(in map[string]uint64) map[string]interface{} { + var m = map[string]interface{}{} + for k, v := range in { + m[k] = v + } + return m +} + +func TestGather(t *testing.T) { + + setup() + var acc testutil.Accumulator + + err := command.Gather(&acc) + assert.NoError(t, err) + assert.Len(t, acc.Metrics, 2) + + expectedFieldsEth1 := toStringMapInterface(interfaceMap["eth1"].Stat) + expectedTagsEth1 := map[string]string{ + "interface": "eth1", + "driver": "driver1", + } + acc.AssertContainsTaggedFields(t, pluginName, expectedFieldsEth1, expectedTagsEth1) + expectedFieldsEth2 := toStringMapInterface(interfaceMap["eth2"].Stat) + expectedTagsEth2 := map[string]string{ + "interface": "eth2", + "driver": "driver1", + } + acc.AssertContainsTaggedFields(t, pluginName, expectedFieldsEth2, expectedTagsEth2) +} + +func TestGatherIncludeInterfaces(t *testing.T) { + + setup() + var acc testutil.Accumulator + + command.InterfaceInclude = append(command.InterfaceInclude, "eth1") + + err := command.Gather(&acc) + assert.NoError(t, err) + assert.Len(t, acc.Metrics, 1) + + // Should contain eth1 + expectedFieldsEth1 := toStringMapInterface(interfaceMap["eth1"].Stat) + expectedTagsEth1 := map[string]string{ + "interface": "eth1", + "driver": "driver1", + } + acc.AssertContainsTaggedFields(t, pluginName, expectedFieldsEth1, expectedTagsEth1) + + // Should not contain eth2 + expectedFieldsEth2 := toStringMapInterface(interfaceMap["eth2"].Stat) + expectedTagsEth2 := map[string]string{ + "interface": "eth2", + "driver": "driver1", + } + acc.AssertDoesNotContainsTaggedFields(t, pluginName, expectedFieldsEth2, expectedTagsEth2) +} + +func TestGatherIgnoreInterfaces(t *testing.T) { + + setup() + var acc testutil.Accumulator + + command.InterfaceExclude = append(command.InterfaceExclude, "eth1") + + err := command.Gather(&acc) + assert.NoError(t, err) + assert.Len(t, acc.Metrics, 1) + + // Should not contain eth1 + expectedFieldsEth1 := toStringMapInterface(interfaceMap["eth1"].Stat) + expectedTagsEth1 := map[string]string{ + "interface": "eth1", + "driver": "driver1", + } + acc.AssertDoesNotContainsTaggedFields(t, pluginName, expectedFieldsEth1, expectedTagsEth1) + + // Should contain eth2 + expectedFieldsEth2 := toStringMapInterface(interfaceMap["eth2"].Stat) + expectedTagsEth2 := map[string]string{ + "interface": "eth2", + "driver": "driver1", + } + acc.AssertContainsTaggedFields(t, pluginName, expectedFieldsEth2, expectedTagsEth2) + +} diff --git a/plugins/inputs/eventhub/README.md b/plugins/inputs/eventhub/README.md new file mode 100644 index 0000000000000..e5d99dfc52688 --- /dev/null +++ b/plugins/inputs/eventhub/README.md @@ -0,0 +1,67 @@ +# Azure Event Hubs input plugin + +This plugin provides a consumer for use with Azure Event Hubs and Azure IoT Hub. The implementation is in essence a wrapper for [Microsoft Azure Event Hubs Client for Golang](https://github.com/Azure/azure-event-hubs-go). + +## Configuration + +```toml +[[inputs.eventhub]] + ## The default behavior is to create a new Event Hub client from enviroment variables. + ## This requires one of the following sets of enviroment variables to be set: + ## + ## 1) Expected Environment Variables: + ## - "EVENTHUB_NAMESPACE" + ## - "EVENTHUB_NAME" + ## - "EVENTHUB_CONNECTION_STRING" + ## + ## 2) Expected Environment Variables: + ## - "EVENTHUB_NAMESPACE" + ## - "EVENTHUB_NAME" + ## - "EVENTHUB_KEY_NAME" + ## - "EVENTHUB_KEY_VALUE" + + ## Uncommenting the option below will create an Event Hub client based solely on the connection string. + ## This can either be the associated envirnoment variable or hardcoded directly. + # connection_string = "$EVENTHUB_CONNECTION_STRING" + + ## Set persistence directory to a valid folder to use a file persister instead of an in-memory persister + # persistence_dir = "" + + ## Change the default consumer group + # consumer_group = "" + + ## By default the event hub receives all messages present on the broker. + ## Alternative modes can be set below. The timestamp should be in RFC3339 format. + # from_timestamp = "" + # starting_offset = "" + # latest = true + + ## Set a custom prefetch count for the receiver(s) + # prefetch_count = 1000 + + ## Add an epoch to the receiver(s) + # epoch = 0 + + ## Change to set a custom user agent, "telegraf" is used by default + # user_agent = "telegraf" + + ## To consume from a specific partition, set the partition_ids option. + ## An empty array will result in receiving from all partitions. + # partition_ids = ["0","1"] + + ## Max undelivered messages + # max_undelivered_messages = 1000 + + ## Data format to consume. + ## Each data format has its own unique set of configuration options, read + ## more about them here: + ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md + data_format = "influx" +``` +## Testing + +The main focus for development of this plugin is Azure IoT hub: + +1. Create an Azure IoT Hub by following any of the guides provided here: https://docs.microsoft.com/en-us/azure/iot-hub/ +2. Create a device, for example a [simulated Raspberry Pi](https://docs.microsoft.com/en-us/azure/iot-hub/iot-hub-raspberry-pi-web-simulator-get-started) +3. The connection string needed for the plugin is located under *Shared access policies*, both the *iothubowner* and *service* policies should work \ No newline at end of file diff --git a/plugins/inputs/eventhub/eventhub.go b/plugins/inputs/eventhub/eventhub.go new file mode 100644 index 0000000000000..2a6eb499d036f --- /dev/null +++ b/plugins/inputs/eventhub/eventhub.go @@ -0,0 +1,323 @@ +package eventhub + +// TODO: move duplicate receiver code to function +// TODO: investigate why waitgroup inhibits exiting telegraf +// TODO: (optional) move receiveroptions to function and combine with 'move duplicate receiver code to function' +// TODO: (optional) change tracking mux to semaphore? +// TODO: (optional) handler to seperate onMessage function for readability (add requirements to EventHub struct)? +// TODO: (optional) Test authentication with AAD TokenProvider environment variables? +// TODO: (optional) Event Processor Host, only appicable for multiple Telegraf instances? + +import ( + "context" + "log" + "sync" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" + "github.com/influxdata/telegraf/plugins/parsers" + + eventhub "github.com/Azure/azure-event-hubs-go/v3" + "github.com/Azure/azure-event-hubs-go/v3/persist" +) + +// EventHub is the top level struct for this plugin +type EventHub struct { + // Configuration + ConnectionString string `toml:"connection_string"` + PersistenceDir string `toml:"persistence_dir"` + ConsumerGroup string `toml:"consumer_group"` + FromTimestamp string `toml:"from_timestamp"` + StartingOffset string `toml:"starting_offset"` + Latest bool `toml:"latest"` + PrefetchCount uint32 `toml:"prefetch_count"` + Epoch int64 `toml:"epoch"` + UserAgent string `toml:"user_agent"` + PartitionIDs []string `toml:"partition_ids"` + MaxUndeliveredMessages int `toml:"max_undelivered_messages"` + + // Azure + hub *eventhub.Hub + cancel context.CancelFunc + + // Influx + parser parsers.Parser + + // Metrics tracking + acc telegraf.TrackingAccumulator + tracker MessageTracker + // wg sync.WaitGroup +} + +// MessageTracker is a struct with a lock and list of tracked messages +type MessageTracker struct { + messages map[telegraf.TrackingID][]telegraf.Metric + mux sync.Mutex +} + +// SampleConfig is provided here +func (*EventHub) SampleConfig() string { + return ` + ## The default behavior is to create a new Event Hub client from enviroment variables. + ## This requires one of the following sets of enviroment variables to be set: + ## + ## 1) Expected Environment Variables: + ## - "EVENTHUB_NAMESPACE" + ## - "EVENTHUB_NAME" + ## - "EVENTHUB_CONNECTION_STRING" + ## + ## 2) Expected Environment Variables: + ## - "EVENTHUB_NAMESPACE" + ## - "EVENTHUB_NAME" + ## - "EVENTHUB_KEY_NAME" + ## - "EVENTHUB_KEY_VALUE" + + ## Uncommenting the option below will create an Event Hub client based solely on the connection string. + ## This can either be the associated envirnoment variable or hardcoded directly. + # connection_string = "$EVENTHUB_CONNECTION_STRING" + + ## Set persistence directory to a valid folder to use a file persister instead of an in-memory persister + # persistence_dir = "" + + ## Change the default consumer group + # consumer_group = "" + + ## By default the event hub receives all messages present on the broker. + ## Alternative modes can be set below. The timestamp should be in RFC3339 format. + ## The 3 options below only apply if no valid persister is read from memory or file (e.g. first run). + # from_timestamp = "" + # starting_offset = "" + # latest = true + + ## Set a custom prefetch count for the receiver(s) + # prefetch_count = 1000 + + ## Add an epoch to the receiver(s) + # epoch = 0 + + ## Change to set a custom user agent, "telegraf" is used by default + # user_agent = "telegraf" + + ## To consume from a specific partition, set the partition_ids option. + ## An empty array will result in receiving from all partitions. + # partition_ids = ["0","1"] + + ## Max undelivered messages + # max_undelivered_messages = 1000 + + ## Data format to consume. + ## Each data format has its own unique set of configuration options, read + ## more about them here: + ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md + data_format = "influx" + ` +} + +// Description of the plugin +func (*EventHub) Description() string { + return "Azure Event Hubs service input plugin" +} + +// SetParser sets the parser +func (e *EventHub) SetParser(parser parsers.Parser) { + e.parser = parser +} + +// Gather function is unused +func (*EventHub) Gather(telegraf.Accumulator) error { + return nil +} + +// Start the EventHub ServiceInput +func (e *EventHub) Start(acc telegraf.Accumulator) error { + + // Set hub options + hubOpts := []eventhub.HubOption{} + + if e.PersistenceDir != "" { + persister, err := persist.NewFilePersister(e.PersistenceDir) + + if err != nil { + return err + } + + hubOpts = append(hubOpts, eventhub.HubWithOffsetPersistence(persister)) + } + + if e.UserAgent != "" { + hubOpts = append(hubOpts, eventhub.HubWithUserAgent(e.UserAgent)) + } else { + hubOpts = append(hubOpts, eventhub.HubWithUserAgent("telegraf")) + } + + // Create event hub connection + var err error + if e.ConnectionString != "" { + e.hub, err = eventhub.NewHubFromConnectionString(e.ConnectionString, hubOpts...) + } else { + e.hub, err = eventhub.NewHubFromEnvironment(hubOpts...) + } + + if err != nil { + return err + } + + // Init metric tracking + e.acc = acc.WithTracking(e.MaxUndeliveredMessages) + e.tracker = MessageTracker{messages: make(map[telegraf.TrackingID][]telegraf.Metric)} + + // Start tracking + // e.wg.Add(1) + go e.startTracking() + + var ctx context.Context + ctx, e.cancel = context.WithCancel(context.Background()) + + // Get runtime information + runtimeinfo, err := e.hub.GetRuntimeInformation(ctx) + + if err != nil { + return err + } + + // Handler function to handle event hub events + handler := func(c context.Context, event *eventhub.Event) error { + + metrics, err := e.parser.Parse(event.Data) + + if err != nil { + log.Printf("E! [inputs.eventhub] %s", err) + return err + } + + log.Printf("D! [inputs.eventhub] %d metrics found after parsing", len(metrics)) + + id := e.acc.AddTrackingMetricGroup(metrics) + + e.tracker.mux.Lock() + e.tracker.messages[id] = metrics + e.tracker.mux.Unlock() + + return nil + } + + // Set receiver options + receiveOpts := []eventhub.ReceiveOption{} + + if e.ConsumerGroup != "" { + receiveOpts = append(receiveOpts, eventhub.ReceiveWithConsumerGroup(e.ConsumerGroup)) + } + + if e.FromTimestamp != "" { + ts, err := time.Parse(time.RFC3339, e.FromTimestamp) + + if err != nil { + return err + } + + receiveOpts = append(receiveOpts, eventhub.ReceiveFromTimestamp(ts)) + + } else if e.StartingOffset != "" { + receiveOpts = append(receiveOpts, eventhub.ReceiveWithStartingOffset(e.StartingOffset)) + } else if e.Latest { + receiveOpts = append(receiveOpts, eventhub.ReceiveWithLatestOffset()) + } + + if e.PrefetchCount != 0 { + receiveOpts = append(receiveOpts, eventhub.ReceiveWithPrefetchCount(e.PrefetchCount)) + } + + if e.Epoch != 0 { + receiveOpts = append(receiveOpts, eventhub.ReceiveWithEpoch(e.Epoch)) + } + + if len(e.PartitionIDs) == 0 { + // Default behavior: receive from all partitions + + for _, partitionID := range runtimeinfo.PartitionIDs { + + _, err = e.hub.Receive(ctx, partitionID, handler, receiveOpts...) + + if err != nil { + return err + } + } + } else { + // Custom behavior: receive from a subset of partitions + // Explicit check for valid partition selection, built in error handling is unreliable + + // Create map of valid partitions + idlist := make(map[string]bool) + + for _, partitionID := range runtimeinfo.PartitionIDs { + idlist[partitionID] = false + } + + // Loop over selected partitions + for _, partitionID := range e.PartitionIDs { + + // Check if partition exists on event hub + if _, ok := idlist[partitionID]; ok { + _, err = e.hub.Receive(ctx, partitionID, handler, receiveOpts...) + + if err != nil { + log.Printf("E! [inputs.eventhub] error creating receiver for partition %v", partitionID) + return err + } + } else { + log.Printf("E! [inputs.eventhub] selected partition with ID \"%s\" not found on event hub", partitionID) + } + } + } + + return nil +} + +// startTracking monitors the message tracker and delivery info +func (e *EventHub) startTracking() { + // defer e.wg.Done() + + for DeliveryInfo := range e.acc.Delivered() { + + log.Printf("D! [inputs.eventhub] tracking:: ID: %v - delivered: %v", DeliveryInfo.ID(), DeliveryInfo.Delivered()) + log.Printf("D! [inputs.eventhub] tracking:: message queue length: %d", len(e.tracker.messages)) + + if DeliveryInfo.Delivered() { + e.tracker.mux.Lock() + delete(e.tracker.messages, DeliveryInfo.ID()) + e.tracker.mux.Unlock() + + log.Printf("D! [inputs.eventhub] tracking:: deleted ID %d from tracked message queue", DeliveryInfo.ID()) + } else { + log.Printf("E! [inputs.eventhub] tracking:: undelivered message ID %d, retrying", DeliveryInfo.ID()) + + e.tracker.mux.Lock() + id := e.acc.AddTrackingMetricGroup(e.tracker.messages[DeliveryInfo.ID()]) + e.tracker.messages[id] = e.tracker.messages[DeliveryInfo.ID()] + delete(e.tracker.messages, DeliveryInfo.ID()) + e.tracker.mux.Unlock() + } + } +} + +// Stop the EventHub ServiceInput +func (e *EventHub) Stop() { + e.cancel() + // e.wg.Wait() + err := e.hub.Close(context.Background()) + + if err != nil { + log.Printf("E! [inputs.eventhub] error closing Azure EventHub connection: %s", err) + } + + log.Printf("D! [inputs.eventhub] event hub connection closed") +} + +func init() { + inputs.Add("eventhub", func() telegraf.Input { + return &EventHub{ + MaxUndeliveredMessages: 1000, // default value + } + }) +} diff --git a/plugins/inputs/exec/README.md b/plugins/inputs/exec/README.md index f4e9172424739..a8544e1d12c93 100644 --- a/plugins/inputs/exec/README.md +++ b/plugins/inputs/exec/README.md @@ -50,8 +50,16 @@ It can be paired with the following configuration and will be run at the `interv ### Common Issues: -#### Q: My script works when I run it by hand, but not when Telegraf is running as a service. +#### My script works when I run it by hand, but not when Telegraf is running as a service. This may be related to the Telegraf service running as a different user. The official packages run Telegraf as the `telegraf` user and group on Linux systems. + +#### With a PowerShell on Windows, the output of the script appears to be truncated. + +You may need to set a variable in your script to increase the numer of columns +available for output: +``` +$host.UI.RawUI.BufferSize = new-object System.Management.Automation.Host.Size(1024,50) +``` diff --git a/plugins/inputs/exec/exec.go b/plugins/inputs/exec/exec.go index 2d3643ad0d2c5..cb4420b0f246f 100644 --- a/plugins/inputs/exec/exec.go +++ b/plugins/inputs/exec/exec.go @@ -10,13 +10,12 @@ import ( "sync" "time" - "github.com/kballard/go-shellquote" - "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/plugins/inputs" "github.com/influxdata/telegraf/plugins/parsers" "github.com/influxdata/telegraf/plugins/parsers/nagios" + "github.com/kballard/go-shellquote" ) const sampleConfig = ` @@ -50,7 +49,7 @@ type Exec struct { parser parsers.Parser runner Runner - log telegraf.Logger + Log telegraf.Logger `toml:"-"` } func NewExec() *Exec { @@ -161,7 +160,7 @@ func (e *Exec) ProcessCommand(command string, acc telegraf.Accumulator, wg *sync if isNagios { metrics, err = nagios.TryAddState(runErr, metrics) if err != nil { - e.log.Errorf("failed to add nagios state: %s", err) + e.Log.Errorf("Failed to add nagios state: %s", err) } } diff --git a/plugins/inputs/exec/exec_test.go b/plugins/inputs/exec/exec_test.go index 0523a181d009d..d0fcc71f668e5 100644 --- a/plugins/inputs/exec/exec_test.go +++ b/plugins/inputs/exec/exec_test.go @@ -96,7 +96,7 @@ func TestExec(t *testing.T) { MetricName: "exec", }) e := &Exec{ - log: testutil.Logger{}, + Log: testutil.Logger{}, runner: newRunnerMock([]byte(validJson), nil, nil), Commands: []string{"testcommand arg1"}, parser: parser, @@ -126,7 +126,7 @@ func TestExecMalformed(t *testing.T) { MetricName: "exec", }) e := &Exec{ - log: testutil.Logger{}, + Log: testutil.Logger{}, runner: newRunnerMock([]byte(malformedJson), nil, nil), Commands: []string{"badcommand arg1"}, parser: parser, @@ -143,7 +143,7 @@ func TestCommandError(t *testing.T) { MetricName: "exec", }) e := &Exec{ - log: testutil.Logger{}, + Log: testutil.Logger{}, runner: newRunnerMock(nil, nil, fmt.Errorf("exit status code 1")), Commands: []string{"badcommand"}, parser: parser, diff --git a/plugins/inputs/file/README.md b/plugins/inputs/file/README.md index 4358b67ad2668..24139973b0dad 100644 --- a/plugins/inputs/file/README.md +++ b/plugins/inputs/file/README.md @@ -7,6 +7,7 @@ Files will always be read in their entirety, if you wish to tail/follow a file use the [tail input plugin](/plugins/inputs/tail) instead. ### Configuration: + ```toml [[inputs.file]] ## Files to parse each interval. @@ -22,4 +23,8 @@ use the [tail input plugin](/plugins/inputs/tail) instead. ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md data_format = "influx" + + ## Name a tag containing the name of the file the data was parsed from. Leave empty + ## to disable. + # file_tag = "" ``` diff --git a/plugins/inputs/file/file.go b/plugins/inputs/file/file.go index b93a7ba9925d0..86059528326d1 100644 --- a/plugins/inputs/file/file.go +++ b/plugins/inputs/file/file.go @@ -3,6 +3,7 @@ package file import ( "fmt" "io/ioutil" + "path/filepath" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal/globpath" @@ -11,8 +12,9 @@ import ( ) type File struct { - Files []string `toml:"files"` - parser parsers.Parser + Files []string `toml:"files"` + FileTag string `toml:"file_tag"` + parser parsers.Parser filenames []string } @@ -31,6 +33,10 @@ const sampleConfig = ` ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md data_format = "influx" + + ## Name a tag containing the name of the file the data was parsed from. Leave empty + ## to disable. + # file_tag = "" ` // SampleConfig returns the default configuration of the Input @@ -54,6 +60,9 @@ func (f *File) Gather(acc telegraf.Accumulator) error { } for _, m := range metrics { + if f.FileTag != "" { + m.AddTag(f.FileTag, filepath.Base(k)) + } acc.AddFields(m.Name(), m.Fields(), m.Tags(), m.Time()) } } diff --git a/plugins/inputs/file/file_test.go b/plugins/inputs/file/file_test.go index 43322c2e84cf9..19341fc08627c 100644 --- a/plugins/inputs/file/file_test.go +++ b/plugins/inputs/file/file_test.go @@ -21,6 +21,34 @@ func TestRefreshFilePaths(t *testing.T) { require.NoError(t, err) assert.Equal(t, 2, len(r.filenames)) } + +func TestFileTag(t *testing.T) { + acc := testutil.Accumulator{} + wd, err := os.Getwd() + require.NoError(t, err) + r := File{ + Files: []string{filepath.Join(wd, "dev/testfiles/json_a.log")}, + FileTag: "filename", + } + + parserConfig := parsers.Config{ + DataFormat: "json", + } + nParser, err := parsers.NewParser(&parserConfig) + assert.NoError(t, err) + r.parser = nParser + + err = r.Gather(&acc) + require.NoError(t, err) + + for _, m := range acc.Metrics { + for key, value := range m.Tags { + assert.Equal(t, r.FileTag, key) + assert.Equal(t, filepath.Base(r.Files[0]), value) + } + } +} + func TestJSONParserCompile(t *testing.T) { var acc testutil.Accumulator wd, _ := os.Getwd() diff --git a/plugins/inputs/filecount/README.md b/plugins/inputs/filecount/README.md index 49e28caa62177..81fc75908e798 100644 --- a/plugins/inputs/filecount/README.md +++ b/plugins/inputs/filecount/README.md @@ -27,6 +27,9 @@ Reports the number and total size of files in specified directories. ## Only count regular files. Defaults to true. regular_only = true + ## Follow all symlinks while walking the directory tree. Defaults to false. + follow_symlinks = false + ## Only count files that are at least this size. If size is ## a negative number, only count files that are smaller than the ## absolute value of size. Acceptable units are B, KiB, MiB, KB, ... diff --git a/plugins/inputs/filecount/filecount.go b/plugins/inputs/filecount/filecount.go index 965f41d2cc380..30815541c8448 100644 --- a/plugins/inputs/filecount/filecount.go +++ b/plugins/inputs/filecount/filecount.go @@ -1,7 +1,6 @@ package filecount import ( - "log" "os" "path/filepath" "time" @@ -36,6 +35,9 @@ const sampleConfig = ` ## Only count regular files. Defaults to true. regular_only = true + ## Follow all symlinks while walking the directory tree. Defaults to false. + follow_symlinks = false + ## Only count files that are at least this size. If size is ## a negative number, only count files that are smaller than the ## absolute value of size. Acceptable units are B, KiB, MiB, KB, ... @@ -49,16 +51,18 @@ const sampleConfig = ` ` type FileCount struct { - Directory string // deprecated in 1.9 - Directories []string - Name string - Recursive bool - RegularOnly bool - Size internal.Size - MTime internal.Duration `toml:"mtime"` - fileFilters []fileFilterFunc - globPaths []globpath.GlobPath - Fs fileSystem + Directory string // deprecated in 1.9 + Directories []string + Name string + Recursive bool + RegularOnly bool + FollowSymlinks bool + Size internal.Size + MTime internal.Duration `toml:"mtime"` + fileFilters []fileFilterFunc + globPaths []globpath.GlobPath + Fs fileSystem + Log telegraf.Logger } func (_ *FileCount) Description() string { @@ -208,9 +212,10 @@ func (fc *FileCount) count(acc telegraf.Accumulator, basedir string, glob globpa Callback: walkFn, PostChildrenCallback: postChildrenFn, Unsorted: true, + FollowSymbolicLinks: fc.FollowSymlinks, ErrorCallback: func(osPathname string, err error) godirwalk.ErrorAction { if os.IsPermission(errors.Cause(err)) { - log.Println("D! [inputs.filecount]", err) + fc.Log.Debug(err) return godirwalk.SkipNode } return godirwalk.Halt @@ -292,15 +297,16 @@ func (fc *FileCount) initGlobPaths(acc telegraf.Accumulator) { func NewFileCount() *FileCount { return &FileCount{ - Directory: "", - Directories: []string{}, - Name: "*", - Recursive: true, - RegularOnly: true, - Size: internal.Size{Size: 0}, - MTime: internal.Duration{Duration: 0}, - fileFilters: nil, - Fs: osFS{}, + Directory: "", + Directories: []string{}, + Name: "*", + Recursive: true, + RegularOnly: true, + FollowSymlinks: false, + Size: internal.Size{Size: 0}, + MTime: internal.Duration{Duration: 0}, + fileFilters: nil, + Fs: osFS{}, } } diff --git a/plugins/inputs/filecount/filecount_test.go b/plugins/inputs/filecount/filecount_test.go index 9cd7c747cfb61..96d8f0c3b7eec 100644 --- a/plugins/inputs/filecount/filecount_test.go +++ b/plugins/inputs/filecount/filecount_test.go @@ -102,7 +102,6 @@ func TestSizeFilter(t *testing.T) { } func TestMTimeFilter(t *testing.T) { - mtime := time.Date(2011, time.December, 14, 18, 25, 5, 0, time.UTC) fileAge := time.Since(mtime) - (60 * time.Second) @@ -119,6 +118,19 @@ func TestMTimeFilter(t *testing.T) { fileCountEquals(t, fc, len(matches), 0) } +// The library dependency karrick/godirwalk completely abstracts out the +// behavior of the FollowSymlinks plugin input option. However, it should at +// least behave identically when enabled on a filesystem with no symlinks. +func TestFollowSymlinks(t *testing.T) { + fc := getNoFilterFileCount() + fc.FollowSymlinks = true + matches := []string{"foo", "bar", "baz", "qux", + "subdir/", "subdir/quux", "subdir/quuz", + "subdir/nested2", "subdir/nested2/qux"} + + fileCountEquals(t, fc, len(matches), 5096) +} + // Paths with a trailing slash will not exactly match paths produced during the // walk as these paths are cleaned before being returned from godirwalk. #6329 func TestDirectoryWithTrailingSlash(t *testing.T) { @@ -144,6 +156,7 @@ func TestDirectoryWithTrailingSlash(t *testing.T) { "size_bytes": 5096, }, time.Unix(0, 0), + telegraf.Gauge, ), } @@ -152,6 +165,7 @@ func TestDirectoryWithTrailingSlash(t *testing.T) { func getNoFilterFileCount() FileCount { return FileCount{ + Log: testutil.Logger{}, Directories: []string{getTestdataDir()}, Name: "*", Recursive: true, diff --git a/plugins/inputs/filestat/README.md b/plugins/inputs/filestat/README.md index 3102c13b077ea..840cafb53c06a 100644 --- a/plugins/inputs/filestat/README.md +++ b/plugins/inputs/filestat/README.md @@ -11,6 +11,7 @@ The filestat plugin gathers metrics about file existence, size, and other stats. ## These accept standard unix glob matching rules, but with the addition of ## ** as a "super asterisk". See https://github.com/gobwas/glob. files = ["/etc/telegraf/telegraf.conf", "/var/log/**.log"] + ## If true, read the entire file and calculate an md5 checksum. md5 = false ``` diff --git a/plugins/inputs/filestat/filestat.go b/plugins/inputs/filestat/filestat.go index 692e58c53e946..bf8ea6c160361 100644 --- a/plugins/inputs/filestat/filestat.go +++ b/plugins/inputs/filestat/filestat.go @@ -4,7 +4,6 @@ import ( "crypto/md5" "fmt" "io" - "log" "os" "github.com/influxdata/telegraf" @@ -23,6 +22,7 @@ const sampleConfig = ` ## See https://github.com/gobwas/glob for more examples ## files = ["/var/log/**.log"] + ## If true, read the entire file and calculate an md5 checksum. md5 = false ` @@ -31,6 +31,8 @@ type FileStat struct { Md5 bool Files []string + Log telegraf.Logger + // maps full file paths to globmatch obj globs map[string]*globpath.GlobPath } @@ -41,11 +43,11 @@ func NewFileStat() *FileStat { } } -func (_ *FileStat) Description() string { +func (*FileStat) Description() string { return "Read stats about given file(s)" } -func (_ *FileStat) SampleConfig() string { return sampleConfig } +func (*FileStat) SampleConfig() string { return sampleConfig } func (f *FileStat) Gather(acc telegraf.Accumulator) error { var err error @@ -86,7 +88,7 @@ func (f *FileStat) Gather(acc telegraf.Accumulator) error { } if fileInfo == nil { - log.Printf("E! Unable to get info for file [%s], possible permissions issue", + f.Log.Errorf("Unable to get info for file %q, possible permissions issue", fileName) } else { fields["size_bytes"] = fileInfo.Size() diff --git a/plugins/inputs/filestat/filestat_test.go b/plugins/inputs/filestat/filestat_test.go index 7fdf6cde841dc..a38d3b0aacdc4 100644 --- a/plugins/inputs/filestat/filestat_test.go +++ b/plugins/inputs/filestat/filestat_test.go @@ -14,6 +14,7 @@ import ( func TestGatherNoMd5(t *testing.T) { dir := getTestdataDir() fs := NewFileStat() + fs.Log = testutil.Logger{} fs.Files = []string{ dir + "log1.log", dir + "log2.log", @@ -44,6 +45,7 @@ func TestGatherNoMd5(t *testing.T) { func TestGatherExplicitFiles(t *testing.T) { dir := getTestdataDir() fs := NewFileStat() + fs.Log = testutil.Logger{} fs.Md5 = true fs.Files = []string{ dir + "log1.log", @@ -77,6 +79,7 @@ func TestGatherExplicitFiles(t *testing.T) { func TestGatherGlob(t *testing.T) { dir := getTestdataDir() fs := NewFileStat() + fs.Log = testutil.Logger{} fs.Md5 = true fs.Files = []string{ dir + "*.log", @@ -103,6 +106,7 @@ func TestGatherGlob(t *testing.T) { func TestGatherSuperAsterisk(t *testing.T) { dir := getTestdataDir() fs := NewFileStat() + fs.Log = testutil.Logger{} fs.Md5 = true fs.Files = []string{ dir + "**", @@ -136,6 +140,7 @@ func TestGatherSuperAsterisk(t *testing.T) { func TestModificationTime(t *testing.T) { dir := getTestdataDir() fs := NewFileStat() + fs.Log = testutil.Logger{} fs.Files = []string{ dir + "log1.log", } @@ -153,6 +158,7 @@ func TestModificationTime(t *testing.T) { func TestNoModificationTime(t *testing.T) { fs := NewFileStat() + fs.Log = testutil.Logger{} fs.Files = []string{ "/non/existant/file", } diff --git a/plugins/inputs/haproxy/README.md b/plugins/inputs/haproxy/README.md index 35b59524de6b6..86fbb986b696a 100644 --- a/plugins/inputs/haproxy/README.md +++ b/plugins/inputs/haproxy/README.md @@ -15,6 +15,10 @@ or [HTTP statistics page](https://cbonte.github.io/haproxy-dconv/1.9/management. ## Make sure you specify the complete path to the stats endpoint ## including the protocol, ie http://10.10.3.33:1936/haproxy?stats + ## Credentials for basic HTTP authentication + # username = "admin" + # password = "admin" + ## If no servers are specified, then default to 127.0.0.1:1936/haproxy?stats servers = ["http://myhaproxy.com:1936/haproxy?stats"] diff --git a/plugins/inputs/http/README.md b/plugins/inputs/http/README.md index 240fd90c98b20..9cd136bd03b35 100644 --- a/plugins/inputs/http/README.md +++ b/plugins/inputs/http/README.md @@ -40,6 +40,9 @@ The HTTP input plugin collects metrics from one or more HTTP(S) endpoints. The ## Amount of time allowed to complete the HTTP request # timeout = "5s" + ## List of success status codes + # success_status_codes = [200] + ## Data format to consume. ## Each data format has its own unique set of configuration options, read ## more about them here: diff --git a/plugins/inputs/http/http.go b/plugins/inputs/http/http.go index 34db9d287549f..13c9cd170afbb 100644 --- a/plugins/inputs/http/http.go +++ b/plugins/inputs/http/http.go @@ -29,6 +29,8 @@ type HTTP struct { Password string `toml:"password"` tls.ClientConfig + SuccessStatusCodes []int `toml:"success_status_codes"` + Timeout internal.Duration `toml:"timeout"` client *http.Client @@ -71,6 +73,9 @@ var sampleConfig = ` ## Amount of time allowed to complete the HTTP request # timeout = "5s" + ## List of success status codes + # success_status_codes = [200] + ## Data format to consume. ## Each data format has its own unique set of configuration options, read ## more about them here: @@ -101,6 +106,11 @@ func (h *HTTP) Init() error { }, Timeout: h.Timeout.Duration, } + + // Set default as [200] + if len(h.SuccessStatusCodes) == 0 { + h.SuccessStatusCodes = []int{200} + } return nil } @@ -143,6 +153,7 @@ func (h *HTTP) gatherURL( if err != nil { return err } + defer body.Close() request, err := http.NewRequest(h.Method, url, body) if err != nil { @@ -171,12 +182,19 @@ func (h *HTTP) gatherURL( } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("Received status code %d (%s), expected %d (%s)", + responseHasSuccessCode := false + for _, statusCode := range h.SuccessStatusCodes { + if resp.StatusCode == statusCode { + responseHasSuccessCode = true + break + } + } + + if !responseHasSuccessCode { + return fmt.Errorf("received status code %d (%s), expected any value out of %v", resp.StatusCode, http.StatusText(resp.StatusCode), - http.StatusOK, - http.StatusText(http.StatusOK)) + h.SuccessStatusCodes) } b, err := ioutil.ReadAll(resp.Body) @@ -199,16 +217,16 @@ func (h *HTTP) gatherURL( return nil } -func makeRequestBodyReader(contentEncoding, body string) (io.Reader, error) { - var err error +func makeRequestBodyReader(contentEncoding, body string) (io.ReadCloser, error) { var reader io.Reader = strings.NewReader(body) if contentEncoding == "gzip" { - reader, err = internal.CompressWithGzip(reader) + rc, err := internal.CompressWithGzip(reader) if err != nil { return nil, err } + return rc, nil } - return reader, nil + return ioutil.NopCloser(reader), nil } func init() { diff --git a/plugins/inputs/http/http_test.go b/plugins/inputs/http/http_test.go index 21eff62650f7c..993eda7321c0f 100644 --- a/plugins/inputs/http/http_test.go +++ b/plugins/inputs/http/http_test.go @@ -106,6 +106,30 @@ func TestInvalidStatusCode(t *testing.T) { require.Error(t, acc.GatherError(plugin.Gather)) } +func TestSuccessStatusCodes(t *testing.T) { + fakeServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusAccepted) + })) + defer fakeServer.Close() + + url := fakeServer.URL + "/endpoint" + plugin := &plugin.HTTP{ + URLs: []string{url}, + SuccessStatusCodes: []int{200, 202}, + } + + metricName := "metricName" + p, _ := parsers.NewParser(&parsers.Config{ + DataFormat: "json", + MetricName: metricName, + }) + plugin.SetParser(p) + + var acc testutil.Accumulator + plugin.Init() + require.NoError(t, acc.GatherError(plugin.Gather)) +} + func TestMethod(t *testing.T) { fakeServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method == "POST" { diff --git a/plugins/inputs/http_listener_v2/http_listener_v2.go b/plugins/inputs/http_listener_v2/http_listener_v2.go index 5427b384d2c03..21d35fab9f0a0 100644 --- a/plugins/inputs/http_listener_v2/http_listener_v2.go +++ b/plugins/inputs/http_listener_v2/http_listener_v2.go @@ -5,7 +5,6 @@ import ( "crypto/subtle" "crypto/tls" "io/ioutil" - "log" "net" "net/http" "net/url" @@ -48,6 +47,7 @@ type HTTPListenerV2 struct { tlsint.ServerConfig TimeFunc + Log telegraf.Logger wg sync.WaitGroup @@ -162,7 +162,7 @@ func (h *HTTPListenerV2) Start(acc telegraf.Accumulator) error { server.Serve(h.listener) }() - log.Printf("I! [inputs.http_listener_v2] Listening on %s", listener.Addr().String()) + h.Log.Infof("Listening on %s", listener.Addr().String()) return nil } @@ -219,7 +219,7 @@ func (h *HTTPListenerV2) serveWrite(res http.ResponseWriter, req *http.Request) metrics, err := h.Parse(bytes) if err != nil { - log.Printf("D! [inputs.http_listener_v2] Parse error: %v", err) + h.Log.Debugf("Parse error: %s", err.Error()) badRequest(res) return } @@ -239,7 +239,7 @@ func (h *HTTPListenerV2) collectBody(res http.ResponseWriter, req *http.Request) var err error body, err = gzip.NewReader(req.Body) if err != nil { - log.Println("D! " + err.Error()) + h.Log.Debug(err.Error()) badRequest(res) return nil, false } @@ -261,7 +261,7 @@ func (h *HTTPListenerV2) collectQuery(res http.ResponseWriter, req *http.Request query, err := url.QueryUnescape(rawQuery) if err != nil { - log.Printf("D! [inputs.http_listener_v2] Error parsing query: %v", err) + h.Log.Debugf("Error parsing query: %s", err.Error()) badRequest(res) return nil, false } diff --git a/plugins/inputs/http_listener_v2/http_listener_v2_test.go b/plugins/inputs/http_listener_v2/http_listener_v2_test.go index c27b022b24d45..c9e96b92db5d0 100644 --- a/plugins/inputs/http_listener_v2/http_listener_v2_test.go +++ b/plugins/inputs/http_listener_v2/http_listener_v2_test.go @@ -46,6 +46,7 @@ func newTestHTTPListenerV2() *HTTPListenerV2 { parser, _ := parsers.NewInfluxParser() listener := &HTTPListenerV2{ + Log: testutil.Logger{}, ServiceAddress: "localhost:0", Path: "/write", Methods: []string{"POST"}, @@ -68,6 +69,7 @@ func newTestHTTPSListenerV2() *HTTPListenerV2 { parser, _ := parsers.NewInfluxParser() listener := &HTTPListenerV2{ + Log: testutil.Logger{}, ServiceAddress: "localhost:0", Path: "/write", Methods: []string{"POST"}, @@ -231,6 +233,7 @@ func TestWriteHTTPExactMaxBodySize(t *testing.T) { parser, _ := parsers.NewInfluxParser() listener := &HTTPListenerV2{ + Log: testutil.Logger{}, ServiceAddress: "localhost:0", Path: "/write", Methods: []string{"POST"}, @@ -253,6 +256,7 @@ func TestWriteHTTPVerySmallMaxBody(t *testing.T) { parser, _ := parsers.NewInfluxParser() listener := &HTTPListenerV2{ + Log: testutil.Logger{}, ServiceAddress: "localhost:0", Path: "/write", Methods: []string{"POST"}, diff --git a/plugins/inputs/http_response/http_response.go b/plugins/inputs/http_response/http_response.go index b863190d725b6..24c22f72f9c48 100644 --- a/plugins/inputs/http_response/http_response.go +++ b/plugins/inputs/http_response/http_response.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "io/ioutil" - "log" "net" "net/http" "net/url" @@ -34,6 +33,8 @@ type HTTPResponse struct { Interface string tls.ClientConfig + Log telegraf.Logger + compiledStringMatch *regexp.Regexp client *http.Client } @@ -242,7 +243,7 @@ func (h *HTTPResponse) httpGather(u string) (map[string]interface{}, map[string] // HTTP error codes do not generate errors in the net/http library if err != nil { // Log error - log.Printf("D! Network error while polling %s: %s", u, err.Error()) + h.Log.Debugf("Network error while polling %s: %s", u, err.Error()) // Get error details netErr := setError(err, fields, tags) @@ -271,7 +272,7 @@ func (h *HTTPResponse) httpGather(u string) (map[string]interface{}, map[string] bodyBytes, err := ioutil.ReadAll(resp.Body) if err != nil { - log.Printf("D! Failed to read body of HTTP Response : %s", err) + h.Log.Debugf("Failed to read body of HTTP Response : %s", err.Error()) setResult("body_read_error", fields, tags) fields["content_length"] = len(bodyBytes) if h.ResponseStringMatch != "" { @@ -322,7 +323,7 @@ func (h *HTTPResponse) Gather(acc telegraf.Accumulator) error { if h.Address == "" { h.URLs = []string{"http://localhost"} } else { - log.Printf("W! [inputs.http_response] 'address' deprecated in telegraf 1.12, please use 'urls'") + h.Log.Warn("'address' deprecated in telegraf 1.12, please use 'urls'") h.URLs = []string{h.Address} } } diff --git a/plugins/inputs/http_response/http_response_test.go b/plugins/inputs/http_response/http_response_test.go index 5ba586c5902d3..530c81901db24 100644 --- a/plugins/inputs/http_response/http_response_test.go +++ b/plugins/inputs/http_response/http_response_test.go @@ -150,6 +150,7 @@ func TestHeaders(t *testing.T) { defer ts.Close() h := &HTTPResponse{ + Log: testutil.Logger{}, Address: ts.URL, Method: "GET", ResponseTimeout: internal.Duration{Duration: time.Second * 2}, @@ -185,6 +186,7 @@ func TestFields(t *testing.T) { defer ts.Close() h := &HTTPResponse{ + Log: testutil.Logger{}, Address: ts.URL + "/good", Body: "{ 'test': 'data'}", Method: "GET", @@ -246,6 +248,7 @@ func TestInterface(t *testing.T) { require.NoError(t, err) h := &HTTPResponse{ + Log: testutil.Logger{}, Address: ts.URL + "/good", Body: "{ 'test': 'data'}", Method: "GET", @@ -284,6 +287,7 @@ func TestRedirects(t *testing.T) { defer ts.Close() h := &HTTPResponse{ + Log: testutil.Logger{}, Address: ts.URL + "/redirect", Body: "{ 'test': 'data'}", Method: "GET", @@ -314,6 +318,7 @@ func TestRedirects(t *testing.T) { checkOutput(t, &acc, expectedFields, expectedTags, absentFields, nil) h = &HTTPResponse{ + Log: testutil.Logger{}, Address: ts.URL + "/badredirect", Body: "{ 'test': 'data'}", Method: "GET", @@ -350,6 +355,7 @@ func TestMethod(t *testing.T) { defer ts.Close() h := &HTTPResponse{ + Log: testutil.Logger{}, Address: ts.URL + "/mustbepostmethod", Body: "{ 'test': 'data'}", Method: "POST", @@ -380,6 +386,7 @@ func TestMethod(t *testing.T) { checkOutput(t, &acc, expectedFields, expectedTags, absentFields, nil) h = &HTTPResponse{ + Log: testutil.Logger{}, Address: ts.URL + "/mustbepostmethod", Body: "{ 'test': 'data'}", Method: "GET", @@ -411,6 +418,7 @@ func TestMethod(t *testing.T) { //check that lowercase methods work correctly h = &HTTPResponse{ + Log: testutil.Logger{}, Address: ts.URL + "/mustbepostmethod", Body: "{ 'test': 'data'}", Method: "head", @@ -447,6 +455,7 @@ func TestBody(t *testing.T) { defer ts.Close() h := &HTTPResponse{ + Log: testutil.Logger{}, Address: ts.URL + "/musthaveabody", Body: "{ 'test': 'data'}", Method: "GET", @@ -477,6 +486,7 @@ func TestBody(t *testing.T) { checkOutput(t, &acc, expectedFields, expectedTags, absentFields, nil) h = &HTTPResponse{ + Log: testutil.Logger{}, Address: ts.URL + "/musthaveabody", Method: "GET", ResponseTimeout: internal.Duration{Duration: time.Second * 20}, @@ -510,6 +520,7 @@ func TestStringMatch(t *testing.T) { defer ts.Close() h := &HTTPResponse{ + Log: testutil.Logger{}, Address: ts.URL + "/good", Body: "{ 'test': 'data'}", Method: "GET", @@ -547,6 +558,7 @@ func TestStringMatchJson(t *testing.T) { defer ts.Close() h := &HTTPResponse{ + Log: testutil.Logger{}, Address: ts.URL + "/jsonresponse", Body: "{ 'test': 'data'}", Method: "GET", @@ -584,6 +596,7 @@ func TestStringMatchFail(t *testing.T) { defer ts.Close() h := &HTTPResponse{ + Log: testutil.Logger{}, Address: ts.URL + "/good", Body: "{ 'test': 'data'}", Method: "GET", @@ -626,6 +639,7 @@ func TestTimeout(t *testing.T) { defer ts.Close() h := &HTTPResponse{ + Log: testutil.Logger{}, Address: ts.URL + "/twosecondnap", Body: "{ 'test': 'data'}", Method: "GET", @@ -659,6 +673,7 @@ func TestBadRegex(t *testing.T) { defer ts.Close() h := &HTTPResponse{ + Log: testutil.Logger{}, Address: ts.URL + "/good", Body: "{ 'test': 'data'}", Method: "GET", @@ -682,6 +697,7 @@ func TestBadRegex(t *testing.T) { func TestNetworkErrors(t *testing.T) { // DNS error h := &HTTPResponse{ + Log: testutil.Logger{}, Address: "https://nonexistent.nonexistent", // Any non-resolvable URL works here Body: "", Method: "GET", @@ -708,6 +724,7 @@ func TestNetworkErrors(t *testing.T) { // Connecton failed h = &HTTPResponse{ + Log: testutil.Logger{}, Address: "https:/nonexistent.nonexistent", // Any non-routable IP works here Body: "", Method: "GET", @@ -739,6 +756,7 @@ func TestContentLength(t *testing.T) { defer ts.Close() h := &HTTPResponse{ + Log: testutil.Logger{}, URLs: []string{ts.URL + "/good"}, Body: "{ 'test': 'data'}", Method: "GET", @@ -769,6 +787,7 @@ func TestContentLength(t *testing.T) { checkOutput(t, &acc, expectedFields, expectedTags, absentFields, nil) h = &HTTPResponse{ + Log: testutil.Logger{}, URLs: []string{ts.URL + "/musthaveabody"}, Body: "{ 'test': 'data'}", Method: "GET", diff --git a/plugins/inputs/icinga2/README.md b/plugins/inputs/icinga2/README.md index 697c6c59cdf9a..14708cd41ff50 100644 --- a/plugins/inputs/icinga2/README.md +++ b/plugins/inputs/icinga2/README.md @@ -11,10 +11,10 @@ services and hosts. You can read Icinga2's documentation for their remote API ```toml # Description [[inputs.icinga2]] - ## Required Icinga2 server address (default: "https://localhost:5665") + ## Required Icinga2 server address # server = "https://localhost:5665" - ## Required Icinga2 object type ("services" or "hosts, default "services") + ## Required Icinga2 object type ("services" or "hosts") # object_type = "services" ## Credentials for basic HTTP authentication diff --git a/plugins/inputs/icinga2/icinga2.go b/plugins/inputs/icinga2/icinga2.go index 82120da2c2e6f..67b9bcab96f3a 100644 --- a/plugins/inputs/icinga2/icinga2.go +++ b/plugins/inputs/icinga2/icinga2.go @@ -3,7 +3,6 @@ package icinga2 import ( "encoding/json" "fmt" - "log" "net/http" "net/url" "time" @@ -22,6 +21,8 @@ type Icinga2 struct { ResponseTimeout internal.Duration tls.ClientConfig + Log telegraf.Logger + client *http.Client } @@ -49,10 +50,10 @@ var levels = []string{"ok", "warning", "critical", "unknown"} type ObjectType string var sampleConfig = ` - ## Required Icinga2 server address (default: "https://localhost:5665") + ## Required Icinga2 server address # server = "https://localhost:5665" - - ## Required Icinga2 object type ("services" or "hosts, default "services") + + ## Required Icinga2 object type ("services" or "hosts") # object_type = "services" ## Credentials for basic HTTP authentication @@ -80,25 +81,27 @@ func (i *Icinga2) SampleConfig() string { func (i *Icinga2) GatherStatus(acc telegraf.Accumulator, checks []Object) { for _, check := range checks { - fields := make(map[string]interface{}) - tags := make(map[string]string) - url, err := url.Parse(i.Server) if err != nil { - log.Fatal(err) + i.Log.Error(err.Error()) + continue } state := int64(check.Attrs.State) - fields["name"] = check.Attrs.Name - fields["state_code"] = state + fields := map[string]interface{}{ + "name": check.Attrs.Name, + "state_code": state, + } - tags["display_name"] = check.Attrs.DisplayName - tags["check_command"] = check.Attrs.CheckCommand - tags["state"] = levels[state] - tags["source"] = url.Hostname() - tags["scheme"] = url.Scheme - tags["port"] = url.Port() + tags := map[string]string{ + "display_name": check.Attrs.DisplayName, + "check_command": check.Attrs.CheckCommand, + "state": levels[state], + "source": url.Hostname(), + "scheme": url.Scheme, + "port": url.Port(), + } acc.AddFields(fmt.Sprintf("icinga2_%s", i.ObjectType), fields, tags) } @@ -165,8 +168,9 @@ func (i *Icinga2) Gather(acc telegraf.Accumulator) error { func init() { inputs.Add("icinga2", func() telegraf.Input { return &Icinga2{ - Server: "https://localhost:5665", - ObjectType: "services", + Server: "https://localhost:5665", + ObjectType: "services", + ResponseTimeout: internal.Duration{Duration: time.Second * 5}, } }) } diff --git a/plugins/inputs/icinga2/icinga2_test.go b/plugins/inputs/icinga2/icinga2_test.go index e62a8d4236189..a908af7d50fa5 100644 --- a/plugins/inputs/icinga2/icinga2_test.go +++ b/plugins/inputs/icinga2/icinga2_test.go @@ -32,6 +32,7 @@ func TestGatherServicesStatus(t *testing.T) { json.Unmarshal([]byte(s), &checks) icinga2 := new(Icinga2) + icinga2.Log = testutil.Logger{} icinga2.ObjectType = "services" icinga2.Server = "https://localhost:5665" @@ -86,6 +87,7 @@ func TestGatherHostsStatus(t *testing.T) { var acc testutil.Accumulator icinga2 := new(Icinga2) + icinga2.Log = testutil.Logger{} icinga2.ObjectType = "hosts" icinga2.Server = "https://localhost:5665" diff --git a/plugins/inputs/infiniband/README.md b/plugins/inputs/infiniband/README.md new file mode 100644 index 0000000000000..bc5b03543c375 --- /dev/null +++ b/plugins/inputs/infiniband/README.md @@ -0,0 +1,58 @@ +# InfiniBand Input Plugin + +This plugin gathers statistics for all InfiniBand devices and ports on the +system. These are the counters that can be found in +`/sys/class/infiniband//port//counters/` + +**Supported Platforms**: Linux + +### Configuration + +```toml +[[inputs.infiniband]] + # no configuration +``` + +### Metrics + +Actual metrics depend on the InfiniBand devices, the plugin uses a simple +mapping from counter -> counter value. + +[Information about the counters][counters] collected is provided by Mellanox. + +[counters]: https://community.mellanox.com/s/article/understanding-mlx5-linux-counters-and-status-parameters + +- infiniband + - tags: + - device + - port + - fields: + - excessive_buffer_overrun_errors (integer) + - link_downed (integer) + - link_error_recovery (integer) + - local_link_integrity_errors (integer) + - multicast_rcv_packets (integer) + - multicast_xmit_packets (integer) + - port_rcv_constraint_errors (integer) + - port_rcv_data (integer) + - port_rcv_errors (integer) + - port_rcv_packets (integer) + - port_rcv_remote_physical_errors (integer) + - port_rcv_switch_relay_errors (integer) + - port_xmit_constraint_errors (integer) + - port_xmit_data (integer) + - port_xmit_discards (integer) + - port_xmit_packets (integer) + - port_xmit_wait (integer) + - symbol_error (integer) + - unicast_rcv_packets (integer) + - unicast_xmit_packets (integer) + - VL15_dropped (integer) + + + +### Example Output + +``` +infiniband,device=mlx5_0,port=1 VL15_dropped=0i,excessive_buffer_overrun_errors=0i,link_downed=0i,link_error_recovery=0i,local_link_integrity_errors=0i,multicast_rcv_packets=0i,multicast_xmit_packets=0i,port_rcv_constraint_errors=0i,port_rcv_data=237159415345822i,port_rcv_errors=0i,port_rcv_packets=801977655075i,port_rcv_remote_physical_errors=0i,port_rcv_switch_relay_errors=0i,port_xmit_constraint_errors=0i,port_xmit_data=238334949937759i,port_xmit_discards=0i,port_xmit_packets=803162651391i,port_xmit_wait=4294967295i,symbol_error=0i,unicast_rcv_packets=801977655075i,unicast_xmit_packets=803162651391i 1573125558000000000 +``` diff --git a/plugins/inputs/infiniband/infiniband.go b/plugins/inputs/infiniband/infiniband.go new file mode 100644 index 0000000000000..65e1d6c712998 --- /dev/null +++ b/plugins/inputs/infiniband/infiniband.go @@ -0,0 +1,22 @@ +package infiniband + +import ( + "github.com/influxdata/telegraf" +) + +// Stores the configuration values for the infiniband plugin - as there are no +// config values, this is intentionally empty +type Infiniband struct { + Log telegraf.Logger `toml:"-"` +} + +// Sample configuration for plugin +var InfinibandConfig = `` + +func (_ *Infiniband) SampleConfig() string { + return InfinibandConfig +} + +func (_ *Infiniband) Description() string { + return "Gets counters from all InfiniBand cards and ports installed" +} diff --git a/plugins/inputs/infiniband/infiniband_linux.go b/plugins/inputs/infiniband/infiniband_linux.go new file mode 100644 index 0000000000000..48cd8a428900d --- /dev/null +++ b/plugins/inputs/infiniband/infiniband_linux.go @@ -0,0 +1,59 @@ +// +build linux + +package infiniband + +import ( + "fmt" + "github.com/Mellanox/rdmamap" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" + "strconv" +) + +// Gather statistics from our infiniband cards +func (_ *Infiniband) Gather(acc telegraf.Accumulator) error { + + rdmaDevices := rdmamap.GetRdmaDeviceList() + + if len(rdmaDevices) == 0 { + return fmt.Errorf("no InfiniBand devices found in /sys/class/infiniband/") + } + + for _, dev := range rdmaDevices { + devicePorts := rdmamap.GetPorts(dev) + for _, port := range devicePorts { + portInt, err := strconv.Atoi(port) + if err != nil { + return err + } + + stats, err := rdmamap.GetRdmaSysfsStats(dev, portInt) + if err != nil { + return err + } + + addStats(dev, port, stats, acc) + } + } + + return nil +} + +// Add the statistics to the accumulator +func addStats(dev string, port string, stats []rdmamap.RdmaStatEntry, acc telegraf.Accumulator) { + + // Allow users to filter by card and port + tags := map[string]string{"device": dev, "port": port} + fields := make(map[string]interface{}) + + for _, entry := range stats { + fields[entry.Name] = entry.Value + } + + acc.AddFields("infiniband", fields, tags) +} + +// Initialise plugin +func init() { + inputs.Add("infiniband", func() telegraf.Input { return &Infiniband{} }) +} diff --git a/plugins/inputs/infiniband/infiniband_notlinux.go b/plugins/inputs/infiniband/infiniband_notlinux.go new file mode 100644 index 0000000000000..5b19672d975d8 --- /dev/null +++ b/plugins/inputs/infiniband/infiniband_notlinux.go @@ -0,0 +1,23 @@ +// +build !linux + +package infiniband + +import ( + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" +) + +func (i *Infiniband) Init() error { + i.Log.Warn("Current platform is not supported") + return nil +} + +func (_ *Infiniband) Gather(acc telegraf.Accumulator) error { + return nil +} + +func init() { + inputs.Add("infiniband", func() telegraf.Input { + return &Infiniband{} + }) +} diff --git a/plugins/inputs/infiniband/infiniband_test.go b/plugins/inputs/infiniband/infiniband_test.go new file mode 100644 index 0000000000000..6c4bb24587f4a --- /dev/null +++ b/plugins/inputs/infiniband/infiniband_test.go @@ -0,0 +1,134 @@ +// +build linux + +package infiniband + +import ( + "github.com/Mellanox/rdmamap" + "github.com/influxdata/telegraf/testutil" + "testing" +) + +func TestInfiniband(t *testing.T) { + fields := map[string]interface{}{ + "excessive_buffer_overrun_errors": uint64(0), + "link_downed": uint64(0), + "link_error_recovery": uint64(0), + "local_link_integrity_errors": uint64(0), + "multicast_rcv_packets": uint64(0), + "multicast_xmit_packets": uint64(0), + "port_rcv_constraint_errors": uint64(0), + "port_rcv_data": uint64(237159415345822), + "port_rcv_errors": uint64(0), + "port_rcv_packets": uint64(801977655075), + "port_rcv_remote_physical_errors": uint64(0), + "port_rcv_switch_relay_errors": uint64(0), + "port_xmit_constraint_errors": uint64(0), + "port_xmit_data": uint64(238334949937759), + "port_xmit_discards": uint64(0), + "port_xmit_packets": uint64(803162651391), + "port_xmit_wait": uint64(4294967295), + "symbol_error": uint64(0), + "unicast_rcv_packets": uint64(801977655075), + "unicast_xmit_packets": uint64(803162651391), + "VL15_dropped": uint64(0), + } + + tags := map[string]string{ + "device": "m1x5_0", + "port": "1", + } + + sample_rdmastats_entries := []rdmamap.RdmaStatEntry{ + { + Name: "excessive_buffer_overrun_errors", + Value: uint64(0), + }, + { + Name: "link_downed", + Value: uint64(0), + }, + { + Name: "link_error_recovery", + Value: uint64(0), + }, + { + Name: "local_link_integrity_errors", + Value: uint64(0), + }, + { + Name: "multicast_rcv_packets", + Value: uint64(0), + }, + { + Name: "multicast_xmit_packets", + Value: uint64(0), + }, + { + Name: "port_rcv_constraint_errors", + Value: uint64(0), + }, + { + Name: "port_rcv_data", + Value: uint64(237159415345822), + }, + { + Name: "port_rcv_errors", + Value: uint64(0), + }, + { + Name: "port_rcv_packets", + Value: uint64(801977655075), + }, + { + Name: "port_rcv_remote_physical_errors", + Value: uint64(0), + }, + { + Name: "port_rcv_switch_relay_errors", + Value: uint64(0), + }, + { + Name: "port_xmit_constraint_errors", + Value: uint64(0), + }, + { + Name: "port_xmit_data", + Value: uint64(238334949937759), + }, + { + Name: "port_xmit_discards", + Value: uint64(0), + }, + { + Name: "port_xmit_packets", + Value: uint64(803162651391), + }, + { + Name: "port_xmit_wait", + Value: uint64(4294967295), + }, + { + Name: "symbol_error", + Value: uint64(0), + }, + { + Name: "unicast_rcv_packets", + Value: uint64(801977655075), + }, + { + Name: "unicast_xmit_packets", + Value: uint64(803162651391), + }, + { + Name: "VL15_dropped", + Value: uint64(0), + }, + } + + var acc testutil.Accumulator + + addStats("m1x5_0", "1", sample_rdmastats_entries, &acc) + + acc.AssertContainsTaggedFields(t, "infiniband", fields, tags) + +} diff --git a/plugins/inputs/influxdb/README.md b/plugins/inputs/influxdb/README.md index 2bab123f81c0e..711503245bc1c 100644 --- a/plugins/inputs/influxdb/README.md +++ b/plugins/inputs/influxdb/README.md @@ -20,6 +20,10 @@ InfluxDB-formatted endpoints. See below for more information. "http://localhost:8086/debug/vars" ] + ## Username and password to send using HTTP Basic Authentication. + # username = "" + # password = "" + ## Optional TLS Config # tls_ca = "/etc/telegraf/ca.pem" # tls_cert = "/etc/telegraf/cert.pem" diff --git a/plugins/inputs/influxdb/influxdb.go b/plugins/inputs/influxdb/influxdb.go index 0bb3ead5ee642..96389a0138b35 100644 --- a/plugins/inputs/influxdb/influxdb.go +++ b/plugins/inputs/influxdb/influxdb.go @@ -1,9 +1,10 @@ package influxdb import ( + "bytes" "encoding/json" "errors" - "fmt" + "io" "net/http" "sync" "time" @@ -14,9 +15,28 @@ import ( "github.com/influxdata/telegraf/plugins/inputs" ) +const ( + maxErrorResponseBodyLength = 1024 +) + +type APIError struct { + StatusCode int + Reason string + Description string `json:"error"` +} + +func (e *APIError) Error() string { + if e.Description != "" { + return e.Reason + ": " + e.Description + } + return e.Reason +} + type InfluxDB struct { - URLs []string `toml:"urls"` - Timeout internal.Duration + URLs []string `toml:"urls"` + Username string `toml:"username"` + Password string `toml:"password"` + Timeout internal.Duration `toml:"timeout"` tls.ClientConfig client *http.Client @@ -38,6 +58,10 @@ func (*InfluxDB) SampleConfig() string { "http://localhost:8086/debug/vars" ] + ## Username and password to send using HTTP Basic Authentication. + # username = "" + # password = "" + ## Optional TLS Config # tls_ca = "/etc/telegraf/ca.pem" # tls_cert = "/etc/telegraf/cert.pem" @@ -75,7 +99,7 @@ func (i *InfluxDB) Gather(acc telegraf.Accumulator) error { go func(url string) { defer wg.Done() if err := i.gatherURL(acc, url); err != nil { - acc.AddError(fmt.Errorf("[url=%s]: %s", url, err)) + acc.AddError(err) } }(u) } @@ -135,12 +159,27 @@ func (i *InfluxDB) gatherURL( shardCounter := 0 now := time.Now() - resp, err := i.client.Get(url) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + + if i.Username != "" || i.Password != "" { + req.SetBasicAuth(i.Username, i.Password) + } + + req.Header.Set("User-Agent", "Telegraf/"+internal.Version()) + + resp, err := i.client.Do(req) if err != nil { return err } defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return readResponseError(resp) + } + // It would be nice to be able to decode into a map[string]point, but // we'll get a decoder error like: // `json: cannot unmarshal array into Go value of type influxdb.point` @@ -255,6 +294,27 @@ func (i *InfluxDB) gatherURL( return nil } +func readResponseError(resp *http.Response) error { + apiError := &APIError{ + StatusCode: resp.StatusCode, + Reason: resp.Status, + } + + var buf bytes.Buffer + r := io.LimitReader(resp.Body, maxErrorResponseBodyLength) + _, err := buf.ReadFrom(r) + if err != nil { + return apiError + } + + err = json.Unmarshal(buf.Bytes(), apiError) + if err != nil { + return apiError + } + + return apiError +} + func init() { inputs.Add("influxdb", func() telegraf.Input { return &InfluxDB{ diff --git a/plugins/inputs/influxdb/influxdb_test.go b/plugins/inputs/influxdb/influxdb_test.go index f24ecc24c11cf..9225c45b09d68 100644 --- a/plugins/inputs/influxdb/influxdb_test.go +++ b/plugins/inputs/influxdb/influxdb_test.go @@ -1,6 +1,7 @@ package influxdb_test import ( + "fmt" "net/http" "net/http/httptest" "testing" @@ -178,6 +179,31 @@ func TestErrorHandling404(t *testing.T) { require.Error(t, acc.GatherError(plugin.Gather)) } +func TestErrorResponse(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + w.Write([]byte(`{"error": "unable to parse authentication credentials"}`)) + })) + defer ts.Close() + + plugin := &influxdb.InfluxDB{ + URLs: []string{ts.URL}, + } + + var acc testutil.Accumulator + err := plugin.Gather(&acc) + require.NoError(t, err) + + expected := []error{ + &influxdb.APIError{ + StatusCode: http.StatusUnauthorized, + Reason: fmt.Sprintf("%d %s", http.StatusUnauthorized, http.StatusText(http.StatusUnauthorized)), + Description: "unable to parse authentication credentials", + }, + } + require.Equal(t, expected, acc.Errors) +} + const basicJSON = ` { "_1": { diff --git a/plugins/inputs/influxdb_listener/http_listener.go b/plugins/inputs/influxdb_listener/http_listener.go index 5383fd2aad6d9..aeb2b589f650f 100644 --- a/plugins/inputs/influxdb_listener/http_listener.go +++ b/plugins/inputs/influxdb_listener/http_listener.go @@ -8,7 +8,6 @@ import ( "encoding/json" "fmt" "io" - "log" "net" "net/http" "sync" @@ -75,6 +74,8 @@ type HTTPListener struct { BuffersCreated selfstat.Stat AuthFailures selfstat.Stat + Log telegraf.Logger + longLines selfstat.Stat } @@ -202,7 +203,7 @@ func (h *HTTPListener) Start(acc telegraf.Accumulator) error { server.Serve(h.listener) }() - log.Printf("I! Started HTTP listener service on %s\n", h.ServiceAddress) + h.Log.Infof("Started HTTP listener service on %s", h.ServiceAddress) return nil } @@ -215,7 +216,7 @@ func (h *HTTPListener) Stop() { h.listener.Close() h.wg.Wait() - log.Println("I! Stopped HTTP listener service on ", h.ServiceAddress) + h.Log.Infof("Stopped HTTP listener service on %s", h.ServiceAddress) } func (h *HTTPListener) ServeHTTP(res http.ResponseWriter, req *http.Request) { @@ -274,7 +275,7 @@ func (h *HTTPListener) serveWrite(res http.ResponseWriter, req *http.Request) { var err error body, err = gzip.NewReader(req.Body) if err != nil { - log.Println("D! " + err.Error()) + h.Log.Debug(err.Error()) badRequest(res, err.Error()) return } @@ -290,7 +291,7 @@ func (h *HTTPListener) serveWrite(res http.ResponseWriter, req *http.Request) { for { n, err := io.ReadFull(body, buf[bufStart:]) if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF { - log.Println("D! " + err.Error()) + h.Log.Debug(err.Error()) // problem reading the request body badRequest(res, err.Error()) return @@ -326,7 +327,7 @@ func (h *HTTPListener) serveWrite(res http.ResponseWriter, req *http.Request) { // finished reading the request body err = h.parse(buf[:n+bufStart], now, precision, db) if err != nil { - log.Println("D! "+err.Error(), bufStart+n) + h.Log.Debugf("%s: %s", err.Error(), bufStart+n) return400 = true } if return400 { @@ -348,7 +349,7 @@ func (h *HTTPListener) serveWrite(res http.ResponseWriter, req *http.Request) { if i == -1 { h.longLines.Incr(1) // drop any line longer than the max buffer size - log.Printf("D! http_listener received a single line longer than the maximum of %d bytes", + h.Log.Debugf("Http_listener received a single line longer than the maximum of %d bytes", len(buf)) hangingBytes = true return400 = true @@ -356,7 +357,7 @@ func (h *HTTPListener) serveWrite(res http.ResponseWriter, req *http.Request) { continue } if err := h.parse(buf[:i+1], now, precision, db); err != nil { - log.Println("D! " + err.Error()) + h.Log.Debug(err.Error()) return400 = true } // rotate the bit remaining after the last newline to the front of the buffer diff --git a/plugins/inputs/influxdb_listener/http_listener_test.go b/plugins/inputs/influxdb_listener/http_listener_test.go index 6d14e6539603b..771bb5faf5dd4 100644 --- a/plugins/inputs/influxdb_listener/http_listener_test.go +++ b/plugins/inputs/influxdb_listener/http_listener_test.go @@ -44,6 +44,7 @@ var ( func newTestHTTPListener() *HTTPListener { listener := &HTTPListener{ + Log: testutil.Logger{}, ServiceAddress: "localhost:0", TimeFunc: time.Now, } @@ -59,6 +60,7 @@ func newTestHTTPAuthListener() *HTTPListener { func newTestHTTPSListener() *HTTPListener { listener := &HTTPListener{ + Log: testutil.Logger{}, ServiceAddress: "localhost:0", ServerConfig: *pki.TLSServerConfig(), TimeFunc: time.Now, @@ -220,6 +222,7 @@ func TestWriteHTTPNoNewline(t *testing.T) { func TestWriteHTTPMaxLineSizeIncrease(t *testing.T) { listener := &HTTPListener{ + Log: testutil.Logger{}, ServiceAddress: "localhost:0", MaxLineSize: internal.Size{Size: 128 * 1000}, TimeFunc: time.Now, @@ -238,6 +241,7 @@ func TestWriteHTTPMaxLineSizeIncrease(t *testing.T) { func TestWriteHTTPVerySmallMaxBody(t *testing.T) { listener := &HTTPListener{ + Log: testutil.Logger{}, ServiceAddress: "localhost:0", MaxBodySize: internal.Size{Size: 4096}, TimeFunc: time.Now, @@ -255,6 +259,7 @@ func TestWriteHTTPVerySmallMaxBody(t *testing.T) { func TestWriteHTTPVerySmallMaxLineSize(t *testing.T) { listener := &HTTPListener{ + Log: testutil.Logger{}, ServiceAddress: "localhost:0", MaxLineSize: internal.Size{Size: 70}, TimeFunc: time.Now, @@ -282,6 +287,7 @@ func TestWriteHTTPVerySmallMaxLineSize(t *testing.T) { func TestWriteHTTPLargeLinesSkipped(t *testing.T) { listener := &HTTPListener{ + Log: testutil.Logger{}, ServiceAddress: "localhost:0", MaxLineSize: internal.Size{Size: 100}, TimeFunc: time.Now, diff --git a/plugins/inputs/influxdb_listener/influxdb_listener_test.go b/plugins/inputs/influxdb_listener/influxdb_listener_test.go new file mode 100644 index 0000000000000..5badc12131e37 --- /dev/null +++ b/plugins/inputs/influxdb_listener/influxdb_listener_test.go @@ -0,0 +1,114 @@ +package http_listener + +import ( + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/plugins/parsers/influx" + "github.com/influxdata/telegraf/selfstat" + "github.com/influxdata/telegraf/testutil" +) + +// newListener is the minimal HTTPListener construction to serve writes. +func newListener() *HTTPListener { + listener := &HTTPListener{ + TimeFunc: time.Now, + acc: &testutil.NopAccumulator{}, + BytesRecv: selfstat.Register("http_listener", "bytes_received", map[string]string{}), + handler: influx.NewMetricHandler(), + pool: NewPool(200, DEFAULT_MAX_LINE_SIZE), + MaxLineSize: internal.Size{ + Size: DEFAULT_MAX_LINE_SIZE, + }, + MaxBodySize: internal.Size{ + Size: DEFAULT_MAX_BODY_SIZE, + }, + } + listener.parser = influx.NewParser(listener.handler) + return listener +} + +func BenchmarkHTTPListener_serveWrite(b *testing.B) { + res := httptest.NewRecorder() + addr := "http://localhost/write?db=mydb" + + benchmarks := []struct { + name string + lines string + }{ + { + name: "single line, tag, and field", + lines: lines(1, 1, 1), + }, + { + name: "single line, 10 tags and fields", + lines: lines(1, 10, 10), + }, + { + name: "single line, 100 tags and fields", + lines: lines(1, 100, 100), + }, + { + name: "1k lines, single tag and field", + lines: lines(1000, 1, 1), + }, + { + name: "1k lines, 10 tags and fields", + lines: lines(1000, 10, 10), + }, + { + name: "10k lines, 10 tags and fields", + lines: lines(10000, 10, 10), + }, + { + name: "100k lines, 10 tags and fields", + lines: lines(100000, 10, 10), + }, + } + + for _, bm := range benchmarks { + b.Run(bm.name, func(b *testing.B) { + listener := newListener() + + b.ResetTimer() + for n := 0; n < b.N; n++ { + req, err := http.NewRequest("POST", addr, strings.NewReader(bm.lines)) + if err != nil { + b.Error(err) + } + listener.serveWrite(res, req) + if res.Code != http.StatusNoContent { + b.Errorf("unexpected status %d", res.Code) + } + } + }) + } +} + +func lines(lines, numTags, numFields int) string { + lp := make([]string, lines) + for i := 0; i < lines; i++ { + tags := make([]string, numTags) + for j := 0; j < numTags; j++ { + tags[j] = fmt.Sprintf("t%d=v%d", j, j) + } + + fields := make([]string, numFields) + for k := 0; k < numFields; k++ { + fields[k] = fmt.Sprintf("f%d=%d", k, k) + } + + lp[i] = fmt.Sprintf("m%d,%s %s", + i, + strings.Join(tags, ","), + strings.Join(fields, ","), + ) + } + + return strings.Join(lp, "\n") +} diff --git a/plugins/inputs/ipmi_sensor/README.md b/plugins/inputs/ipmi_sensor/README.md index fb2e8f26e0c07..6c93bd15ed16a 100644 --- a/plugins/inputs/ipmi_sensor/README.md +++ b/plugins/inputs/ipmi_sensor/README.md @@ -27,6 +27,11 @@ ipmitool -I lan -H SERVER -U USERID -P PASSW0RD sdr ## optionally specify the path to the ipmitool executable # path = "/usr/bin/ipmitool" ## + ## Setting 'use_sudo' to true will make use of sudo to run ipmitool. + ## Sudo must be configured to allow the telegraf user to run ipmitool + ## without a password. + # use_sudo = false + ## ## optionally force session privilege level. Can be CALLBACK, USER, OPERATOR, ADMINISTRATOR # privilege = "ADMINISTRATOR" ## @@ -86,6 +91,21 @@ ipmi device node. When using udev you can create the device node giving ``` KERNEL=="ipmi*", MODE="660", GROUP="telegraf" ``` +Alternatively, it is possible to use sudo. You will need the following in your telegraf config: +```toml +[[inputs.ipmi_sensor]] + use_sudo = true +``` + +You will also need to update your sudoers file: + +```bash +$ visudo +# Add the following line: +Cmnd_Alias IPMITOOL = /usr/bin/ipmitool * +telegraf ALL=(root) NOPASSWD: IPMITOOL +Defaults!IPMITOOL !logfile, !syslog, !pam_session +``` ### Example Output diff --git a/plugins/inputs/ipmi_sensor/ipmi.go b/plugins/inputs/ipmi_sensor/ipmi.go index 2ec51525b7736..9ac842b896ac2 100644 --- a/plugins/inputs/ipmi_sensor/ipmi.go +++ b/plugins/inputs/ipmi_sensor/ipmi.go @@ -32,12 +32,18 @@ type Ipmi struct { Servers []string Timeout internal.Duration MetricVersion int + UseSudo bool } var sampleConfig = ` ## optionally specify the path to the ipmitool executable # path = "/usr/bin/ipmitool" ## + ## Setting 'use_sudo' to true will make use of sudo to run ipmitool. + ## Sudo must be configured to allow the telegraf user to run ipmitool + ## without a password. + # use_sudo = false + ## ## optionally force session privilege level. Can be CALLBACK, USER, OPERATOR, ADMINISTRATOR # privilege = "ADMINISTRATOR" ## @@ -112,7 +118,13 @@ func (m *Ipmi) parse(acc telegraf.Accumulator, server string) error { if m.MetricVersion == 2 { opts = append(opts, "elist") } - cmd := execCommand(m.Path, opts...) + name := m.Path + if m.UseSudo { + // -n - avoid prompting the user for input of any kind + opts = append([]string{"-n", name}, opts...) + name = "sudo" + } + cmd := execCommand(name, opts...) out, err := internal.CombinedOutputTimeout(cmd, m.Timeout.Duration) timestamp := time.Now() if err != nil { diff --git a/plugins/inputs/ipvs/ipvs.go b/plugins/inputs/ipvs/ipvs.go index 2d3ad02787e4a..5e3ae0d5637b0 100644 --- a/plugins/inputs/ipvs/ipvs.go +++ b/plugins/inputs/ipvs/ipvs.go @@ -3,21 +3,21 @@ package ipvs import ( - "errors" "fmt" - "log" "math/bits" "strconv" "syscall" "github.com/docker/libnetwork/ipvs" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/common/logrus" "github.com/influxdata/telegraf/plugins/inputs" ) // IPVS holds the state for this input plugin type IPVS struct { handle *ipvs.Handle + Log telegraf.Logger } // Description returns a description string @@ -35,7 +35,7 @@ func (i *IPVS) Gather(acc telegraf.Accumulator) error { if i.handle == nil { h, err := ipvs.New("") // TODO: make the namespace configurable if err != nil { - return errors.New("Unable to open IPVS handle") + return fmt.Errorf("unable to open IPVS handle: %v", err) } i.handle = h } @@ -44,7 +44,7 @@ func (i *IPVS) Gather(acc telegraf.Accumulator) error { if err != nil { i.handle.Close() i.handle = nil // trigger a reopen on next call to gather - return errors.New("Failed to list IPVS services") + return fmt.Errorf("failed to list IPVS services: %v", err) } for _, s := range services { fields := map[string]interface{}{ @@ -61,7 +61,7 @@ func (i *IPVS) Gather(acc telegraf.Accumulator) error { destinations, err := i.handle.GetDestinations(s) if err != nil { - log.Println("E! Failed to list destinations for a virtual server") + i.Log.Errorf("Failed to list destinations for a virtual server: %v", err) continue // move on to the next virtual server } @@ -148,5 +148,8 @@ func addressFamilyToString(af uint16) string { } func init() { - inputs.Add("ipvs", func() telegraf.Input { return &IPVS{} }) + inputs.Add("ipvs", func() telegraf.Input { + logrus.InstallHook() + return &IPVS{} + }) } diff --git a/plugins/inputs/jenkins/README.md b/plugins/inputs/jenkins/README.md index 79f55e6aac0f4..2bbfd157e7cbd 100644 --- a/plugins/inputs/jenkins/README.md +++ b/plugins/inputs/jenkins/README.md @@ -8,7 +8,7 @@ This plugin does not require a plugin on jenkins and it makes use of Jenkins API ```toml [[inputs.jenkins]] - ## The Jenkins URL + ## The Jenkins URL in the format "schema://host:port" url = "http://my-jenkins-instance:8080" # username = "admin" # password = "admin" @@ -59,22 +59,27 @@ This plugin does not require a plugin on jenkins and it makes use of Jenkins API - temp_path - node_name - status ("online", "offline") + - source + - port - fields: - - disk_available - - temp_available - - memory_available - - memory_total - - swap_available - - swap_total - - response_time + - disk_available (Bytes) + - temp_available (Bytes) + - memory_available (Bytes) + - memory_total (Bytes) + - swap_available (Bytes) + - swap_total (Bytes) + - response_time (ms) + - num_executors - jenkins_job - tags: - name - parents - result + - source + - port - fields: - - duration + - duration (ms) - result_code (0 = SUCCESS, 1 = FAILURE, 2 = NOT_BUILD, 3 = UNSTABLE, 4 = ABORTED) ### Sample Queries: @@ -91,7 +96,8 @@ SELECT mean("duration") AS "mean_duration" FROM "jenkins_job" WHERE time > now() ``` $ ./telegraf --config telegraf.conf --input-filter jenkins --test -jenkins_node,arch=Linux\ (amd64),disk_path=/var/jenkins_home,temp_path=/tmp,host=myhost,node_name=master swap_total=4294963200,memory_available=586711040,memory_total=6089498624,status=online,response_time=1000i,disk_available=152392036352,temp_available=152392036352,swap_available=3503263744 1516031535000000000 -jenkins_job,host=myhost,name=JOB1,parents=apps/br1,result=SUCCESS duration=2831i,result_code=0i 1516026630000000000 -jenkins_job,host=myhost,name=JOB2,parents=apps/br2,result=SUCCESS duration=2285i,result_code=0i 1516027230000000000 +jenkins_node,arch=Linux\ (amd64),disk_path=/var/jenkins_home,temp_path=/tmp,host=myhost,node_name=master,source=my-jenkins-instance,port=8080 swap_total=4294963200,memory_available=586711040,memory_total=6089498624,status=online,response_time=1000i,disk_available=152392036352,temp_available=152392036352,swap_available=3503263744,num_executors=2i 1516031535000000000 +jenkins_job,host=myhost,name=JOB1,parents=apps/br1,result=SUCCESS,source=my-jenkins-instance,port=8080 duration=2831i,result_code=0i 1516026630000000000 +jenkins_job,host=myhost,name=JOB2,parents=apps/br2,result=SUCCESS,source=my-jenkins-instance,port=8080 duration=2285i,result_code=0i 1516027230000000000 ``` + diff --git a/plugins/inputs/jenkins/jenkins.go b/plugins/inputs/jenkins/jenkins.go index cfa0a38e4c104..7a2b19d956959 100644 --- a/plugins/inputs/jenkins/jenkins.go +++ b/plugins/inputs/jenkins/jenkins.go @@ -2,10 +2,9 @@ package jenkins import ( "context" - "errors" "fmt" - "log" "net/http" + "net/url" "strconv" "strings" "sync" @@ -23,12 +22,16 @@ type Jenkins struct { URL string Username string Password string + Source string + Port string // HTTP Timeout specified as a string - 3s, 1m, 1h ResponseTimeout internal.Duration tls.ClientConfig client *client + Log telegraf.Logger + MaxConnections int `toml:"max_connections"` MaxBuildAge internal.Duration `toml:"max_build_age"` MaxSubJobDepth int `toml:"max_subjob_depth"` @@ -43,7 +46,7 @@ type Jenkins struct { } const sampleConfig = ` - ## The Jenkins URL + ## The Jenkins URL in the format "schema://host:port" url = "http://my-jenkins-instance:8080" # username = "admin" # password = "admin" @@ -137,6 +140,22 @@ func (j *Jenkins) newHTTPClient() (*http.Client, error) { func (j *Jenkins) initialize(client *http.Client) error { var err error + // init jenkins tags + u, err := url.Parse(j.URL) + if err != nil { + return err + } + if u.Port() == "" { + if u.Scheme == "http" { + j.Port = "80" + } else if u.Scheme == "https" { + j.Port = "443" + } + } else { + j.Port = u.Port() + } + j.Source = u.Hostname() + // init job filter j.jobFilter, err = filter.Compile(j.JobExclude) if err != nil { @@ -179,27 +198,39 @@ func (j *Jenkins) gatherNodeData(n node, acc telegraf.Accumulator) error { return nil } - tags["arch"] = n.MonitorData.HudsonNodeMonitorsArchitectureMonitor + monitorData := n.MonitorData + + if monitorData.HudsonNodeMonitorsArchitectureMonitor != "" { + tags["arch"] = monitorData.HudsonNodeMonitorsArchitectureMonitor + } tags["status"] = "online" if n.Offline { tags["status"] = "offline" } - monitorData := n.MonitorData - if monitorData.HudsonNodeMonitorsArchitectureMonitor == "" { - return errors.New("empty monitor data, please check your permission") + + tags["source"] = j.Source + tags["port"] = j.Port + + fields := make(map[string]interface{}) + fields["num_executors"] = n.NumExecutors + + if monitorData.HudsonNodeMonitorsResponseTimeMonitor != nil { + fields["response_time"] = monitorData.HudsonNodeMonitorsResponseTimeMonitor.Average + } + if monitorData.HudsonNodeMonitorsDiskSpaceMonitor != nil { + tags["disk_path"] = monitorData.HudsonNodeMonitorsDiskSpaceMonitor.Path + fields["disk_available"] = monitorData.HudsonNodeMonitorsDiskSpaceMonitor.Size } - tags["disk_path"] = monitorData.HudsonNodeMonitorsDiskSpaceMonitor.Path - tags["temp_path"] = monitorData.HudsonNodeMonitorsTemporarySpaceMonitor.Path - - fields := map[string]interface{}{ - "response_time": monitorData.HudsonNodeMonitorsResponseTimeMonitor.Average, - "disk_available": monitorData.HudsonNodeMonitorsDiskSpaceMonitor.Size, - "temp_available": monitorData.HudsonNodeMonitorsTemporarySpaceMonitor.Size, - "swap_available": monitorData.HudsonNodeMonitorsSwapSpaceMonitor.SwapAvailable, - "memory_available": monitorData.HudsonNodeMonitorsSwapSpaceMonitor.MemoryAvailable, - "swap_total": monitorData.HudsonNodeMonitorsSwapSpaceMonitor.SwapTotal, - "memory_total": monitorData.HudsonNodeMonitorsSwapSpaceMonitor.MemoryTotal, + if monitorData.HudsonNodeMonitorsTemporarySpaceMonitor != nil { + tags["temp_path"] = monitorData.HudsonNodeMonitorsTemporarySpaceMonitor.Path + fields["temp_available"] = monitorData.HudsonNodeMonitorsTemporarySpaceMonitor.Size + } + if monitorData.HudsonNodeMonitorsSwapSpaceMonitor != nil { + fields["swap_available"] = monitorData.HudsonNodeMonitorsSwapSpaceMonitor.SwapAvailable + fields["memory_available"] = monitorData.HudsonNodeMonitorsSwapSpaceMonitor.MemoryAvailable + fields["swap_total"] = monitorData.HudsonNodeMonitorsSwapSpaceMonitor.SwapTotal + fields["memory_total"] = monitorData.HudsonNodeMonitorsSwapSpaceMonitor.MemoryTotal } acc.AddFields(measurementNode, fields, tags) @@ -304,7 +335,7 @@ func (j *Jenkins) getJobDetail(jr jobRequest, acc telegraf.Accumulator) error { } if build.Building { - log.Printf("D! Ignore running build on %s, build %v", jr.name, number) + j.Log.Debugf("Ignore running build on %s, build %v", jr.name, number) return nil } @@ -317,7 +348,7 @@ func (j *Jenkins) getJobDetail(jr jobRequest, acc telegraf.Accumulator) error { return nil } - gatherJobBuild(jr, build, acc) + j.gatherJobBuild(jr, build, acc) return nil } @@ -326,24 +357,18 @@ type nodeResponse struct { } type node struct { - DisplayName string `json:"displayName"` - Offline bool `json:"offline"` - MonitorData monitorData `json:"monitorData"` + DisplayName string `json:"displayName"` + Offline bool `json:"offline"` + NumExecutors int `json:"numExecutors"` + MonitorData monitorData `json:"monitorData"` } type monitorData struct { - HudsonNodeMonitorsArchitectureMonitor string `json:"hudson.node_monitors.ArchitectureMonitor"` - HudsonNodeMonitorsDiskSpaceMonitor nodeSpaceMonitor `json:"hudson.node_monitors.DiskSpaceMonitor"` - HudsonNodeMonitorsResponseTimeMonitor struct { - Average int64 `json:"average"` - } `json:"hudson.node_monitors.ResponseTimeMonitor"` - HudsonNodeMonitorsSwapSpaceMonitor struct { - SwapAvailable float64 `json:"availableSwapSpace"` - SwapTotal float64 `json:"totalSwapSpace"` - MemoryAvailable float64 `json:"availablePhysicalMemory"` - MemoryTotal float64 `json:"totalPhysicalMemory"` - } `json:"hudson.node_monitors.SwapSpaceMonitor"` - HudsonNodeMonitorsTemporarySpaceMonitor nodeSpaceMonitor `json:"hudson.node_monitors.TemporarySpaceMonitor"` + HudsonNodeMonitorsArchitectureMonitor string `json:"hudson.node_monitors.ArchitectureMonitor"` + HudsonNodeMonitorsDiskSpaceMonitor *nodeSpaceMonitor `json:"hudson.node_monitors.DiskSpaceMonitor"` + HudsonNodeMonitorsResponseTimeMonitor *responseTimeMonitor `json:"hudson.node_monitors.ResponseTimeMonitor"` + HudsonNodeMonitorsSwapSpaceMonitor *swapSpaceMonitor `json:"hudson.node_monitors.SwapSpaceMonitor"` + HudsonNodeMonitorsTemporarySpaceMonitor *nodeSpaceMonitor `json:"hudson.node_monitors.TemporarySpaceMonitor"` } type nodeSpaceMonitor struct { @@ -351,6 +376,17 @@ type nodeSpaceMonitor struct { Size float64 `json:"size"` } +type responseTimeMonitor struct { + Average int64 `json:"average"` +} + +type swapSpaceMonitor struct { + SwapAvailable float64 `json:"availableSwapSpace"` + SwapTotal float64 `json:"totalSwapSpace"` + MemoryAvailable float64 `json:"availablePhysicalMemory"` + MemoryTotal float64 `json:"totalPhysicalMemory"` +} + type jobResponse struct { LastBuild jobBuild `json:"lastBuild"` Jobs []innerJob `json:"jobs"` @@ -410,8 +446,8 @@ func (jr jobRequest) parentsString() string { return strings.Join(jr.parents, "/") } -func gatherJobBuild(jr jobRequest, b *buildResponse, acc telegraf.Accumulator) { - tags := map[string]string{"name": jr.name, "parents": jr.parentsString(), "result": b.Result} +func (j *Jenkins) gatherJobBuild(jr jobRequest, b *buildResponse, acc telegraf.Accumulator) { + tags := map[string]string{"name": jr.name, "parents": jr.parentsString(), "result": b.Result, "source": j.Source, "port": j.Port} fields := make(map[string]interface{}) fields["duration"] = b.Duration fields["result_code"] = mapResultCode(b.Result) diff --git a/plugins/inputs/jenkins/jenkins_test.go b/plugins/inputs/jenkins/jenkins_test.go index 7724fc0e3a139..6233bb83f3fbf 100644 --- a/plugins/inputs/jenkins/jenkins_test.go +++ b/plugins/inputs/jenkins/jenkins_test.go @@ -1,3 +1,4 @@ +// Test Suite package jenkins import ( @@ -107,7 +108,7 @@ func TestGatherNodeData(t *testing.T) { wantErr: true, }, { - name: "bad empty monitor data", + name: "empty monitor data", input: mockHandler{ responseMap: map[string]interface{}{ "/api/json": struct{}{}, @@ -119,7 +120,9 @@ func TestGatherNodeData(t *testing.T) { }, }, }, - wantErr: true, + output: &testutil.Accumulator{ + Metrics: []*testutil.Metric{}, + }, }, { name: "filtered nodes", @@ -135,7 +138,6 @@ func TestGatherNodeData(t *testing.T) { }, }, }, - { name: "normal data collection", input: mockHandler{ @@ -147,25 +149,18 @@ func TestGatherNodeData(t *testing.T) { DisplayName: "master", MonitorData: monitorData{ HudsonNodeMonitorsArchitectureMonitor: "linux", - HudsonNodeMonitorsResponseTimeMonitor: struct { - Average int64 `json:"average"` - }{ + HudsonNodeMonitorsResponseTimeMonitor: &responseTimeMonitor{ Average: 10032, }, - HudsonNodeMonitorsDiskSpaceMonitor: nodeSpaceMonitor{ + HudsonNodeMonitorsDiskSpaceMonitor: &nodeSpaceMonitor{ Path: "/path/1", Size: 123, }, - HudsonNodeMonitorsTemporarySpaceMonitor: nodeSpaceMonitor{ + HudsonNodeMonitorsTemporarySpaceMonitor: &nodeSpaceMonitor{ Path: "/path/2", Size: 245, }, - HudsonNodeMonitorsSwapSpaceMonitor: struct { - SwapAvailable float64 `json:"availableSwapSpace"` - SwapTotal float64 `json:"totalSwapSpace"` - MemoryAvailable float64 `json:"availablePhysicalMemory"` - MemoryTotal float64 `json:"totalPhysicalMemory"` - }{ + HudsonNodeMonitorsSwapSpaceMonitor: &swapSpaceMonitor{ SwapAvailable: 212, SwapTotal: 500, MemoryAvailable: 101, @@ -187,6 +182,7 @@ func TestGatherNodeData(t *testing.T) { "status": "online", "disk_path": "/path/1", "temp_path": "/path/2", + "source": "127.0.0.1", }, Fields: map[string]interface{}{ "response_time": int64(10032), @@ -201,41 +197,75 @@ func TestGatherNodeData(t *testing.T) { }, }, }, + { + name: "slave is offline", + input: mockHandler{ + responseMap: map[string]interface{}{ + "/api/json": struct{}{}, + "/computer/api/json": nodeResponse{ + Computers: []node{ + { + DisplayName: "slave", + MonitorData: monitorData{}, + NumExecutors: 1, + Offline: true, + }, + }, + }, + }, + }, + output: &testutil.Accumulator{ + Metrics: []*testutil.Metric{ + { + Tags: map[string]string{ + "node_name": "slave", + "status": "offline", + }, + Fields: map[string]interface{}{ + "num_executors": 1, + }, + }, + }, + }, + }, } for _, test := range tests { - ts := httptest.NewServer(test.input) - defer ts.Close() - j := &Jenkins{ - URL: ts.URL, - ResponseTimeout: internal.Duration{Duration: time.Microsecond}, - NodeExclude: []string{"ignore-1", "ignore-2"}, - } - te := j.initialize(&http.Client{Transport: &http.Transport{}}) - acc := new(testutil.Accumulator) - j.gatherNodesData(acc) - if err := acc.FirstError(); err != nil { - te = err - } + t.Run(test.name, func(t *testing.T) { + ts := httptest.NewServer(test.input) + defer ts.Close() + j := &Jenkins{ + Log: testutil.Logger{}, + URL: ts.URL, + ResponseTimeout: internal.Duration{Duration: time.Microsecond}, + NodeExclude: []string{"ignore-1", "ignore-2"}, + } + te := j.initialize(&http.Client{Transport: &http.Transport{}}) + acc := new(testutil.Accumulator) + j.gatherNodesData(acc) + if err := acc.FirstError(); err != nil { + te = err + } - if !test.wantErr && te != nil { - t.Fatalf("%s: failed %s, expected to be nil", test.name, te.Error()) - } else if test.wantErr && te == nil { - t.Fatalf("%s: expected err, got nil", test.name) - } - if test.output == nil && len(acc.Metrics) > 0 { - t.Fatalf("%s: collected extra data", test.name) - } else if test.output != nil && len(test.output.Metrics) > 0 { - for k, m := range test.output.Metrics[0].Tags { - if acc.Metrics[0].Tags[k] != m { - t.Fatalf("%s: tag %s metrics unmatch Expected %s, got %s\n", test.name, k, m, acc.Metrics[0].Tags[k]) - } + if !test.wantErr && te != nil { + t.Fatalf("%s: failed %s, expected to be nil", test.name, te.Error()) + } else if test.wantErr && te == nil { + t.Fatalf("%s: expected err, got nil", test.name) } - for k, m := range test.output.Metrics[0].Fields { - if acc.Metrics[0].Fields[k] != m { - t.Fatalf("%s: field %s metrics unmatch Expected %v(%T), got %v(%T)\n", test.name, k, m, m, acc.Metrics[0].Fields[k], acc.Metrics[0].Fields[k]) + if test.output == nil && len(acc.Metrics) > 0 { + t.Fatalf("%s: collected extra data", test.name) + } else if test.output != nil && len(test.output.Metrics) > 0 { + for k, m := range test.output.Metrics[0].Tags { + if acc.Metrics[0].Tags[k] != m { + t.Fatalf("%s: tag %s metrics unmatch Expected %s, got %s\n", test.name, k, m, acc.Metrics[0].Tags[k]) + } + } + for k, m := range test.output.Metrics[0].Fields { + if acc.Metrics[0].Fields[k] != m { + t.Fatalf("%s: field %s metrics unmatch Expected %v(%T), got %v(%T)\n", test.name, k, m, m, acc.Metrics[0].Fields[k], acc.Metrics[0].Fields[k]) + } } } - } + }) } } @@ -258,6 +288,7 @@ func TestInitialize(t *testing.T) { { name: "bad jenkins config", input: &Jenkins{ + Log: testutil.Logger{}, URL: "http://a bad url", ResponseTimeout: internal.Duration{Duration: time.Microsecond}, }, @@ -266,6 +297,7 @@ func TestInitialize(t *testing.T) { { name: "has filter", input: &Jenkins{ + Log: testutil.Logger{}, URL: ts.URL, ResponseTimeout: internal.Duration{Duration: time.Microsecond}, JobExclude: []string{"job1", "job2"}, @@ -275,31 +307,34 @@ func TestInitialize(t *testing.T) { { name: "default config", input: &Jenkins{ + Log: testutil.Logger{}, URL: ts.URL, ResponseTimeout: internal.Duration{Duration: time.Microsecond}, }, output: &Jenkins{ + Log: testutil.Logger{}, MaxConnections: 5, MaxSubJobPerLayer: 10, }, }, } for _, test := range tests { - te := test.input.initialize(mockClient) - if !test.wantErr && te != nil { - t.Fatalf("%s: failed %s, expected to be nil", test.name, te.Error()) - } else if test.wantErr && te == nil { - t.Fatalf("%s: expected err, got nil", test.name) - } - if test.output != nil { - if test.input.client == nil { - t.Fatalf("%s: failed %s, jenkins instance shouldn't be nil", test.name, te.Error()) + t.Run(test.name, func(t *testing.T) { + te := test.input.initialize(mockClient) + if !test.wantErr && te != nil { + t.Fatalf("%s: failed %s, expected to be nil", test.name, te.Error()) + } else if test.wantErr && te == nil { + t.Fatalf("%s: expected err, got nil", test.name) } - if test.input.MaxConnections != test.output.MaxConnections { - t.Fatalf("%s: different MaxConnections Expected %d, got %d\n", test.name, test.output.MaxConnections, test.input.MaxConnections) + if test.output != nil { + if test.input.client == nil { + t.Fatalf("%s: failed %s, jenkins instance shouldn't be nil", test.name, te.Error()) + } + if test.input.MaxConnections != test.output.MaxConnections { + t.Fatalf("%s: different MaxConnections Expected %d, got %d\n", test.name, test.output.MaxConnections, test.input.MaxConnections) + } } - } - + }) } } @@ -567,49 +602,51 @@ func TestGatherJobs(t *testing.T) { }, } for _, test := range tests { - ts := httptest.NewServer(test.input) - defer ts.Close() - j := &Jenkins{ - URL: ts.URL, - MaxBuildAge: internal.Duration{Duration: time.Hour}, - ResponseTimeout: internal.Duration{Duration: time.Microsecond}, - JobExclude: []string{ - "ignore-1", - "apps/ignore-all/*", - "apps/k8s-cloud/PR-ignore2", - }, - } - te := j.initialize(&http.Client{Transport: &http.Transport{}}) - acc := new(testutil.Accumulator) - acc.SetDebug(true) - j.gatherJobs(acc) - if err := acc.FirstError(); err != nil { - te = err - } - if !test.wantErr && te != nil { - t.Fatalf("%s: failed %s, expected to be nil", test.name, te.Error()) - } else if test.wantErr && te == nil { - t.Fatalf("%s: expected err, got nil", test.name) - } + t.Run(test.name, func(t *testing.T) { + ts := httptest.NewServer(test.input) + defer ts.Close() + j := &Jenkins{ + Log: testutil.Logger{}, + URL: ts.URL, + MaxBuildAge: internal.Duration{Duration: time.Hour}, + ResponseTimeout: internal.Duration{Duration: time.Microsecond}, + JobExclude: []string{ + "ignore-1", + "apps/ignore-all/*", + "apps/k8s-cloud/PR-ignore2", + }, + } + te := j.initialize(&http.Client{Transport: &http.Transport{}}) + acc := new(testutil.Accumulator) + j.gatherJobs(acc) + if err := acc.FirstError(); err != nil { + te = err + } + if !test.wantErr && te != nil { + t.Fatalf("%s: failed %s, expected to be nil", test.name, te.Error()) + } else if test.wantErr && te == nil { + t.Fatalf("%s: expected err, got nil", test.name) + } - if test.output != nil && len(test.output.Metrics) > 0 { - // sort metrics - sort.Slice(acc.Metrics, func(i, j int) bool { - return strings.Compare(acc.Metrics[i].Tags["name"], acc.Metrics[j].Tags["name"]) < 0 - }) - for i := range test.output.Metrics { - for k, m := range test.output.Metrics[i].Tags { - if acc.Metrics[i].Tags[k] != m { - t.Fatalf("%s: tag %s metrics unmatch Expected %s, got %s\n", test.name, k, m, acc.Metrics[i].Tags[k]) + if test.output != nil && len(test.output.Metrics) > 0 { + // sort metrics + sort.Slice(acc.Metrics, func(i, j int) bool { + return strings.Compare(acc.Metrics[i].Tags["name"], acc.Metrics[j].Tags["name"]) < 0 + }) + for i := range test.output.Metrics { + for k, m := range test.output.Metrics[i].Tags { + if acc.Metrics[i].Tags[k] != m { + t.Fatalf("%s: tag %s metrics unmatch Expected %s, got %s\n", test.name, k, m, acc.Metrics[i].Tags[k]) + } } - } - for k, m := range test.output.Metrics[i].Fields { - if acc.Metrics[i].Fields[k] != m { - t.Fatalf("%s: field %s metrics unmatch Expected %v(%T), got %v(%T)\n", test.name, k, m, m, acc.Metrics[i].Fields[k], acc.Metrics[0].Fields[k]) + for k, m := range test.output.Metrics[i].Fields { + if acc.Metrics[i].Fields[k] != m { + t.Fatalf("%s: field %s metrics unmatch Expected %v(%T), got %v(%T)\n", test.name, k, m, m, acc.Metrics[i].Fields[k], acc.Metrics[0].Fields[k]) + } } } - } - } + } + }) } } diff --git a/plugins/inputs/jti_openconfig_telemetry/openconfig_telemetry.go b/plugins/inputs/jti_openconfig_telemetry/openconfig_telemetry.go index c30ef9bf4a7f2..39f9bb58a2cb1 100644 --- a/plugins/inputs/jti_openconfig_telemetry/openconfig_telemetry.go +++ b/plugins/inputs/jti_openconfig_telemetry/openconfig_telemetry.go @@ -2,7 +2,6 @@ package jti_openconfig_telemetry import ( "fmt" - "log" "net" "regexp" "strings" @@ -34,6 +33,8 @@ type OpenConfigTelemetry struct { EnableTLS bool `toml:"enable_tls"` internaltls.ClientConfig + Log telegraf.Logger + sensorsConfig []sensorConfig grpcClientConns []*grpc.ClientConn wg *sync.WaitGroup @@ -243,7 +244,7 @@ func (m *OpenConfigTelemetry) splitSensorConfig() int { } if len(spathSplit) == 0 { - log.Printf("E! No sensors are specified") + m.Log.Error("No sensors are specified") continue } @@ -257,7 +258,7 @@ func (m *OpenConfigTelemetry) splitSensorConfig() int { } if len(spathSplit) == 0 { - log.Printf("E! No valid sensors are specified") + m.Log.Error("No valid sensors are specified") continue } @@ -294,13 +295,13 @@ func (m *OpenConfigTelemetry) collectData(ctx context.Context, rpcStatus, _ := status.FromError(err) // If service is currently unavailable and may come back later, retry if rpcStatus.Code() != codes.Unavailable { - acc.AddError(fmt.Errorf("E! Could not subscribe to %s: %v", grpcServer, + acc.AddError(fmt.Errorf("could not subscribe to %s: %v", grpcServer, err)) return } else { // Retry with delay. If delay is not provided, use default if m.RetryDelay.Duration > 0 { - log.Printf("D! Retrying %s with timeout %v", grpcServer, + m.Log.Debugf("Retrying %s with timeout %v", grpcServer, m.RetryDelay.Duration) time.Sleep(m.RetryDelay.Duration) continue @@ -314,11 +315,11 @@ func (m *OpenConfigTelemetry) collectData(ctx context.Context, if err != nil { // If we encounter error in the stream, break so we can retry // the connection - acc.AddError(fmt.Errorf("E! Failed to read from %s: %v", err, grpcServer)) + acc.AddError(fmt.Errorf("failed to read from %s: %s", grpcServer, err)) break } - log.Printf("D! Received from %s: %v", grpcServer, r) + m.Log.Debugf("Received from %s: %v", grpcServer, r) // Create a point and add to batch tags := make(map[string]string) @@ -329,7 +330,7 @@ func (m *OpenConfigTelemetry) collectData(ctx context.Context, dgroups := m.extractData(r, grpcServer) // Print final data collection - log.Printf("D! Available collection for %s is: %v", grpcServer, dgroups) + m.Log.Debugf("Available collection for %s is: %v", grpcServer, dgroups) tnow := time.Now() // Iterate through data groups and add them @@ -349,10 +350,9 @@ func (m *OpenConfigTelemetry) collectData(ctx context.Context, } func (m *OpenConfigTelemetry) Start(acc telegraf.Accumulator) error { - // Build sensors config if m.splitSensorConfig() == 0 { - return fmt.Errorf("E! No valid sensor configuration available") + return fmt.Errorf("no valid sensor configuration available") } // Parse TLS config @@ -376,15 +376,15 @@ func (m *OpenConfigTelemetry) Start(acc telegraf.Accumulator) error { // Extract device address and port grpcServer, grpcPort, err := net.SplitHostPort(server) if err != nil { - log.Printf("E! Invalid server address: %v", err) + m.Log.Errorf("Invalid server address: %s", err.Error()) continue } grpcClientConn, err = grpc.Dial(server, opts...) if err != nil { - log.Printf("E! Failed to connect to %s: %v", server, err) + m.Log.Errorf("Failed to connect to %s: %s", server, err.Error()) } else { - log.Printf("D! Opened a new gRPC session to %s on port %s", grpcServer, grpcPort) + m.Log.Debugf("Opened a new gRPC session to %s on port %s", grpcServer, grpcPort) } // Add to the list of client connections @@ -396,13 +396,13 @@ func (m *OpenConfigTelemetry) Start(acc telegraf.Accumulator) error { &authentication.LoginRequest{UserName: m.Username, Password: m.Password, ClientId: m.ClientID}) if loginErr != nil { - log.Printf("E! Could not initiate login check for %s: %v", server, loginErr) + m.Log.Errorf("Could not initiate login check for %s: %v", server, loginErr) continue } // Check if the user is authenticated. Bail if auth error if !loginReply.Result { - log.Printf("E! Failed to authenticate the user for %s", server) + m.Log.Errorf("Failed to authenticate the user for %s", server) continue } } diff --git a/plugins/inputs/jti_openconfig_telemetry/openconfig_telemetry_test.go b/plugins/inputs/jti_openconfig_telemetry/openconfig_telemetry_test.go index 8b0abd88377d9..a3df62e1bb0c0 100644 --- a/plugins/inputs/jti_openconfig_telemetry/openconfig_telemetry_test.go +++ b/plugins/inputs/jti_openconfig_telemetry/openconfig_telemetry_test.go @@ -17,6 +17,7 @@ import ( ) var cfg = &OpenConfigTelemetry{ + Log: testutil.Logger{}, Servers: []string{"127.0.0.1:50051"}, SampleFrequency: internal.Duration{Duration: time.Second * 2}, } diff --git a/plugins/inputs/kafka_consumer/README.md b/plugins/inputs/kafka_consumer/README.md index efd3ffad6d34c..dec39cc32871b 100644 --- a/plugins/inputs/kafka_consumer/README.md +++ b/plugins/inputs/kafka_consumer/README.md @@ -34,16 +34,23 @@ and use the old zookeeper connection method. ## Use TLS but skip chain & host verification # insecure_skip_verify = false - ## Optional SASL Config + ## SASL authentication credentials. These settings should typically be used + ## with TLS encryption enabled using the "enable_tls" option. # sasl_username = "kafka" # sasl_password = "secret" + ## SASL protocol version. When connecting to Azure EventHub set to 0. + # sasl_version = 1 + ## Name of the consumer group. # consumer_group = "telegraf_metrics_consumers" ## Initial offset position; one of "oldest" or "newest". # offset = "oldest" + ## Consumer group partition assignment strategy; one of "range", "roundrobin" or "sticky". + # balance_strategy = "range" + ## Maximum length of a message to consume, in bytes (default 0/unlimited); ## larger messages are dropped max_message_len = 1000000 diff --git a/plugins/inputs/kafka_consumer/kafka_consumer.go b/plugins/inputs/kafka_consumer/kafka_consumer.go index 997988ca6b915..5cd6a9771041b 100644 --- a/plugins/inputs/kafka_consumer/kafka_consumer.go +++ b/plugins/inputs/kafka_consumer/kafka_consumer.go @@ -10,6 +10,7 @@ import ( "github.com/Shopify/sarama" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal/tls" + "github.com/influxdata/telegraf/plugins/common/kafka" "github.com/influxdata/telegraf/plugins/inputs" "github.com/influxdata/telegraf/plugins/parsers" ) @@ -33,22 +34,30 @@ const sampleConfig = ` # version = "" ## Optional TLS Config + # enable_tls = true # tls_ca = "/etc/telegraf/ca.pem" # tls_cert = "/etc/telegraf/cert.pem" # tls_key = "/etc/telegraf/key.pem" ## Use TLS but skip chain & host verification # insecure_skip_verify = false - ## Optional SASL Config + ## SASL authentication credentials. These settings should typically be used + ## with TLS encryption enabled using the "enable_tls" option. # sasl_username = "kafka" # sasl_password = "secret" + ## SASL protocol version. When connecting to Azure EventHub set to 0. + # sasl_version = 1 + ## Name of the consumer group. # consumer_group = "telegraf_metrics_consumers" ## Initial offset position; one of "oldest" or "newest". # offset = "oldest" + ## Consumer group partition assignment strategy; one of "range", "roundrobin" or "sticky". + # balance_strategy = "range" + ## Maximum length of a message to consume, in bytes (default 0/unlimited); ## larger messages are dropped max_message_len = 1000000 @@ -86,14 +95,19 @@ type KafkaConsumer struct { MaxMessageLen int `toml:"max_message_len"` MaxUndeliveredMessages int `toml:"max_undelivered_messages"` Offset string `toml:"offset"` + BalanceStrategy string `toml:"balance_strategy"` Topics []string `toml:"topics"` TopicTag string `toml:"topic_tag"` Version string `toml:"version"` SASLPassword string `toml:"sasl_password"` SASLUsername string `toml:"sasl_username"` + SASLVersion *int `toml:"sasl_version"` + EnableTLS *bool `toml:"enable_tls"` tls.ClientConfig + Log telegraf.Logger `toml:"-"` + ConsumerCreator ConsumerGroupCreator `toml:"-"` consumer ConsumerGroup config *sarama.Config @@ -154,6 +168,10 @@ func (k *KafkaConsumer) Init() error { config.Version = version } + if k.EnableTLS != nil && *k.EnableTLS { + config.Net.TLS.Enable = true + } + tlsConfig, err := k.ClientConfig.TLSConfig() if err != nil { return err @@ -161,13 +179,25 @@ func (k *KafkaConsumer) Init() error { if tlsConfig != nil { config.Net.TLS.Config = tlsConfig - config.Net.TLS.Enable = true + + // To maintain backwards compatibility, if the enable_tls option is not + // set TLS is enabled if a non-default TLS config is used. + if k.EnableTLS == nil { + k.Log.Warnf("Use of deprecated configuration: enable_tls should be set when using TLS") + config.Net.TLS.Enable = true + } } if k.SASLUsername != "" && k.SASLPassword != "" { config.Net.SASL.User = k.SASLUsername config.Net.SASL.Password = k.SASLPassword config.Net.SASL.Enable = true + + version, err := kafka.SASLVersion(config.Version, k.SASLVersion) + if err != nil { + return err + } + config.Net.SASL.Version = version } if k.ClientID != "" { @@ -185,6 +215,17 @@ func (k *KafkaConsumer) Init() error { return fmt.Errorf("invalid offset %q", k.Offset) } + switch strings.ToLower(k.BalanceStrategy) { + case "range", "": + config.Consumer.Group.Rebalance.Strategy = sarama.BalanceStrategyRange + case "roundrobin": + config.Consumer.Group.Rebalance.Strategy = sarama.BalanceStrategyRoundRobin + case "sticky": + config.Consumer.Group.Rebalance.Strategy = sarama.BalanceStrategySticky + default: + return fmt.Errorf("invalid balance strategy %q", k.BalanceStrategy) + } + if k.ConsumerCreator == nil { k.ConsumerCreator = &SaramaCreator{} } diff --git a/plugins/inputs/kafka_consumer/kafka_consumer_test.go b/plugins/inputs/kafka_consumer/kafka_consumer_test.go index 3aa0efa506db3..0c80635785b68 100644 --- a/plugins/inputs/kafka_consumer/kafka_consumer_test.go +++ b/plugins/inputs/kafka_consumer/kafka_consumer_test.go @@ -7,6 +7,7 @@ import ( "github.com/Shopify/sarama" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal/tls" "github.com/influxdata/telegraf/plugins/parsers/value" "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/require" @@ -68,6 +69,7 @@ func TestInit(t *testing.T) { name: "parses valid version string", plugin: &KafkaConsumer{ Version: "1.0.0", + Log: testutil.Logger{}, }, check: func(t *testing.T, plugin *KafkaConsumer) { require.Equal(t, plugin.config.Version, sarama.V1_0_0_0) @@ -77,6 +79,7 @@ func TestInit(t *testing.T) { name: "invalid version string", plugin: &KafkaConsumer{ Version: "100", + Log: testutil.Logger{}, }, initError: true, }, @@ -84,6 +87,7 @@ func TestInit(t *testing.T) { name: "custom client_id", plugin: &KafkaConsumer{ ClientID: "custom", + Log: testutil.Logger{}, }, check: func(t *testing.T, plugin *KafkaConsumer) { require.Equal(t, plugin.config.ClientID, "custom") @@ -93,6 +97,7 @@ func TestInit(t *testing.T) { name: "custom offset", plugin: &KafkaConsumer{ Offset: "newest", + Log: testutil.Logger{}, }, check: func(t *testing.T, plugin *KafkaConsumer) { require.Equal(t, plugin.config.Consumer.Offsets.Initial, sarama.OffsetNewest) @@ -102,9 +107,54 @@ func TestInit(t *testing.T) { name: "invalid offset", plugin: &KafkaConsumer{ Offset: "middle", + Log: testutil.Logger{}, }, initError: true, }, + { + name: "default tls without tls config", + plugin: &KafkaConsumer{ + Log: testutil.Logger{}, + }, + check: func(t *testing.T, plugin *KafkaConsumer) { + require.False(t, plugin.config.Net.TLS.Enable) + }, + }, + { + name: "default tls with a tls config", + plugin: &KafkaConsumer{ + ClientConfig: tls.ClientConfig{ + InsecureSkipVerify: true, + }, + Log: testutil.Logger{}, + }, + check: func(t *testing.T, plugin *KafkaConsumer) { + require.True(t, plugin.config.Net.TLS.Enable) + }, + }, + { + name: "disable tls", + plugin: &KafkaConsumer{ + EnableTLS: func() *bool { v := false; return &v }(), + ClientConfig: tls.ClientConfig{ + InsecureSkipVerify: true, + }, + Log: testutil.Logger{}, + }, + check: func(t *testing.T, plugin *KafkaConsumer) { + require.False(t, plugin.config.Net.TLS.Enable) + }, + }, + { + name: "enable tls", + plugin: &KafkaConsumer{ + EnableTLS: func() *bool { v := true; return &v }(), + Log: testutil.Logger{}, + }, + check: func(t *testing.T, plugin *KafkaConsumer) { + require.True(t, plugin.config.Net.TLS.Enable) + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -125,6 +175,7 @@ func TestStartStop(t *testing.T) { cg := &FakeConsumerGroup{errors: make(chan error)} plugin := &KafkaConsumer{ ConsumerCreator: &FakeCreator{ConsumerGroup: cg}, + Log: testutil.Logger{}, } err := plugin.Init() require.NoError(t, err) diff --git a/plugins/inputs/kafka_consumer_legacy/README.md b/plugins/inputs/kafka_consumer_legacy/README.md index 31976788bc75f..8fc7ed295a085 100644 --- a/plugins/inputs/kafka_consumer_legacy/README.md +++ b/plugins/inputs/kafka_consumer_legacy/README.md @@ -13,12 +13,16 @@ from the same topic in parallel. [[inputs.kafka_consumer]] ## topic(s) to consume topics = ["telegraf"] + ## an array of Zookeeper connection strings zookeeper_peers = ["localhost:2181"] + ## Zookeeper Chroot zookeeper_chroot = "" + ## the name of the consumer group consumer_group = "telegraf_metrics_consumers" + ## Offset (must be either "oldest" or "newest") offset = "oldest" diff --git a/plugins/inputs/kafka_consumer_legacy/kafka_consumer_legacy.go b/plugins/inputs/kafka_consumer_legacy/kafka_consumer_legacy.go index d9558d5bd080a..939fc8850ef5f 100644 --- a/plugins/inputs/kafka_consumer_legacy/kafka_consumer_legacy.go +++ b/plugins/inputs/kafka_consumer_legacy/kafka_consumer_legacy.go @@ -2,7 +2,6 @@ package kafka_consumer_legacy import ( "fmt" - "log" "strings" "sync" @@ -30,6 +29,8 @@ type Kafka struct { Offset string parser parsers.Parser + Log telegraf.Logger + sync.Mutex // channel for all incoming kafka messages @@ -49,12 +50,16 @@ type Kafka struct { var sampleConfig = ` ## topic(s) to consume topics = ["telegraf"] + ## an array of Zookeeper connection strings zookeeper_peers = ["localhost:2181"] + ## Zookeeper Chroot zookeeper_chroot = "" + ## the name of the consumer group consumer_group = "telegraf_metrics_consumers" + ## Offset (must be either "oldest" or "newest") offset = "oldest" @@ -96,7 +101,7 @@ func (k *Kafka) Start(acc telegraf.Accumulator) error { case "newest": config.Offsets.Initial = sarama.OffsetNewest default: - log.Printf("I! WARNING: Kafka consumer invalid offset '%s', using 'oldest'\n", + k.Log.Infof("WARNING: Kafka consumer invalid offset '%s', using 'oldest'\n", k.Offset) config.Offsets.Initial = sarama.OffsetOldest } @@ -121,7 +126,7 @@ func (k *Kafka) Start(acc telegraf.Accumulator) error { // Start the kafka message reader go k.receiver() - log.Printf("I! Started the kafka consumer service, peers: %v, topics: %v\n", + k.Log.Infof("Started the kafka consumer service, peers: %v, topics: %v\n", k.ZookeeperPeers, k.Topics) return nil } diff --git a/plugins/inputs/kafka_consumer_legacy/kafka_consumer_legacy_integration_test.go b/plugins/inputs/kafka_consumer_legacy/kafka_consumer_legacy_integration_test.go index 60404cfac13fc..31bea2210b741 100644 --- a/plugins/inputs/kafka_consumer_legacy/kafka_consumer_legacy_integration_test.go +++ b/plugins/inputs/kafka_consumer_legacy/kafka_consumer_legacy_integration_test.go @@ -37,6 +37,7 @@ func TestReadsMetricsFromKafka(t *testing.T) { // Start the Kafka Consumer k := &Kafka{ + Log: testutil.Logger{}, ConsumerGroup: "telegraf_test_consumers", Topics: []string{testTopic}, ZookeeperPeers: zkPeers, diff --git a/plugins/inputs/kafka_consumer_legacy/kafka_consumer_legacy_test.go b/plugins/inputs/kafka_consumer_legacy/kafka_consumer_legacy_test.go index 38bc48290a37e..8037f49a053b5 100644 --- a/plugins/inputs/kafka_consumer_legacy/kafka_consumer_legacy_test.go +++ b/plugins/inputs/kafka_consumer_legacy/kafka_consumer_legacy_test.go @@ -21,6 +21,7 @@ const ( func newTestKafka() (*Kafka, chan *sarama.ConsumerMessage) { in := make(chan *sarama.ConsumerMessage, 1000) k := Kafka{ + Log: testutil.Logger{}, ConsumerGroup: "test", Topics: []string{"telegraf"}, ZookeeperPeers: []string{"localhost:2181"}, diff --git a/plugins/inputs/kapacitor/README.md b/plugins/inputs/kapacitor/README.md index 2ff4eab88af57..8a6f3477f5320 100644 --- a/plugins/inputs/kapacitor/README.md +++ b/plugins/inputs/kapacitor/README.md @@ -25,63 +25,247 @@ The Kapacitor plugin will collect metrics from the given Kapacitor instances. ### Measurements & Fields -- kapacitor - - num_enabled_tasks, integer - - num_subscriptions, integer - - num_tasks, integer -- kapacitor_edges - - collected, integer - - emitted, integer -- kapacitor_ingress - - points_received, integer -- kapacitor_memstats - - alloc_bytes, integer - - buck_hash_sys_bytes, integer - - frees, integer - - gcc_pu_fraction, float - - gc_sys_bytes, integer - - heap_alloc_bytes, integer - - heap_idle_bytes, integer - - heap_inuse_bytes, integer - - heap_objects, integer - - heap_released_bytes, integer - - heap_sys_bytes, integer - - last_gc_ns, integer - - lookups, integer - - mallocs, integer - - mcache_in_use_bytes, integer - - mcache_sys_bytes, integer - - mspan_in_use_bytes, integer - - mspan_sys_bytes, integer - - next_gc_ns, integer - - num_gc, integer - - other_sys_bytes, integer - - pause_total_ns, integer - - stack_in_use_bytes, integer - - stack_sys_bytes, integer - - sys_bytes, integer - - total_alloc_bytes, integer -- kapacitor_nodes - - alerts_triggered, integer - - avg_exec_time_ns, integer - - batches_queried, integer - - crits_triggered, integer - - eval_errors, integer - - fields_defaulted, integer - - infos_triggered, integer - - oks_triggered, integer - - points_queried, integer - - points_written, integer - - query_errors, integer - - tags_defaulted, integer - - warns_triggered, integer - - write_errors, integer +- [kapacitor](#kapacitor) + - [num_enabled_tasks](#num_enabled_tasks) _(integer)_ + - [num_subscriptions](#num_subscriptions) _(integer)_ + - [num_tasks](#num_tasks) _(integer)_ +- [kapacitor_edges](#kapacitor_edges) + - [collected](#collected) _(integer)_ + - [emitted](#emitted) _(integer)_ +- [kapacitor_ingress](#kapacitor_ingress) + - [points_received](#points_received) _(integer)_ +- [kapacitor_load](#kapacitor_load) + - [errors](#errors) _(integer)_ +- [kapacitor_memstats](#kapacitor_memstats) + - [alloc_bytes](#alloc_bytes) _(integer)_ + - [buck_hash_sys_bytes](#buck_hash_sys_bytes) _(integer)_ + - [frees](#frees) _(integer)_ + - [gc_sys_bytes](#gc_sys_bytes) _(integer)_ + - [gcc_pu_fraction](#gcc_pu_fraction) _(float)_ + - [heap_alloc_bytes](#heap_alloc_bytes) _(integer)_ + - [heap_idle_bytes](#heap_idle_bytes) _(integer)_ + - [heap_in_use_bytes](#heap_in_use_bytes) _(integer)_ + - [heap_objects](#heap_objects) _(integer)_ + - [heap_released_bytes](#heap_released_bytes) _(integer)_ + - [heap_sys_bytes](#heap_sys_bytes) _(integer)_ + - [last_gc_ns](#last_gc_ns) _(integer)_ + - [lookups](#lookups) _(integer)_ + - [mallocs](#mallocs) _(integer)_ + - [mcache_in_use_bytes](#mcache_in_use_bytes) _(integer)_ + - [mcache_sys_bytes](#mcache_sys_bytes) _(integer)_ + - [mspan_in_use_bytes](#mspan_in_use_bytes) _(integer)_ + - [mspan_sys_bytes](#mspan_sys_bytes) _(integer)_ + - [next_gc_ns](#next_gc_ns) _(integer)_ + - [num_gc](#num_gc) _(integer)_ + - [other_sys_bytes](#other_sys_bytes) _(integer)_ + - [pause_total_ns](#pause_total_ns) _(integer)_ + - [stack_in_use_bytes](#stack_in_use_bytes) _(integer)_ + - [stack_sys_bytes](#stack_sys_bytes) _(integer)_ + - [sys_bytes](#sys_bytes) _(integer)_ + - [total_alloc_bytes](#total_alloc_bytes) _(integer)_ +- [kapacitor_nodes](#kapacitor_nodes) + - [alerts_inhibited](#alerts_inhibited) _(integer)_ + - [alerts_triggered](#alerts_triggered) _(integer)_ + - [avg_exec_time_ns](#avg_exec_time_ns) _(integer)_ + - [crits_triggered](#crits_triggered) _(integer)_ + - [errors](#errors) _(integer)_ + - [infos_triggered](#infos_triggered) _(integer)_ + - [oks_triggered](#oks_triggered) _(integer)_ + - [points_written](#points_written) _(integer)_ + - [warns_triggered](#warns_triggered) _(integer)_ + - [write_errors](#write_errors) _(integer)_ +- [kapacitor_topics](#kapacitor_topics) + - [collected](#collected) _(integer)_ + + +--- + +### kapacitor +The `kapacitor` measurement stores fields with information related to +[Kapacitor tasks](https://docs.influxdata.com/kapacitor/latest/introduction/getting-started/#kapacitor-tasks) +and [subscriptions](https://docs.influxdata.com/kapacitor/latest/administration/subscription-management/). + +#### num_enabled_tasks +The number of enabled Kapacitor tasks. + +#### num_subscriptions +The number of Kapacitor/InfluxDB subscriptions. + +#### num_tasks +The total number of Kapacitor tasks. + +--- + +### kapacitor_edges +The `kapacitor_edges` measurement stores fields with information related to +[edges](https://docs.influxdata.com/kapacitor/latest/tick/introduction/#pipelines) +in Kapacitor TICKscripts. + +#### collected +The number of messages collected by TICKscript edges. + +#### emitted +The number of messages emitted by TICKscript edges. + +--- + +### kapacitor_ingress +The `kapacitor_ingress` measurement stores fields with information related to data +coming into Kapacitor. + +#### points_received +The number of points received by Kapacitor. + +--- + +### kapacitor_load +The `kapacitor_load` measurement stores fields with information related to the +[Kapacitor Load Directory service](https://docs.influxdata.com/kapacitor/latest/guides/load_directory/). + +#### errors +The number of errors reported from the load directory service. + +--- + +### kapacitor_memstats +The `kapacitor_memstats` measurement stores fields related to Kapacitor memory usage. + +#### alloc_bytes +The number of bytes of memory allocated by Kapacitor that are still in use. + +#### buck_hash_sys_bytes +The number of bytes of memory used by the profiling bucket hash table. + +#### frees +The number of heap objects freed. + +#### gc_sys_bytes +The number of bytes of memory used for garbage collection system metadata. + +#### gcc_pu_fraction +The fraction of Kapacitor's available CPU time used by garbage collection since +Kapacitor started. + +#### heap_alloc_bytes +The number of reachable and unreachable heap objects garbage collection has +not freed. + +#### heap_idle_bytes +The number of heap bytes waiting to be used. + +#### heap_in_use_bytes +The number of heap bytes in use. + +#### heap_objects +The number of allocated objects. + +#### heap_released_bytes +The number of heap bytes released to the operating system. + +#### heap_sys_bytes +The number of heap bytes obtained from `system`. + +#### last_gc_ns +The nanosecond epoch time of the last garbage collection. + +#### lookups +The total number of pointer lookups. + +#### mallocs +The total number of mallocs. + +#### mcache_in_use_bytes +The number of bytes in use by mcache structures. + +#### mcache_sys_bytes +The number of bytes used for mcache structures obtained from `system`. + +#### mspan_in_use_bytes +The number of bytes in use by mspan structures. + +#### mspan_sys_bytes +The number of bytes used for mspan structures obtained from `system`. + +#### next_gc_ns +The nanosecond epoch time of the next garbage collection. + +#### num_gc +The number of completed garbage collection cycles. + +#### other_sys_bytes +The number of bytes used for other system allocations. + +#### pause_total_ns +The total number of nanoseconds spent in garbage collection "stop-the-world" +pauses since Kapacitor started. + +#### stack_in_use_bytes +The number of bytes in use by the stack allocator. + +#### stack_sys_bytes +The number of bytes obtained from `system` for stack allocator. + +#### sys_bytes +The number of bytes of memory obtained from `system`. + +#### total_alloc_bytes +The total number of bytes allocated, even if freed. + +--- + +### kapacitor_nodes +The `kapacitor_nodes` measurement stores fields related to events that occur in +[TICKscript nodes](https://docs.influxdata.com/kapacitor/latest/nodes/). + +#### alerts_inhibited +The total number of alerts inhibited by TICKscripts. + +#### alerts_triggered +The total number of alerts triggered by TICKscripts. + +#### avg_exec_time_ns +The average execution time of TICKscripts in nanoseconds. + +#### crits_triggered +The number of critical (`crit`) alerts triggered by TICKscripts. + +#### errors +The number of errors caused caused by TICKscripts. + +#### infos_triggered +The number of info (`info`) alerts triggered by TICKscripts. + +#### oks_triggered +The number of ok (`ok`) alerts triggered by TICKscripts. + +#### points_written +The number of points written to InfluxDB or back to Kapacitor. + +#### warns_triggered +The number of warning (`warn`) alerts triggered by TICKscripts. + +#### working_cardinality +The total number of unique series processed. + +#### write_errors +The number of errors that occurred when writing to InfluxDB or other write endpoints. + +--- + +### kapacitor_topics +The `kapacitor_topics` measurement stores fields related to +Kapacitor topics](https://docs.influxdata.com/kapacitor/latest/working/using_alert_topics/). + +#### collected +The number of events collected by Kapacitor topics. + +--- *Note:* The Kapacitor variables `host`, `cluster_id`, and `server_id` are currently not recorded due to the potential high cardinality of these values. -### Example Output: +## Example Output ``` $ telegraf --config /etc/telegraf.conf --input-filter kapacitor --test diff --git a/plugins/inputs/kinesis_consumer/kinesis_consumer.go b/plugins/inputs/kinesis_consumer/kinesis_consumer.go index b9b98243b3100..aec806da1888a 100644 --- a/plugins/inputs/kinesis_consumer/kinesis_consumer.go +++ b/plugins/inputs/kinesis_consumer/kinesis_consumer.go @@ -3,7 +3,6 @@ package kinesis_consumer import ( "context" "fmt" - "log" "math/big" "strings" "sync" @@ -40,6 +39,8 @@ type ( DynamoDB *DynamoDB `toml:"checkpoint_dynamodb"` MaxUndeliveredMessages int `toml:"max_undelivered_messages"` + Log telegraf.Logger + cons *consumer.Consumer parser parsers.Parser cancel context.CancelFunc @@ -220,7 +221,7 @@ func (k *KinesisConsumer) connect(ac telegraf.Accumulator) error { }) if err != nil { k.cancel() - log.Printf("E! [inputs.kinesis_consumer] Scan encounterred an error - %s", err.Error()) + k.Log.Errorf("Scan encounterred an error: %s", err.Error()) k.cons = nil } }() @@ -285,7 +286,7 @@ func (k *KinesisConsumer) onDelivery(ctx context.Context) { k.lastSeqNum = strToBint(sequenceNum) k.checkpoint.Set(chk.streamName, chk.shardID, sequenceNum) } else { - log.Println("D! [inputs.kinesis_consumer] Metric group failed to process") + k.Log.Debug("Metric group failed to process") } } } diff --git a/plugins/inputs/kube_inventory/README.md b/plugins/inputs/kube_inventory/README.md index d24ca95bdf00a..f017b18c6800b 100644 --- a/plugins/inputs/kube_inventory/README.md +++ b/plugins/inputs/kube_inventory/README.md @@ -1,17 +1,25 @@ # Kube_Inventory Plugin + This plugin generates metrics derived from the state of the following Kubernetes resources: - - daemonsets - - deployments - - nodes - - persistentvolumes - - persistentvolumeclaims - - pods (containers) - - statefulsets + +- daemonsets +- deployments +- nodes +- persistentvolumes +- persistentvolumeclaims +- pods (containers) +- statefulsets + +Kubernetes is a fast moving project, with a new minor release every 3 months. As +such, we will aim to maintain support only for versions that are supported by +the major cloud providers; this is roughly 4 release / 2 years. + +**This plugin supports Kubernetes 1.11 and later.** #### Series Cardinality Warning This plugin may produce a high number of series which, when not controlled -for, will cause high load on your database. Use the following techniques to +for, will cause high load on your database. Use the following techniques to avoid cardinality issues: - Use [metric filtering][] options to exclude unneeded measurements and tags. @@ -33,6 +41,8 @@ avoid cardinality issues: # namespace = "default" ## Use bearer token for authorization. ('bearer_token' takes priority) + ## If both of these are empty, we'll use the default serviceaccount: + ## at: /run/secrets/kubernetes.io/serviceaccount/token # bearer_token = "/path/to/bearer/token" ## OR # bearer_token_string = "abc_123" @@ -61,6 +71,7 @@ avoid cardinality issues: #### Kubernetes Permissions If using [RBAC authorization](https://kubernetes.io/docs/reference/access-authn-authz/rbac/), you will need to create a cluster role to list "persistentvolumes" and "nodes". You will then need to make an [aggregated ClusterRole](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#aggregated-clusterroles) that will eventually be bound to a user or group. + ```yaml --- kind: ClusterRole @@ -70,9 +81,9 @@ metadata: labels: rbac.authorization.k8s.io/aggregate-view-telegraf: "true" rules: -- apiGroups: [""] - resources: ["persistentvolumes","nodes"] - verbs: ["get","list"] + - apiGroups: [""] + resources: ["persistentvolumes", "nodes"] + verbs: ["get", "list"] --- kind: ClusterRole @@ -81,14 +92,15 @@ metadata: name: influx:telegraf aggregationRule: clusterRoleSelectors: - - matchLabels: - rbac.authorization.k8s.io/aggregate-view-telegraf: "true" - - matchLabels: - rbac.authorization.k8s.io/aggregate-to-view: "true" + - matchLabels: + rbac.authorization.k8s.io/aggregate-view-telegraf: "true" + - matchLabels: + rbac.authorization.k8s.io/aggregate-to-view: "true" rules: [] # Rules are automatically filled in by the controller manager. ``` Bind the newly created aggregated ClusterRole with the following config file, updating the subjects as needed. + ```yaml --- apiVersion: rbac.authorization.k8s.io/v1 @@ -100,15 +112,14 @@ roleRef: kind: ClusterRole name: influx:telegraf subjects: -- kind: ServiceAccount - name: telegraf - namespace: default + - kind: ServiceAccount + name: telegraf + namespace: default ``` - ### Metrics: -+ kubernetes_daemonset +- kubernetes_daemonset - tags: - daemonset_name - namespace @@ -122,7 +133,7 @@ subjects: - number_unavailable - updated_number_scheduled -- kubernetes_deployment +* kubernetes_deployment - tags: - deployment_name - namespace @@ -131,7 +142,7 @@ subjects: - replicas_unavailable - created -+ kubernetes_endpoints +- kubernetes_endpoints - tags: - endpoint_name - namespace @@ -139,14 +150,14 @@ subjects: - node_name - port_name - port_protocol - - kind (*varies) + - kind (\*varies) - fields: - created - generation - ready - port -- kubernetes_ingress +* kubernetes_ingress - tags: - ingress_name - namespace @@ -161,7 +172,7 @@ subjects: - backend_service_port - tls -+ kubernetes_node +- kubernetes_node - tags: - node_name - fields: @@ -172,7 +183,7 @@ subjects: - allocatable_memory_bytes - allocatable_pods -- kubernetes_persistentvolume +* kubernetes_persistentvolume - tags: - pv_name - phase @@ -180,7 +191,7 @@ subjects: - fields: - phase_type (int, [see below](#pv-phase_type)) -+ kubernetes_persistentvolumeclaim +- kubernetes_persistentvolumeclaim - tags: - pvc_name - namespace @@ -189,7 +200,7 @@ subjects: - fields: - phase_type (int, [see below](#pvc-phase_type)) -- kubernetes_pod_container +* kubernetes_pod_container - tags: - container_name - namespace @@ -204,7 +215,7 @@ subjects: - resource_limits_cpu_units - resource_limits_memory_bytes -+ kubernetes_service +- kubernetes_service - tags: - service_name - namespace @@ -218,7 +229,7 @@ subjects: - port - target_port -- kubernetes_statefulset +* kubernetes_statefulset - tags: - statefulset_name - namespace @@ -236,25 +247,25 @@ subjects: The persistentvolume "phase" is saved in the `phase` tag with a correlated numeric field called `phase_type` corresponding with that tag value. -|Tag value |Corresponding field value| ------------|-------------------------| -|bound | 0 | -|failed | 1 | -|pending | 2 | -|released | 3 | -|available | 4 | -|unknown | 5 | +| Tag value | Corresponding field value | +| --------- | ------------------------- | +| bound | 0 | +| failed | 1 | +| pending | 2 | +| released | 3 | +| available | 4 | +| unknown | 5 | #### pvc `phase_type` The persistentvolumeclaim "phase" is saved in the `phase` tag with a correlated numeric field called `phase_type` corresponding with that tag value. -|Tag value |Corresponding field value| ------------|-------------------------| -|bound | 0 | -|lost | 1 | -|pending | 2 | -|unknown | 3 | +| Tag value | Corresponding field value | +| --------- | ------------------------- | +| bound | 0 | +| lost | 1 | +| pending | 2 | +| unknown | 3 | ### Example Output: @@ -271,7 +282,6 @@ kubernetes_pod_container,container_name=telegraf,namespace=default,node_name=ip- kubernetes_statefulset,namespace=default,statefulset_name=etcd replicas_updated=3i,spec_replicas=3i,observed_generation=1i,created=1544101669000000000i,generation=1i,replicas=3i,replicas_current=3i,replicas_ready=3i 1547597616000000000 ``` - [metric filtering]: https://github.com/influxdata/telegraf/blob/master/docs/CONFIGURATION.md#metric-filtering [retention policy]: https://docs.influxdata.com/influxdb/latest/guides/downsampling_and_retention/ [max-series-per-database]: https://docs.influxdata.com/influxdb/latest/administration/config/#max-series-per-database-1000000 diff --git a/plugins/inputs/kube_inventory/client.go b/plugins/inputs/kube_inventory/client.go index 5bb2baf5ce412..d16428c40d480 100644 --- a/plugins/inputs/kube_inventory/client.go +++ b/plugins/inputs/kube_inventory/client.go @@ -5,9 +5,8 @@ import ( "time" "github.com/ericchiang/k8s" - "github.com/ericchiang/k8s/apis/apps/v1beta1" - "github.com/ericchiang/k8s/apis/apps/v1beta2" - "github.com/ericchiang/k8s/apis/core/v1" + v1APPS "github.com/ericchiang/k8s/apis/apps/v1" + v1 "github.com/ericchiang/k8s/apis/core/v1" v1beta1EXT "github.com/ericchiang/k8s/apis/extensions/v1beta1" "github.com/influxdata/telegraf/internal/tls" @@ -48,15 +47,15 @@ func newClient(baseURL, namespace, bearerToken string, timeout time.Duration, tl }, nil } -func (c *client) getDaemonSets(ctx context.Context) (*v1beta2.DaemonSetList, error) { - list := new(v1beta2.DaemonSetList) +func (c *client) getDaemonSets(ctx context.Context) (*v1APPS.DaemonSetList, error) { + list := new(v1APPS.DaemonSetList) ctx, cancel := context.WithTimeout(ctx, c.timeout) defer cancel() return list, c.List(ctx, c.namespace, list) } -func (c *client) getDeployments(ctx context.Context) (*v1beta1.DeploymentList, error) { - list := &v1beta1.DeploymentList{} +func (c *client) getDeployments(ctx context.Context) (*v1APPS.DeploymentList, error) { + list := &v1APPS.DeploymentList{} ctx, cancel := context.WithTimeout(ctx, c.timeout) defer cancel() return list, c.List(ctx, c.namespace, list) @@ -111,8 +110,8 @@ func (c *client) getServices(ctx context.Context) (*v1.ServiceList, error) { return list, c.List(ctx, c.namespace, list) } -func (c *client) getStatefulSets(ctx context.Context) (*v1beta1.StatefulSetList, error) { - list := new(v1beta1.StatefulSetList) +func (c *client) getStatefulSets(ctx context.Context) (*v1APPS.StatefulSetList, error) { + list := new(v1APPS.StatefulSetList) ctx, cancel := context.WithTimeout(ctx, c.timeout) defer cancel() return list, c.List(ctx, c.namespace, list) diff --git a/plugins/inputs/kube_inventory/daemonset.go b/plugins/inputs/kube_inventory/daemonset.go index 92c7bc195763e..15df586d6041d 100644 --- a/plugins/inputs/kube_inventory/daemonset.go +++ b/plugins/inputs/kube_inventory/daemonset.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/ericchiang/k8s/apis/apps/v1beta2" + "github.com/ericchiang/k8s/apis/apps/v1" "github.com/influxdata/telegraf" ) @@ -23,7 +23,7 @@ func collectDaemonSets(ctx context.Context, acc telegraf.Accumulator, ki *Kubern } } -func (ki *KubernetesInventory) gatherDaemonSet(d v1beta2.DaemonSet, acc telegraf.Accumulator) error { +func (ki *KubernetesInventory) gatherDaemonSet(d v1.DaemonSet, acc telegraf.Accumulator) error { fields := map[string]interface{}{ "generation": d.Metadata.GetGeneration(), "current_number_scheduled": d.Status.GetCurrentNumberScheduled(), diff --git a/plugins/inputs/kube_inventory/daemonset_test.go b/plugins/inputs/kube_inventory/daemonset_test.go index 3f11df1ca108d..bf4e934d312cd 100644 --- a/plugins/inputs/kube_inventory/daemonset_test.go +++ b/plugins/inputs/kube_inventory/daemonset_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/ericchiang/k8s/apis/apps/v1beta2" + "github.com/ericchiang/k8s/apis/apps/v1" metav1 "github.com/ericchiang/k8s/apis/meta/v1" "github.com/influxdata/telegraf/testutil" @@ -24,7 +24,7 @@ func TestDaemonSet(t *testing.T) { name: "no daemon set", handler: &mockHandler{ responseMap: map[string]interface{}{ - "/daemonsets/": &v1beta2.DaemonSetList{}, + "/daemonsets/": &v1.DaemonSetList{}, }, }, hasError: false, @@ -33,10 +33,10 @@ func TestDaemonSet(t *testing.T) { name: "collect daemonsets", handler: &mockHandler{ responseMap: map[string]interface{}{ - "/daemonsets/": &v1beta2.DaemonSetList{ - Items: []*v1beta2.DaemonSet{ + "/daemonsets/": &v1.DaemonSetList{ + Items: []*v1.DaemonSet{ { - Status: &v1beta2.DaemonSetStatus{ + Status: &v1.DaemonSetStatus{ CurrentNumberScheduled: toInt32Ptr(3), DesiredNumberScheduled: toInt32Ptr(5), NumberAvailable: toInt32Ptr(2), @@ -90,7 +90,7 @@ func TestDaemonSet(t *testing.T) { client: cli, } acc := new(testutil.Accumulator) - for _, dset := range ((v.handler.responseMap["/daemonsets/"]).(*v1beta2.DaemonSetList)).Items { + for _, dset := range ((v.handler.responseMap["/daemonsets/"]).(*v1.DaemonSetList)).Items { err := ks.gatherDaemonSet(*dset, acc) if err != nil { t.Errorf("Failed to gather daemonset - %s", err.Error()) diff --git a/plugins/inputs/kube_inventory/deployment.go b/plugins/inputs/kube_inventory/deployment.go index 2d72e8d03a4f0..5a0eb0b197155 100644 --- a/plugins/inputs/kube_inventory/deployment.go +++ b/plugins/inputs/kube_inventory/deployment.go @@ -4,8 +4,7 @@ import ( "context" "time" - "github.com/ericchiang/k8s/apis/apps/v1beta1" - + v1 "github.com/ericchiang/k8s/apis/apps/v1" "github.com/influxdata/telegraf" ) @@ -23,7 +22,7 @@ func collectDeployments(ctx context.Context, acc telegraf.Accumulator, ki *Kuber } } -func (ki *KubernetesInventory) gatherDeployment(d v1beta1.Deployment, acc telegraf.Accumulator) error { +func (ki *KubernetesInventory) gatherDeployment(d v1.Deployment, acc telegraf.Accumulator) error { fields := map[string]interface{}{ "replicas_available": d.Status.GetAvailableReplicas(), "replicas_unavailable": d.Status.GetUnavailableReplicas(), diff --git a/plugins/inputs/kube_inventory/deployment_test.go b/plugins/inputs/kube_inventory/deployment_test.go index 0429b84fa1d87..21b7bfd026429 100644 --- a/plugins/inputs/kube_inventory/deployment_test.go +++ b/plugins/inputs/kube_inventory/deployment_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/ericchiang/k8s/apis/apps/v1beta1" + "github.com/ericchiang/k8s/apis/apps/v1" metav1 "github.com/ericchiang/k8s/apis/meta/v1" "github.com/ericchiang/k8s/util/intstr" "github.com/influxdata/telegraf/testutil" @@ -37,7 +37,7 @@ func TestDeployment(t *testing.T) { name: "no deployments", handler: &mockHandler{ responseMap: map[string]interface{}{ - "/deployments/": &v1beta1.DeploymentList{}, + "/deployments/": &v1.DeploymentList{}, }, }, hasError: false, @@ -46,19 +46,19 @@ func TestDeployment(t *testing.T) { name: "collect deployments", handler: &mockHandler{ responseMap: map[string]interface{}{ - "/deployments/": &v1beta1.DeploymentList{ - Items: []*v1beta1.Deployment{ + "/deployments/": &v1.DeploymentList{ + Items: []*v1.Deployment{ { - Status: &v1beta1.DeploymentStatus{ + Status: &v1.DeploymentStatus{ Replicas: toInt32Ptr(3), AvailableReplicas: toInt32Ptr(1), UnavailableReplicas: toInt32Ptr(4), UpdatedReplicas: toInt32Ptr(2), ObservedGeneration: toInt64Ptr(9121), }, - Spec: &v1beta1.DeploymentSpec{ - Strategy: &v1beta1.DeploymentStrategy{ - RollingUpdate: &v1beta1.RollingUpdateDeployment{ + Spec: &v1.DeploymentSpec{ + Strategy: &v1.DeploymentStrategy{ + RollingUpdate: &v1.RollingUpdateDeployment{ MaxUnavailable: &intstr.IntOrString{ IntVal: toInt32Ptr(30), }, @@ -98,7 +98,7 @@ func TestDeployment(t *testing.T) { client: cli, } acc := new(testutil.Accumulator) - for _, deployment := range ((v.handler.responseMap["/deployments/"]).(*v1beta1.DeploymentList)).Items { + for _, deployment := range ((v.handler.responseMap["/deployments/"]).(*v1.DeploymentList)).Items { err := ks.gatherDeployment(*deployment, acc) if err != nil { t.Errorf("Failed to gather deployment - %s", err.Error()) diff --git a/plugins/inputs/kube_inventory/ingress_test.go b/plugins/inputs/kube_inventory/ingress_test.go index e3b44512cc11f..2d111801a96f3 100644 --- a/plugins/inputs/kube_inventory/ingress_test.go +++ b/plugins/inputs/kube_inventory/ingress_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/ericchiang/k8s/apis/core/v1" + v1 "github.com/ericchiang/k8s/apis/core/v1" v1beta1EXT "github.com/ericchiang/k8s/apis/extensions/v1beta1" metav1 "github.com/ericchiang/k8s/apis/meta/v1" "github.com/influxdata/telegraf/testutil" diff --git a/plugins/inputs/kube_inventory/kube_state.go b/plugins/inputs/kube_inventory/kube_state.go index b69ad47acead8..5aa51b6c5f8cd 100644 --- a/plugins/inputs/kube_inventory/kube_state.go +++ b/plugins/inputs/kube_inventory/kube_state.go @@ -19,6 +19,10 @@ import ( "github.com/influxdata/telegraf/plugins/inputs" ) +const ( + defaultServiceAccountPath = "/run/secrets/kubernetes.io/serviceaccount/token" +) + // KubernetesInventory represents the config object for the plugin. type KubernetesInventory struct { URL string `toml:"url"` @@ -42,6 +46,8 @@ var sampleConfig = ` # namespace = "default" ## Use bearer token for authorization. ('bearer_token' takes priority) + ## If both of these are empty, we'll use the default serviceaccount: + ## at: /run/secrets/kubernetes.io/serviceaccount/token # bearer_token = "/path/to/bearer/token" ## OR # bearer_token_string = "abc_123" @@ -77,14 +83,32 @@ func (ki *KubernetesInventory) Description() string { return "Read metrics from the Kubernetes api" } -// Gather collects kubernetes metrics from a given URL. -func (ki *KubernetesInventory) Gather(acc telegraf.Accumulator) (err error) { - if ki.client == nil { - if ki.client, err = ki.initClient(); err != nil { +func (ki *KubernetesInventory) Init() error { + // If neither are provided, use the default service account. + if ki.BearerToken == "" && ki.BearerTokenString == "" { + ki.BearerToken = defaultServiceAccountPath + } + + if ki.BearerToken != "" { + token, err := ioutil.ReadFile(ki.BearerToken) + if err != nil { return err } + ki.BearerTokenString = strings.TrimSpace(string(token)) + } + + var err error + ki.client, err = newClient(ki.URL, ki.Namespace, ki.BearerTokenString, ki.ResponseTimeout.Duration, ki.ClientConfig) + + if err != nil { + return err } + return nil +} + +// Gather collects kubernetes metrics from a given URL. +func (ki *KubernetesInventory) Gather(acc telegraf.Accumulator) (err error) { resourceFilter, err := filter.NewIncludeExcludeFilter(ki.ResourceInclude, ki.ResourceExclude) if err != nil { return err @@ -114,23 +138,11 @@ var availableCollectors = map[string]func(ctx context.Context, acc telegraf.Accu "endpoints": collectEndpoints, "ingress": collectIngress, "nodes": collectNodes, - "persistentvolumes": collectPersistentVolumes, - "persistentvolumeclaims": collectPersistentVolumeClaims, "pods": collectPods, "services": collectServices, "statefulsets": collectStatefulSets, -} - -func (ki *KubernetesInventory) initClient() (*client, error) { - if ki.BearerToken != "" { - token, err := ioutil.ReadFile(ki.BearerToken) - if err != nil { - return nil, err - } - ki.BearerTokenString = strings.TrimSpace(string(token)) - } - - return newClient(ki.URL, ki.Namespace, ki.BearerTokenString, ki.ResponseTimeout.Duration, ki.ClientConfig) + "persistentvolumes": collectPersistentVolumes, + "persistentvolumeclaims": collectPersistentVolumeClaims, } func atoi(s string) int64 { @@ -144,12 +156,12 @@ func atoi(s string) int64 { func convertQuantity(s string, m float64) int64 { q, err := resource.ParseQuantity(s) if err != nil { - log.Printf("E! Failed to parse quantity - %v", err) + log.Printf("D! [inputs.kube_inventory] failed to parse quantity: %s", err.Error()) return 0 } f, err := strconv.ParseFloat(fmt.Sprint(q.AsDec()), 64) if err != nil { - log.Printf("E! Failed to parse float - %v", err) + log.Printf("D! [inputs.kube_inventory] failed to parse float: %s", err.Error()) return 0 } if m < 1 { diff --git a/plugins/inputs/kube_inventory/statefulset.go b/plugins/inputs/kube_inventory/statefulset.go index 407aaac2fce08..c95e566c20564 100644 --- a/plugins/inputs/kube_inventory/statefulset.go +++ b/plugins/inputs/kube_inventory/statefulset.go @@ -4,7 +4,7 @@ import ( "context" "time" - "github.com/ericchiang/k8s/apis/apps/v1beta1" + "github.com/ericchiang/k8s/apis/apps/v1" "github.com/influxdata/telegraf" ) @@ -23,7 +23,7 @@ func collectStatefulSets(ctx context.Context, acc telegraf.Accumulator, ki *Kube } } -func (ki *KubernetesInventory) gatherStatefulSet(s v1beta1.StatefulSet, acc telegraf.Accumulator) error { +func (ki *KubernetesInventory) gatherStatefulSet(s v1.StatefulSet, acc telegraf.Accumulator) error { status := s.Status fields := map[string]interface{}{ "created": time.Unix(s.Metadata.CreationTimestamp.GetSeconds(), int64(s.Metadata.CreationTimestamp.GetNanos())).UnixNano(), diff --git a/plugins/inputs/kube_inventory/statefulset_test.go b/plugins/inputs/kube_inventory/statefulset_test.go index 6e94ad150ce0f..1a971b7b69c6e 100644 --- a/plugins/inputs/kube_inventory/statefulset_test.go +++ b/plugins/inputs/kube_inventory/statefulset_test.go @@ -4,7 +4,7 @@ import ( "testing" "time" - "github.com/ericchiang/k8s/apis/apps/v1beta1" + "github.com/ericchiang/k8s/apis/apps/v1" metav1 "github.com/ericchiang/k8s/apis/meta/v1" "github.com/influxdata/telegraf/testutil" @@ -24,7 +24,7 @@ func TestStatefulSet(t *testing.T) { name: "no statefulsets", handler: &mockHandler{ responseMap: map[string]interface{}{ - "/statefulsets/": &v1beta1.StatefulSetList{}, + "/statefulsets/": &v1.StatefulSetList{}, }, }, hasError: false, @@ -33,17 +33,17 @@ func TestStatefulSet(t *testing.T) { name: "collect statefulsets", handler: &mockHandler{ responseMap: map[string]interface{}{ - "/statefulsets/": &v1beta1.StatefulSetList{ - Items: []*v1beta1.StatefulSet{ + "/statefulsets/": &v1.StatefulSetList{ + Items: []*v1.StatefulSet{ { - Status: &v1beta1.StatefulSetStatus{ + Status: &v1.StatefulSetStatus{ Replicas: toInt32Ptr(2), CurrentReplicas: toInt32Ptr(4), ReadyReplicas: toInt32Ptr(1), UpdatedReplicas: toInt32Ptr(3), ObservedGeneration: toInt64Ptr(119), }, - Spec: &v1beta1.StatefulSetSpec{ + Spec: &v1.StatefulSetSpec{ Replicas: toInt32Ptr(3), }, Metadata: &metav1.ObjectMeta{ @@ -90,7 +90,7 @@ func TestStatefulSet(t *testing.T) { client: cli, } acc := new(testutil.Accumulator) - for _, ss := range ((v.handler.responseMap["/statefulsets/"]).(*v1beta1.StatefulSetList)).Items { + for _, ss := range ((v.handler.responseMap["/statefulsets/"]).(*v1.StatefulSetList)).Items { err := ks.gatherStatefulSet(*ss, acc) if err != nil { t.Errorf("Failed to gather ss - %s", err.Error()) diff --git a/plugins/inputs/kubernetes/README.md b/plugins/inputs/kubernetes/README.md index d53d94e97aa6a..2d38f23d96ff8 100644 --- a/plugins/inputs/kubernetes/README.md +++ b/plugins/inputs/kubernetes/README.md @@ -1,17 +1,29 @@ # Kubernetes Input Plugin -This input plugin talks to the kubelet api using the `/stats/summary` endpoint to gather metrics about the running pods and containers for a single host. It is assumed that this plugin is running as part of a `daemonset` within a kubernetes installation. This means that telegraf is running on every node within the cluster. Therefore, you should configure this plugin to talk to its locally running kubelet. +The Kubernetes plugin talks to the Kubelet API and gathers metrics about the +running pods and containers for a single host. It is assumed that this plugin +is running as part of a `daemonset` within a kubernetes installation. This +means that telegraf is running on every node within the cluster. Therefore, you +should configure this plugin to talk to its locally running kubelet. To find the ip address of the host you are running on you can issue a command like the following: + ``` $ curl -s $API_URL/api/v1/namespaces/$POD_NAMESPACE/pods/$HOSTNAME --header "Authorization: Bearer $TOKEN" --insecure | jq -r '.status.hostIP' ``` + In this case we used the downward API to pass in the `$POD_NAMESPACE` and `$HOSTNAME` is the hostname of the pod which is set by the kubernetes API. +Kubernetes is a fast moving project, with a new minor release every 3 months. As +such, we will aim to maintain support only for versions that are supported by +the major cloud providers; this is roughly 4 release / 2 years. + +**This plugin supports Kubernetes 1.11 and later.** + #### Series Cardinality Warning This plugin may produce a high number of series which, when not controlled -for, will cause high load on your database. Use the following techniques to +for, will cause high load on your database. Use the following techniques to avoid cardinality issues: - Use [metric filtering][] options to exclude unneeded measurements and tags. @@ -30,10 +42,17 @@ avoid cardinality issues: url = "http://127.0.0.1:10255" ## Use bearer token for authorization. ('bearer_token' takes priority) + ## If both of these are empty, we'll use the default serviceaccount: + ## at: /run/secrets/kubernetes.io/serviceaccount/token # bearer_token = "/path/to/bearer/token" ## OR # bearer_token_string = "abc_123" + ## Pod labels to be added as tags. An empty array for both include and + ## exclude will include all labels. + # label_include = [] + # label_exclude = ["*"] + ## Set response_timeout (default 5 seconds) # response_timeout = "5s" @@ -80,7 +99,7 @@ Architecture][k8s-telegraf] or view the Helm charts: - runtime_image_fs_capacity_bytes - runtime_image_fs_used_bytes -+ kubernetes_pod_container +* kubernetes_pod_container - tags: - container_name - namespace @@ -112,7 +131,7 @@ Architecture][k8s-telegraf] or view the Helm charts: - capacity_bytes - used_bytes -+ kubernetes_pod_network +* kubernetes_pod_network - tags: - namespace - node_name @@ -141,7 +160,7 @@ kubernetes_system_container [series cardinality]: https://docs.influxdata.com/influxdb/latest/query_language/spec/#show-cardinality [influx-docs]: https://docs.influxdata.com/influxdb/latest/ [k8s-telegraf]: https://www.influxdata.com/blog/monitoring-kubernetes-architecture/ -[Telegraf]: https://github.com/helm/charts/tree/master/stable/telegraf -[InfluxDB]: https://github.com/helm/charts/tree/master/stable/influxdb -[Chronograf]: https://github.com/helm/charts/tree/master/stable/chronograf -[Kapacitor]: https://github.com/helm/charts/tree/master/stable/kapacitor +[telegraf]: https://github.com/helm/charts/tree/master/stable/telegraf +[influxdb]: https://github.com/helm/charts/tree/master/stable/influxdb +[chronograf]: https://github.com/helm/charts/tree/master/stable/chronograf +[kapacitor]: https://github.com/helm/charts/tree/master/stable/kapacitor diff --git a/plugins/inputs/kubernetes/kubernetes.go b/plugins/inputs/kubernetes/kubernetes.go index 4e6e17ef11d98..412db1dc334ee 100644 --- a/plugins/inputs/kubernetes/kubernetes.go +++ b/plugins/inputs/kubernetes/kubernetes.go @@ -10,6 +10,7 @@ import ( "time" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/filter" "github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/internal/tls" "github.com/influxdata/telegraf/plugins/inputs" @@ -23,6 +24,11 @@ type Kubernetes struct { BearerToken string `toml:"bearer_token"` BearerTokenString string `toml:"bearer_token_string"` + LabelInclude []string `toml:"label_include"` + LabelExclude []string `toml:"label_exclude"` + + labelFilter filter.Filter + // HTTP Timeout specified as a string - 3s, 1m, 1h ResponseTimeout internal.Duration @@ -36,10 +42,17 @@ var sampleConfig = ` url = "http://127.0.0.1:10255" ## Use bearer token for authorization. ('bearer_token' takes priority) + ## If both of these are empty, we'll use the default serviceaccount: + ## at: /run/secrets/kubernetes.io/serviceaccount/token # bearer_token = "/path/to/bearer/token" ## OR # bearer_token_string = "abc_123" + ## Pod labels to be added as tags. An empty array for both include and + ## exclude will include all labels. + # label_include = [] + # label_exclude = ["*"] + ## Set response_timeout (default 5 seconds) # response_timeout = "5s" @@ -52,12 +65,16 @@ var sampleConfig = ` ` const ( - summaryEndpoint = `%s/stats/summary` + summaryEndpoint = `%s/stats/summary` + defaultServiceAccountPath = "/run/secrets/kubernetes.io/serviceaccount/token" ) func init() { inputs.Add("kubernetes", func() telegraf.Input { - return &Kubernetes{} + return &Kubernetes{ + LabelInclude: []string{}, + LabelExclude: []string{"*"}, + } }) } @@ -71,6 +88,30 @@ func (k *Kubernetes) Description() string { return "Read metrics from the kubernetes kubelet api" } +func (k *Kubernetes) Init() error { + + // If neither are provided, use the default service account. + if k.BearerToken == "" && k.BearerTokenString == "" { + k.BearerToken = defaultServiceAccountPath + } + + if k.BearerToken != "" { + token, err := ioutil.ReadFile(k.BearerToken) + if err != nil { + return err + } + k.BearerTokenString = strings.TrimSpace(string(token)) + } + + labelFilter, err := filter.NewIncludeExcludeFilter(k.LabelInclude, k.LabelExclude) + if err != nil { + return err + } + k.labelFilter = labelFilter + + return nil +} + //Gather collects kubernetes metrics from a given URL func (k *Kubernetes) Gather(acc telegraf.Accumulator) error { acc.AddError(k.gatherSummary(k.URL, acc)) @@ -87,56 +128,19 @@ func buildURL(endpoint string, base string) (*url.URL, error) { } func (k *Kubernetes) gatherSummary(baseURL string, acc telegraf.Accumulator) error { - url := fmt.Sprintf("%s/stats/summary", baseURL) - var req, err = http.NewRequest("GET", url, nil) - var resp *http.Response - - tlsCfg, err := k.ClientConfig.TLSConfig() + summaryMetrics := &SummaryMetrics{} + err := k.LoadJson(fmt.Sprintf("%s/stats/summary", baseURL), summaryMetrics) if err != nil { return err } - if k.RoundTripper == nil { - // Set default values - if k.ResponseTimeout.Duration < time.Second { - k.ResponseTimeout.Duration = time.Second * 5 - } - k.RoundTripper = &http.Transport{ - TLSHandshakeTimeout: 5 * time.Second, - TLSClientConfig: tlsCfg, - ResponseHeaderTimeout: k.ResponseTimeout.Duration, - } - } - - if k.BearerToken != "" { - token, err := ioutil.ReadFile(k.BearerToken) - if err != nil { - return err - } - req.Header.Set("Authorization", "Bearer "+strings.TrimSpace(string(token))) - } else if k.BearerTokenString != "" { - req.Header.Set("Authorization", "Bearer "+k.BearerTokenString) - } - req.Header.Add("Accept", "application/json") - - resp, err = k.RoundTripper.RoundTrip(req) - if err != nil { - return fmt.Errorf("error making HTTP request to %s: %s", url, err) - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("%s returned HTTP status %s", url, resp.Status) - } - - summaryMetrics := &SummaryMetrics{} - err = json.NewDecoder(resp.Body).Decode(summaryMetrics) + podInfos, err := k.gatherPodInfo(baseURL) if err != nil { - return fmt.Errorf(`Error parsing response: %s`, err) + return err } buildSystemContainerMetrics(summaryMetrics, acc) buildNodeMetrics(summaryMetrics, acc) - buildPodMetrics(summaryMetrics, acc) + buildPodMetrics(baseURL, summaryMetrics, podInfos, k.labelFilter, acc) return nil } @@ -188,7 +192,56 @@ func buildNodeMetrics(summaryMetrics *SummaryMetrics, acc telegraf.Accumulator) acc.AddFields("kubernetes_node", fields, tags) } -func buildPodMetrics(summaryMetrics *SummaryMetrics, acc telegraf.Accumulator) { +func (k *Kubernetes) gatherPodInfo(baseURL string) ([]Metadata, error) { + var podApi Pods + err := k.LoadJson(fmt.Sprintf("%s/pods", baseURL), &podApi) + if err != nil { + return nil, err + } + var podInfos []Metadata + for _, podMetadata := range podApi.Items { + podInfos = append(podInfos, podMetadata.Metadata) + } + return podInfos, nil +} + +func (k *Kubernetes) LoadJson(url string, v interface{}) error { + var req, err = http.NewRequest("GET", url, nil) + var resp *http.Response + tlsCfg, err := k.ClientConfig.TLSConfig() + if err != nil { + return err + } + if k.RoundTripper == nil { + if k.ResponseTimeout.Duration < time.Second { + k.ResponseTimeout.Duration = time.Second * 5 + } + k.RoundTripper = &http.Transport{ + TLSHandshakeTimeout: 5 * time.Second, + TLSClientConfig: tlsCfg, + ResponseHeaderTimeout: k.ResponseTimeout.Duration, + } + } + req.Header.Set("Authorization", "Bearer "+k.BearerTokenString) + req.Header.Add("Accept", "application/json") + resp, err = k.RoundTripper.RoundTrip(req) + if err != nil { + return fmt.Errorf("error making HTTP request to %s: %s", url, err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("%s returned HTTP status %s", url, resp.Status) + } + + err = json.NewDecoder(resp.Body).Decode(v) + if err != nil { + return fmt.Errorf(`Error parsing response: %s`, err) + } + + return nil +} + +func buildPodMetrics(baseURL string, summaryMetrics *SummaryMetrics, podInfo []Metadata, labelFilter filter.Filter, acc telegraf.Accumulator) { for _, pod := range summaryMetrics.Pods { for _, container := range pod.Containers { tags := map[string]string{ @@ -197,6 +250,16 @@ func buildPodMetrics(summaryMetrics *SummaryMetrics, acc telegraf.Accumulator) { "container_name": container.Name, "pod_name": pod.PodRef.Name, } + for _, info := range podInfo { + if info.Name == pod.PodRef.Name && info.Namespace == pod.PodRef.Namespace { + for k, v := range info.Labels { + if labelFilter.Match(k) { + tags[k] = v + } + } + } + } + fields := make(map[string]interface{}) fields["cpu_usage_nanocores"] = container.CPU.UsageNanoCores fields["cpu_usage_core_nanoseconds"] = container.CPU.UsageCoreNanoSeconds diff --git a/plugins/inputs/kubernetes/kubernetes_pods.go b/plugins/inputs/kubernetes/kubernetes_pods.go new file mode 100644 index 0000000000000..672608e54fe25 --- /dev/null +++ b/plugins/inputs/kubernetes/kubernetes_pods.go @@ -0,0 +1,17 @@ +package kubernetes + +type Pods struct { + Kind string `json:"kind"` + ApiVersion string `json:"apiVersion"` + Items []Item `json:"items"` +} + +type Item struct { + Metadata Metadata `json:"metadata"` +} + +type Metadata struct { + Name string `json:"name"` + Namespace string `json:"namespace"` + Labels map[string]string `json:"labels"` +} diff --git a/plugins/inputs/kubernetes/kubernetes_test.go b/plugins/inputs/kubernetes/kubernetes_test.go index 081bca03aa536..faf40be3e1000 100644 --- a/plugins/inputs/kubernetes/kubernetes_test.go +++ b/plugins/inputs/kubernetes/kubernetes_test.go @@ -2,6 +2,7 @@ package kubernetes import ( "fmt" + "github.com/influxdata/telegraf/filter" "net/http" "net/http/httptest" "testing" @@ -12,13 +13,23 @@ import ( func TestKubernetesStats(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusOK) - fmt.Fprintln(w, response) + if r.RequestURI == "/stats/summary" { + w.WriteHeader(http.StatusOK) + fmt.Fprintln(w, responseStatsSummery) + } + if r.RequestURI == "/pods" { + w.WriteHeader(http.StatusOK) + fmt.Fprintln(w, responsePods) + } + })) defer ts.Close() + labelFilter, _ := filter.NewIncludeExcludeFilter([]string{"app", "superkey"}, nil) + k := &Kubernetes{ - URL: ts.URL, + URL: ts.URL, + labelFilter: labelFilter, } var acc testutil.Accumulator @@ -89,6 +100,8 @@ func TestKubernetesStats(t *testing.T) { "container_name": "foocontainer", "namespace": "foons", "pod_name": "foopod", + "app": "foo", + "superkey": "foobar", } acc.AssertContainsTaggedFields(t, "kubernetes_pod_container", fields, tags) @@ -112,6 +125,8 @@ func TestKubernetesStats(t *testing.T) { "container_name": "stopped-container", "namespace": "foons", "pod_name": "stopped-pod", + "app": "foo-stop", + "superkey": "superfoo", } acc.AssertContainsTaggedFields(t, "kubernetes_pod_container", fields, tags) @@ -143,7 +158,39 @@ func TestKubernetesStats(t *testing.T) { } -var response = ` +var responsePods = ` +{ + "kind": "PodList", + "apiVersion": "v1", + "metadata": {}, + "items": [ + { + "metadata": { + "name": "foopod", + "namespace": "foons", + "labels": { + "superkey": "foobar", + "app": "foo", + "exclude": "exclude0" + } + } + }, + { + "metadata": { + "name": "stopped-pod", + "namespace": "foons", + "labels": { + "superkey": "superfoo", + "app": "foo-stop", + "exclude": "exclude1" + } + } + } + ] +} +` + +var responseStatsSummery = ` { "node": { "nodeName": "node1", diff --git a/plugins/inputs/logparser/README.md b/plugins/inputs/logparser/README.md index efd50952f6f66..22250ff4522f8 100644 --- a/plugins/inputs/logparser/README.md +++ b/plugins/inputs/logparser/README.md @@ -105,6 +105,7 @@ Patterns that convert all captures to tags will result in points that can't be w - ts-rfc3339nano ("2006-01-02T15:04:05.999999999Z07:00") - ts-httpd ("02/Jan/2006:15:04:05 -0700") - ts-epoch (seconds since unix epoch, may contain decimal) + - ts-epochmilli (milliseconds since unix epoch, may contain decimal) - ts-epochnano (nanoseconds since unix epoch) - ts-syslog ("Jan 02 15:04:05", parsed time is set to the current year) - ts-"CUSTOM" diff --git a/plugins/inputs/logparser/logparser.go b/plugins/inputs/logparser/logparser.go index c132ba7a2ccb2..0ce3ede04aa1d 100644 --- a/plugins/inputs/logparser/logparser.go +++ b/plugins/inputs/logparser/logparser.go @@ -4,7 +4,6 @@ package logparser import ( "fmt" - "log" "strings" "sync" @@ -14,7 +13,6 @@ import ( "github.com/influxdata/telegraf/internal/globpath" "github.com/influxdata/telegraf/plugins/inputs" "github.com/influxdata/telegraf/plugins/parsers" - // Parsers ) const ( @@ -48,6 +46,8 @@ type LogParserPlugin struct { FromBeginning bool WatchMethod string + Log telegraf.Logger + tailers map[string]*tail.Tail offsets map[string]int64 lines chan logEntry @@ -207,7 +207,7 @@ func (l *LogParserPlugin) tailNewfiles(fromBeginning bool) error { for _, filepath := range l.Files { g, err := globpath.Compile(filepath) if err != nil { - log.Printf("E! [inputs.logparser] Error Glob %s failed to compile, %s", filepath, err) + l.Log.Errorf("Glob %q failed to compile: %s", filepath, err) continue } files := g.Match() @@ -221,7 +221,7 @@ func (l *LogParserPlugin) tailNewfiles(fromBeginning bool) error { var seek *tail.SeekInfo if !fromBeginning { if offset, ok := l.offsets[file]; ok { - log.Printf("D! [inputs.tail] using offset %d for file: %v", offset, file) + l.Log.Debugf("Using offset %d for file: %v", offset, file) seek = &tail.SeekInfo{ Whence: 0, Offset: offset, @@ -248,7 +248,7 @@ func (l *LogParserPlugin) tailNewfiles(fromBeginning bool) error { continue } - log.Printf("D! [inputs.logparser] tail added for file: %v", file) + l.Log.Debugf("Tail added for file: %v", file) // create a goroutine for each "tailer" l.wg.Add(1) @@ -269,7 +269,7 @@ func (l *LogParserPlugin) receiver(tailer *tail.Tail) { for line = range tailer.Lines { if line.Err != nil { - log.Printf("E! [inputs.logparser] Error tailing file %s, Error: %s", + l.Log.Errorf("Error tailing file %s, Error: %s", tailer.Filename, line.Err) continue } @@ -315,7 +315,7 @@ func (l *LogParserPlugin) parser() { l.acc.AddFields(m.Name(), m.Fields(), tags, m.Time()) } } else { - log.Println("E! [inputs.logparser] Error parsing log line: " + err.Error()) + l.Log.Errorf("Error parsing log line: %s", err.Error()) } } @@ -332,7 +332,7 @@ func (l *LogParserPlugin) Stop() { offset, err := t.Tell() if err == nil { l.offsets[t.Filename] = offset - log.Printf("D! [inputs.logparser] recording offset %d for file: %v", offset, t.Filename) + l.Log.Debugf("Recording offset %d for file: %v", offset, t.Filename) } else { l.acc.AddError(fmt.Errorf("error recording offset for file %s", t.Filename)) } @@ -340,10 +340,10 @@ func (l *LogParserPlugin) Stop() { err := t.Stop() //message for a stopped tailer - log.Printf("D! [inputs.logparser] tail dropped for file: %v", t.Filename) + l.Log.Debugf("Tail dropped for file: %v", t.Filename) if err != nil { - log.Printf("E! [inputs.logparser] Error stopping tail on file %s", t.Filename) + l.Log.Errorf("Error stopping tail on file %s", t.Filename) } } close(l.done) diff --git a/plugins/inputs/logparser/logparser_test.go b/plugins/inputs/logparser/logparser_test.go index 90ae39161f0cf..142f78d464963 100644 --- a/plugins/inputs/logparser/logparser_test.go +++ b/plugins/inputs/logparser/logparser_test.go @@ -6,14 +6,17 @@ import ( "runtime" "strings" "testing" + "time" + "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/testutil" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestStartNoParsers(t *testing.T) { logparser := &LogParserPlugin{ + Log: testutil.Logger{}, FromBeginning: true, Files: []string{"testdata/*.log"}, } @@ -26,6 +29,7 @@ func TestGrokParseLogFilesNonExistPattern(t *testing.T) { thisdir := getCurrentDir() logparser := &LogParserPlugin{ + Log: testutil.Logger{}, FromBeginning: true, Files: []string{thisdir + "testdata/*.log"}, GrokConfig: GrokConfig{ @@ -43,9 +47,10 @@ func TestGrokParseLogFiles(t *testing.T) { thisdir := getCurrentDir() logparser := &LogParserPlugin{ + Log: testutil.Logger{}, GrokConfig: GrokConfig{ MeasurementName: "logparser_grok", - Patterns: []string{"%{TEST_LOG_A}", "%{TEST_LOG_B}"}, + Patterns: []string{"%{TEST_LOG_A}", "%{TEST_LOG_B}", "%{TEST_LOG_C}"}, CustomPatternFiles: []string{thisdir + "testdata/test-patterns"}, }, FromBeginning: true, @@ -53,32 +58,56 @@ func TestGrokParseLogFiles(t *testing.T) { } acc := testutil.Accumulator{} - assert.NoError(t, logparser.Start(&acc)) - acc.Wait(2) + require.NoError(t, logparser.Start(&acc)) + acc.Wait(3) logparser.Stop() - acc.AssertContainsTaggedFields(t, "logparser_grok", - map[string]interface{}{ - "clientip": "192.168.1.1", - "myfloat": float64(1.25), - "response_time": int64(5432), - "myint": int64(101), - }, - map[string]string{ - "response_code": "200", - "path": thisdir + "testdata/test_a.log", - }) + expected := []telegraf.Metric{ + testutil.MustMetric( + "logparser_grok", + map[string]string{ + "response_code": "200", + "path": thisdir + "testdata/test_a.log", + }, + map[string]interface{}{ + "clientip": "192.168.1.1", + "myfloat": float64(1.25), + "response_time": int64(5432), + "myint": int64(101), + }, + time.Unix(0, 0), + ), + testutil.MustMetric( + "logparser_grok", + map[string]string{ + "path": thisdir + "testdata/test_b.log", + }, + map[string]interface{}{ + "myfloat": 1.25, + "mystring": "mystring", + "nomodifier": "nomodifier", + }, + time.Unix(0, 0), + ), + testutil.MustMetric( + "logparser_grok", + map[string]string{ + "path": thisdir + "testdata/test_c.log", + "response_code": "200", + }, + map[string]interface{}{ + "clientip": "192.168.1.1", + "myfloat": 1.25, + "myint": 101, + "response_time": 5432, + }, + time.Unix(0, 0), + ), + } - acc.AssertContainsTaggedFields(t, "logparser_grok", - map[string]interface{}{ - "myfloat": 1.25, - "mystring": "mystring", - "nomodifier": "nomodifier", - }, - map[string]string{ - "path": thisdir + "testdata/test_b.log", - }) + testutil.RequireMetricsEqual(t, expected, acc.GetTelegrafMetrics(), + testutil.IgnoreTime(), testutil.SortMetrics()) } func TestGrokParseLogFilesAppearLater(t *testing.T) { @@ -89,6 +118,7 @@ func TestGrokParseLogFilesAppearLater(t *testing.T) { thisdir := getCurrentDir() logparser := &LogParserPlugin{ + Log: testutil.Logger{}, FromBeginning: true, Files: []string{emptydir + "/*.log"}, GrokConfig: GrokConfig{ @@ -128,6 +158,7 @@ func TestGrokParseLogFilesOneBad(t *testing.T) { thisdir := getCurrentDir() logparser := &LogParserPlugin{ + Log: testutil.Logger{}, FromBeginning: true, Files: []string{thisdir + "testdata/test_a.log"}, GrokConfig: GrokConfig{ @@ -157,6 +188,40 @@ func TestGrokParseLogFilesOneBad(t *testing.T) { }) } +func TestGrokParseLogFiles_TimestampInEpochMilli(t *testing.T) { + thisdir := getCurrentDir() + + logparser := &LogParserPlugin{ + Log: testutil.Logger{}, + GrokConfig: GrokConfig{ + MeasurementName: "logparser_grok", + Patterns: []string{"%{TEST_LOG_C}"}, + CustomPatternFiles: []string{thisdir + "testdata/test-patterns"}, + }, + FromBeginning: true, + Files: []string{thisdir + "testdata/test_c.log"}, + } + + acc := testutil.Accumulator{} + acc.SetDebug(true) + assert.NoError(t, logparser.Start(&acc)) + acc.Wait(1) + + logparser.Stop() + + acc.AssertContainsTaggedFields(t, "logparser_grok", + map[string]interface{}{ + "clientip": "192.168.1.1", + "myfloat": float64(1.25), + "response_time": int64(5432), + "myint": int64(101), + }, + map[string]string{ + "response_code": "200", + "path": thisdir + "testdata/test_c.log", + }) +} + func getCurrentDir() string { _, filename, _, _ := runtime.Caller(1) return strings.Replace(filename, "logparser_test.go", "", 1) diff --git a/plugins/inputs/logparser/testdata/test-patterns b/plugins/inputs/logparser/testdata/test-patterns index ba995fbd1770f..45970a9c8ed12 100644 --- a/plugins/inputs/logparser/testdata/test-patterns +++ b/plugins/inputs/logparser/testdata/test-patterns @@ -12,3 +12,7 @@ TEST_LOG_B \[%{TEST_TIMESTAMP:timestamp:ts-"02/01/2006--15:04:05"}\] %{NUMBER:my TEST_TIMESTAMP %{MONTHDAY}/%{MONTHNUM}/%{YEAR}--%{TIME} TEST_LOG_BAD \[%{TEST_TIMESTAMP:timestamp:ts-"02/01/2006--15:04:05"}\] %{NUMBER:myfloat:float} %{WORD:mystring:int} %{WORD:dropme:drop} %{WORD:nomodifier} + +# Test C log line: +# 1568723594631 1.25 200 192.168.1.1 5.432µs 101 +TEST_LOG_C %{POSINT:timestamp:ts-epochmilli} %{NUMBER:myfloat:float} %{RESPONSE_CODE} %{IPORHOST:clientip} %{RESPONSE_TIME} %{NUMBER:myint:int} diff --git a/plugins/inputs/logparser/testdata/test_c.log b/plugins/inputs/logparser/testdata/test_c.log new file mode 100644 index 0000000000000..f814c0c30dfe8 --- /dev/null +++ b/plugins/inputs/logparser/testdata/test_c.log @@ -0,0 +1 @@ +1568723594631 1.25 200 192.168.1.1 5.432µs 101 diff --git a/plugins/inputs/mailchimp/chimp_api.go b/plugins/inputs/mailchimp/chimp_api.go index db0004ce2264f..6e4ec2d4f120e 100644 --- a/plugins/inputs/mailchimp/chimp_api.go +++ b/plugins/inputs/mailchimp/chimp_api.go @@ -134,7 +134,7 @@ func runChimp(api *ChimpAPI, params ReportsParams) ([]byte, error) { req.URL.RawQuery = params.String() req.Header.Set("User-Agent", "Telegraf-MailChimp-Plugin") if api.Debug { - log.Printf("D! Request URL: %s", req.URL.String()) + log.Printf("D! [inputs.mailchimp] request URL: %s", req.URL.String()) } resp, err := client.Do(req) @@ -148,7 +148,7 @@ func runChimp(api *ChimpAPI, params ReportsParams) ([]byte, error) { return nil, err } if api.Debug { - log.Printf("D! Response Body:%s", string(body)) + log.Printf("D! [inputs.mailchimp] response Body: %q", string(body)) } if err = chimpErrorCheck(body); err != nil { diff --git a/plugins/inputs/mem/README.md b/plugins/inputs/mem/README.md index 8425468256d5a..87280d8d2b299 100644 --- a/plugins/inputs/mem/README.md +++ b/plugins/inputs/mem/README.md @@ -43,6 +43,8 @@ Available fields are dependent on platform. - mapped (integer) - page_tables (integer) - shared (integer) + - sreclaimable (integer) + - sunreclaim (integer) - swap_cached (integer) - swap_free (integer) - swap_total (integer) @@ -54,5 +56,5 @@ Available fields are dependent on platform. ### Example Output: ``` -mem active=11347566592i,available=18705133568i,available_percent=89.4288960571006,buffered=1976709120i,cached=13975572480i,commit_limit=14753067008i,committed_as=2872422400i,dirty=87461888i,free=1352400896i,high_free=0i,high_total=0i,huge_page_size=2097152i,huge_pages_free=0i,huge_pages_total=0i,inactive=6201593856i,low_free=0i,low_total=0i,mapped=310427648i,page_tables=14397440i,shared=200781824i,slab=1937526784i,swap_cached=0i,swap_free=4294963200i,swap_total=4294963200i,total=20916207616i,used=3611525120i,used_percent=17.26663449848977,vmalloc_chunk=0i,vmalloc_total=35184372087808i,vmalloc_used=0i,wired=0i,write_back=0i,write_back_tmp=0i 1536704085000000000 +mem active=9299595264i,available=16818249728i,available_percent=80.41654254645131,buffered=2383761408i,cached=13316689920i,commit_limit=14751920128i,committed_as=11781156864i,dirty=122880i,free=1877688320i,high_free=0i,high_total=0i,huge_page_size=2097152i,huge_pages_free=0i,huge_pages_total=0i,inactive=7549939712i,low_free=0i,low_total=0i,mapped=416763904i,page_tables=19787776i,shared=670679040i,slab=2081071104i,sreclaimable=1923395584i,sunreclaim=157675520i,swap_cached=1302528i,swap_free=4286128128i,swap_total=4294963200i,total=20913917952i,used=3335778304i,used_percent=15.95004011996231,vmalloc_chunk=0i,vmalloc_total=35184372087808i,vmalloc_used=0i,wired=0i,write_back=0i,write_back_tmp=0i 1574712869000000000 ``` diff --git a/plugins/inputs/mem/memory.go b/plugins/inputs/mem/memory.go index a7d887cbe8bec..daae390b81077 100644 --- a/plugins/inputs/mem/memory.go +++ b/plugins/inputs/mem/memory.go @@ -50,6 +50,8 @@ func (s *MemStats) Gather(acc telegraf.Accumulator) error { "mapped": vm.Mapped, "page_tables": vm.PageTables, "shared": vm.Shared, + "sreclaimable": vm.SReclaimable, + "sunreclaim": vm.SUnreclaim, "swap_cached": vm.SwapCached, "swap_free": vm.SwapFree, "swap_total": vm.SwapTotal, diff --git a/plugins/inputs/mem/memory_test.go b/plugins/inputs/mem/memory_test.go index 06f2f6ea97fd0..653010fa8d795 100644 --- a/plugins/inputs/mem/memory_test.go +++ b/plugins/inputs/mem/memory_test.go @@ -40,6 +40,8 @@ func TestMemStats(t *testing.T) { Mapped: 42236, PageTables: 1236, Shared: 0, + SReclaimable: 1923022848, + SUnreclaim: 157728768, SwapCached: 0, SwapFree: 524280, SwapTotal: 524280, @@ -81,6 +83,8 @@ func TestMemStats(t *testing.T) { "mapped": uint64(42236), "page_tables": uint64(1236), "shared": uint64(0), + "sreclaimable": uint64(1923022848), + "sunreclaim": uint64(157728768), "swap_cached": uint64(0), "swap_free": uint64(524280), "swap_total": uint64(524280), diff --git a/plugins/inputs/mesos/README.md b/plugins/inputs/mesos/README.md index b9a46eaa9a632..2845881880d95 100644 --- a/plugins/inputs/mesos/README.md +++ b/plugins/inputs/mesos/README.md @@ -10,8 +10,10 @@ For more information, please check the [Mesos Observability Metrics](http://meso [[inputs.mesos]] ## Timeout, in ms. timeout = 100 + ## A list of Mesos masters. masters = ["http://localhost:5050"] + ## Master metrics groups to be collected, by default, all enabled. master_collections = [ "resources", @@ -26,8 +28,10 @@ For more information, please check the [Mesos Observability Metrics](http://meso "registrar", "allocator", ] + ## A list of Mesos slaves, default is [] # slaves = [] + ## Slave metrics groups to be collected, by default, all enabled. # slave_collections = [ # "resources", diff --git a/plugins/inputs/mesos/mesos.go b/plugins/inputs/mesos/mesos.go index 3e0e256915646..741dd73dc443e 100644 --- a/plugins/inputs/mesos/mesos.go +++ b/plugins/inputs/mesos/mesos.go @@ -32,9 +32,10 @@ type Mesos struct { MasterCols []string `toml:"master_collections"` Slaves []string SlaveCols []string `toml:"slave_collections"` - //SlaveTasks bool tls.ClientConfig + Log telegraf.Logger + initialized bool client *http.Client masterURLs []*url.URL @@ -49,8 +50,10 @@ var allMetrics = map[Role][]string{ var sampleConfig = ` ## Timeout, in ms. timeout = 100 + ## A list of Mesos masters. masters = ["http://localhost:5050"] + ## Master metrics groups to be collected, by default, all enabled. master_collections = [ "resources", @@ -65,8 +68,10 @@ var sampleConfig = ` "registrar", "allocator", ] + ## A list of Mesos slaves, default is [] # slaves = [] + ## Slave metrics groups to be collected, by default, all enabled. # slave_collections = [ # "resources", @@ -110,7 +115,7 @@ func parseURL(s string, role Role) (*url.URL, error) { } s = "http://" + host + ":" + port - log.Printf("W! [inputs.mesos] Using %q as connection URL; please update your configuration to use an URL", s) + log.Printf("W! [inputs.mesos] using %q as connection URL; please update your configuration to use an URL", s) } return url.Parse(s) @@ -126,7 +131,7 @@ func (m *Mesos) initialize() error { } if m.Timeout == 0 { - log.Println("I! [inputs.mesos] Missing timeout value, setting default value (100ms)") + m.Log.Info("Missing timeout value, setting default value (100ms)") m.Timeout = 100 } @@ -191,17 +196,6 @@ func (m *Mesos) Gather(acc telegraf.Accumulator) error { wg.Done() return }(slave) - - // if !m.SlaveTasks { - // continue - // } - - // wg.Add(1) - // go func(c string) { - // acc.AddError(m.gatherSlaveTaskMetrics(slave, acc)) - // wg.Done() - // return - // }(v) } wg.Wait() @@ -487,7 +481,7 @@ func getMetrics(role Role, group string) []string { ret, ok := m[group] if !ok { - log.Printf("I! [mesos] Unknown %s metrics group: %s\n", role, group) + log.Printf("I! [inputs.mesos] unknown role %q metrics group: %s", role, group) return []string{} } diff --git a/plugins/inputs/mesos/mesos_test.go b/plugins/inputs/mesos/mesos_test.go index 066d5b971f6a4..e25f250c8f8d4 100644 --- a/plugins/inputs/mesos/mesos_test.go +++ b/plugins/inputs/mesos/mesos_test.go @@ -349,6 +349,7 @@ func TestMesosMaster(t *testing.T) { var acc testutil.Accumulator m := Mesos{ + Log: testutil.Logger{}, Masters: []string{masterTestServer.Listener.Addr().String()}, Timeout: 10, } @@ -364,6 +365,7 @@ func TestMesosMaster(t *testing.T) { func TestMasterFilter(t *testing.T) { m := Mesos{ + Log: testutil.Logger{}, MasterCols: []string{ "resources", "master", "registrar", "allocator", }, @@ -416,6 +418,7 @@ func TestMesosSlave(t *testing.T) { var acc testutil.Accumulator m := Mesos{ + Log: testutil.Logger{}, Masters: []string{}, Slaves: []string{slaveTestServer.Listener.Addr().String()}, // SlaveTasks: true, @@ -433,6 +436,7 @@ func TestMesosSlave(t *testing.T) { func TestSlaveFilter(t *testing.T) { m := Mesos{ + Log: testutil.Logger{}, SlaveCols: []string{ "resources", "agent", "tasks", }, diff --git a/plugins/inputs/mongodb/README.md b/plugins/inputs/mongodb/README.md index 5772f4fc31a48..5449e3b4b5dc4 100644 --- a/plugins/inputs/mongodb/README.md +++ b/plugins/inputs/mongodb/README.md @@ -55,6 +55,7 @@ by running Telegraf with the `--debug` argument. - mongodb - tags: - hostname + - node_type - fields: - active_reads (integer) - active_writes (integer) @@ -74,8 +75,14 @@ by running Telegraf with the `--debug` argument. - flushes (integer) - flushes_total_time_ns (integer) - getmores (integer) - - inserts (integer + - inserts (integer) - jumbo_chunks (integer) + - latency_commands_count (integer) + - latency_commands (integer) + - latency_reads_count (integer) + - latency_reads (integer) + - latency_writes_count (integer) + - latency_writes (integer) - member_status (string) - net_in_bytes_count (integer) - net_out_bytes_count (integer) @@ -102,6 +109,7 @@ by running Telegraf with the `--debug` argument. - ttl_deletes (integer) - ttl_passes (integer) - updates (integer) + - uptime_ns (integer) - vsize_megabytes (integer) - wtcache_app_threads_page_read_count (integer) - wtcache_app_threads_page_read_time (integer) @@ -183,7 +191,8 @@ by running Telegraf with the `--debug` argument. ### Example Output: ``` -mongodb,hostname=127.0.0.1:27017 active_reads=0i,active_writes=0i,commands=1335i,commands_per_sec=7i,connections_available=814i,connections_current=5i,connections_total_created=0i,cursor_no_timeout=0i,cursor_no_timeout_count=0i,cursor_pinned=0i,cursor_pinned_count=1i,cursor_timed_out=0i,cursor_timed_out_count=0i,cursor_total=0i,cursor_total_count=1i,deletes=0i,deletes_per_sec=0i,document_deleted=0i,document_inserted=0i,document_returned=13i,document_updated=0i,flushes=5i,flushes_per_sec=0i,getmores=269i,getmores_per_sec=0i,inserts=0i,inserts_per_sec=0i,jumbo_chunks=0i,member_status="PRI",net_in_bytes=986i,net_in_bytes_count=358006i,net_out_bytes=23906i,net_out_bytes_count=661507i,open_connections=5i,percent_cache_dirty=0,percent_cache_used=0,queries=18i,queries_per_sec=3i,queued_reads=0i,queued_writes=0i,repl_commands=0i,repl_commands_per_sec=0i,repl_deletes=0i,repl_deletes_per_sec=0i,repl_getmores=0i,repl_getmores_per_sec=0i,repl_inserts=0i,repl_inserts_per_sec=0i,repl_lag=0i,repl_oplog_window_sec=24355215i,repl_queries=0i,repl_queries_per_sec=0i,repl_updates=0i,repl_updates_per_sec=0i,resident_megabytes=62i,state="PRIMARY",total_available=0i,total_created=0i,total_in_use=0i,total_refreshing=0i,ttl_deletes=0i,ttl_deletes_per_sec=0i,ttl_passes=23i,ttl_passes_per_sec=0i,updates=0i,updates_per_sec=0i,vsize_megabytes=713i,wtcache_app_threads_page_read_count=13i,wtcache_app_threads_page_read_time=74i,wtcache_app_threads_page_write_count=0i,wtcache_bytes_read_into=55271i,wtcache_bytes_written_from=125402i,wtcache_current_bytes=117050i,wtcache_max_bytes_configured=1073741824i,wtcache_pages_evicted_by_app_thread=0i,wtcache_pages_queued_for_eviction=0i,wtcache_server_evicting_pages=0i,wtcache_tracked_dirty_bytes=0i,wtcache_worker_thread_evictingpages=0i 1547159491000000000 +mongodb,hostname=127.0.0.1:27017 active_reads=0i,active_writes=0i,commands=1335i,commands_per_sec=7i,connections_available=814i,connections_current=5i,connections_total_created=0i,cursor_no_timeout=0i,cursor_no_timeout_count=0i,cursor_pinned=0i,cursor_pinned_count=1i,cursor_timed_out=0i,cursor_timed_out_count=0i,cursor_total=0i,cursor_total_count=1i,deletes=0i,deletes_per_sec=0i,document_deleted=0i,document_inserted=0i,document_returned=13i,document_updated=0i,flushes=5i,flushes_per_sec=0i,getmores=269i,getmores_per_sec=0i,inserts=0i,inserts_per_sec=0i,jumbo_chunks=0i,latency_commands_count=0i,latency_commands=0i,latency_reads_count=0i,latency_reads=0i,latency_writes_count=0i,latency_writes=0i,member_status="PRI",net_in_bytes=986i,net_in_bytes_count=358006i,net_out_bytes=23906i,net_out_bytes_count=661507i,open_connections=5i,percent_cache_dirty=0,percent_cache_used=0,queries=18i,queries_per_sec=3i,queued_reads=0i,queued_writes=0i,repl_commands=0i,repl_commands_per_sec=0i,repl_deletes=0i,repl_deletes_per_sec=0i,repl_getmores=0i,repl_getmores_per_sec=0i,repl_inserts=0i,repl_inserts_per_sec=0i,repl_lag=0i,repl_oplog_window_sec=24355215i,repl_queries=0i,repl_queries_per_sec=0i,repl_updates=0i,repl_updates_per_sec=0i,resident_megabytes=62i,state="PRIMARY",total_available=0i,total_created=0i,total_in_use=0i,total_refreshing=0i,ttl_deletes=0i,ttl_deletes_per_sec=0i,ttl_passes=23i,ttl_passes_per_sec=0i,updates=0i,updates_per_sec=0i,vsize_megabytes=713i,wtcache_app_threads_page_read_count=13i,wtcache_app_threads_page_read_time=74i,wtcache_app_threads_page_write_count=0i,wtcache_bytes_read_into=55271i,wtcache_bytes_written_from=125402i,wtcache_current_bytes=117050i,wtcache_max_bytes_configured=1073741824i,wtcache_pages_evicted_by_app_thread=0i,wtcache_pages_queued_for_eviction=0i,wtcache_server_evicting_pages=0i,wtcache_tracked_dirty_bytes=0i,wtcache_worker_thread_evictingpages=0i 1547159491000000000 +mongodb,hostname=127.0.0.1:27017,node_type=PRI active_reads=0i,active_writes=0i,commands=1335i,commands_per_sec=7i,connections_available=814i,connections_current=5i,connections_total_created=0i,cursor_no_timeout=0i,cursor_no_timeout_count=0i,cursor_pinned=0i,cursor_pinned_count=1i,cursor_timed_out=0i,cursor_timed_out_count=0i,cursor_total=0i,cursor_total_count=1i,deletes=0i,deletes_per_sec=0i,document_deleted=0i,document_inserted=0i,document_returned=13i,document_updated=0i,flushes=5i,flushes_per_sec=0i,getmores=269i,getmores_per_sec=0i,inserts=0i,inserts_per_sec=0i,jumbo_chunks=0i,member_status="PRI",net_in_bytes=986i,net_in_bytes_count=358006i,net_out_bytes=23906i,net_out_bytes_count=661507i,open_connections=5i,percent_cache_dirty=0,percent_cache_used=0,queries=18i,queries_per_sec=3i,queued_reads=0i,queued_writes=0i,repl_commands=0i,repl_commands_per_sec=0i,repl_deletes=0i,repl_deletes_per_sec=0i,repl_getmores=0i,repl_getmores_per_sec=0i,repl_inserts=0i,repl_inserts_per_sec=0i,repl_lag=0i,repl_oplog_window_sec=24355215i,repl_queries=0i,repl_queries_per_sec=0i,repl_updates=0i,repl_updates_per_sec=0i,resident_megabytes=62i,state="PRIMARY",total_available=0i,total_created=0i,total_in_use=0i,total_refreshing=0i,ttl_deletes=0i,ttl_deletes_per_sec=0i,ttl_passes=23i,ttl_passes_per_sec=0i,updates=0i,updates_per_sec=0i,vsize_megabytes=713i,wtcache_app_threads_page_read_count=13i,wtcache_app_threads_page_read_time=74i,wtcache_app_threads_page_write_count=0i,wtcache_bytes_read_into=55271i,wtcache_bytes_written_from=125402i,wtcache_current_bytes=117050i,wtcache_max_bytes_configured=1073741824i,wtcache_pages_evicted_by_app_thread=0i,wtcache_pages_queued_for_eviction=0i,wtcache_server_evicting_pages=0i,wtcache_tracked_dirty_bytes=0i,wtcache_worker_thread_evictingpages=0i 1547159491000000000 mongodb_db_stats,db_name=admin,hostname=127.0.0.1:27017 avg_obj_size=241,collections=2i,data_size=723i,index_size=49152i,indexes=3i,num_extents=0i,objects=3i,ok=1i,storage_size=53248i,type="db_stat" 1547159491000000000 mongodb_db_stats,db_name=local,hostname=127.0.0.1:27017 avg_obj_size=813.9705882352941,collections=6i,data_size=55350i,index_size=102400i,indexes=5i,num_extents=0i,objects=68i,ok=1i,storage_size=204800i,type="db_stat" 1547159491000000000 mongodb_col_stats,collection=foo,db_name=local,hostname=127.0.0.1:27017 size=375005928i,avg_obj_size=5494,type="col_stat",storage_size=249307136i,total_index_size=2138112i,ok=1i,count=68251i 1547159491000000000 diff --git a/plugins/inputs/mongodb/mongodb.go b/plugins/inputs/mongodb/mongodb.go index 14fcce12e763f..967ccbe5f5c81 100644 --- a/plugins/inputs/mongodb/mongodb.go +++ b/plugins/inputs/mongodb/mongodb.go @@ -4,7 +4,6 @@ import ( "crypto/tls" "crypto/x509" "fmt" - "log" "net" "net/url" "strings" @@ -25,6 +24,8 @@ type MongoDB struct { GatherColStats bool ColStatsDbs []string tlsint.ClientConfig + + Log telegraf.Logger } type Ssl struct { @@ -82,24 +83,27 @@ func (m *MongoDB) Gather(acc telegraf.Accumulator) error { // Preserve backwards compatibility for hostnames without a // scheme, broken in go 1.8. Remove in Telegraf 2.0 serv = "mongodb://" + serv - log.Printf("W! [inputs.mongodb] Using %q as connection URL; please update your configuration to use an URL", serv) + m.Log.Warnf("Using %q as connection URL; please update your configuration to use an URL", serv) m.Servers[i] = serv } u, err := url.Parse(serv) if err != nil { - acc.AddError(fmt.Errorf("Unable to parse address %q: %s", serv, err)) + m.Log.Errorf("Unable to parse address %q: %s", serv, err.Error()) continue } if u.Host == "" { - acc.AddError(fmt.Errorf("Unable to parse address %q", serv)) + m.Log.Errorf("Unable to parse address %q", serv) continue } wg.Add(1) go func(srv *Server) { defer wg.Done() - acc.AddError(m.gatherServer(srv, acc)) + err := m.gatherServer(srv, acc) + if err != nil { + m.Log.Errorf("Error in plugin: %v", err) + } }(m.getMongoServer(u)) } @@ -110,6 +114,7 @@ func (m *MongoDB) Gather(acc telegraf.Accumulator) error { func (m *MongoDB) getMongoServer(url *url.URL) *Server { if _, ok := m.mongos[url.Host]; !ok { m.mongos[url.Host] = &Server{ + Log: m.Log, Url: url, } } @@ -126,8 +131,7 @@ func (m *MongoDB) gatherServer(server *Server, acc telegraf.Accumulator) error { } dialInfo, err := mgo.ParseURL(dialAddrs[0]) if err != nil { - return fmt.Errorf("Unable to parse URL (%s), %s\n", - dialAddrs[0], err.Error()) + return fmt.Errorf("unable to parse URL %q: %s", dialAddrs[0], err.Error()) } dialInfo.Direct = true dialInfo.Timeout = 5 * time.Second @@ -169,7 +173,7 @@ func (m *MongoDB) gatherServer(server *Server, acc telegraf.Accumulator) error { sess, err := mgo.DialWithInfo(dialInfo) if err != nil { - return fmt.Errorf("Unable to connect to MongoDB, %s\n", err.Error()) + return fmt.Errorf("unable to connect to MongoDB: %s", err.Error()) } server.Session = sess } diff --git a/plugins/inputs/mongodb/mongodb_data.go b/plugins/inputs/mongodb/mongodb_data.go index 6f999cbd77afe..09bacdae19fca 100644 --- a/plugins/inputs/mongodb/mongodb_data.go +++ b/plugins/inputs/mongodb/mongodb_data.go @@ -38,6 +38,7 @@ func NewMongodbData(statLine *StatLine, tags map[string]string) *MongodbData { } var DefaultStats = map[string]string{ + "uptime_ns": "UptimeNanos", "inserts": "InsertCnt", "inserts_per_sec": "Insert", "queries": "QueryCnt", @@ -85,6 +86,15 @@ var DefaultStats = map[string]string{ "connections_total_created": "TotalCreatedC", } +var DefaultLatencyStats = map[string]string{ + "latency_writes_count": "WriteOpsCnt", + "latency_writes": "WriteLatency", + "latency_reads_count": "ReadOpsCnt", + "latency_reads": "ReadLatency", + "latency_commands_count": "CommandOpsCnt", + "latency_commands": "CommandLatency", +} + var DefaultReplStats = map[string]string{ "repl_inserts": "InsertRCnt", "repl_inserts_per_sec": "InsertR", @@ -228,6 +238,11 @@ func (d *MongodbData) AddDefaultStats() { d.addStat(statLine, DefaultStats) if d.StatLine.NodeType != "" { d.addStat(statLine, DefaultReplStats) + d.Tags["node_type"] = d.StatLine.NodeType + } + + if d.StatLine.ReadLatency > 0 { + d.addStat(statLine, DefaultLatencyStats) } if d.StatLine.OplogStats != nil { @@ -246,6 +261,7 @@ func (d *MongodbData) AddDefaultStats() { d.add(key, floatVal) } d.addStat(statLine, WiredTigerExtStats) + d.add("page_faults", d.StatLine.FaultsCnt) } } diff --git a/plugins/inputs/mongodb/mongodb_data_test.go b/plugins/inputs/mongodb/mongodb_data_test.go index 527e7ab93e9f5..711b3eef24857 100644 --- a/plugins/inputs/mongodb/mongodb_data_test.go +++ b/plugins/inputs/mongodb/mongodb_data_test.go @@ -16,6 +16,7 @@ func TestAddNonReplStats(t *testing.T) { &StatLine{ StorageEngine: "", Time: time.Now(), + UptimeNanos: 0, Insert: 0, Query: 0, Update: 0, @@ -99,6 +100,7 @@ func TestAddWiredTigerStats(t *testing.T) { PagesQueuedForEviction: 0, ServerEvictingPages: 0, WorkerThreadEvictingPages: 0, + FaultsCnt: 204, }, tags, ) @@ -115,6 +117,8 @@ func TestAddWiredTigerStats(t *testing.T) { for key := range WiredTigerExtStats { assert.True(t, acc.HasFloatField("mongodb", key) || acc.HasInt64Field("mongodb", key), key) } + + assert.True(t, acc.HasInt64Field("mongodb", "page_faults")) } func TestAddShardStats(t *testing.T) { @@ -138,6 +142,29 @@ func TestAddShardStats(t *testing.T) { } } +func TestAddLatencyStats(t *testing.T) { + d := NewMongodbData( + &StatLine{ + CommandOpsCnt: 73, + CommandLatency: 364, + ReadOpsCnt: 113, + ReadLatency: 201, + WriteOpsCnt: 7, + WriteLatency: 55, + }, + tags, + ) + + var acc testutil.Accumulator + + d.AddDefaultStats() + d.flush(&acc) + + for key := range DefaultLatencyStats { + assert.True(t, acc.HasInt64Field("mongodb", key)) + } +} + func TestAddShardHostStats(t *testing.T) { expectedHosts := []string{"hostA", "hostB"} hostStatLines := map[string]ShardHostStatLine{} @@ -189,6 +216,7 @@ func TestStateTag(t *testing.T) { ) stateTags := make(map[string]string) + stateTags["node_type"] = "PRI" var acc testutil.Accumulator @@ -235,6 +263,7 @@ func TestStateTag(t *testing.T) { "resident_megabytes": int64(0), "updates": int64(0), "updates_per_sec": int64(0), + "uptime_ns": int64(0), "vsize_megabytes": int64(0), "ttl_deletes": int64(0), "ttl_deletes_per_sec": int64(0), diff --git a/plugins/inputs/mongodb/mongodb_server.go b/plugins/inputs/mongodb/mongodb_server.go index e6e66a2a4aac0..be3916b5ea2b6 100644 --- a/plugins/inputs/mongodb/mongodb_server.go +++ b/plugins/inputs/mongodb/mongodb_server.go @@ -1,7 +1,7 @@ package mongodb import ( - "log" + "fmt" "net/url" "strings" "time" @@ -15,6 +15,8 @@ type Server struct { Url *url.URL Session *mgo.Session lastResult *MongoStatus + + Log telegraf.Logger } func (s *Server) getDefaultTags() map[string]string { @@ -31,11 +33,11 @@ func IsAuthorization(err error) bool { return strings.Contains(err.Error(), "not authorized") } -func authLogLevel(err error) string { +func (s *Server) authLog(err error) { if IsAuthorization(err) { - return "D!" + s.Log.Debug(err.Error()) } else { - return "E!" + s.Log.Error(err.Error()) } } @@ -158,30 +160,30 @@ func (s *Server) gatherCollectionStats(colStatsDbs []string) (*ColStats, error) } results := &ColStats{} - for _, db_name := range names { - if stringInSlice(db_name, colStatsDbs) || len(colStatsDbs) == 0 { + for _, dbName := range names { + if stringInSlice(dbName, colStatsDbs) || len(colStatsDbs) == 0 { var colls []string - colls, err = s.Session.DB(db_name).CollectionNames() + colls, err = s.Session.DB(dbName).CollectionNames() if err != nil { - log.Printf("E! [inputs.mongodb] Error getting collection names: %v", err) + s.Log.Errorf("Error getting collection names: %s", err.Error()) continue } - for _, col_name := range colls { - col_stat_line := &ColStatsData{} - err = s.Session.DB(db_name).Run(bson.D{ + for _, colName := range colls { + colStatLine := &ColStatsData{} + err = s.Session.DB(dbName).Run(bson.D{ { Name: "collStats", - Value: col_name, + Value: colName, }, - }, col_stat_line) + }, colStatLine) if err != nil { - log.Printf("%s [inputs.mongodb] Error getting col stats from %q: %v", authLogLevel(err), col_name, err) + s.authLog(fmt.Errorf("error getting col stats from %q: %v", colName, err)) continue } collection := &Collection{ - Name: col_name, - DbName: db_name, - ColStatsData: col_stat_line, + Name: colName, + DbName: dbName, + ColStatsData: colStatLine, } results.Collections = append(results.Collections, *collection) } @@ -203,7 +205,7 @@ func (s *Server) gatherData(acc telegraf.Accumulator, gatherDbStats bool, gather // member of a replica set. replSetStatus, err := s.gatherReplSetStatus() if err != nil { - log.Printf("D! [inputs.mongodb] Unable to gather replica set status: %v", err) + s.Log.Debugf("Unable to gather replica set status: %s", err.Error()) } // Gather the oplog if we are a member of a replica set. Non-replica set @@ -212,19 +214,18 @@ func (s *Server) gatherData(acc telegraf.Accumulator, gatherDbStats bool, gather if replSetStatus != nil { oplogStats, err = s.gatherOplogStats() if err != nil { - return err + s.authLog(fmt.Errorf("Unable to get oplog stats: %v", err)) } } clusterStatus, err := s.gatherClusterStatus() if err != nil { - log.Printf("D! [inputs.mongodb] Unable to gather cluster status: %v", err) + s.Log.Debugf("Unable to gather cluster status: %s", err.Error()) } shardStats, err := s.gatherShardConnPoolStats() if err != nil { - log.Printf("%s [inputs.mongodb] Unable to gather shard connection pool stats: %v", - authLogLevel(err), err) + s.authLog(fmt.Errorf("unable to gather shard connection pool stats: %s", err.Error())) } var collectionStats *ColStats @@ -246,7 +247,7 @@ func (s *Server) gatherData(acc telegraf.Accumulator, gatherDbStats bool, gather for _, name := range names { db, err := s.gatherDBStats(name) if err != nil { - log.Printf("D! [inputs.mongodb] Error getting db stats from %q: %v", name, err) + s.Log.Debugf("Error getting db stats from %q: %s", name, err.Error()) } dbStats.Dbs = append(dbStats.Dbs, *db) } diff --git a/plugins/inputs/mongodb/mongostat.go b/plugins/inputs/mongodb/mongostat.go index 44c071a2f526f..985627c87d1d4 100644 --- a/plugins/inputs/mongodb/mongostat.go +++ b/plugins/inputs/mongodb/mongostat.go @@ -58,6 +58,7 @@ type ServerStatus struct { Network *NetworkStats `bson:"network"` Opcounters *OpcountStats `bson:"opcounters"` OpcountersRepl *OpcountStats `bson:"opcountersRepl"` + OpLatencies *OpLatenciesStats `bson:"opLatencies"` RecordStats *DBRecordStats `bson:"recordStats"` Mem *MemStats `bson:"mem"` Repl *ReplStatus `bson:"repl"` @@ -252,7 +253,7 @@ type FlushStats struct { type ConnectionStats struct { Current int64 `bson:"current"` Available int64 `bson:"available"` - TotalCreated int64 `bson:"total_created"` + TotalCreated int64 `bson:"totalCreated"` } // DurTiming stores information related to journaling. @@ -314,6 +315,19 @@ type OpcountStats struct { Command int64 `bson:"command"` } +// OpLatenciesStats stores information related to operation latencies for the database as a whole +type OpLatenciesStats struct { + Reads *LatencyStats `bson:"reads"` + Writes *LatencyStats `bson:"writes"` + Commands *LatencyStats `bson:"commands"` +} + +// LatencyStats lists total latency in microseconds and count of operations, enabling you to obtain an average +type LatencyStats struct { + Latency int64 `bson:"latency"` + Ops int64 `bson:"ops"` +} + // MetricsStats stores information related to metrics type MetricsStats struct { TTL *TTLStats `bson:"ttl"` @@ -477,6 +491,8 @@ type StatLine struct { IsMongos bool Host string + UptimeNanos int64 + // The time at which this StatLine was generated. Time time.Time @@ -491,6 +507,14 @@ type StatLine struct { GetMore, GetMoreCnt int64 Command, CommandCnt int64 + // OpLatency fields + WriteOpsCnt int64 + WriteLatency int64 + ReadOpsCnt int64 + ReadLatency int64 + CommandOpsCnt int64 + CommandLatency int64 + // TTL fields Passes, PassesCnt int64 DeletedDocuments, DeletedDocumentsCnt int64 @@ -659,6 +683,8 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec Faults: -1, } + returnVal.UptimeNanos = 1000 * 1000 * newStat.UptimeMillis + // set connection info returnVal.CurrentC = newStat.Connections.Current returnVal.AvailableC = newStat.Connections.Available @@ -680,6 +706,21 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec returnVal.Command, returnVal.CommandCnt = diff(newStat.Opcounters.Command, oldStat.Opcounters.Command, sampleSecs) } + if newStat.OpLatencies != nil { + if newStat.OpLatencies.Reads != nil { + returnVal.ReadOpsCnt = newStat.OpLatencies.Reads.Ops + returnVal.ReadLatency = newStat.OpLatencies.Reads.Latency + } + if newStat.OpLatencies.Writes != nil { + returnVal.WriteOpsCnt = newStat.OpLatencies.Writes.Ops + returnVal.WriteLatency = newStat.OpLatencies.Writes.Latency + } + if newStat.OpLatencies.Commands != nil { + returnVal.CommandOpsCnt = newStat.OpLatencies.Commands.Ops + returnVal.CommandLatency = newStat.OpLatencies.Commands.Latency + } + } + if newStat.Metrics != nil && oldStat.Metrics != nil { if newStat.Metrics.TTL != nil && oldStat.Metrics.TTL != nil { returnVal.Passes, returnVal.PassesCnt = diff(newStat.Metrics.TTL.Passes, oldStat.Metrics.TTL.Passes, sampleSecs) @@ -983,21 +1024,23 @@ func NewStatLine(oldMongo, newMongo MongoStatus, key string, all bool, sampleSec } // Set shard stats - newShardStats := *newMongo.ShardStats - returnVal.TotalInUse = newShardStats.TotalInUse - returnVal.TotalAvailable = newShardStats.TotalAvailable - returnVal.TotalCreated = newShardStats.TotalCreated - returnVal.TotalRefreshing = newShardStats.TotalRefreshing - returnVal.ShardHostStatsLines = map[string]ShardHostStatLine{} - for host, stats := range newShardStats.Hosts { - shardStatLine := &ShardHostStatLine{ - InUse: stats.InUse, - Available: stats.Available, - Created: stats.Created, - Refreshing: stats.Refreshing, - } + if newMongo.ShardStats != nil { + newShardStats := *newMongo.ShardStats + returnVal.TotalInUse = newShardStats.TotalInUse + returnVal.TotalAvailable = newShardStats.TotalAvailable + returnVal.TotalCreated = newShardStats.TotalCreated + returnVal.TotalRefreshing = newShardStats.TotalRefreshing + returnVal.ShardHostStatsLines = map[string]ShardHostStatLine{} + for host, stats := range newShardStats.Hosts { + shardStatLine := &ShardHostStatLine{ + InUse: stats.InUse, + Available: stats.Available, + Created: stats.Created, + Refreshing: stats.Refreshing, + } - returnVal.ShardHostStatsLines[host] = *shardStatLine + returnVal.ShardHostStatsLines[host] = *shardStatLine + } } return returnVal diff --git a/plugins/inputs/mongodb/mongostat_test.go b/plugins/inputs/mongodb/mongostat_test.go new file mode 100644 index 0000000000000..5506602a9e692 --- /dev/null +++ b/plugins/inputs/mongodb/mongostat_test.go @@ -0,0 +1,205 @@ +package mongodb + +import ( + "testing" + //"time" + + //"github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/assert" +) + +func TestLatencyStats(t *testing.T) { + + sl := NewStatLine( + MongoStatus{ + ServerStatus: &ServerStatus{ + Connections: &ConnectionStats{}, + Mem: &MemStats{ + Bits: 0, + Resident: 0, + Virtual: 0, + Supported: false, + Mapped: 0, + MappedWithJournal: 0, + }, + }, + }, + MongoStatus{ + ServerStatus: &ServerStatus{ + Connections: &ConnectionStats{}, + Mem: &MemStats{ + Bits: 0, + Resident: 0, + Virtual: 0, + Supported: false, + Mapped: 0, + MappedWithJournal: 0, + }, + OpLatencies: &OpLatenciesStats{ + Reads: &LatencyStats{ + Ops: 0, + Latency: 0, + }, + Writes: &LatencyStats{ + Ops: 0, + Latency: 0, + }, + Commands: &LatencyStats{ + Ops: 0, + Latency: 0, + }, + }, + }, + }, + "foo", + true, + 60, + ) + + assert.Equal(t, sl.CommandLatency, int64(0)) + assert.Equal(t, sl.ReadLatency, int64(0)) + assert.Equal(t, sl.WriteLatency, int64(0)) + assert.Equal(t, sl.CommandOpsCnt, int64(0)) + assert.Equal(t, sl.ReadOpsCnt, int64(0)) + assert.Equal(t, sl.WriteOpsCnt, int64(0)) +} + +func TestLatencyStatsDiffZero(t *testing.T) { + + sl := NewStatLine( + MongoStatus{ + ServerStatus: &ServerStatus{ + Connections: &ConnectionStats{}, + Mem: &MemStats{ + Bits: 0, + Resident: 0, + Virtual: 0, + Supported: false, + Mapped: 0, + MappedWithJournal: 0, + }, + OpLatencies: &OpLatenciesStats{ + Reads: &LatencyStats{ + Ops: 0, + Latency: 0, + }, + Writes: &LatencyStats{ + Ops: 0, + Latency: 0, + }, + Commands: &LatencyStats{ + Ops: 0, + Latency: 0, + }, + }, + }, + }, + MongoStatus{ + ServerStatus: &ServerStatus{ + Connections: &ConnectionStats{}, + Mem: &MemStats{ + Bits: 0, + Resident: 0, + Virtual: 0, + Supported: false, + Mapped: 0, + MappedWithJournal: 0, + }, + OpLatencies: &OpLatenciesStats{ + Reads: &LatencyStats{ + Ops: 0, + Latency: 0, + }, + Writes: &LatencyStats{ + Ops: 0, + Latency: 0, + }, + Commands: &LatencyStats{ + Ops: 0, + Latency: 0, + }, + }, + }, + }, + "foo", + true, + 60, + ) + + assert.Equal(t, sl.CommandLatency, int64(0)) + assert.Equal(t, sl.ReadLatency, int64(0)) + assert.Equal(t, sl.WriteLatency, int64(0)) + assert.Equal(t, sl.CommandOpsCnt, int64(0)) + assert.Equal(t, sl.ReadOpsCnt, int64(0)) + assert.Equal(t, sl.WriteOpsCnt, int64(0)) +} + +func TestLatencyStatsDiff(t *testing.T) { + + sl := NewStatLine( + MongoStatus{ + ServerStatus: &ServerStatus{ + Connections: &ConnectionStats{}, + Mem: &MemStats{ + Bits: 0, + Resident: 0, + Virtual: 0, + Supported: false, + Mapped: 0, + MappedWithJournal: 0, + }, + OpLatencies: &OpLatenciesStats{ + Reads: &LatencyStats{ + Ops: 4189041956, + Latency: 2255922322753, + }, + Writes: &LatencyStats{ + Ops: 1691019457, + Latency: 494478256915, + }, + Commands: &LatencyStats{ + Ops: 1019150402, + Latency: 59177710371, + }, + }, + }, + }, + MongoStatus{ + ServerStatus: &ServerStatus{ + Connections: &ConnectionStats{}, + Mem: &MemStats{ + Bits: 0, + Resident: 0, + Virtual: 0, + Supported: false, + Mapped: 0, + MappedWithJournal: 0, + }, + OpLatencies: &OpLatenciesStats{ + Reads: &LatencyStats{ + Ops: 4189049884, + Latency: 2255946760057, + }, + Writes: &LatencyStats{ + Ops: 1691021287, + Latency: 494479456987, + }, + Commands: &LatencyStats{ + Ops: 1019152861, + Latency: 59177981552, + }, + }, + }, + }, + "foo", + true, + 60, + ) + + assert.Equal(t, sl.CommandLatency, int64(59177981552)) + assert.Equal(t, sl.ReadLatency, int64(2255946760057)) + assert.Equal(t, sl.WriteLatency, int64(494479456987)) + assert.Equal(t, sl.CommandOpsCnt, int64(1019152861)) + assert.Equal(t, sl.ReadOpsCnt, int64(4189049884)) + assert.Equal(t, sl.WriteOpsCnt, int64(1691021287)) +} diff --git a/plugins/inputs/mqtt_consumer/README.md b/plugins/inputs/mqtt_consumer/README.md index 9e60679f65f39..ddb5a073bf76d 100644 --- a/plugins/inputs/mqtt_consumer/README.md +++ b/plugins/inputs/mqtt_consumer/README.md @@ -45,7 +45,7 @@ and creates metrics using one of the supported [input data formats][]. # max_undelivered_messages = 1000 ## Persistent session disables clearing of the client session on connection. - ## In order for this option to work you must also set client_id to identity + ## In order for this option to work you must also set client_id to identify ## the client. To receive messages that arrived while the client is offline, ## also set the qos option to 1 or 2 and don't forget to also set the QoS when ## publishing. diff --git a/plugins/inputs/mqtt_consumer/mqtt_consumer.go b/plugins/inputs/mqtt_consumer/mqtt_consumer.go index 7e3b43d44e22e..5f54f4bb47552 100644 --- a/plugins/inputs/mqtt_consumer/mqtt_consumer.go +++ b/plugins/inputs/mqtt_consumer/mqtt_consumer.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "log" "strings" "time" @@ -61,6 +60,8 @@ type MQTTConsumer struct { ClientID string `toml:"client_id"` tls.ClientConfig + Log telegraf.Logger + clientFactory ClientFactory client Client opts *mqtt.ClientOptions @@ -113,7 +114,7 @@ var sampleConfig = ` # max_undelivered_messages = 1000 ## Persistent session disables clearing of the client session on connection. - ## In order for this option to work you must also set client_id to identity + ## In order for this option to work you must also set client_id to identify ## the client. To receive messages that arrived while the client is offline, ## also set the qos option to 1 or 2 and don't forget to also set the QoS when ## publishing. @@ -186,6 +187,7 @@ func (m *MQTTConsumer) Start(acc telegraf.Accumulator) error { m.state = Disconnected m.acc = acc.WithTracking(m.MaxUndeliveredMessages) + m.sem = make(semaphore, m.MaxUndeliveredMessages) m.ctx, m.cancel = context.WithCancel(context.Background()) m.client = m.clientFactory(m.opts) @@ -212,9 +214,8 @@ func (m *MQTTConsumer) connect() error { return err } - log.Printf("I! [inputs.mqtt_consumer] Connected %v", m.Servers) + m.Log.Infof("Connected %v", m.Servers) m.state = Connected - m.sem = make(semaphore, m.MaxUndeliveredMessages) m.messages = make(map[telegraf.TrackingID]bool) // Presistent sessions should skip subscription if a session is present, as @@ -223,7 +224,7 @@ func (m *MQTTConsumer) connect() error { SessionPresent() bool } if t, ok := token.(sessionPresent); ok && t.SessionPresent() { - log.Printf("D! [inputs.mqtt_consumer] Session found %v", m.Servers) + m.Log.Debugf("Session found %v", m.Servers) return nil } @@ -244,7 +245,7 @@ func (m *MQTTConsumer) connect() error { func (m *MQTTConsumer) onConnectionLost(c mqtt.Client, err error) { m.acc.AddError(fmt.Errorf("connection lost: %v", err)) - log.Printf("D! [inputs.mqtt_consumer] Disconnected %v", m.Servers) + m.Log.Debugf("Disconnected %v", m.Servers) m.state = Disconnected return } @@ -253,12 +254,12 @@ func (m *MQTTConsumer) recvMessage(c mqtt.Client, msg mqtt.Message) { for { select { case track := <-m.acc.Delivered(): + <-m.sem _, ok := m.messages[track.ID()] if !ok { // Added by a previous connection continue } - <-m.sem // No ack, MQTT does not support durable handling delete(m.messages, track.ID()) case m.sem <- empty{}: @@ -292,9 +293,9 @@ func (m *MQTTConsumer) onMessage(acc telegraf.TrackingAccumulator, msg mqtt.Mess func (m *MQTTConsumer) Stop() { if m.state == Connected { - log.Printf("D! [inputs.mqtt_consumer] Disconnecting %v", m.Servers) + m.Log.Debugf("Disconnecting %v", m.Servers) m.client.Disconnect(200) - log.Printf("D! [inputs.mqtt_consumer] Disconnected %v", m.Servers) + m.Log.Debugf("Disconnected %v", m.Servers) m.state = Disconnected } m.cancel() @@ -303,7 +304,7 @@ func (m *MQTTConsumer) Stop() { func (m *MQTTConsumer) Gather(acc telegraf.Accumulator) error { if m.state == Disconnected { m.state = Connecting - log.Printf("D! [inputs.mqtt_consumer] Connecting %v", m.Servers) + m.Log.Debugf("Connecting %v", m.Servers) m.connect() } @@ -346,7 +347,7 @@ func (m *MQTTConsumer) createOpts() (*mqtt.ClientOptions, error) { for _, server := range m.Servers { // Preserve support for host:port style servers; deprecated in Telegraf 1.4.4 if !strings.Contains(server, "://") { - log.Printf("W! [inputs.mqtt_consumer] Server %q should be updated to use `scheme://host:port` format", server) + m.Log.Warnf("Server %q should be updated to use `scheme://host:port` format", server) if tlsCfg == nil { server = "tcp://" + server } else { diff --git a/plugins/inputs/mqtt_consumer/mqtt_consumer_test.go b/plugins/inputs/mqtt_consumer/mqtt_consumer_test.go index cbc6ee9869a1c..4884fc0508107 100644 --- a/plugins/inputs/mqtt_consumer/mqtt_consumer_test.go +++ b/plugins/inputs/mqtt_consumer/mqtt_consumer_test.go @@ -102,6 +102,7 @@ func TestLifecycleSanity(t *testing.T) { }, } }) + plugin.Log = testutil.Logger{} plugin.Servers = []string{"tcp://127.0.0.1"} parser := &FakeParser{} @@ -124,10 +125,12 @@ func TestRandomClientID(t *testing.T) { var err error m1 := New(nil) + m1.Log = testutil.Logger{} err = m1.Init() require.NoError(t, err) m2 := New(nil) + m2.Log = testutil.Logger{} err = m2.Init() require.NoError(t, err) @@ -137,6 +140,7 @@ func TestRandomClientID(t *testing.T) { // PersistentSession requires ClientID func TestPersistentClientIDFail(t *testing.T) { plugin := New(nil) + plugin.Log = testutil.Logger{} plugin.PersistentSession = true err := plugin.Init() @@ -255,6 +259,7 @@ func TestTopicTag(t *testing.T) { plugin := New(func(o *mqtt.ClientOptions) Client { return client }) + plugin.Log = testutil.Logger{} plugin.Topics = []string{"telegraf"} plugin.TopicTag = tt.topicTag() @@ -295,6 +300,7 @@ func TestAddRouteCalledForEachTopic(t *testing.T) { plugin := New(func(o *mqtt.ClientOptions) Client { return client }) + plugin.Log = testutil.Logger{} plugin.Topics = []string{"a", "b"} err := plugin.Init() @@ -325,6 +331,7 @@ func TestSubscribeCalledIfNoSession(t *testing.T) { plugin := New(func(o *mqtt.ClientOptions) Client { return client }) + plugin.Log = testutil.Logger{} plugin.Topics = []string{"b"} err := plugin.Init() @@ -355,6 +362,7 @@ func TestSubscribeNotCalledIfSession(t *testing.T) { plugin := New(func(o *mqtt.ClientOptions) Client { return client }) + plugin.Log = testutil.Logger{} plugin.Topics = []string{"b"} err := plugin.Init() diff --git a/plugins/inputs/mysql/README.md b/plugins/inputs/mysql/README.md index 564d75e614046..3e07229da7e15 100644 --- a/plugins/inputs/mysql/README.md +++ b/plugins/inputs/mysql/README.md @@ -21,10 +21,9 @@ This plugin gathers the statistic data from MySQL server ### Configuration ```toml -# Read metrics from one or many mysql servers [[inputs.mysql]] ## specify servers via a url matching: - ## [username[:password]@][protocol[(address)]]/[?tls=[true|false|skip-verify]] + ## [username[:password]@][protocol[(address)]]/[?tls=[true|false|skip-verify|custom]] ## see https://github.com/go-sql-driver/mysql#dsn-data-source-name ## e.g. ## servers = ["user:passwd@tcp(127.0.0.1:3306)/?tls=false"] @@ -32,60 +31,80 @@ This plugin gathers the statistic data from MySQL server # ## If no servers are specified, then localhost is used as the host. servers = ["tcp(127.0.0.1:3306)/"] - ## the limits for metrics form perf_events_statements - perf_events_statements_digest_text_limit = 120 - perf_events_statements_limit = 250 - perf_events_statements_time_limit = 86400 - # - ## if the list is empty, then metrics are gathered from all database tables - table_schema_databases = [] - # + + ## Selects the metric output format. + ## + ## This option exists to maintain backwards compatibility, if you have + ## existing metrics do not set or change this value until you are ready to + ## migrate to the new format. + ## + ## If you do not have existing metrics from this plugin set to the latest + ## version. + ## + ## Telegraf >=1.6: metric_version = 2 + ## <1.6: metric_version = 1 (or unset) + metric_version = 2 + + ## if the list is empty, then metrics are gathered from all databasee tables + # table_schema_databases = [] + ## gather metrics from INFORMATION_SCHEMA.TABLES for databases provided above list - gather_table_schema = false - # + # gather_table_schema = false + ## gather thread state counts from INFORMATION_SCHEMA.PROCESSLIST - gather_process_list = true - # - ## gather thread state counts from INFORMATION_SCHEMA.USER_STATISTICS - gather_user_statistics = true - # + # gather_process_list = false + + ## gather user statistics from INFORMATION_SCHEMA.USER_STATISTICS + # gather_user_statistics = false + ## gather auto_increment columns and max values from information schema - gather_info_schema_auto_inc = true - # + # gather_info_schema_auto_inc = false + ## gather metrics from INFORMATION_SCHEMA.INNODB_METRICS - gather_innodb_metrics = true - # + # gather_innodb_metrics = false + ## gather metrics from SHOW SLAVE STATUS command output - gather_slave_status = true - # + # gather_slave_status = false + ## gather metrics from SHOW BINARY LOGS command output - gather_binary_logs = false - # + # gather_binary_logs = false + + ## gather metrics from SHOW GLOBAL VARIABLES command output + # gather_global_variables = true + ## gather metrics from PERFORMANCE_SCHEMA.TABLE_IO_WAITS_SUMMARY_BY_TABLE - gather_table_io_waits = false - # + # gather_table_io_waits = false + ## gather metrics from PERFORMANCE_SCHEMA.TABLE_LOCK_WAITS - gather_table_lock_waits = false - # + # gather_table_lock_waits = false + ## gather metrics from PERFORMANCE_SCHEMA.TABLE_IO_WAITS_SUMMARY_BY_INDEX_USAGE - gather_index_io_waits = false - # + # gather_index_io_waits = false + ## gather metrics from PERFORMANCE_SCHEMA.EVENT_WAITS - gather_event_waits = false - # + # gather_event_waits = false + ## gather metrics from PERFORMANCE_SCHEMA.FILE_SUMMARY_BY_EVENT_NAME - gather_file_events_stats = false - # + # gather_file_events_stats = false + ## gather metrics from PERFORMANCE_SCHEMA.EVENTS_STATEMENTS_SUMMARY_BY_DIGEST - gather_perf_events_statements = false - # + # gather_perf_events_statements = false + + ## the limits for metrics form perf_events_statements + # perf_events_statements_digest_text_limit = 120 + # perf_events_statements_limit = 250 + # perf_events_statements_time_limit = 86400 + ## Some queries we may want to run less often (such as SHOW GLOBAL VARIABLES) - interval_slow = "30m" + ## example: interval_slow = "30m" + # interval_slow = "" ## Optional TLS Config (will be used if tls=custom parameter specified in server uri) - tls_ca = "/etc/telegraf/ca.pem" - tls_cert = "/etc/telegraf/cert.pem" - tls_key = "/etc/telegraf/key.pem" + # tls_ca = "/etc/telegraf/ca.pem" + # tls_cert = "/etc/telegraf/cert.pem" + # tls_key = "/etc/telegraf/key.pem" + ## Use TLS but skip chain & host verification + # insecure_skip_verify = false ``` #### Metric Version diff --git a/plugins/inputs/mysql/mysql.go b/plugins/inputs/mysql/mysql.go index 0516e22b73bf8..a2dc56505692a 100644 --- a/plugins/inputs/mysql/mysql.go +++ b/plugins/inputs/mysql/mysql.go @@ -9,12 +9,12 @@ import ( "sync" "time" + "github.com/go-sql-driver/mysql" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal/tls" "github.com/influxdata/telegraf/plugins/inputs" "github.com/influxdata/telegraf/plugins/inputs/mysql/v1" - - "github.com/go-sql-driver/mysql" + "github.com/influxdata/telegraf/plugins/inputs/mysql/v2" ) type Mysql struct { @@ -36,12 +36,18 @@ type Mysql struct { GatherTableSchema bool `toml:"gather_table_schema"` GatherFileEventsStats bool `toml:"gather_file_events_stats"` GatherPerfEventsStatements bool `toml:"gather_perf_events_statements"` + GatherGlobalVars bool `toml:"gather_global_variables"` IntervalSlow string `toml:"interval_slow"` MetricVersion int `toml:"metric_version"` + + Log telegraf.Logger `toml:"-"` tls.ClientConfig + lastT time.Time + initDone bool + scanIntervalSlow uint32 } -var sampleConfig = ` +const sampleConfig = ` ## specify servers via a url matching: ## [username[:password]@][protocol[(address)]]/[?tls=[true|false|skip-verify|custom]] ## see https://github.com/go-sql-driver/mysql#dsn-data-source-name @@ -65,55 +71,59 @@ var sampleConfig = ` ## <1.6: metric_version = 1 (or unset) metric_version = 2 - ## the limits for metrics form perf_events_statements - perf_events_statements_digest_text_limit = 120 - perf_events_statements_limit = 250 - perf_events_statements_time_limit = 86400 - # ## if the list is empty, then metrics are gathered from all databasee tables - table_schema_databases = [] - # + # table_schema_databases = [] + ## gather metrics from INFORMATION_SCHEMA.TABLES for databases provided above list - gather_table_schema = false - # + # gather_table_schema = false + ## gather thread state counts from INFORMATION_SCHEMA.PROCESSLIST - gather_process_list = true - # + # gather_process_list = false + ## gather user statistics from INFORMATION_SCHEMA.USER_STATISTICS - gather_user_statistics = true - # + # gather_user_statistics = false + ## gather auto_increment columns and max values from information schema - gather_info_schema_auto_inc = true - # + # gather_info_schema_auto_inc = false + ## gather metrics from INFORMATION_SCHEMA.INNODB_METRICS - gather_innodb_metrics = true - # + # gather_innodb_metrics = false + ## gather metrics from SHOW SLAVE STATUS command output - gather_slave_status = true - # + # gather_slave_status = false + ## gather metrics from SHOW BINARY LOGS command output - gather_binary_logs = false - # + # gather_binary_logs = false + + ## gather metrics from PERFORMANCE_SCHEMA.GLOBAL_VARIABLES + # gather_global_variables = true + ## gather metrics from PERFORMANCE_SCHEMA.TABLE_IO_WAITS_SUMMARY_BY_TABLE - gather_table_io_waits = false - # + # gather_table_io_waits = false + ## gather metrics from PERFORMANCE_SCHEMA.TABLE_LOCK_WAITS - gather_table_lock_waits = false - # + # gather_table_lock_waits = false + ## gather metrics from PERFORMANCE_SCHEMA.TABLE_IO_WAITS_SUMMARY_BY_INDEX_USAGE - gather_index_io_waits = false - # + # gather_index_io_waits = false + ## gather metrics from PERFORMANCE_SCHEMA.EVENT_WAITS - gather_event_waits = false - # + # gather_event_waits = false + ## gather metrics from PERFORMANCE_SCHEMA.FILE_SUMMARY_BY_EVENT_NAME - gather_file_events_stats = false - # + # gather_file_events_stats = false + ## gather metrics from PERFORMANCE_SCHEMA.EVENTS_STATEMENTS_SUMMARY_BY_DIGEST - gather_perf_events_statements = false - # + # gather_perf_events_statements = false + + ## the limits for metrics form perf_events_statements + # perf_events_statements_digest_text_limit = 120 + # perf_events_statements_limit = 250 + # perf_events_statements_time_limit = 86400 + ## Some queries we may want to run less often (such as SHOW GLOBAL VARIABLES) - interval_slow = "30m" + ## example: interval_slow = "30m" + # interval_slow = "" ## Optional TLS Config (will be used if tls=custom parameter specified in server uri) # tls_ca = "/etc/telegraf/ca.pem" @@ -123,7 +133,13 @@ var sampleConfig = ` # insecure_skip_verify = false ` -var defaultTimeout = time.Second * time.Duration(5) +const ( + defaultTimeout = 5 * time.Second + defaultPerfEventsStatementsDigestTextLimit = 120 + defaultPerfEventsStatementsLimit = 250 + defaultPerfEventsStatementsTimeLimit = 86400 + defaultGatherGlobalVars = true +) func (m *Mysql) SampleConfig() string { return sampleConfig @@ -133,21 +149,16 @@ func (m *Mysql) Description() string { return "Read metrics from one or many mysql servers" } -var ( - localhost = "" - lastT time.Time - initDone = false - scanIntervalSlow uint32 -) +const localhost = "" func (m *Mysql) InitMysql() { if len(m.IntervalSlow) > 0 { interval, err := time.ParseDuration(m.IntervalSlow) if err == nil && interval.Seconds() >= 1.0 { - scanIntervalSlow = uint32(interval.Seconds()) + m.scanIntervalSlow = uint32(interval.Seconds()) } } - initDone = true + m.initDone = true } func (m *Mysql) Gather(acc telegraf.Accumulator) error { @@ -156,7 +167,7 @@ func (m *Mysql) Gather(acc telegraf.Accumulator) error { return m.gatherServer(localhost, acc) } // Initialise additional query intervals - if !initDone { + if !m.initDone { m.InitMysql() } @@ -184,6 +195,7 @@ func (m *Mysql) Gather(acc telegraf.Accumulator) error { return nil } +// These are const but can't be declared as such because golang doesn't allow const maps var ( // status counter generalThreadStates = map[string]uint32{ @@ -424,14 +436,16 @@ func (m *Mysql) gatherServer(serv string, acc telegraf.Accumulator) error { return err } - // Global Variables may be gathered less often - if len(m.IntervalSlow) > 0 { - if uint32(time.Since(lastT).Seconds()) >= scanIntervalSlow { - err = m.gatherGlobalVariables(db, serv, acc) - if err != nil { - return err + if m.GatherGlobalVars { + // Global Variables may be gathered less often + if len(m.IntervalSlow) > 0 { + if uint32(time.Since(m.lastT).Seconds()) >= m.scanIntervalSlow { + err = m.gatherGlobalVariables(db, serv, acc) + if err != nil { + return err + } + m.lastT = time.Now() } - lastT = time.Now() } } @@ -550,14 +564,20 @@ func (m *Mysql) gatherGlobalVariables(db *sql.DB, serv string, acc telegraf.Accu return err } key = strings.ToLower(key) + // parse mysql version and put into field and tag if strings.Contains(key, "version") { fields[key] = string(val) tags[key] = string(val) } - if value, ok := m.parseValue(val); ok { + + value, err := m.parseGlobalVariables(key, val) + if err != nil { + m.Log.Debugf("Error parsing global variable %q: %v", key, err) + } else { fields[key] = value } + // Send 20 fields at a time if len(fields) >= 20 { acc.AddFields("mysql_variables", fields, tags) @@ -571,6 +591,18 @@ func (m *Mysql) gatherGlobalVariables(db *sql.DB, serv string, acc telegraf.Accu return nil } +func (m *Mysql) parseGlobalVariables(key string, value sql.RawBytes) (interface{}, error) { + if m.MetricVersion < 2 { + v, ok := v1.ParseValue(value) + if ok { + return v, nil + } + return v, fmt.Errorf("could not parse value: %q", string(value)) + } else { + return v2.ConvertGlobalVariables(key, value) + } +} + // gatherSlaveStatuses can be used to get replication analytics // When the server is slave, then it returns only one row. // If the multi-source replication is set, then everything works differently @@ -744,7 +776,10 @@ func (m *Mysql) gatherGlobalStatuses(db *sql.DB, serv string, acc telegraf.Accum } } else { key = strings.ToLower(key) - if value, ok := m.parseValue(val); ok { + value, err := v2.ConvertGlobalStatus(key, val) + if err != nil { + m.Log.Debugf("Error parsing global status: %v", err) + } else { fields[key] = value } } @@ -1735,6 +1770,11 @@ func getDSNTag(dsn string) string { func init() { inputs.Add("mysql", func() telegraf.Input { - return &Mysql{} + return &Mysql{ + PerfEventsStatementsDigestTextLimit: defaultPerfEventsStatementsDigestTextLimit, + PerfEventsStatementsLimit: defaultPerfEventsStatementsLimit, + PerfEventsStatementsTimeLimit: defaultPerfEventsStatementsTimeLimit, + GatherGlobalVars: defaultGatherGlobalVars, + } }) } diff --git a/plugins/inputs/mysql/mysql_test.go b/plugins/inputs/mysql/mysql_test.go index b4983ba0e028f..be9c338bf7b0e 100644 --- a/plugins/inputs/mysql/mysql_test.go +++ b/plugins/inputs/mysql/mysql_test.go @@ -26,6 +26,54 @@ func TestMysqlDefaultsToLocal(t *testing.T) { assert.True(t, acc.HasMeasurement("mysql")) } +func TestMysqlMultipleInstances(t *testing.T) { + // Invoke Gather() from two separate configurations and + // confirm they don't interfere with each other + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + testServer := "root@tcp(127.0.0.1:3306)/?tls=false" + m := &Mysql{ + Servers: []string{testServer}, + IntervalSlow: "30s", + } + + var acc, acc2 testutil.Accumulator + err := m.Gather(&acc) + require.NoError(t, err) + assert.True(t, acc.HasMeasurement("mysql")) + // acc should have global variables + assert.True(t, acc.HasMeasurement("mysql_variables")) + + m2 := &Mysql{ + Servers: []string{testServer}, + } + err = m2.Gather(&acc2) + require.NoError(t, err) + assert.True(t, acc2.HasMeasurement("mysql")) + // acc2 should not have global variables + assert.False(t, acc2.HasMeasurement("mysql_variables")) +} + +func TestMysqlMultipleInits(t *testing.T) { + m := &Mysql{ + IntervalSlow: "30s", + } + m2 := &Mysql{} + + m.InitMysql() + assert.True(t, m.initDone) + assert.False(t, m2.initDone) + assert.Equal(t, m.scanIntervalSlow, uint32(30)) + assert.Equal(t, m2.scanIntervalSlow, uint32(0)) + + m2.InitMysql() + assert.True(t, m.initDone) + assert.True(t, m2.initDone) + assert.Equal(t, m.scanIntervalSlow, uint32(30)) + assert.Equal(t, m2.scanIntervalSlow, uint32(0)) +} + func TestMysqlGetDSNTag(t *testing.T) { tests := []struct { input string diff --git a/plugins/inputs/mysql/v2/convert.go b/plugins/inputs/mysql/v2/convert.go new file mode 100644 index 0000000000000..a3ac3e976d6a3 --- /dev/null +++ b/plugins/inputs/mysql/v2/convert.go @@ -0,0 +1,103 @@ +package v2 + +import ( + "bytes" + "database/sql" + "fmt" + "strconv" +) + +type ConversionFunc func(value sql.RawBytes) (interface{}, error) + +func ParseInt(value sql.RawBytes) (interface{}, error) { + v, err := strconv.ParseInt(string(value), 10, 64) + + // Ignore ErrRange. When this error is set the returned value is "the + // maximum magnitude integer of the appropriate bitSize and sign." + if err, ok := err.(*strconv.NumError); ok && err.Err == strconv.ErrRange { + return v, nil + } + + return v, err +} + +func ParseBoolAsInteger(value sql.RawBytes) (interface{}, error) { + if bytes.EqualFold(value, []byte("YES")) || bytes.EqualFold(value, []byte("ON")) { + return int64(1), nil + } + + return int64(0), nil +} + +func ParseGTIDMode(value sql.RawBytes) (interface{}, error) { + // https://dev.mysql.com/doc/refman/8.0/en/replication-mode-change-online-concepts.html + v := string(value) + switch v { + case "OFF": + return int64(0), nil + case "ON": + return int64(1), nil + case "OFF_PERMISSIVE": + return int64(0), nil + case "ON_PERMISSIVE": + return int64(1), nil + default: + return nil, fmt.Errorf("unrecognized gtid_mode: %q", v) + } +} + +func ParseValue(value sql.RawBytes) (interface{}, error) { + if bytes.EqualFold(value, []byte("YES")) || bytes.Compare(value, []byte("ON")) == 0 { + return 1, nil + } + + if bytes.EqualFold(value, []byte("NO")) || bytes.Compare(value, []byte("OFF")) == 0 { + return 0, nil + } + + if val, err := strconv.ParseInt(string(value), 10, 64); err == nil { + return val, nil + } + if val, err := strconv.ParseFloat(string(value), 64); err == nil { + return val, nil + } + + if len(string(value)) > 0 { + return string(value), nil + } + + return nil, fmt.Errorf("unconvertible value: %q", string(value)) +} + +var GlobalStatusConversions = map[string]ConversionFunc{ + "ssl_ctx_verify_depth": ParseInt, + "ssl_verify_depth": ParseInt, +} + +var GlobalVariableConversions = map[string]ConversionFunc{ + "gtid_mode": ParseGTIDMode, +} + +func ConvertGlobalStatus(key string, value sql.RawBytes) (interface{}, error) { + if bytes.Equal(value, []byte("")) { + return nil, nil + } + + if conv, ok := GlobalStatusConversions[key]; ok { + return conv(value) + } + + return ParseValue(value) +} + +func ConvertGlobalVariables(key string, value sql.RawBytes) (interface{}, error) { + if bytes.Equal(value, []byte("")) { + return nil, nil + } + + if conv, ok := GlobalVariableConversions[key]; ok { + return conv(value) + } + + return ParseValue(value) +} diff --git a/plugins/inputs/mysql/v2/convert_test.go b/plugins/inputs/mysql/v2/convert_test.go new file mode 100644 index 0000000000000..47189c18d1576 --- /dev/null +++ b/plugins/inputs/mysql/v2/convert_test.go @@ -0,0 +1,86 @@ +package v2 + +import ( + "database/sql" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestConvertGlobalStatus(t *testing.T) { + tests := []struct { + name string + key string + value sql.RawBytes + expected interface{} + expectedErr error + }{ + { + name: "default", + key: "ssl_ctx_verify_depth", + value: []byte("0"), + expected: int64(0), + expectedErr: nil, + }, + { + name: "overflow int64", + key: "ssl_ctx_verify_depth", + value: []byte("18446744073709551615"), + expected: int64(9223372036854775807), + expectedErr: nil, + }, + { + name: "defined variable but unset", + key: "ssl_ctx_verify_depth", + value: []byte(""), + expected: nil, + expectedErr: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual, err := ConvertGlobalStatus(tt.key, tt.value) + require.Equal(t, tt.expectedErr, err) + require.Equal(t, tt.expected, actual) + }) + } +} + +func TestCovertGlobalVariables(t *testing.T) { + tests := []struct { + name string + key string + value sql.RawBytes + expected interface{} + expectedErr error + }{ + { + name: "boolean type mysql<=5.6", + key: "gtid_mode", + value: []byte("ON"), + expected: int64(1), + expectedErr: nil, + }, + { + name: "enum type mysql>=5.7", + key: "gtid_mode", + value: []byte("ON_PERMISSIVE"), + expected: int64(1), + expectedErr: nil, + }, + { + name: "defined variable but unset", + key: "ssl_ctx_verify_depth", + value: []byte(""), + expected: nil, + expectedErr: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual, err := ConvertGlobalVariables(tt.key, tt.value) + require.Equal(t, tt.expectedErr, err) + require.Equal(t, tt.expected, actual) + }) + } +} diff --git a/plugins/inputs/nats/nats.go b/plugins/inputs/nats/nats.go index ba1cc803c6d45..83e262ec8a10c 100644 --- a/plugins/inputs/nats/nats.go +++ b/plugins/inputs/nats/nats.go @@ -1,20 +1,18 @@ -// +build !freebsd +// +build !freebsd freebsd,cgo package nats import ( + "encoding/json" "io/ioutil" "net/http" "net/url" "path" "time" - "encoding/json" - "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/plugins/inputs" - gnatsd "github.com/nats-io/gnatsd/server" ) diff --git a/plugins/inputs/nats/nats_freebsd.go b/plugins/inputs/nats/nats_freebsd.go index c23a6eec5ab91..08d08ba760df0 100644 --- a/plugins/inputs/nats/nats_freebsd.go +++ b/plugins/inputs/nats/nats_freebsd.go @@ -1,3 +1,3 @@ -// +build freebsd +// +build freebsd,!cgo package nats diff --git a/plugins/inputs/nats/nats_test.go b/plugins/inputs/nats/nats_test.go index ef387f7e4a649..ece22288ff9af 100644 --- a/plugins/inputs/nats/nats_test.go +++ b/plugins/inputs/nats/nats_test.go @@ -1,4 +1,4 @@ -// +build !freebsd +// +build !freebsd freebsd,cgo package nats diff --git a/plugins/inputs/nats_consumer/README.md b/plugins/inputs/nats_consumer/README.md index 205578a17df13..7c1abab0bd3b6 100644 --- a/plugins/inputs/nats_consumer/README.md +++ b/plugins/inputs/nats_consumer/README.md @@ -12,8 +12,10 @@ instances of telegraf can read from a NATS cluster in parallel. [[inputs.nats_consumer]] ## urls of NATS servers servers = ["nats://localhost:4222"] + ## subject(s) to consume subjects = ["telegraf"] + ## name a queue group queue_group = "telegraf_consumers" diff --git a/plugins/inputs/nats_consumer/nats_consumer.go b/plugins/inputs/nats_consumer/nats_consumer.go index b82e3f3a6f05d..eff72696492cf 100644 --- a/plugins/inputs/nats_consumer/nats_consumer.go +++ b/plugins/inputs/nats_consumer/nats_consumer.go @@ -3,7 +3,6 @@ package natsconsumer import ( "context" "fmt" - "log" "sync" "github.com/influxdata/telegraf" @@ -40,6 +39,8 @@ type natsConsumer struct { Password string `toml:"password"` tls.ClientConfig + Log telegraf.Logger + // Client pending limits: PendingMessageLimit int `toml:"pending_message_limit"` PendingBytesLimit int `toml:"pending_bytes_limit"` @@ -68,6 +69,7 @@ var sampleConfig = ` ## subject(s) to consume subjects = ["telegraf"] + ## name a queue group queue_group = "telegraf_consumers" @@ -198,7 +200,7 @@ func (n *natsConsumer) Start(acc telegraf.Accumulator) error { go n.receiver(ctx) }() - log.Printf("I! Started the NATS consumer service, nats: %v, subjects: %v, queue: %v\n", + n.Log.Infof("Started the NATS consumer service, nats: %v, subjects: %v, queue: %v", n.conn.ConnectedUrl(), n.Subjects, n.QueueGroup) return nil @@ -216,21 +218,21 @@ func (n *natsConsumer) receiver(ctx context.Context) { case <-n.acc.Delivered(): <-sem case err := <-n.errs: - n.acc.AddError(err) + n.Log.Error(err) case sem <- empty{}: select { case <-ctx.Done(): return case err := <-n.errs: <-sem - n.acc.AddError(err) + n.Log.Error(err) case <-n.acc.Delivered(): <-sem <-sem case msg := <-n.in: metrics, err := n.parser.Parse(msg.Data) if err != nil { - n.acc.AddError(fmt.Errorf("subject: %s, error: %s", msg.Subject, err.Error())) + n.Log.Errorf("Subject: %s, error: %s", msg.Subject, err.Error()) <-sem continue } @@ -244,8 +246,8 @@ func (n *natsConsumer) receiver(ctx context.Context) { func (n *natsConsumer) clean() { for _, sub := range n.subs { if err := sub.Unsubscribe(); err != nil { - n.acc.AddError(fmt.Errorf("Error unsubscribing from subject %s in queue %s: %s\n", - sub.Subject, sub.Queue, err.Error())) + n.Log.Errorf("Error unsubscribing from subject %s in queue %s: %s", + sub.Subject, sub.Queue, err.Error()) } } diff --git a/plugins/inputs/net_response/README.md b/plugins/inputs/net_response/README.md index dcfb341d50dac..2c492408beef2 100644 --- a/plugins/inputs/net_response/README.md +++ b/plugins/inputs/net_response/README.md @@ -43,7 +43,6 @@ verify text in the response. - result - fields: - response_time (float, seconds) - - success (int) # success 0, failure 1 - result_code (int, success = 0, timeout = 1, connection_failed = 2, read_failed = 3, string_mismatch = 4) - result_type (string) **DEPRECATED in 1.7; use result tag** - string_found (boolean) **DEPRECATED in 1.4; use result tag** diff --git a/plugins/inputs/nginx_plus_api/README.md b/plugins/inputs/nginx_plus_api/README.md index e90645e4372ae..4ec63b2e80b53 100644 --- a/plugins/inputs/nginx_plus_api/README.md +++ b/plugins/inputs/nginx_plus_api/README.md @@ -29,6 +29,24 @@ Nginx Plus is a commercial version of the open source web server Nginx. The use | nginx_plus_stream_upstream_peer | nginx_plus_api_stream_upstream_peers | | nginx.stream.zone | nginx_plus_api_stream_server_zones | +### Measurements by API version + +| Measurement | API version (api_version) | +|--------------------------------------|---------------------------| +| nginx_plus_api_processes | >= 3 | +| nginx_plus_api_connections | >= 3 | +| nginx_plus_api_ssl | >= 3 | +| nginx_plus_api_http_requests | >= 3 | +| nginx_plus_api_http_server_zones | >= 3 | +| nginx_plus_api_http_upstreams | >= 3 | +| nginx_plus_api_http_upstream_peers | >= 3 | +| nginx_plus_api_http_caches | >= 3 | +| nginx_plus_api_stream_upstreams | >= 3 | +| nginx_plus_api_stream_upstream_peers | >= 3 | +| nginx_plus_api_stream_server_zones | >= 3 | +| nginx_plus_api_http_location_zones | >= 5 | +| nginx_plus_api_resolver_zones | >= 5 | + ### Measurements & Fields: - nginx_plus_api_processes @@ -129,7 +147,29 @@ Nginx Plus is a commercial version of the open source web server Nginx. The use - connections - received - sent - +- nginx_plus_api_location_zones + - requests + - responses_1xx + - responses_2xx + - responses_3xx + - responses_4xx + - responses_5xx + - responses_total + - received + - sent + - discarded +- nginx_plus_api_resolver_zones + - name + - srv + - addr + - noerror + - formerr + - servfail + - nxdomain + - notimp + - refused + - timedout + - unknown ### Tags: @@ -142,7 +182,7 @@ Nginx Plus is a commercial version of the open source web server Nginx. The use - source - port -- nginx_plus_api_http_server_zones, nginx_plus_api_upstream_server_zones +- nginx_plus_api_http_server_zones, nginx_plus_api_upstream_server_zones, nginx_plus_api_http_location_zones, nginx_plus_api_resolver_zones - source - port - zone @@ -174,41 +214,40 @@ When run with: It produces: ``` -> nginx_plus_api_processes,host=localhost,port=80,source=localhost respawned=0i 1539163505000000000 -> nginx_plus_api_connections,host=localhost,port=80,source=localhost accepted=120890747i,active=6i,dropped=0i,idle=67i 1539163505000000000 -> nginx_plus_api_ssl,host=localhost,port=80,source=localhost handshakes=2983938i,handshakes_failed=54350i,session_reuses=2485267i 1539163506000000000 -> nginx_plus_api_http_requests,host=localhost,port=80,source=localhost current=12i,total=175270198i 1539163506000000000 -> nginx_plus_api_http_server_zones,host=localhost,port=80,source=localhost,zone=hg.nginx.org discarded=45i,processing=0i,received=35723884i,requests=134102i,responses_1xx=0i,responses_2xx=96890i,responses_3xx=6892i,responses_4xx=30270i,responses_5xx=5i,responses_total=134057i,sent=3681826618i 1539163506000000000 -> nginx_plus_api_http_server_zones,host=localhost,port=80,source=localhost,zone=trac.nginx.org discarded=4034i,processing=9i,received=282399663i,requests=336129i,responses_1xx=0i,responses_2xx=101264i,responses_3xx=25454i,responses_4xx=68961i,responses_5xx=136407i,responses_total=332086i,sent=2346677493i 1539163506000000000 -> nginx_plus_api_http_server_zones,host=localhost,port=80,source=localhost,zone=lxr.nginx.org discarded=4i,processing=1i,received=7223569i,requests=29661i,responses_1xx=0i,responses_2xx=28584i,responses_3xx=73i,responses_4xx=390i,responses_5xx=609i,responses_total=29656i,sent=5811238975i 1539163506000000000 -> nginx_plus_api_http_upstreams,host=localhost,port=80,source=localhost,upstream=trac-backend keepalive=0i,zombies=0i 1539163506000000000 -> nginx_plus_api_http_upstream_peers,host=localhost,id=0,port=80,source=localhost,upstream=trac-backend,upstream_address=10.0.0.1:8080 active=0i,backup=false,downtime=53870i,fails=5i,header_time=421i,healthchecks_checks=17275i,healthchecks_fails=0i,healthchecks_last_passed=true,healthchecks_unhealthy=0i,received=1885213684i,requests=88476i,response_time=423i,responses_1xx=0i,responses_2xx=50997i,responses_3xx=205i,responses_4xx=34344i,responses_5xx=2076i,responses_total=87622i,sent=189938404i,state="up",unavail=5i,weight=1i 1539163506000000000 -> nginx_plus_api_http_upstream_peers,host=localhost,id=1,port=80,source=localhost,upstream=trac-backend,upstream_address=10.0.0.1:8081 active=0i,backup=true,downtime=173957231i,fails=0i,healthchecks_checks=17394i,healthchecks_fails=17394i,healthchecks_last_passed=false,healthchecks_unhealthy=1i,received=0i,requests=0i,responses_1xx=0i,responses_2xx=0i,responses_3xx=0i,responses_4xx=0i,responses_5xx=0i,responses_total=0i,sent=0i,state="unhealthy",unavail=0i,weight=1i 1539163506000000000 -> nginx_plus_api_http_upstreams,host=localhost,port=80,source=localhost,upstream=hg-backend keepalive=0i,zombies=0i 1539163506000000000 -> nginx_plus_api_http_upstream_peers,host=localhost,id=0,port=80,source=localhost,upstream=hg-backend,upstream_address=10.0.0.1:8088 active=0i,backup=false,downtime=0i,fails=0i,header_time=22i,healthchecks_checks=17319i,healthchecks_fails=0i,healthchecks_last_passed=true,healthchecks_unhealthy=0i,received=3724240605i,requests=89563i,response_time=44i,responses_1xx=0i,responses_2xx=81996i,responses_3xx=6886i,responses_4xx=639i,responses_5xx=5i,responses_total=89526i,sent=31597952i,state="up",unavail=0i,weight=5i 1539163506000000000 -> nginx_plus_api_http_upstream_peers,host=localhost,id=1,port=80,source=localhost,upstream=hg-backend,upstream_address=10.0.0.1:8089 active=0i,backup=true,downtime=173957231i,fails=0i,healthchecks_checks=17394i,healthchecks_fails=17394i,healthchecks_last_passed=false,healthchecks_unhealthy=1i,received=0i,requests=0i,responses_1xx=0i,responses_2xx=0i,responses_3xx=0i,responses_4xx=0i,responses_5xx=0i,responses_total=0i,sent=0i,state="unhealthy",unavail=0i,weight=1i 1539163506000000000 -> nginx_plus_api_http_upstreams,host=localhost,port=80,source=localhost,upstream=lxr-backend keepalive=0i,zombies=0i 1539163506000000000 -> nginx_plus_api_http_upstream_peers,host=localhost,id=0,port=80,source=localhost,upstream=lxr-backend,upstream_address=unix:/tmp/cgi.sock active=0i,backup=false,downtime=0i,fails=609i,header_time=111i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,received=6220215064i,requests=28278i,response_time=172i,responses_1xx=0i,responses_2xx=27665i,responses_3xx=0i,responses_4xx=0i,responses_5xx=0i,responses_total=27665i,sent=21337016i,state="up",unavail=0i,weight=1i 1539163506000000000 -> nginx_plus_api_http_upstream_peers,host=localhost,id=1,port=80,source=localhost,upstream=lxr-backend,upstream_address=unix:/tmp/cgib.sock active=0i,backup=true,downtime=0i,fails=0i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,max_conns=42i,received=0i,requests=0i,responses_1xx=0i,responses_2xx=0i,responses_3xx=0i,responses_4xx=0i,responses_5xx=0i,responses_total=0i,sent=0i,state="up",unavail=0i,weight=1i 1539163506000000000 -> nginx_plus_api_http_upstreams,host=localhost,port=80,source=localhost,upstream=demo-backend keepalive=0i,zombies=0i 1539163506000000000 -> nginx_plus_api_http_upstream_peers,host=localhost,id=0,port=80,source=localhost,upstream=demo-backend,upstream_address=10.0.0.2:15431 active=0i,backup=false,downtime=0i,fails=0i,healthchecks_checks=173640i,healthchecks_fails=0i,healthchecks_last_passed=true,healthchecks_unhealthy=0i,received=0i,requests=0i,responses_1xx=0i,responses_2xx=0i,responses_3xx=0i,responses_4xx=0i,responses_5xx=0i,responses_total=0i,sent=0i,state="up",unavail=0i,weight=1i 1539163506000000000 -> nginx_plus_api_http_caches,cache=http_cache,host=localhost,port=80,source=localhost bypass_bytes=0i,bypass_bytes_written=0i,bypass_responses=0i,bypass_responses_written=0i,cold=false,expired_bytes=133671410i,expired_bytes_written=129210272i,expired_responses=15721i,expired_responses_written=15213i,hit_bytes=2459840828i,hit_responses=231195i,max_size=536870912i,miss_bytes=18742246i,miss_bytes_written=85199i,miss_responses=2816i,miss_responses_written=69i,revalidated_bytes=0i,revalidated_responses=0i,size=774144i,stale_bytes=0i,stale_responses=0i,updating_bytes=0i,updating_responses=0i 1539163506000000000 -> nginx_plus_api_stream_server_zones,host=localhost,port=80,source=localhost,zone=postgresql_loadbalancer connections=173639i,processing=0i,received=17884817i,sent=33685966i 1539163506000000000 -> nginx_plus_api_stream_server_zones,host=localhost,port=80,source=localhost,zone=dns_loadbalancer connections=97255i,processing=0i,received=2699082i,sent=16566552i 1539163506000000000 -> nginx_plus_api_stream_upstreams,host=localhost,port=80,source=localhost,upstream=postgresql_backends zombies=0i 1539163507000000000 -> nginx_plus_api_stream_upstream_peers,host=localhost,id=0,port=80,source=localhost,upstream=postgresql_backends,upstream_address=10.0.0.2:15432 active=0i,backup=false,connect_time=4i,connections=57880i,downtime=0i,fails=0i,first_byte_time=10i,healthchecks_checks=34781i,healthchecks_fails=0i,healthchecks_last_passed=true,healthchecks_unhealthy=0i,received=11228720i,response_time=10i,sent=5961640i,state="up",unavail=0i,weight=1i 1539163507000000000 -> nginx_plus_api_stream_upstream_peers,host=localhost,id=1,port=80,source=localhost,upstream=postgresql_backends,upstream_address=10.0.0.2:15433 active=0i,backup=false,connect_time=3i,connections=57880i,downtime=0i,fails=0i,first_byte_time=9i,healthchecks_checks=34781i,healthchecks_fails=0i,healthchecks_last_passed=true,healthchecks_unhealthy=0i,received=11228720i,response_time=10i,sent=5961640i,state="up",unavail=0i,weight=1i 1539163507000000000 -> nginx_plus_api_stream_upstream_peers,host=localhost,id=2,port=80,source=localhost,upstream=postgresql_backends,upstream_address=10.0.0.2:15434 active=0i,backup=false,connect_time=2i,connections=57879i,downtime=0i,fails=0i,first_byte_time=9i,healthchecks_checks=34781i,healthchecks_fails=0i,healthchecks_last_passed=true,healthchecks_unhealthy=0i,received=11228526i,response_time=9i,sent=5961537i,state="up",unavail=0i,weight=1i 1539163507000000000 -> nginx_plus_api_stream_upstream_peers,host=localhost,id=3,port=80,source=localhost,upstream=postgresql_backends,upstream_address=10.0.0.2:15435 active=0i,backup=false,connections=0i,downtime=0i,fails=0i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,received=0i,sent=0i,state="down",unavail=0i,weight=1i 1539163507000000000 -> nginx_plus_api_stream_upstreams,host=localhost,port=80,source=localhost,upstream=dns_udp_backends zombies=0i 1539163507000000000 -> nginx_plus_api_stream_upstream_peers,host=localhost,id=0,port=80,source=localhost,upstream=dns_udp_backends,upstream_address=10.0.0.5:53 active=0i,backup=false,connect_time=0i,connections=64837i,downtime=0i,fails=0i,first_byte_time=17i,healthchecks_checks=34761i,healthchecks_fails=0i,healthchecks_last_passed=true,healthchecks_unhealthy=0i,received=10996616i,response_time=17i,sent=1791693i,state="up",unavail=0i,weight=2i 1539163507000000000 -> nginx_plus_api_stream_upstream_peers,host=localhost,id=1,port=80,source=localhost,upstream=dns_udp_backends,upstream_address=10.0.0.2:53 active=0i,backup=false,connect_time=0i,connections=32418i,downtime=0i,fails=0i,first_byte_time=17i,healthchecks_checks=34761i,healthchecks_fails=0i,healthchecks_last_passed=true,healthchecks_unhealthy=0i,received=5569936i,response_time=17i,sent=907389i,state="up",unavail=0i,weight=1i 1539163507000000000 -> nginx_plus_api_stream_upstream_peers,host=localhost,id=2,port=80,source=localhost,upstream=dns_udp_backends,upstream_address=10.0.0.7:53 active=0i,backup=false,connections=0i,downtime=0i,fails=0i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,received=0i,sent=0i,state="down",unavail=0i,weight=1i 1539163507000000000 -> nginx_plus_api_stream_upstreams,host=localhost,port=80,source=localhost,upstream=unused_tcp_backends zombies=0i 1539163507000000000 -> nginx_plus_api_stream_upstream_peers,host=localhost,id=1,port=80,source=localhost,upstream=unused_tcp_backends,upstream_address=95.211.80.227:80 active=0i,backup=false,connections=0i,downtime=0i,fails=0i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,received=0i,sent=0i,state="down",unavail=0i,weight=1i 1539163507000000000 -> nginx_plus_api_stream_upstream_peers,host=localhost,id=2,port=80,source=localhost,upstream=unused_tcp_backends,upstream_address=206.251.255.63:80 active=0i,backup=false,connections=0i,downtime=0i,fails=0i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,received=0i,sent=0i,state="down",unavail=0i,weight=1i 1539163507000000000 -> nginx_plus_api_stream_upstream_peers,host=localhost,id=3,port=80,source=localhost,upstream=unused_tcp_backends,upstream_address=[2001:1af8:4060:a004:21::e3]:80 active=0i,backup=false,connections=0i,downtime=0i,fails=0i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,received=0i,sent=0i,state="down",unavail=0i,weight=1i 1539163507000000000 -> nginx_plus_api_stream_upstream_peers,host=localhost,id=4,port=80,source=localhost,upstream=unused_tcp_backends,upstream_address=[2606:7100:1:69::3f]:80 active=0i,backup=false,connections=0i,downtime=0i,fails=0i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,received=0i,sent=0i,state="down",unavail=0i,weight=1i 1539163507000000000 +> nginx_plus_api_processes,port=80,source=demo.nginx.com respawned=0i 1570696321000000000 +> nginx_plus_api_connections,port=80,source=demo.nginx.com accepted=68998606i,active=7i,dropped=0i,idle=57i 1570696322000000000 +> nginx_plus_api_ssl,port=80,source=demo.nginx.com handshakes=9398978i,handshakes_failed=289353i,session_reuses=1004389i 1570696322000000000 +> nginx_plus_api_http_requests,port=80,source=demo.nginx.com current=51i,total=264649353i 1570696322000000000 +> nginx_plus_api_http_server_zones,port=80,source=demo.nginx.com,zone=hg.nginx.org discarded=5i,processing=0i,received=24123604i,requests=60138i,responses_1xx=0i,responses_2xx=59353i,responses_3xx=531i,responses_4xx=249i,responses_5xx=0i,responses_total=60133i,sent=830165221i 1570696322000000000 +> nginx_plus_api_http_server_zones,port=80,source=demo.nginx.com,zone=trac.nginx.org discarded=250i,processing=0i,received=2184618i,requests=12404i,responses_1xx=0i,responses_2xx=8579i,responses_3xx=2513i,responses_4xx=583i,responses_5xx=479i,responses_total=12154i,sent=139384159i 1570696322000000000 +> nginx_plus_api_http_server_zones,port=80,source=demo.nginx.com,zone=lxr.nginx.org discarded=1i,processing=0i,received=1011701i,requests=4523i,responses_1xx=0i,responses_2xx=4332i,responses_3xx=28i,responses_4xx=39i,responses_5xx=123i,responses_total=4522i,sent=72631354i 1570696322000000000 +> nginx_plus_api_http_upstreams,port=80,source=demo.nginx.com,upstream=trac-backend keepalive=0i,zombies=0i 1570696322000000000 +> nginx_plus_api_http_upstream_peers,id=0,port=80,source=demo.nginx.com,upstream=trac-backend,upstream_address=10.0.0.1:8080 active=0i,backup=false,downtime=0i,fails=0i,header_time=235i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,received=88581178i,requests=3180i,response_time=235i,responses_1xx=0i,responses_2xx=3168i,responses_3xx=5i,responses_4xx=6i,responses_5xx=0i,responses_total=3179i,sent=1321720i,state="up",unavail=0i,weight=1i 1570696322000000000 +> nginx_plus_api_http_upstream_peers,id=1,port=80,source=demo.nginx.com,upstream=trac-backend,upstream_address=10.0.0.1:8081 active=0i,backup=true,downtime=0i,fails=0i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,received=0i,requests=0i,responses_1xx=0i,responses_2xx=0i,responses_3xx=0i,responses_4xx=0i,responses_5xx=0i,responses_total=0i,sent=0i,state="up",unavail=0i,weight=1i 1570696322000000000 +> nginx_plus_api_http_upstreams,port=80,source=demo.nginx.com,upstream=hg-backend keepalive=0i,zombies=0i 1570696322000000000 +> nginx_plus_api_http_upstream_peers,id=0,port=80,source=demo.nginx.com,upstream=hg-backend,upstream_address=10.0.0.1:8088 active=0i,backup=false,downtime=0i,fails=0i,header_time=22i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,received=909402572i,requests=18514i,response_time=88i,responses_1xx=0i,responses_2xx=17799i,responses_3xx=531i,responses_4xx=179i,responses_5xx=0i,responses_total=18509i,sent=10608107i,state="up",unavail=0i,weight=5i 1570696322000000000 +> nginx_plus_api_http_upstream_peers,id=1,port=80,source=demo.nginx.com,upstream=hg-backend,upstream_address=10.0.0.1:8089 active=0i,backup=true,downtime=0i,fails=0i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,received=0i,requests=0i,responses_1xx=0i,responses_2xx=0i,responses_3xx=0i,responses_4xx=0i,responses_5xx=0i,responses_total=0i,sent=0i,state="up",unavail=0i,weight=1i 1570696322000000000 +> nginx_plus_api_http_upstreams,port=80,source=demo.nginx.com,upstream=lxr-backend keepalive=0i,zombies=0i 1570696322000000000 +> nginx_plus_api_http_upstream_peers,id=0,port=80,source=demo.nginx.com,upstream=lxr-backend,upstream_address=unix:/tmp/cgi.sock active=0i,backup=false,downtime=0i,fails=123i,header_time=91i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,received=71782888i,requests=4354i,response_time=91i,responses_1xx=0i,responses_2xx=4230i,responses_3xx=0i,responses_4xx=0i,responses_5xx=0i,responses_total=4230i,sent=3088656i,state="up",unavail=0i,weight=1i 1570696322000000000 +> nginx_plus_api_http_upstream_peers,id=1,port=80,source=demo.nginx.com,upstream=lxr-backend,upstream_address=unix:/tmp/cgib.sock active=0i,backup=true,downtime=0i,fails=0i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,max_conns=42i,received=0i,requests=0i,responses_1xx=0i,responses_2xx=0i,responses_3xx=0i,responses_4xx=0i,responses_5xx=0i,responses_total=0i,sent=0i,state="up",unavail=0i,weight=1i 1570696322000000000 +> nginx_plus_api_http_upstreams,port=80,source=demo.nginx.com,upstream=demo-backend keepalive=0i,zombies=0i 1570696322000000000 +> nginx_plus_api_http_upstream_peers,id=0,port=80,source=demo.nginx.com,upstream=demo-backend,upstream_address=10.0.0.2:15431 active=0i,backup=false,downtime=0i,fails=0i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,received=0i,requests=0i,responses_1xx=0i,responses_2xx=0i,responses_3xx=0i,responses_4xx=0i,responses_5xx=0i,responses_total=0i,sent=0i,state="up",unavail=0i,weight=1i 1570696322000000000 +> nginx_plus_api_http_caches,cache=http_cache,port=80,source=demo.nginx.com bypass_bytes=0i,bypass_bytes_written=0i,bypass_responses=0i,bypass_responses_written=0i,cold=false,expired_bytes=381518640i,expired_bytes_written=363449785i,expired_responses=42114i,expired_responses_written=39954i,hit_bytes=6321885979i,hit_responses=596730i,max_size=536870912i,miss_bytes=48512185i,miss_bytes_written=155600i,miss_responses=6052i,miss_responses_written=136i,revalidated_bytes=0i,revalidated_responses=0i,size=765952i,stale_bytes=0i,stale_responses=0i,updating_bytes=0i,updating_responses=0i 1570696323000000000 +> nginx_plus_api_stream_server_zones,port=80,source=demo.nginx.com,zone=postgresql_loadbalancer connections=0i,processing=0i,received=0i,sent=0i 1570696323000000000 +> nginx_plus_api_stream_server_zones,port=80,source=demo.nginx.com,zone=dns_loadbalancer connections=0i,processing=0i,received=0i,sent=0i 1570696323000000000 +> nginx_plus_api_stream_upstreams,port=80,source=demo.nginx.com,upstream=postgresql_backends zombies=0i 1570696323000000000 +> nginx_plus_api_stream_upstream_peers,id=0,port=80,source=demo.nginx.com,upstream=postgresql_backends,upstream_address=10.0.0.2:15432 active=0i,backup=false,connections=0i,downtime=0i,fails=0i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,received=0i,sent=0i,state="up",unavail=0i,weight=1i 1570696323000000000 +> nginx_plus_api_stream_upstream_peers,id=1,port=80,source=demo.nginx.com,upstream=postgresql_backends,upstream_address=10.0.0.2:15433 active=0i,backup=false,connections=0i,downtime=0i,fails=0i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,received=0i,sent=0i,state="up",unavail=0i,weight=1i 1570696323000000000 +> nginx_plus_api_stream_upstream_peers,id=2,port=80,source=demo.nginx.com,upstream=postgresql_backends,upstream_address=10.0.0.2:15434 active=0i,backup=false,connections=0i,downtime=0i,fails=0i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,received=0i,sent=0i,state="up",unavail=0i,weight=1i 1570696323000000000 +> nginx_plus_api_stream_upstream_peers,id=3,port=80,source=demo.nginx.com,upstream=postgresql_backends,upstream_address=10.0.0.2:15435 active=0i,backup=false,connections=0i,downtime=0i,fails=0i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,received=0i,sent=0i,state="down",unavail=0i,weight=1i 1570696323000000000 +> nginx_plus_api_stream_upstreams,port=80,source=demo.nginx.com,upstream=dns_udp_backends zombies=0i 1570696323000000000 +> nginx_plus_api_stream_upstream_peers,id=0,port=80,source=demo.nginx.com,upstream=dns_udp_backends,upstream_address=10.0.0.5:53 active=0i,backup=false,connections=0i,downtime=0i,fails=0i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,received=0i,sent=0i,state="up",unavail=0i,weight=2i 1570696323000000000 +> nginx_plus_api_stream_upstream_peers,id=1,port=80,source=demo.nginx.com,upstream=dns_udp_backends,upstream_address=10.0.0.2:53 active=0i,backup=false,connections=0i,downtime=0i,fails=0i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,received=0i,sent=0i,state="up",unavail=0i,weight=1i 1570696323000000000 +> nginx_plus_api_stream_upstream_peers,id=2,port=80,source=demo.nginx.com,upstream=dns_udp_backends,upstream_address=10.0.0.7:53 active=0i,backup=false,connections=0i,downtime=0i,fails=0i,healthchecks_checks=0i,healthchecks_fails=0i,healthchecks_unhealthy=0i,received=0i,sent=0i,state="down",unavail=0i,weight=1i 1570696323000000000 +> nginx_plus_api_stream_upstreams,port=80,source=demo.nginx.com,upstream=unused_tcp_backends zombies=0i 1570696323000000000 +> nginx_plus_api_http_location_zones,port=80,source=demo.nginx.com,zone=swagger discarded=0i,received=1622i,requests=8i,responses_1xx=0i,responses_2xx=7i,responses_3xx=0i,responses_4xx=1i,responses_5xx=0i,responses_total=8i,sent=638333i 1570696323000000000 +> nginx_plus_api_http_location_zones,port=80,source=demo.nginx.com,zone=api-calls discarded=64i,received=337530181i,requests=1726513i,responses_1xx=0i,responses_2xx=1726428i,responses_3xx=0i,responses_4xx=21i,responses_5xx=0i,responses_total=1726449i,sent=1902577668i 1570696323000000000 +> nginx_plus_api_resolver_zones,port=80,source=demo.nginx.com,zone=resolver1 addr=0i,formerr=0i,name=0i,noerror=0i,notimp=0i,nxdomain=0i,refused=0i,servfail=0i,srv=0i,timedout=0i,unknown=0i 1570696324000000000 ``` ### Reference material diff --git a/plugins/inputs/nginx_plus_api/nginx_plus_api.go b/plugins/inputs/nginx_plus_api/nginx_plus_api.go index 3487dd512aa1e..addb813e32625 100644 --- a/plugins/inputs/nginx_plus_api/nginx_plus_api.go +++ b/plugins/inputs/nginx_plus_api/nginx_plus_api.go @@ -31,10 +31,13 @@ const ( connectionsPath = "connections" sslPath = "ssl" - httpRequestsPath = "http/requests" - httpServerZonesPath = "http/server_zones" - httpUpstreamsPath = "http/upstreams" - httpCachesPath = "http/caches" + httpRequestsPath = "http/requests" + httpServerZonesPath = "http/server_zones" + httpLocationZonesPath = "http/location_zones" + httpUpstreamsPath = "http/upstreams" + httpCachesPath = "http/caches" + + resolverZonesPath = "resolvers" streamServerZonesPath = "stream/server_zones" streamUpstreamsPath = "stream/upstreams" diff --git a/plugins/inputs/nginx_plus_api/nginx_plus_api_metrics.go b/plugins/inputs/nginx_plus_api/nginx_plus_api_metrics.go index 1936591c94579..6aaaff2d344c7 100644 --- a/plugins/inputs/nginx_plus_api/nginx_plus_api_metrics.go +++ b/plugins/inputs/nginx_plus_api/nginx_plus_api_metrics.go @@ -29,6 +29,11 @@ func (n *NginxPlusApi) gatherMetrics(addr *url.URL, acc telegraf.Accumulator) { addError(acc, n.gatherHttpCachesMetrics(addr, acc)) addError(acc, n.gatherStreamServerZonesMetrics(addr, acc)) addError(acc, n.gatherStreamUpstreamsMetrics(addr, acc)) + + if n.ApiVersion >= 5 { + addError(acc, n.gatherHttpLocationZonesMetrics(addr, acc)) + addError(acc, n.gatherResolverZonesMetrics(addr, acc)) + } } func addError(acc telegraf.Accumulator, err error) { @@ -221,6 +226,53 @@ func (n *NginxPlusApi) gatherHttpServerZonesMetrics(addr *url.URL, acc telegraf. return nil } +// Added in 5 API version +func (n *NginxPlusApi) gatherHttpLocationZonesMetrics(addr *url.URL, acc telegraf.Accumulator) error { + body, err := n.gatherUrl(addr, httpLocationZonesPath) + if err != nil { + return err + } + + var httpLocationZones HttpLocationZones + + if err := json.Unmarshal(body, &httpLocationZones); err != nil { + return err + } + + tags := getTags(addr) + + for zoneName, zone := range httpLocationZones { + zoneTags := map[string]string{} + for k, v := range tags { + zoneTags[k] = v + } + zoneTags["zone"] = zoneName + acc.AddFields( + "nginx_plus_api_http_location_zones", + func() map[string]interface{} { + result := map[string]interface{}{ + "requests": zone.Requests, + "responses_1xx": zone.Responses.Responses1xx, + "responses_2xx": zone.Responses.Responses2xx, + "responses_3xx": zone.Responses.Responses3xx, + "responses_4xx": zone.Responses.Responses4xx, + "responses_5xx": zone.Responses.Responses5xx, + "responses_total": zone.Responses.Total, + "received": zone.Received, + "sent": zone.Sent, + } + if zone.Discarded != nil { + result["discarded"] = *zone.Discarded + } + return result + }(), + zoneTags, + ) + } + + return nil +} + func (n *NginxPlusApi) gatherHttpUpstreamsMetrics(addr *url.URL, acc telegraf.Accumulator) error { body, err := n.gatherUrl(addr, httpUpstreamsPath) if err != nil { @@ -394,6 +446,50 @@ func (n *NginxPlusApi) gatherStreamServerZonesMetrics(addr *url.URL, acc telegra return nil } +// Added in 5 API version +func (n *NginxPlusApi) gatherResolverZonesMetrics(addr *url.URL, acc telegraf.Accumulator) error { + body, err := n.gatherUrl(addr, resolverZonesPath) + if err != nil { + return err + } + + var resolverZones ResolverZones + + if err := json.Unmarshal(body, &resolverZones); err != nil { + return err + } + + tags := getTags(addr) + + for zoneName, resolver := range resolverZones { + zoneTags := map[string]string{} + for k, v := range tags { + zoneTags[k] = v + } + zoneTags["zone"] = zoneName + acc.AddFields( + "nginx_plus_api_resolver_zones", + map[string]interface{}{ + "name": resolver.Requests.Name, + "srv": resolver.Requests.Srv, + "addr": resolver.Requests.Addr, + + "noerror": resolver.Responses.Noerror, + "formerr": resolver.Responses.Formerr, + "servfail": resolver.Responses.Servfail, + "nxdomain": resolver.Responses.Nxdomain, + "notimp": resolver.Responses.Notimp, + "refused": resolver.Responses.Refused, + "timedout": resolver.Responses.Timedout, + "unknown": resolver.Responses.Unknown, + }, + zoneTags, + ) + } + + return nil +} + func (n *NginxPlusApi) gatherStreamUpstreamsMetrics(addr *url.URL, acc telegraf.Accumulator) error { body, err := n.gatherUrl(addr, streamUpstreamsPath) if err != nil { diff --git a/plugins/inputs/nginx_plus_api/nginx_plus_api_metrics_test.go b/plugins/inputs/nginx_plus_api/nginx_plus_api_metrics_test.go index da1806aac0426..584816fe7c853 100644 --- a/plugins/inputs/nginx_plus_api/nginx_plus_api_metrics_test.go +++ b/plugins/inputs/nginx_plus_api/nginx_plus_api_metrics_test.go @@ -35,6 +35,45 @@ const sslPayload = ` } ` +const resolverZonesPayload = ` +{ + "resolver_zone1": { + "requests": { + "name": 25460, + "srv": 130, + "addr": 2580 + }, + "responses": { + "noerror": 26499, + "formerr": 0, + "servfail": 3, + "nxdomain": 0, + "notimp": 0, + "refused": 0, + "timedout": 243, + "unknown": 478 + } + }, + "resolver_zone2": { + "requests": { + "name": 325460, + "srv": 1130, + "addr": 12580 + }, + "responses": { + "noerror": 226499, + "formerr": 0, + "servfail": 283, + "nxdomain": 0, + "notimp": 0, + "refused": 0, + "timedout": 743, + "unknown": 1478 + } + } +} +` + const httpRequestsPayload = ` { "total": 10624511, @@ -77,6 +116,39 @@ const httpServerZonesPayload = ` } ` +const httpLocationZonesPayload = ` +{ + "site1": { + "requests": 736395, + "responses": { + "1xx": 0, + "2xx": 727290, + "3xx": 4614, + "4xx": 934, + "5xx": 1535, + "total": 734373 + }, + "discarded": 2020, + "received": 180157219, + "sent": 20183175459 + }, + "site2": { + "requests": 185307, + "responses": { + "1xx": 0, + "2xx": 112674, + "3xx": 45383, + "4xx": 2504, + "5xx": 4419, + "total": 164980 + }, + "discarded": 20326, + "received": 51575327, + "sent": 2983241510 + } +} +` + const httpUpstreamsPayload = ` { "trac-backend": { @@ -591,6 +663,58 @@ func TestGatherHttpServerZonesMetrics(t *testing.T) { }) } +func TestGatherHttpLocationZonesMetrics(t *testing.T) { + ts, n := prepareEndpoint(t, httpLocationZonesPath, defaultApiVersion, httpLocationZonesPayload) + defer ts.Close() + + var acc testutil.Accumulator + addr, host, port := prepareAddr(t, ts) + + require.NoError(t, n.gatherHttpLocationZonesMetrics(addr, &acc)) + + acc.AssertContainsTaggedFields( + t, + "nginx_plus_api_http_location_zones", + map[string]interface{}{ + "discarded": int64(2020), + "received": int64(180157219), + "requests": int64(736395), + "responses_1xx": int64(0), + "responses_2xx": int64(727290), + "responses_3xx": int64(4614), + "responses_4xx": int64(934), + "responses_5xx": int64(1535), + "responses_total": int64(734373), + "sent": int64(20183175459), + }, + map[string]string{ + "source": host, + "port": port, + "zone": "site1", + }) + + acc.AssertContainsTaggedFields( + t, + "nginx_plus_api_http_location_zones", + map[string]interface{}{ + "discarded": int64(20326), + "received": int64(51575327), + "requests": int64(185307), + "responses_1xx": int64(0), + "responses_2xx": int64(112674), + "responses_3xx": int64(45383), + "responses_4xx": int64(2504), + "responses_5xx": int64(4419), + "responses_total": int64(164980), + "sent": int64(2983241510), + }, + map[string]string{ + "source": host, + "port": port, + "zone": "site2", + }) +} + func TestHatherHttpUpstreamsMetrics(t *testing.T) { ts, n := prepareEndpoint(t, httpUpstreamsPath, defaultApiVersion, httpUpstreamsPayload) defer ts.Close() @@ -841,6 +965,60 @@ func TestGatherHttpCachesMetrics(t *testing.T) { }) } +func TestGatherResolverZonesMetrics(t *testing.T) { + ts, n := prepareEndpoint(t, resolverZonesPath, defaultApiVersion, resolverZonesPayload) + defer ts.Close() + + var acc testutil.Accumulator + addr, host, port := prepareAddr(t, ts) + + require.NoError(t, n.gatherResolverZonesMetrics(addr, &acc)) + + acc.AssertContainsTaggedFields( + t, + "nginx_plus_api_resolver_zones", + map[string]interface{}{ + "name": int64(25460), + "srv": int64(130), + "addr": int64(2580), + "noerror": int64(26499), + "formerr": int64(0), + "servfail": int64(3), + "nxdomain": int64(0), + "notimp": int64(0), + "refused": int64(0), + "timedout": int64(243), + "unknown": int64(478), + }, + map[string]string{ + "source": host, + "port": port, + "zone": "resolver_zone1", + }) + + acc.AssertContainsTaggedFields( + t, + "nginx_plus_api_resolver_zones", + map[string]interface{}{ + "name": int64(325460), + "srv": int64(1130), + "addr": int64(12580), + "noerror": int64(226499), + "formerr": int64(0), + "servfail": int64(283), + "nxdomain": int64(0), + "notimp": int64(0), + "refused": int64(0), + "timedout": int64(743), + "unknown": int64(1478), + }, + map[string]string{ + "source": host, + "port": port, + "zone": "resolver_zone2", + }) +} + func TestGatherStreamUpstreams(t *testing.T) { ts, n := prepareEndpoint(t, streamUpstreamsPath, defaultApiVersion, streamUpstreamsPayload) defer ts.Close() @@ -1023,6 +1201,7 @@ func TestGatherStreamServerZonesMetrics(t *testing.T) { "zone": "dns", }) } + func TestUnavailableEndpoints(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNotFound) diff --git a/plugins/inputs/nginx_plus_api/nginx_plus_api_types.go b/plugins/inputs/nginx_plus_api/nginx_plus_api_types.go index b8240f8444b7d..868bc04e445eb 100644 --- a/plugins/inputs/nginx_plus_api/nginx_plus_api_types.go +++ b/plugins/inputs/nginx_plus_api/nginx_plus_api_types.go @@ -17,6 +17,24 @@ type Ssl struct { // added in version 6 SessionReuses int64 `json:"session_reuses"` } +type ResolverZones map[string]struct { + Requests struct { + Name int64 `json:"name"` + Srv int64 `json:"srv"` + Addr int64 `json:"addr"` + } `json:"requests"` + Responses struct { + Noerror int64 `json:"noerror"` + Formerr int64 `json:"formerr"` + Servfail int64 `json:"servfail"` + Nxdomain int64 `json:"nxdomain"` + Notimp int64 `json:"notimp"` + Refused int64 `json:"refused"` + Timedout int64 `json:"timedout"` + Unknown int64 `json:"unknown"` + } `json:"responses"` +} + type HttpRequests struct { Total int64 `json:"total"` Current int64 `json:"current"` @@ -40,6 +58,14 @@ type HttpServerZones map[string]struct { Sent int64 `json:"sent"` } +type HttpLocationZones map[string]struct { + Requests int64 `json:"requests"` + Responses ResponseStats `json:"responses"` + Discarded *int64 `json:"discarded"` // added in version 6 + Received int64 `json:"received"` + Sent int64 `json:"sent"` +} + type HealthCheckStats struct { Checks int64 `json:"checks"` Fails int64 `json:"fails"` diff --git a/plugins/inputs/nsq_consumer/README.md b/plugins/inputs/nsq_consumer/README.md index 0dae26e8c9584..d1e7194bbd7e0 100644 --- a/plugins/inputs/nsq_consumer/README.md +++ b/plugins/inputs/nsq_consumer/README.md @@ -10,8 +10,10 @@ of the supported [input data formats][]. [[inputs.nsq_consumer]] ## Server option still works but is deprecated, we just prepend it to the nsqd array. # server = "localhost:4150" + ## An array representing the NSQD TCP HTTP Endpoints nsqd = ["localhost:4150"] + ## An array representing the NSQLookupd HTTP Endpoints nsqlookupd = ["localhost:4161"] topic = "telegraf" diff --git a/plugins/inputs/nsq_consumer/nsq_consumer.go b/plugins/inputs/nsq_consumer/nsq_consumer.go index de7572316a375..2c25cce7d8114 100644 --- a/plugins/inputs/nsq_consumer/nsq_consumer.go +++ b/plugins/inputs/nsq_consumer/nsq_consumer.go @@ -2,7 +2,6 @@ package nsq_consumer import ( "context" - "log" "sync" "github.com/influxdata/telegraf" @@ -18,10 +17,12 @@ const ( type empty struct{} type semaphore chan empty -type logger struct{} +type logger struct { + log telegraf.Logger +} func (l *logger) Output(calldepth int, s string) error { - log.Println("D! [inputs.nsq_consumer] " + s) + l.log.Debug(s) return nil } @@ -39,6 +40,8 @@ type NSQConsumer struct { parser parsers.Parser consumer *nsq.Consumer + Log telegraf.Logger + mu sync.Mutex messages map[telegraf.TrackingID]*nsq.Message wg sync.WaitGroup @@ -48,8 +51,10 @@ type NSQConsumer struct { var sampleConfig = ` ## Server option still works but is deprecated, we just prepend it to the nsqd array. # server = "localhost:4150" + ## An array representing the NSQD TCP HTTP Endpoints nsqd = ["localhost:4150"] + ## An array representing the NSQLookupd HTTP Endpoints nsqlookupd = ["localhost:4161"] topic = "telegraf" @@ -98,7 +103,7 @@ func (n *NSQConsumer) Start(ac telegraf.Accumulator) error { n.cancel = cancel n.connect() - n.consumer.SetLogger(&logger{}, nsq.LogLevelInfo) + n.consumer.SetLogger(&logger{log: n.Log}, nsq.LogLevelInfo) n.consumer.AddHandler(nsq.HandlerFunc(func(message *nsq.Message) error { metrics, err := n.parser.Parse(message.Body) if err != nil { diff --git a/plugins/inputs/nsq_consumer/nsq_consumer_test.go b/plugins/inputs/nsq_consumer/nsq_consumer_test.go index 6558dfba29b57..1e8264d065fd8 100644 --- a/plugins/inputs/nsq_consumer/nsq_consumer_test.go +++ b/plugins/inputs/nsq_consumer/nsq_consumer_test.go @@ -36,6 +36,7 @@ func TestReadsMetricsFromNSQ(t *testing.T) { newMockNSQD(script, addr.String()) consumer := &NSQConsumer{ + Log: testutil.Logger{}, Server: "127.0.0.1:4155", Topic: "telegraf", Channel: "consume", diff --git a/plugins/inputs/ntpq/README.md b/plugins/inputs/ntpq/README.md index f6ee8e2af28e5..e691200ddd682 100644 --- a/plugins/inputs/ntpq/README.md +++ b/plugins/inputs/ntpq/README.md @@ -29,7 +29,7 @@ server (RMS of difference of multiple time samples, milliseconds); ```toml # Get standard NTP query metrics, requires ntpq executable [[inputs.ntpq]] - ## If false, set the -n ntpq flag. Can reduce metric gather times. + ## If false, add -n for ntpq command. Can reduce metric gather times. dns_lookup = true ``` diff --git a/plugins/inputs/nvidia_smi/README.md b/plugins/inputs/nvidia_smi/README.md index 8afa7453811be..2173c904eb0c4 100644 --- a/plugins/inputs/nvidia_smi/README.md +++ b/plugins/inputs/nvidia_smi/README.md @@ -55,10 +55,19 @@ SELECT mean("temperature_gpu") FROM "nvidia_smi" WHERE time > now() - 5m GROUP B ### Troubleshooting -As the `telegraf` user run the following command. Adjust the path to `nvidia-smi` if customized. +Check the full output by running `nvidia-smi` binary manually. + +Linux: +``` +sudo -u telegraf -- /usr/bin/nvidia-smi -q -x ``` -/usr/bin/nvidia-smi --format=noheader,nounits,csv --query-gpu=fan.speed,memory.total,memory.used,memory.free,pstate,temperature.gpu,name,uuid,compute_mode,utilization.gpu,utilization.memory,index,power.draw,pcie.link.gen.current,pcie.link.width.current,encoder.stats.sessionCount,encoder.stats.averageFps,encoder.stats.averageLatency,clocks.current.graphics,clocks.current.sm,clocks.current.memory,clocks.current.video + +Windows: ``` +"C:\Program Files\NVIDIA Corporation\NVSMI\nvidia-smi.exe" -q -x +``` + +Please include the output of this command if opening an GitHub issue. ### Example Output ``` diff --git a/plugins/inputs/nvidia_smi/nvidia_smi.go b/plugins/inputs/nvidia_smi/nvidia_smi.go index e2ec1995930c9..b21e390c644d6 100644 --- a/plugins/inputs/nvidia_smi/nvidia_smi.go +++ b/plugins/inputs/nvidia_smi/nvidia_smi.go @@ -1,7 +1,7 @@ package nvidia_smi import ( - "bufio" + "encoding/xml" "fmt" "os" "os/exec" @@ -14,41 +14,12 @@ import ( "github.com/influxdata/telegraf/plugins/inputs" ) -var ( - measurement = "nvidia_smi" - metrics = "fan.speed,memory.total,memory.used,memory.free,pstate,temperature.gpu,name,uuid,compute_mode,utilization.gpu,utilization.memory,index,power.draw,pcie.link.gen.current,pcie.link.width.current,encoder.stats.sessionCount,encoder.stats.averageFps,encoder.stats.averageLatency,clocks.current.graphics,clocks.current.sm,clocks.current.memory,clocks.current.video" - metricNames = [][]string{ - {"fan_speed", "integer"}, - {"memory_total", "integer"}, - {"memory_used", "integer"}, - {"memory_free", "integer"}, - {"pstate", "tag"}, - {"temperature_gpu", "integer"}, - {"name", "tag"}, - {"uuid", "tag"}, - {"compute_mode", "tag"}, - {"utilization_gpu", "integer"}, - {"utilization_memory", "integer"}, - {"index", "tag"}, - {"power_draw", "float"}, - {"pcie_link_gen_current", "integer"}, - {"pcie_link_width_current", "integer"}, - {"encoder_stats_session_count", "integer"}, - {"encoder_stats_average_fps", "integer"}, - {"encoder_stats_average_latency", "integer"}, - {"clocks_current_graphics", "integer"}, - {"clocks_current_sm", "integer"}, - {"clocks_current_memory", "integer"}, - {"clocks_current_video", "integer"}, - } -) +const measurement = "nvidia_smi" // NvidiaSMI holds the methods for this plugin type NvidiaSMI struct { BinPath string Timeout internal.Duration - - metrics string } // Description returns the description of the NvidiaSMI plugin @@ -69,7 +40,6 @@ func (smi *NvidiaSMI) SampleConfig() string { // Gather implements the telegraf interface func (smi *NvidiaSMI) Gather(acc telegraf.Accumulator) error { - if _, err := os.Stat(smi.BinPath); os.IsNotExist(err) { return fmt.Errorf("nvidia-smi binary not at path %s, cannot gather GPU data", smi.BinPath) } @@ -92,93 +62,178 @@ func init() { return &NvidiaSMI{ BinPath: "/usr/bin/nvidia-smi", Timeout: internal.Duration{Duration: 5 * time.Second}, - metrics: metrics, } }) } -func (smi *NvidiaSMI) pollSMI() (string, error) { +func (smi *NvidiaSMI) pollSMI() ([]byte, error) { // Construct and execute metrics query - opts := []string{"--format=noheader,nounits,csv", fmt.Sprintf("--query-gpu=%s", smi.metrics)} - ret, err := internal.CombinedOutputTimeout(exec.Command(smi.BinPath, opts...), smi.Timeout.Duration) + ret, err := internal.CombinedOutputTimeout(exec.Command(smi.BinPath, "-q", "-x"), smi.Timeout.Duration) if err != nil { - return "", err + return nil, err } - return string(ret), nil + return ret, nil } -func gatherNvidiaSMI(ret string, acc telegraf.Accumulator) error { - // First split the lines up and handle each one - scanner := bufio.NewScanner(strings.NewReader(ret)) - for scanner.Scan() { - tags, fields, err := parseLine(scanner.Text()) - if err != nil { - return err - } - acc.AddFields(measurement, fields, tags) +func gatherNvidiaSMI(ret []byte, acc telegraf.Accumulator) error { + smi := &SMI{} + err := xml.Unmarshal(ret, smi) + if err != nil { + return err } - if err := scanner.Err(); err != nil { - return fmt.Errorf("Error scanning text %s", ret) + metrics := smi.genTagsFields() + + for _, metric := range metrics { + acc.AddFields(measurement, metric.fields, metric.tags) } return nil } -func parseLine(line string) (map[string]string, map[string]interface{}, error) { - tags := make(map[string]string, 0) - fields := make(map[string]interface{}, 0) +type metric struct { + tags map[string]string + fields map[string]interface{} +} - // Next split up the comma delimited metrics - met := strings.Split(line, ",") +func (s *SMI) genTagsFields() []metric { + metrics := []metric{} + for i, gpu := range s.GPU { + tags := map[string]string{ + "index": strconv.Itoa(i), + } + fields := map[string]interface{}{} + + setTagIfUsed(tags, "pstate", gpu.PState) + setTagIfUsed(tags, "name", gpu.ProdName) + setTagIfUsed(tags, "uuid", gpu.UUID) + setTagIfUsed(tags, "compute_mode", gpu.ComputeMode) + + setIfUsed("int", fields, "fan_speed", gpu.FanSpeed) + setIfUsed("int", fields, "memory_total", gpu.Memory.Total) + setIfUsed("int", fields, "memory_used", gpu.Memory.Used) + setIfUsed("int", fields, "memory_free", gpu.Memory.Free) + setIfUsed("int", fields, "temperature_gpu", gpu.Temp.GPUTemp) + setIfUsed("int", fields, "utilization_gpu", gpu.Utilization.GPU) + setIfUsed("int", fields, "utilization_memory", gpu.Utilization.Memory) + setIfUsed("int", fields, "pcie_link_gen_current", gpu.PCI.LinkInfo.PCIEGen.CurrentLinkGen) + setIfUsed("int", fields, "pcie_link_width_current", gpu.PCI.LinkInfo.LinkWidth.CurrentLinkWidth) + setIfUsed("int", fields, "encoder_stats_session_count", gpu.Encoder.SessionCount) + setIfUsed("int", fields, "encoder_stats_average_fps", gpu.Encoder.AverageFPS) + setIfUsed("int", fields, "encoder_stats_average_latency", gpu.Encoder.AverageLatency) + setIfUsed("int", fields, "clocks_current_graphics", gpu.Clocks.Graphics) + setIfUsed("int", fields, "clocks_current_sm", gpu.Clocks.SM) + setIfUsed("int", fields, "clocks_current_memory", gpu.Clocks.Memory) + setIfUsed("int", fields, "clocks_current_video", gpu.Clocks.Video) + + setIfUsed("float", fields, "power_draw", gpu.Power.PowerDraw) + metrics = append(metrics, metric{tags, fields}) + } + return metrics +} - // Make sure there are as many metrics in the line as there were queried. - if len(met) == len(metricNames) { - for i, m := range metricNames { - col := strings.TrimSpace(met[i]) +func setTagIfUsed(m map[string]string, k, v string) { + if v != "" { + m[k] = v + } +} - // Handle the tags - if m[1] == "tag" { - tags[m[0]] = col - continue - } +func setIfUsed(t string, m map[string]interface{}, k, v string) { + vals := strings.Fields(v) + if len(vals) < 1 { + return + } - // In some cases we may not be able to get data. - // One such case is when the memory is overclocked. - // nvidia-smi reads the max supported memory clock from the stock value. - // If the current memory clock is greater than the max detected memory clock then we receive [Unknown Error] as a value. - - // For example, the stock max memory clock speed on a 2080 Ti is 7000 MHz which nvidia-smi detects. - // The user has overclocked their memory using an offset of +1000 so under load the memory clock reaches 8000 MHz. - // Now when nvidia-smi tries to read the current memory clock it fails and spits back [Unknown Error] as the value. - // This value will break the parsing logic below unless it is accounted for here. - if strings.Contains(col, "[Not Supported]") || strings.Contains(col, "[Unknown Error]") { - continue - } + val := vals[0] + if k == "pcie_link_width_current" { + val = strings.TrimSuffix(vals[0], "x") + } - // Parse the integers - if m[1] == "integer" { - out, err := strconv.ParseInt(col, 10, 64) - if err != nil { - return tags, fields, err - } - fields[m[0]] = out + switch t { + case "float": + if val != "" { + f, err := strconv.ParseFloat(val, 64) + if err == nil { + m[k] = f } - - // Parse the floats - if m[1] == "float" { - out, err := strconv.ParseFloat(col, 64) - if err != nil { - return tags, fields, err - } - fields[m[0]] = out + } + case "int": + if val != "" { + i, err := strconv.Atoi(val) + if err == nil { + m[k] = i } } - - // Return the tags and fields - return tags, fields, nil } +} + +// SMI defines the structure for the output of _nvidia-smi -q -x_. +type SMI struct { + GPU GPU `xml:"gpu"` +} + +// GPU defines the structure of the GPU portion of the smi output. +type GPU []struct { + FanSpeed string `xml:"fan_speed"` // int + Memory MemoryStats `xml:"fb_memory_usage"` + PState string `xml:"performance_state"` + Temp TempStats `xml:"temperature"` + ProdName string `xml:"product_name"` + UUID string `xml:"uuid"` + ComputeMode string `xml:"compute_mode"` + Utilization UtilizationStats `xml:"utilization"` + Power PowerReadings `xml:"power_readings"` + PCI PCI `xml:"pci"` + Encoder EncoderStats `xml:"encoder_stats"` + Clocks ClockStats `xml:"clocks"` +} + +// MemoryStats defines the structure of the memory portions in the smi output. +type MemoryStats struct { + Total string `xml:"total"` // int + Used string `xml:"used"` // int + Free string `xml:"free"` // int +} + +// TempStats defines the structure of the temperature portion of the smi output. +type TempStats struct { + GPUTemp string `xml:"gpu_temp"` // int +} + +// UtilizationStats defines the structure of the utilization portion of the smi output. +type UtilizationStats struct { + GPU string `xml:"gpu_util"` // int + Memory string `xml:"memory_util"` // int +} + +// PowerReadings defines the structure of the power_readings portion of the smi output. +type PowerReadings struct { + PowerDraw string `xml:"power_draw"` // float +} + +// PCI defines the structure of the pci portion of the smi output. +type PCI struct { + LinkInfo struct { + PCIEGen struct { + CurrentLinkGen string `xml:"current_link_gen"` // int + } `xml:"pcie_gen"` + LinkWidth struct { + CurrentLinkWidth string `xml:"current_link_width"` // int + } `xml:"link_widths"` + } `xml:"pci_gpu_link_info"` +} + +// EncoderStats defines the structure of the encoder_stats portion of the smi output. +type EncoderStats struct { + SessionCount string `xml:"session_count"` // int + AverageFPS string `xml:"average_fps"` // int + AverageLatency string `xml:"average_latency"` // int +} - // If the line is empty return an emptyline error - return tags, fields, fmt.Errorf("Different number of metrics returned (%d) than expeced (%d)", len(met), len(metricNames)) +// ClockStats defines the structure of the clocks portion of the smi output. +type ClockStats struct { + Graphics string `xml:"graphics_clock"` // int + SM string `xml:"sm_clock"` // int + Memory string `xml:"mem_clock"` // int + Video string `xml:"video_clock"` // int } diff --git a/plugins/inputs/nvidia_smi/nvidia_smi_test.go b/plugins/inputs/nvidia_smi/nvidia_smi_test.go index a16447d696976..6fd37b570e081 100644 --- a/plugins/inputs/nvidia_smi/nvidia_smi_test.go +++ b/plugins/inputs/nvidia_smi/nvidia_smi_test.go @@ -1,51 +1,137 @@ package nvidia_smi import ( + "io/ioutil" + "path/filepath" "testing" + "time" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/require" ) -func TestParseLineStandard(t *testing.T) { - line := "41, 11264, 1074, 10190, P8, 32, GeForce RTX 2080 Ti, GPU-c97b7f88-c06d-650f-5339-f8dd0c1315c0, Default, 1, 4, 0, 24.33, 1, 16, 0, 0, 0, 300, 300, 405, 540\n" - tags, fields, err := parseLine(line) - if err != nil { - t.Fail() +func TestGatherValidXML(t *testing.T) { + tests := []struct { + name string + filename string + expected []telegraf.Metric + }{ + { + name: "GeForce GTX 1070 Ti", + filename: "gtx-1070-ti.xml", + expected: []telegraf.Metric{ + testutil.MustMetric( + "nvidia_smi", + map[string]string{ + "name": "GeForce GTX 1070 Ti", + "compute_mode": "Default", + "index": "0", + "pstate": "P8", + "uuid": "GPU-f9ba66fc-a7f5-94c5-da19-019ef2f9c665", + }, + map[string]interface{}{ + "clocks_current_graphics": 135, + "clocks_current_memory": 405, + "clocks_current_sm": 135, + "clocks_current_video": 405, + "encoder_stats_average_fps": 0, + "encoder_stats_average_latency": 0, + "encoder_stats_session_count": 0, + "fan_speed": 100, + "memory_free": 4054, + "memory_total": 4096, + "memory_used": 42, + "pcie_link_gen_current": 1, + "pcie_link_width_current": 16, + "temperature_gpu": 39, + "utilization_gpu": 0, + "utilization_memory": 0, + }, + time.Unix(0, 0)), + }, + }, + { + name: "GeForce GTX 1660 Ti", + filename: "gtx-1660-ti.xml", + expected: []telegraf.Metric{ + testutil.MustMetric( + "nvidia_smi", + map[string]string{ + "compute_mode": "Default", + "index": "0", + "name": "Graphics Device", + "pstate": "P8", + "uuid": "GPU-304a277d-3545-63b8-3a36-dfde3c992989", + }, + map[string]interface{}{ + "clocks_current_graphics": 300, + "clocks_current_memory": 405, + "clocks_current_sm": 300, + "clocks_current_video": 540, + "encoder_stats_average_fps": 0, + "encoder_stats_average_latency": 0, + "encoder_stats_session_count": 0, + "fan_speed": 0, + "memory_free": 5912, + "memory_total": 5912, + "memory_used": 0, + "pcie_link_gen_current": 1, + "pcie_link_width_current": 16, + "power_draw": 8.93, + "temperature_gpu": 40, + "utilization_gpu": 0, + "utilization_memory": 1, + }, + time.Unix(0, 0)), + }, + }, + { + name: "Quadro P400", + filename: "quadro-p400.xml", + expected: []telegraf.Metric{ + testutil.MustMetric( + "nvidia_smi", + map[string]string{ + "compute_mode": "Default", + "index": "0", + "name": "Quadro P400", + "pstate": "P8", + "uuid": "GPU-8f750be4-dfbc-23b9-b33f-da729a536494", + }, + map[string]interface{}{ + "clocks_current_graphics": 139, + "clocks_current_memory": 405, + "clocks_current_sm": 139, + "clocks_current_video": 544, + "encoder_stats_average_fps": 0, + "encoder_stats_average_latency": 0, + "encoder_stats_session_count": 0, + "fan_speed": 34, + "memory_free": 1998, + "memory_total": 1998, + "memory_used": 0, + "pcie_link_gen_current": 1, + "pcie_link_width_current": 16, + "temperature_gpu": 33, + "utilization_gpu": 0, + "utilization_memory": 3, + }, + time.Unix(0, 0)), + }, + }, } - if tags["name"] != "GeForce RTX 2080 Ti" { - t.Fail() - } - if temp, ok := fields["temperature_gpu"].(int); ok && temp != 32 { - t.Fail() - } -} - -func TestParseLineEmptyLine(t *testing.T) { - line := "\n" - _, _, err := parseLine(line) - if err == nil { - t.Fail() - } -} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var acc testutil.Accumulator -func TestParseLineBad(t *testing.T) { - line := "the quick brown fox jumped over the lazy dog" - _, _, err := parseLine(line) - if err == nil { - t.Fail() - } -} + octets, err := ioutil.ReadFile(filepath.Join("testdata", tt.filename)) + require.NoError(t, err) -func TestParseLineNotSupported(t *testing.T) { - line := "[Not Supported], 11264, 1074, 10190, P8, 32, GeForce RTX 2080 Ti, GPU-c97b7f88-c06d-650f-5339-f8dd0c1315c0, Default, 1, 4, 0, 24.33, 1, 16, 0, 0, 0, 300, 300, 405, 540\n" - _, fields, err := parseLine(line) - require.NoError(t, err) - require.Equal(t, nil, fields["fan_speed"]) -} + err = gatherNvidiaSMI(octets, &acc) + require.NoError(t, err) -func TestParseLineUnknownError(t *testing.T) { - line := "[Unknown Error], 11264, 1074, 10190, P8, 32, GeForce RTX 2080 Ti, GPU-c97b7f88-c06d-650f-5339-f8dd0c1315c0, Default, 1, 4, 0, 24.33, 1, 16, 0, 0, 0, 300, 300, 405, 540\n" - _, fields, err := parseLine(line) - require.NoError(t, err) - require.Equal(t, nil, fields["fan_speed"]) + testutil.RequireMetricsEqual(t, tt.expected, acc.GetTelegrafMetrics(), testutil.IgnoreTime()) + }) + } } diff --git a/plugins/inputs/nvidia_smi/testdata/gtx-1070-ti.xml b/plugins/inputs/nvidia_smi/testdata/gtx-1070-ti.xml new file mode 100644 index 0000000000000..3e3e3ec870585 --- /dev/null +++ b/plugins/inputs/nvidia_smi/testdata/gtx-1070-ti.xml @@ -0,0 +1,47 @@ + + + + + GeForce GTX 1070 Ti + GPU-f9ba66fc-a7f5-94c5-da19-019ef2f9c665 + + + + 1 + + + 16x + + + + 100 % + P8 + + 4096 MiB + 42 MiB + 4054 MiB + + Default + + 0 % + 0 % + + + 0 + 0 + 0 + + + 39 C + + + N/A + + + 135 MHz + 135 MHz + 405 MHz + 405 MHz + + + diff --git a/plugins/inputs/nvidia_smi/testdata/gtx-1660-ti.xml b/plugins/inputs/nvidia_smi/testdata/gtx-1660-ti.xml new file mode 100644 index 0000000000000..1a6c7d0891935 --- /dev/null +++ b/plugins/inputs/nvidia_smi/testdata/gtx-1660-ti.xml @@ -0,0 +1,189 @@ + + + Fri Mar 29 19:19:44 2019 + 418.43 + 10.1 + 1 + + Graphics Device + GeForce + Disabled + Disabled + Disabled + Disabled + 4000 + + N/A + N/A + + N/A + GPU-304a277d-3545-63b8-3a36-dfde3c992989 + 0 + 90.16.25.00.4C + No + 0x4300 + N/A + + G001.0000.02.04 + 1.1 + N/A + N/A + + + N/A + N/A + + + None + + + N/A + + + 43 + 00 + 0000 + 218410DE + 00000000:43:00.0 + 3FC81458 + + + 3 + 1 + + + 16x + 16x + + + + N/A + N/A + + 0 + 0 + 0 KB/s + 0 KB/s + + 0 % + P8 + + Active + Not Active + Not Active + Not Active + Not Active + Not Active + Not Active + Not Active + Not Active + + + 5912 MiB + 0 MiB + 5912 MiB + + + 256 MiB + 2 MiB + 254 MiB + + Default + + 0 % + 1 % + 0 % + 0 % + + + 0 + 0 + 0 + + + 0 + 0 + 0 + + + N/A + N/A + + + + N/A + N/A + N/A + N/A + + + N/A + N/A + N/A + N/A + + + + + N/A + N/A + + + N/A + N/A + + N/A + + + 40 C + 96 C + 93 C + 91 C + N/A + N/A + + + P8 + Supported + 8.93 W + 130.00 W + 130.00 W + 130.00 W + 70.00 W + 130.00 W + + + 300 MHz + 300 MHz + 405 MHz + 540 MHz + + + N/A + N/A + + + N/A + N/A + + + 2145 MHz + 2145 MHz + 4001 MHz + 1950 MHz + + + N/A + + + N/A + N/A + + N/A + + + + + + + diff --git a/plugins/inputs/nvidia_smi/testdata/quadro-p400.xml b/plugins/inputs/nvidia_smi/testdata/quadro-p400.xml new file mode 100644 index 0000000000000..ca9e2191ec94f --- /dev/null +++ b/plugins/inputs/nvidia_smi/testdata/quadro-p400.xml @@ -0,0 +1,447 @@ + + + Mon Mar 11 17:03:27 2019 + 418.43 + 10.1 + 1 + + Quadro P400 + Quadro + Disabled + Disabled + Disabled + Disabled + 4000 + + N/A + N/A + + 0424418054852 + GPU-8f750be4-dfbc-23b9-b33f-da729a536494 + 0 + 86.07.3B.00.4A + No + 0x4300 + 900-5G212-1701-000 + + G212.0500.00.01 + 1.1 + N/A + N/A + + + N/A + N/A + + + None + + + N/A + + + 43 + 00 + 0000 + 1CB310DE + 00000000:43:00.0 + 11BE10DE + + + 3 + 1 + + + 16x + 16x + + + + N/A + N/A + + 0 + 0 + 0 KB/s + 0 KB/s + + 34 % + P8 + + Active + Not Active + Not Active + Not Active + Not Active + Not Active + Not Active + Not Active + Not Active + + + 1998 MiB + 0 MiB + 1998 MiB + + + 256 MiB + 2 MiB + 254 MiB + + Default + + 0 % + 3 % + 0 % + 0 % + + + 0 + 0 + 0 + + + 0 + 0 + 0 + + + N/A + N/A + + + + + N/A + N/A + N/A + N/A + N/A + N/A + N/A + N/A + + + N/A + N/A + N/A + N/A + N/A + N/A + N/A + N/A + + + + + N/A + N/A + N/A + N/A + N/A + N/A + N/A + N/A + + + N/A + N/A + N/A + N/A + N/A + N/A + N/A + N/A + + + + + + N/A + N/A + + + N/A + N/A + + N/A + + + 33 C + 103 C + 100 C + N/A + N/A + N/A + + + P8 + N/A + N/A + N/A + N/A + N/A + N/A + N/A + + + 139 MHz + 139 MHz + 405 MHz + 544 MHz + + + 1227 MHz + 2005 MHz + + + 1227 MHz + 2005 MHz + + + 1252 MHz + 1252 MHz + 2005 MHz + 1126 MHz + + + 1252 MHz + + + N/A + N/A + + + + 2005 MHz + 1252 MHz + 1240 MHz + 1227 MHz + 1215 MHz + 1202 MHz + 1189 MHz + 1177 MHz + 1164 MHz + 1151 MHz + 1139 MHz + 1126 MHz + 1113 MHz + 1101 MHz + 1088 MHz + 1075 MHz + 1063 MHz + 1050 MHz + 1037 MHz + 1025 MHz + 1012 MHz + 999 MHz + 987 MHz + 974 MHz + 961 MHz + 949 MHz + 936 MHz + 923 MHz + 911 MHz + 898 MHz + 885 MHz + 873 MHz + 860 MHz + 847 MHz + 835 MHz + 822 MHz + 810 MHz + 797 MHz + 784 MHz + 772 MHz + 759 MHz + 746 MHz + 734 MHz + 721 MHz + 708 MHz + 696 MHz + 683 MHz + 670 MHz + 658 MHz + 645 MHz + 632 MHz + 620 MHz + 607 MHz + 594 MHz + 582 MHz + 569 MHz + 556 MHz + 544 MHz + 531 MHz + 518 MHz + 506 MHz + 493 MHz + 480 MHz + 468 MHz + 455 MHz + 442 MHz + 430 MHz + 417 MHz + 405 MHz + 392 MHz + 379 MHz + 367 MHz + 354 MHz + 341 MHz + 329 MHz + 316 MHz + 303 MHz + 291 MHz + 278 MHz + 265 MHz + 253 MHz + 240 MHz + 227 MHz + 215 MHz + 202 MHz + 189 MHz + 177 MHz + 164 MHz + 151 MHz + 139 MHz + + + 810 MHz + 1252 MHz + 1240 MHz + 1227 MHz + 1215 MHz + 1202 MHz + 1189 MHz + 1177 MHz + 1164 MHz + 1151 MHz + 1139 MHz + 1126 MHz + 1113 MHz + 1101 MHz + 1088 MHz + 1075 MHz + 1063 MHz + 1050 MHz + 1037 MHz + 1025 MHz + 1012 MHz + 999 MHz + 987 MHz + 974 MHz + 961 MHz + 949 MHz + 936 MHz + 923 MHz + 911 MHz + 898 MHz + 885 MHz + 873 MHz + 860 MHz + 847 MHz + 835 MHz + 822 MHz + 810 MHz + 797 MHz + 784 MHz + 772 MHz + 759 MHz + 746 MHz + 734 MHz + 721 MHz + 708 MHz + 696 MHz + 683 MHz + 670 MHz + 658 MHz + 645 MHz + 632 MHz + 620 MHz + 607 MHz + 594 MHz + 582 MHz + 569 MHz + 556 MHz + 544 MHz + 531 MHz + 518 MHz + 506 MHz + 493 MHz + 480 MHz + 468 MHz + 455 MHz + 442 MHz + 430 MHz + 417 MHz + 405 MHz + 392 MHz + 379 MHz + 367 MHz + 354 MHz + 341 MHz + 329 MHz + 316 MHz + 303 MHz + 291 MHz + 278 MHz + 265 MHz + 253 MHz + 240 MHz + 227 MHz + 215 MHz + 202 MHz + 189 MHz + 177 MHz + 164 MHz + 151 MHz + 139 MHz + + + 405 MHz + 607 MHz + 594 MHz + 582 MHz + 569 MHz + 556 MHz + 544 MHz + 531 MHz + 518 MHz + 506 MHz + 493 MHz + 480 MHz + 468 MHz + 455 MHz + 442 MHz + 430 MHz + 417 MHz + 405 MHz + 392 MHz + 379 MHz + 367 MHz + 354 MHz + 341 MHz + 329 MHz + 316 MHz + 303 MHz + 291 MHz + 278 MHz + 265 MHz + 253 MHz + 240 MHz + 227 MHz + 215 MHz + 202 MHz + 189 MHz + 177 MHz + 164 MHz + 151 MHz + 139 MHz + + + + + + + + + diff --git a/plugins/inputs/openldap/openldap.go b/plugins/inputs/openldap/openldap.go index 9e69c8a211924..a92a373712853 100644 --- a/plugins/inputs/openldap/openldap.go +++ b/plugins/inputs/openldap/openldap.go @@ -5,11 +5,10 @@ import ( "strconv" "strings" - "gopkg.in/ldap.v2" - "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal/tls" "github.com/influxdata/telegraf/plugins/inputs" + "gopkg.in/ldap.v3" ) type Openldap struct { diff --git a/plugins/inputs/openldap/openldap_test.go b/plugins/inputs/openldap/openldap_test.go index 10835896fbbc8..76d9cc3a9dd42 100644 --- a/plugins/inputs/openldap/openldap_test.go +++ b/plugins/inputs/openldap/openldap_test.go @@ -4,11 +4,10 @@ import ( "strconv" "testing" - "gopkg.in/ldap.v2" - "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "gopkg.in/ldap.v3" ) func TestOpenldapMockResult(t *testing.T) { diff --git a/plugins/inputs/openweathermap/README.md b/plugins/inputs/openweathermap/README.md index d796990495a38..85803f76ab046 100644 --- a/plugins/inputs/openweathermap/README.md +++ b/plugins/inputs/openweathermap/README.md @@ -4,9 +4,11 @@ Collect current weather and forecast data from OpenWeatherMap. To use this plugin you will need an [api key][] (app_id). -City identifiers can be found in the [city list][]. Alternately you can -[search][] by name; the `city_id` can be found as the last digits of the URL: -https://openweathermap.org/city/2643743 +City identifiers can be found in the [city list][]. Alternately you +can [search][] by name; the `city_id` can be found as the last digits +of the URL: https://openweathermap.org/city/2643743. Language +identifiers can be found in the [lang list][]. Documentation for +condition ID, icon, and main is at [weather conditions][]. ### Configuration @@ -18,6 +20,12 @@ https://openweathermap.org/city/2643743 ## City ID's to collect weather data from. city_id = ["5391959"] + ## Language of the description field. Can be one of "ar", "bg", + ## "ca", "cz", "de", "el", "en", "fa", "fi", "fr", "gl", "hr", "hu", + ## "it", "ja", "kr", "la", "lt", "mk", "nl", "pl", "pt", "ro", "ru", + ## "se", "sk", "sl", "es", "tr", "ua", "vi", "zh_cn", "zh_tw" + # lang = "en" + ## APIs to fetch; can contain "weather" or "forecast". fetch = ["weather", "forecast"] @@ -42,27 +50,33 @@ https://openweathermap.org/city/2643743 - tags: - city_id - forecast + - condition_id + - condition_main - fields: - cloudiness (int, percent) - humidity (int, percent) - pressure (float, atmospheric pressure hPa) - - rain (float, rain volume for the last 3 hours in mm) + - rain (float, rain volume for the last 1-3 hours (depending on API response) in mm) - sunrise (int, nanoseconds since unix epoch) - sunset (int, nanoseconds since unix epoch) - temperature (float, degrees) - visibility (int, meters, not available on forecast data) - wind_degrees (float, wind direction in degrees) - wind_speed (float, wind speed in meters/sec or miles/sec) + - condition_description (string, localized long description) + - condition_icon ### Example Output ``` -> weather,city=San\ Francisco,city_id=5391959,country=US,forecast=* cloudiness=40i,humidity=72i,pressure=1013,rain=0,sunrise=1559220629000000000i,sunset=1559273058000000000i,temperature=13.31,visibility=16093i,wind_degrees=280,wind_speed=4.6 1559268695000000000 -> weather,city=San\ Francisco,city_id=5391959,country=US,forecast=3h cloudiness=0i,humidity=86i,pressure=1012.03,rain=0,temperature=10.69,wind_degrees=222.855,wind_speed=2.76 1559271600000000000 -> weather,city=San\ Francisco,city_id=5391959,country=US,forecast=6h cloudiness=11i,humidity=93i,pressure=1012.79,rain=0,temperature=9.34,wind_degrees=212.685,wind_speed=1.85 1559282400000000000 +> weather,city=San\ Francisco,city_id=5391959,condition_id=800,condition_main=Clear,country=US,forecast=* cloudiness=1i,condition_description="clear sky",condition_icon="01d",humidity=35i,pressure=1012,rain=0,sunrise=1570630329000000000i,sunset=1570671689000000000i,temperature=21.52,visibility=16093i,wind_degrees=280,wind_speed=5.7 1570659256000000000 +> weather,city=San\ Francisco,city_id=5391959,condition_id=800,condition_main=Clear,country=US,forecast=3h cloudiness=0i,condition_description="clear sky",condition_icon="01n",humidity=41i,pressure=1010,rain=0,temperature=22.34,wind_degrees=249.393,wind_speed=2.085 1570665600000000000 +> weather,city=San\ Francisco,city_id=5391959,condition_id=800,condition_main=Clear,country=US,forecast=6h cloudiness=0i,condition_description="clear sky",condition_icon="01n",humidity=50i,pressure=1012,rain=0,temperature=17.09,wind_degrees=310.754,wind_speed=3.009 1570676400000000000 ``` [api key]: https://openweathermap.org/appid [city list]: http://bulk.openweathermap.org/sample/city.list.json.gz [search]: https://openweathermap.org/find +[lang list]: https://openweathermap.org/current#multi +[weather conditions]: https://openweathermap.org/weather-conditions diff --git a/plugins/inputs/openweathermap/openweathermap.go b/plugins/inputs/openweathermap/openweathermap.go index c15ee3832edc5..94055a6f8bb6a 100644 --- a/plugins/inputs/openweathermap/openweathermap.go +++ b/plugins/inputs/openweathermap/openweathermap.go @@ -23,20 +23,23 @@ const ( // The limit of locations is 20. owmRequestSeveralCityId int = 20 - defaultBaseURL = "https://api.openweathermap.org/" + defaultBaseUrl = "https://api.openweathermap.org/" defaultResponseTimeout time.Duration = time.Second * 5 defaultUnits string = "metric" + defaultLang string = "en" ) type OpenWeatherMap struct { AppId string `toml:"app_id"` CityId []string `toml:"city_id"` + Lang string `toml:"lang"` Fetch []string `toml:"fetch"` BaseUrl string `toml:"base_url"` ResponseTimeout internal.Duration `toml:"response_timeout"` Units string `toml:"units"` - client *http.Client + client *http.Client + baseUrl *url.URL } var sampleConfig = ` @@ -46,6 +49,12 @@ var sampleConfig = ` ## City ID's to collect weather data from. city_id = ["5391959"] + ## Language of the description field. Can be one of "ar", "bg", + ## "ca", "cz", "de", "el", "en", "fa", "fi", "fr", "gl", "hr", "hu", + ## "it", "ja", "kr", "la", "lt", "mk", "nl", "pl", "pt", "ro", "ru", + ## "se", "sk", "sl", "es", "tr", "ua", "vi", "zh_cn", "zh_tw" + # lang = "en" + ## APIs to fetch; can contain "weather" or "forecast". fetch = ["weather", "forecast"] @@ -76,41 +85,10 @@ func (n *OpenWeatherMap) Gather(acc telegraf.Accumulator) error { var wg sync.WaitGroup var strs []string - base, err := url.Parse(n.BaseUrl) - if err != nil { - return err - } - - // Create an HTTP client that is re-used for each - // collection interval - if n.client == nil { - client, err := n.createHttpClient() - if err != nil { - return err - } - n.client = client - } - - units := n.Units - switch n.Units { - case "imperial", "standard": - break - default: - units = defaultUnits - } - for _, fetch := range n.Fetch { if fetch == "forecast" { - var u *url.URL - for _, city := range n.CityId { - u, err = url.Parse(fmt.Sprintf("/data/2.5/forecast?id=%s&APPID=%s&units=%s", city, n.AppId, units)) - if err != nil { - acc.AddError(fmt.Errorf("unable to parse address '%s': %s", u, err)) - continue - } - - addr := base.ResolveReference(u).String() + addr := n.formatURL("/data/2.5/forecast", city) wg.Add(1) go func() { defer wg.Done() @@ -126,7 +104,6 @@ func (n *OpenWeatherMap) Gather(acc telegraf.Accumulator) error { } else if fetch == "weather" { j := 0 for j < len(n.CityId) { - var u *url.URL strs = make([]string, 0) for i := 0; j < len(n.CityId) && i < owmRequestSeveralCityId; i++ { strs = append(strs, n.CityId[j]) @@ -134,13 +111,7 @@ func (n *OpenWeatherMap) Gather(acc telegraf.Accumulator) error { } cities := strings.Join(strs, ",") - u, err = url.Parse(fmt.Sprintf("/data/2.5/group?id=%s&APPID=%s&units=%s", cities, n.AppId, units)) - if err != nil { - acc.AddError(fmt.Errorf("Unable to parse address '%s': %s", u, err)) - continue - } - - addr := base.ResolveReference(u).String() + addr := n.formatURL("/data/2.5/group", cities) wg.Add(1) go func() { defer wg.Done() @@ -208,6 +179,7 @@ type WeatherEntry struct { Temp float64 `json:"temp"` } `json:"main"` Rain struct { + Rain1 float64 `json:"1h"` Rain3 float64 `json:"3h"` } `json:"rain"` Sys struct { @@ -226,6 +198,12 @@ type WeatherEntry struct { Lon float64 `json:"lon"` } `json:"coord"` Visibility int64 `json:"visibility"` + Weather []struct { + ID int64 `json:"id"` + Main string `json:"main"` + Description string `json:"description"` + Icon string `json:"icon"` + } `json:"weather"` } type Status struct { @@ -250,30 +228,44 @@ func gatherWeatherUrl(r io.Reader) (*Status, error) { return status, nil } +func gatherRain(e WeatherEntry) float64 { + if e.Rain.Rain1 > 0 { + return e.Rain.Rain1 + } + return e.Rain.Rain3 +} + func gatherWeather(acc telegraf.Accumulator, status *Status) { for _, e := range status.List { tm := time.Unix(e.Dt, 0) - acc.AddFields( - "weather", - map[string]interface{}{ - "cloudiness": e.Clouds.All, - "humidity": e.Main.Humidity, - "pressure": e.Main.Pressure, - "rain": e.Rain.Rain3, - "sunrise": time.Unix(e.Sys.Sunrise, 0).UnixNano(), - "sunset": time.Unix(e.Sys.Sunset, 0).UnixNano(), - "temperature": e.Main.Temp, - "visibility": e.Visibility, - "wind_degrees": e.Wind.Deg, - "wind_speed": e.Wind.Speed, - }, - map[string]string{ - "city": e.Name, - "city_id": strconv.FormatInt(e.Id, 10), - "country": e.Sys.Country, - "forecast": "*", - }, - tm) + + fields := map[string]interface{}{ + "cloudiness": e.Clouds.All, + "humidity": e.Main.Humidity, + "pressure": e.Main.Pressure, + "rain": gatherRain(e), + "sunrise": time.Unix(e.Sys.Sunrise, 0).UnixNano(), + "sunset": time.Unix(e.Sys.Sunset, 0).UnixNano(), + "temperature": e.Main.Temp, + "visibility": e.Visibility, + "wind_degrees": e.Wind.Deg, + "wind_speed": e.Wind.Speed, + } + tags := map[string]string{ + "city": e.Name, + "city_id": strconv.FormatInt(e.Id, 10), + "country": e.Sys.Country, + "forecast": "*", + } + + if len(e.Weather) > 0 { + fields["condition_description"] = e.Weather[0].Description + fields["condition_icon"] = e.Weather[0].Icon + tags["condition_id"] = strconv.FormatInt(e.Weather[0].ID, 10) + tags["condition_main"] = e.Weather[0].Main + } + + acc.AddFields("weather", fields, tags, tm) } } @@ -286,20 +278,23 @@ func gatherForecast(acc telegraf.Accumulator, status *Status) { } for i, e := range status.List { tm := time.Unix(e.Dt, 0) + fields := map[string]interface{}{ + "cloudiness": e.Clouds.All, + "humidity": e.Main.Humidity, + "pressure": e.Main.Pressure, + "rain": gatherRain(e), + "temperature": e.Main.Temp, + "wind_degrees": e.Wind.Deg, + "wind_speed": e.Wind.Speed, + } + if len(e.Weather) > 0 { + fields["condition_description"] = e.Weather[0].Description + fields["condition_icon"] = e.Weather[0].Icon + tags["condition_id"] = strconv.FormatInt(e.Weather[0].ID, 10) + tags["condition_main"] = e.Weather[0].Main + } tags["forecast"] = fmt.Sprintf("%dh", (i+1)*3) - acc.AddFields( - "weather", - map[string]interface{}{ - "cloudiness": e.Clouds.All, - "humidity": e.Main.Humidity, - "pressure": e.Main.Pressure, - "rain": e.Rain.Rain3, - "temperature": e.Main.Temp, - "wind_degrees": e.Wind.Deg, - "wind_speed": e.Wind.Speed, - }, - tags, - tm) + acc.AddFields("weather", fields, tags, tm) } } @@ -310,8 +305,59 @@ func init() { } return &OpenWeatherMap{ ResponseTimeout: tmout, - Units: defaultUnits, - BaseUrl: defaultBaseURL, + BaseUrl: defaultBaseUrl, } }) } + +func (n *OpenWeatherMap) Init() error { + var err error + n.baseUrl, err = url.Parse(n.BaseUrl) + if err != nil { + return err + } + + // Create an HTTP client that is re-used for each + // collection interval + n.client, err = n.createHttpClient() + if err != nil { + return err + } + + switch n.Units { + case "imperial", "standard", "metric": + case "": + n.Units = defaultUnits + default: + return fmt.Errorf("unknown units: %s", n.Units) + } + + switch n.Lang { + case "ar", "bg", "ca", "cz", "de", "el", "en", "fa", "fi", "fr", "gl", + "hr", "hu", "it", "ja", "kr", "la", "lt", "mk", "nl", "pl", + "pt", "ro", "ru", "se", "sk", "sl", "es", "tr", "ua", "vi", + "zh_cn", "zh_tw": + case "": + n.Lang = defaultLang + default: + return fmt.Errorf("unknown language: %s", n.Lang) + } + + return nil +} + +func (n *OpenWeatherMap) formatURL(path string, city string) string { + v := url.Values{ + "id": []string{city}, + "APPID": []string{n.AppId}, + "lang": []string{n.Lang}, + "units": []string{n.Units}, + } + + relative := &url.URL{ + Path: path, + RawQuery: v.Encode(), + } + + return n.baseUrl.ResolveReference(relative).String() +} diff --git a/plugins/inputs/openweathermap/openweathermap_test.go b/plugins/inputs/openweathermap/openweathermap_test.go index d59766dd6b7de..9bee1d2e96199 100644 --- a/plugins/inputs/openweathermap/openweathermap_test.go +++ b/plugins/inputs/openweathermap/openweathermap_test.go @@ -106,9 +106,9 @@ const groupWeatherResponse = ` { "cnt": 1, "list": [{ - "clouds": { - "all": 0 - }, + "clouds": { + "all": 0 + }, "coord": { "lat": 48.85, "lon": 2.35 @@ -146,6 +146,145 @@ const groupWeatherResponse = ` } ` +const rainWeatherResponse = ` +{ + "cnt": 2, + "list": [{ + "dt": 1544194800, + "id": 111, + "main": { + "humidity": 87, + "pressure": 1007, + "temp": 9.25 + }, + "name": "Paris", + "sys": { + "country": "FR", + "id": 6550, + "message": 0.002, + "sunrise": 1544167818, + "sunset": 1544198047, + "type": 1 + }, + "visibility": 10000, + "weather": [ + { + "description": "light intensity drizzle", + "icon": "09d", + "id": 300, + "main": "Drizzle" + } + ], + "rain": { + "1h": 1.000 + }, + "wind": { + "deg": 290, + "speed": 8.7 + } + }, + { + "dt": 1544194800, + "id": 222, + "main": { + "humidity": 87, + "pressure": 1007, + "temp": 9.25 + }, + "name": "Paris", + "sys": { + "country": "FR", + "id": 6550, + "message": 0.002, + "sunrise": 1544167818, + "sunset": 1544198047, + "type": 1 + }, + "visibility": 10000, + "weather": [ + { + "description": "light intensity drizzle", + "icon": "09d", + "id": 300, + "main": "Drizzle" + } + ], + "rain": { + "3h": 3.000 + }, + "wind": { + "deg": 290, + "speed": 8.7 + } + }, + { + "dt": 1544194800, + "id": 333, + "main": { + "humidity": 87, + "pressure": 1007, + "temp": 9.25 + }, + "name": "Paris", + "sys": { + "country": "FR", + "id": 6550, + "message": 0.002, + "sunrise": 1544167818, + "sunset": 1544198047, + "type": 1 + }, + "visibility": 10000, + "weather": [ + { + "description": "light intensity drizzle", + "icon": "09d", + "id": 300, + "main": "Drizzle" + } + ], + "rain": { + "1h": 1.300, + "3h": 999 + }, + "wind": { + "deg": 290, + "speed": 8.7 + } + }, + { + "dt": 1544194800, + "id": 444, + "main": { + "humidity": 87, + "pressure": 1007, + "temp": 9.25 + }, + "name": "Paris", + "sys": { + "country": "FR", + "id": 6550, + "message": 0.002, + "sunrise": 1544167818, + "sunset": 1544198047, + "type": 1 + }, + "visibility": 10000, + "weather": [ + { + "description": "light intensity drizzle", + "icon": "09d", + "id": 300, + "main": "Drizzle" + } + ], + "wind": { + "deg": 290, + "speed": 8.7 + } + }] +} +` const batchWeatherResponse = ` { "cnt": 3, @@ -283,6 +422,7 @@ func TestForecastGeneratesMetrics(t *testing.T) { Fetch: []string{"weather", "forecast"}, Units: "metric", } + n.Init() var acc testutil.Accumulator @@ -293,38 +433,46 @@ func TestForecastGeneratesMetrics(t *testing.T) { testutil.MustMetric( "weather", map[string]string{ - "city_id": "2988507", - "forecast": "3h", - "city": "Paris", - "country": "FR", + "city_id": "2988507", + "forecast": "3h", + "city": "Paris", + "country": "FR", + "condition_id": "500", + "condition_main": "Rain", }, map[string]interface{}{ - "cloudiness": int64(88), - "humidity": int64(91), - "pressure": 1018.65, - "temperature": 6.71, - "rain": 0.035, - "wind_degrees": 228.501, - "wind_speed": 3.76, + "cloudiness": int64(88), + "humidity": int64(91), + "pressure": 1018.65, + "temperature": 6.71, + "rain": 0.035, + "wind_degrees": 228.501, + "wind_speed": 3.76, + "condition_description": "light rain", + "condition_icon": "10n", }, time.Unix(1543622400, 0), ), testutil.MustMetric( "weather", map[string]string{ - "city_id": "2988507", - "forecast": "6h", - "city": "Paris", - "country": "FR", + "city_id": "2988507", + "forecast": "6h", + "city": "Paris", + "country": "FR", + "condition_id": "500", + "condition_main": "Rain", }, map[string]interface{}{ - "cloudiness": int64(92), - "humidity": int64(98), - "pressure": 1032.18, - "temperature": 6.38, - "rain": 0.049999999999997, - "wind_degrees": 335.005, - "wind_speed": 2.66, + "cloudiness": int64(92), + "humidity": int64(98), + "pressure": 1032.18, + "temperature": 6.38, + "rain": 0.049999999999997, + "wind_degrees": 335.005, + "wind_speed": 2.66, + "condition_description": "light rain", + "condition_icon": "10n", }, time.Unix(1544043600, 0), ), @@ -358,6 +506,7 @@ func TestWeatherGeneratesMetrics(t *testing.T) { Fetch: []string{"weather"}, Units: "metric", } + n.Init() var acc testutil.Accumulator @@ -368,22 +517,168 @@ func TestWeatherGeneratesMetrics(t *testing.T) { testutil.MustMetric( "weather", map[string]string{ - "city_id": "2988507", - "forecast": "*", - "city": "Paris", - "country": "FR", + "city_id": "2988507", + "forecast": "*", + "city": "Paris", + "country": "FR", + "condition_id": "300", + "condition_main": "Drizzle", }, map[string]interface{}{ - "cloudiness": int64(0), - "humidity": int64(87), - "pressure": 1007.0, - "temperature": 9.25, - "rain": 0.0, - "sunrise": int64(1544167818000000000), - "sunset": int64(1544198047000000000), - "wind_degrees": 290.0, - "wind_speed": 8.7, - "visibility": 10000, + "cloudiness": int64(0), + "humidity": int64(87), + "pressure": 1007.0, + "temperature": 9.25, + "rain": 0.0, + "sunrise": int64(1544167818000000000), + "sunset": int64(1544198047000000000), + "wind_degrees": 290.0, + "wind_speed": 8.7, + "visibility": 10000, + "condition_description": "light intensity drizzle", + "condition_icon": "09d", + }, + time.Unix(1544194800, 0), + ), + } + testutil.RequireMetricsEqual(t, expected, acc.GetTelegrafMetrics()) +} + +// Ensure that results containing "1h", "3h", both, or no rain values are parsed correctly +func TestRainMetrics(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var rsp string + if r.URL.Path == "/data/2.5/group" { + rsp = rainWeatherResponse + w.Header()["Content-Type"] = []string{"application/json"} + } else { + panic("Cannot handle request") + } + + fmt.Fprintln(w, rsp) + })) + defer ts.Close() + + n := &OpenWeatherMap{ + BaseUrl: ts.URL, + AppId: "noappid", + CityId: []string{"111", "222", "333", "444"}, + Fetch: []string{"weather"}, + Units: "metric", + } + n.Init() + + var acc testutil.Accumulator + + err := n.Gather(&acc) + require.NoError(t, err) + + expected := []telegraf.Metric{ + // City with 1h rain value + testutil.MustMetric( + "weather", + map[string]string{ + "city_id": "111", + "forecast": "*", + "city": "Paris", + "country": "FR", + "condition_id": "300", + "condition_main": "Drizzle", + }, + map[string]interface{}{ + "cloudiness": int64(0), + "humidity": int64(87), + "pressure": 1007.0, + "temperature": 9.25, + "rain": 1.0, + "sunrise": int64(1544167818000000000), + "sunset": int64(1544198047000000000), + "wind_degrees": 290.0, + "wind_speed": 8.7, + "visibility": 10000, + "condition_description": "light intensity drizzle", + "condition_icon": "09d", + }, + time.Unix(1544194800, 0), + ), + // City with 3h rain value + testutil.MustMetric( + "weather", + map[string]string{ + "city_id": "222", + "forecast": "*", + "city": "Paris", + "country": "FR", + "condition_id": "300", + "condition_main": "Drizzle", + }, + map[string]interface{}{ + "cloudiness": int64(0), + "humidity": int64(87), + "pressure": 1007.0, + "temperature": 9.25, + "rain": 3.0, + "sunrise": int64(1544167818000000000), + "sunset": int64(1544198047000000000), + "wind_degrees": 290.0, + "wind_speed": 8.7, + "visibility": 10000, + "condition_description": "light intensity drizzle", + "condition_icon": "09d", + }, + time.Unix(1544194800, 0), + ), + // City with both 1h and 3h rain values, prefer the 1h value + testutil.MustMetric( + "weather", + map[string]string{ + "city_id": "333", + "forecast": "*", + "city": "Paris", + "country": "FR", + "condition_id": "300", + "condition_main": "Drizzle", + }, + map[string]interface{}{ + "cloudiness": int64(0), + "humidity": int64(87), + "pressure": 1007.0, + "temperature": 9.25, + "rain": 1.3, + "sunrise": int64(1544167818000000000), + "sunset": int64(1544198047000000000), + "wind_degrees": 290.0, + "wind_speed": 8.7, + "visibility": 10000, + "condition_description": "light intensity drizzle", + "condition_icon": "09d", + }, + time.Unix(1544194800, 0), + ), + // City with no rain values + testutil.MustMetric( + "weather", + map[string]string{ + "city_id": "444", + "forecast": "*", + "city": "Paris", + "country": "FR", + "condition_id": "300", + "condition_main": "Drizzle", + }, + map[string]interface{}{ + "cloudiness": int64(0), + "humidity": int64(87), + "pressure": 1007.0, + "temperature": 9.25, + "rain": 0.0, + "sunrise": int64(1544167818000000000), + "sunset": int64(1544198047000000000), + "wind_degrees": 290.0, + "wind_speed": 8.7, + "visibility": 10000, + "condition_description": "light intensity drizzle", + "condition_icon": "09d", }, time.Unix(1544194800, 0), ), @@ -414,6 +709,7 @@ func TestBatchWeatherGeneratesMetrics(t *testing.T) { Fetch: []string{"weather"}, Units: "metric", } + n.Init() var acc testutil.Accumulator @@ -424,66 +720,78 @@ func TestBatchWeatherGeneratesMetrics(t *testing.T) { testutil.MustMetric( "weather", map[string]string{ - "city_id": "524901", - "forecast": "*", - "city": "Moscow", - "country": "RU", + "city_id": "524901", + "forecast": "*", + "city": "Moscow", + "country": "RU", + "condition_id": "802", + "condition_main": "Clouds", }, map[string]interface{}{ - "cloudiness": 40, - "humidity": int64(46), - "pressure": 1014.0, - "temperature": 9.57, - "wind_degrees": 60.0, - "wind_speed": 5.0, - "rain": 0.0, - "sunrise": int64(1556416455000000000), - "sunset": int64(1556470779000000000), - "visibility": 10000, + "cloudiness": 40, + "humidity": int64(46), + "pressure": 1014.0, + "temperature": 9.57, + "wind_degrees": 60.0, + "wind_speed": 5.0, + "rain": 0.0, + "sunrise": int64(1556416455000000000), + "sunset": int64(1556470779000000000), + "visibility": 10000, + "condition_description": "scattered clouds", + "condition_icon": "03d", }, time.Unix(1556444155, 0), ), testutil.MustMetric( "weather", map[string]string{ - "city_id": "703448", - "forecast": "*", - "city": "Kiev", - "country": "UA", + "city_id": "703448", + "forecast": "*", + "city": "Kiev", + "country": "UA", + "condition_id": "520", + "condition_main": "Rain", }, map[string]interface{}{ - "cloudiness": 0, - "humidity": int64(63), - "pressure": 1009.0, - "temperature": 19.29, - "wind_degrees": 0.0, - "wind_speed": 1.0, - "rain": 0.0, - "sunrise": int64(1556419155000000000), - "sunset": int64(1556471486000000000), - "visibility": 10000, + "cloudiness": 0, + "humidity": int64(63), + "pressure": 1009.0, + "temperature": 19.29, + "wind_degrees": 0.0, + "wind_speed": 1.0, + "rain": 0.0, + "sunrise": int64(1556419155000000000), + "sunset": int64(1556471486000000000), + "visibility": 10000, + "condition_description": "light intensity shower rain", + "condition_icon": "09d", }, time.Unix(1556444155, 0), ), testutil.MustMetric( "weather", map[string]string{ - "city_id": "2643743", - "forecast": "*", - "city": "London", - "country": "GB", + "city_id": "2643743", + "forecast": "*", + "city": "London", + "country": "GB", + "condition_id": "803", + "condition_main": "Clouds", }, map[string]interface{}{ - "cloudiness": 75, - "humidity": int64(66), - "pressure": 1019.0, - "temperature": 10.62, - "wind_degrees": 290.0, - "wind_speed": 6.2, - "rain": 0.072, - "sunrise": int64(1556426319000000000), - "sunset": int64(1556479032000000000), - "visibility": 10000, + "cloudiness": 75, + "humidity": int64(66), + "pressure": 1019.0, + "temperature": 10.62, + "wind_degrees": 290.0, + "wind_speed": 6.2, + "rain": 0.072, + "sunrise": int64(1556426319000000000), + "sunset": int64(1556479032000000000), + "visibility": 10000, + "condition_description": "broken clouds", + "condition_icon": "04d", }, time.Unix(1556444155, 0), ), @@ -492,3 +800,31 @@ func TestBatchWeatherGeneratesMetrics(t *testing.T) { expected, acc.GetTelegrafMetrics(), testutil.SortMetrics()) } + +func TestFormatURL(t *testing.T) { + n := &OpenWeatherMap{ + AppId: "appid", + Units: "units", + Lang: "lang", + BaseUrl: "http://foo.com", + } + n.Init() + + require.Equal(t, + "http://foo.com/data/2.5/forecast?APPID=appid&id=12345&lang=lang&units=units", + n.formatURL("/data/2.5/forecast", "12345")) +} + +func TestDefaultUnits(t *testing.T) { + n := &OpenWeatherMap{} + n.Init() + + require.Equal(t, "metric", n.Units) +} + +func TestDefaultLang(t *testing.T) { + n := &OpenWeatherMap{} + n.Init() + + require.Equal(t, "en", n.Lang) +} diff --git a/plugins/inputs/ping/ping.go b/plugins/inputs/ping/ping.go index ac0e9ebdfdfe2..17767bac397b9 100644 --- a/plugins/inputs/ping/ping.go +++ b/plugins/inputs/ping/ping.go @@ -3,10 +3,12 @@ package ping import ( "context" "errors" + "log" "math" "net" "os/exec" "runtime" + "strings" "sync" "time" @@ -116,31 +118,29 @@ func (*Ping) SampleConfig() string { } func (p *Ping) Gather(acc telegraf.Accumulator) error { - if p.Interface != "" && p.listenAddr != "" { + if p.Interface != "" && p.listenAddr == "" { p.listenAddr = getAddr(p.Interface) } - for _, ip := range p.Urls { - _, err := net.LookupHost(ip) + for _, host := range p.Urls { + _, err := net.LookupHost(host) if err != nil { - acc.AddFields("ping", map[string]interface{}{"result_code": 1}, map[string]string{"ip": ip}) + acc.AddFields("ping", map[string]interface{}{"result_code": 1}, map[string]string{"url": host}) acc.AddError(err) - return nil + continue } - if p.Method == "native" { - p.wg.Add(1) - go func(ip string) { - defer p.wg.Done() - p.pingToURLNative(ip, acc) - }(ip) - } else { - p.wg.Add(1) - go func(ip string) { - defer p.wg.Done() - p.pingToURL(ip, acc) - }(ip) - } + p.wg.Add(1) + go func(host string) { + defer p.wg.Done() + + switch p.Method { + case "native": + p.pingToURLNative(host, acc) + default: + p.pingToURL(host, acc) + } + }(host) } p.wg.Wait() @@ -204,7 +204,11 @@ func (p *Ping) pingToURLNative(destination string, acc telegraf.Accumulator) { host, err := net.ResolveIPAddr(network, destination) if err != nil { - acc.AddFields("ping", map[string]interface{}{"result_code": 1}, map[string]string{"url": destination}) + acc.AddFields( + "ping", + map[string]interface{}{"result_code": 1}, + map[string]string{"url": destination}, + ) acc.AddError(err) return } @@ -243,8 +247,29 @@ func (p *Ping) pingToURLNative(destination string, acc telegraf.Accumulator) { wg := &sync.WaitGroup{} c := ping.Client{} - var i int - for i = 0; i < p.Count; i++ { + var doErr error + var packetsSent int + + type sentReq struct { + err error + sent bool + } + sents := make(chan sentReq) + + r.Add(1) + go func() { + for sent := range sents { + if sent.err != nil { + doErr = sent.err + } + if sent.sent { + packetsSent++ + } + } + r.Done() + }() + + for i := 0; i < p.Count; i++ { select { case <-ctx.Done(): goto finish @@ -260,13 +285,18 @@ func (p *Ping) pingToURLNative(destination string, acc telegraf.Accumulator) { Src: net.ParseIP(p.listenAddr), Seq: seq, }) + + sent := sentReq{err: err, sent: true} if err != nil { - acc.AddFields("ping", map[string]interface{}{"result_code": 2}, map[string]string{"url": destination}) - acc.AddError(err) + if strings.Contains(err.Error(), "not permitted") { + sent.sent = false + } + sents <- sent return } resps <- resp + sents <- sent }(i + 1) } } @@ -274,13 +304,19 @@ func (p *Ping) pingToURLNative(destination string, acc telegraf.Accumulator) { finish: wg.Wait() close(resps) + close(sents) r.Wait() - tags, fields := onFin(i, rsps, destination) + + if doErr != nil && strings.Contains(doErr.Error(), "not permitted") { + log.Printf("D! [inputs.ping] %s", doErr.Error()) + } + + tags, fields := onFin(packetsSent, rsps, doErr, destination) acc.AddFields("ping", fields, tags) } -func onFin(packetsSent int, resps []*ping.Response, destination string) (map[string]string, map[string]interface{}) { +func onFin(packetsSent int, resps []*ping.Response, err error, destination string) (map[string]string, map[string]interface{}) { packetsRcvd := len(resps) tags := map[string]string{"url": destination} @@ -291,10 +327,16 @@ func onFin(packetsSent int, resps []*ping.Response, destination string) (map[str } if packetsSent == 0 { + if err != nil { + fields["result_code"] = 2 + } return tags, fields } if packetsRcvd == 0 { + if err != nil { + fields["result_code"] = 1 + } fields["percent_packet_loss"] = float64(100) return tags, fields } diff --git a/plugins/inputs/ping/ping_test.go b/plugins/inputs/ping/ping_test.go index 56303b1b23dbd..8a1a0a9e1d7ca 100644 --- a/plugins/inputs/ping/ping_test.go +++ b/plugins/inputs/ping/ping_test.go @@ -355,4 +355,5 @@ func TestPingGatherNative(t *testing.T) { assert.NoError(t, acc.GatherError(p.Gather)) assert.True(t, acc.HasPoint("ping", map[string]string{"url": "localhost"}, "packets_transmitted", 5)) + assert.True(t, acc.HasPoint("ping", map[string]string{"url": "localhost"}, "packets_received", 5)) } diff --git a/plugins/inputs/postgresql_extensible/postgresql_extensible.go b/plugins/inputs/postgresql_extensible/postgresql_extensible.go index 05e57583f83b3..9a3457228a26c 100644 --- a/plugins/inputs/postgresql_extensible/postgresql_extensible.go +++ b/plugins/inputs/postgresql_extensible/postgresql_extensible.go @@ -4,11 +4,9 @@ import ( "bytes" "fmt" "io/ioutil" - "log" "os" "strings" - // register in driver. _ "github.com/jackc/pgx/stdlib" "github.com/influxdata/telegraf" @@ -23,6 +21,8 @@ type Postgresql struct { AdditionalTags []string Query query Debug bool + + Log telegraf.Logger } type query []struct { @@ -186,7 +186,7 @@ func (p *Postgresql) Gather(acc telegraf.Accumulator) error { if p.Query[i].Version <= db_version { rows, err := p.DB.Query(sql_query) if err != nil { - acc.AddError(err) + p.Log.Error(err.Error()) continue } @@ -194,7 +194,7 @@ func (p *Postgresql) Gather(acc telegraf.Accumulator) error { // grab the column information from the result if columns, err = rows.Columns(); err != nil { - acc.AddError(err) + p.Log.Error(err.Error()) continue } @@ -209,7 +209,7 @@ func (p *Postgresql) Gather(acc telegraf.Accumulator) error { for rows.Next() { err = p.accRow(meas_name, rows, acc, columns) if err != nil { - acc.AddError(err) + p.Log.Error(err.Error()) break } } @@ -272,7 +272,7 @@ func (p *Postgresql) accRow(meas_name string, row scanner, acc telegraf.Accumula fields := make(map[string]interface{}) COLUMN: for col, val := range columnMap { - log.Printf("D! postgresql_extensible: column: %s = %T: %v\n", col, *val, *val) + p.Log.Debugf("Column: %s = %T: %v\n", col, *val, *val) _, ignore := ignoredColumns[col] if ignore || *val == nil { continue @@ -290,7 +290,7 @@ COLUMN: case int64, int32, int: tags[col] = fmt.Sprintf("%d", v) default: - log.Println("failed to add additional tag", col) + p.Log.Debugf("Failed to add %q as additional tag", col) } continue COLUMN } diff --git a/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go b/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go index 7fbc343028015..757b468f26682 100644 --- a/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go +++ b/plugins/inputs/postgresql_extensible/postgresql_extensible_test.go @@ -13,6 +13,7 @@ import ( func queryRunner(t *testing.T, q query) *testutil.Accumulator { p := &Postgresql{ + Log: testutil.Logger{}, Service: postgresql.Service{ Address: fmt.Sprintf( "host=%s user=postgres sslmode=disable", @@ -232,6 +233,7 @@ func TestPostgresqlIgnoresUnwantedColumns(t *testing.T) { } p := &Postgresql{ + Log: testutil.Logger{}, Service: postgresql.Service{ Address: fmt.Sprintf( "host=%s user=postgres sslmode=disable", @@ -251,7 +253,10 @@ func TestPostgresqlIgnoresUnwantedColumns(t *testing.T) { } func TestAccRow(t *testing.T) { - p := Postgresql{} + p := Postgresql{ + Log: testutil.Logger{}, + } + var acc testutil.Accumulator columns := []string{"datname", "cat"} diff --git a/plugins/inputs/powerdns/README.md b/plugins/inputs/powerdns/README.md index 4b1732782e7ec..2e245eeffba91 100644 --- a/plugins/inputs/powerdns/README.md +++ b/plugins/inputs/powerdns/README.md @@ -14,6 +14,16 @@ The powerdns plugin gathers metrics about PowerDNS using unix socket. unix_sockets = ["/var/run/pdns.controlsocket"] ``` +#### Permissions + +Telegraf will need read access to the powerdns control socket. + +On many systems this can be accomplished by adding the `telegraf` user to the +`pdns` group: +``` +usermod telegraf -a -G pdns +``` + ### Measurements & Fields: - powerdns diff --git a/plugins/inputs/powerdns/powerdns.go b/plugins/inputs/powerdns/powerdns.go index e53373baf1ce4..3c661990cee4c 100644 --- a/plugins/inputs/powerdns/powerdns.go +++ b/plugins/inputs/powerdns/powerdns.go @@ -110,8 +110,8 @@ func parseResponse(metrics string) map[string]interface{} { i, err := strconv.ParseInt(m[1], 10, 64) if err != nil { - log.Printf("E! powerdns: Error parsing integer for metric [%s]: %s", - metric, err) + log.Printf("E! [inputs.powerdns] error parsing integer for metric %q: %s", + metric, err.Error()) continue } values[m[0]] = i diff --git a/plugins/inputs/powerdns_recursor/powerdns_recursor.go b/plugins/inputs/powerdns_recursor/powerdns_recursor.go index 85c7cbccaaeee..d040d8355329d 100644 --- a/plugins/inputs/powerdns_recursor/powerdns_recursor.go +++ b/plugins/inputs/powerdns_recursor/powerdns_recursor.go @@ -18,10 +18,11 @@ import ( ) type PowerdnsRecursor struct { - UnixSockets []string + UnixSockets []string `toml:"unix_sockets"` + SocketDir string `toml:"socket_dir"` + SocketMode string `toml:"socket_mode"` - SocketDir string `toml:"socket_dir"` - SocketMode uint32 `toml:"socket_mode"` + mode uint32 } var defaultTimeout = 5 * time.Second @@ -45,6 +46,18 @@ func (p *PowerdnsRecursor) Description() string { return "Read metrics from one or many PowerDNS Recursor servers" } +func (p *PowerdnsRecursor) Init() error { + if p.SocketMode != "" { + mode, err := strconv.ParseUint(p.SocketMode, 8, 32) + if err != nil { + return fmt.Errorf("could not parse socket_mode: %v", err) + } + + p.mode = uint32(mode) + } + return nil +} + func (p *PowerdnsRecursor) Gather(acc telegraf.Accumulator) error { if len(p.UnixSockets) == 0 { return p.gatherServer("/var/run/pdns_recursor.controlsocket", acc) @@ -79,11 +92,7 @@ func (p *PowerdnsRecursor) gatherServer(address string, acc telegraf.Accumulator if err != nil { return err } - perm := uint32(0666) - if p.SocketMode > 0 { - perm = p.SocketMode - } - if err := os.Chmod(recvSocket, os.FileMode(perm)); err != nil { + if err := os.Chmod(recvSocket, os.FileMode(p.mode)); err != nil { return err } defer conn.Close() @@ -139,8 +148,8 @@ func parseResponse(metrics string) map[string]interface{} { i, err := strconv.ParseInt(m[1], 10, 64) if err != nil { - log.Printf("E! [inputs.powerdns_recursor] Error parsing integer for metric [%s] %v", - metric, err) + log.Printf("E! [inputs.powerdns_recursor] error parsing integer for metric %q: %s", + metric, err.Error()) continue } values[m[0]] = i @@ -151,6 +160,8 @@ func parseResponse(metrics string) map[string]interface{} { func init() { inputs.Add("powerdns_recursor", func() telegraf.Input { - return &PowerdnsRecursor{} + return &PowerdnsRecursor{ + mode: uint32(0666), + } }) } diff --git a/plugins/inputs/powerdns_recursor/powerdns_recursor_test.go b/plugins/inputs/powerdns_recursor/powerdns_recursor_test.go index 629fe81c8cc4e..0ca4daf69d28f 100644 --- a/plugins/inputs/powerdns_recursor/powerdns_recursor_test.go +++ b/plugins/inputs/powerdns_recursor/powerdns_recursor_test.go @@ -139,7 +139,10 @@ func TestPowerdnsRecursorGeneratesMetrics(t *testing.T) { p := &PowerdnsRecursor{ UnixSockets: []string{controlSocket}, SocketDir: "/tmp", + SocketMode: "0666", } + err = p.Init() + require.NoError(t, err) var acc testutil.Accumulator diff --git a/plugins/inputs/processes/README.md b/plugins/inputs/processes/README.md index 4113f0d3af1f7..756326d75246d 100644 --- a/plugins/inputs/processes/README.md +++ b/plugins/inputs/processes/README.md @@ -6,7 +6,9 @@ them by status (zombie, sleeping, running, etc.) On linux this plugin requires access to procfs (/proc), on other OSes it requires access to execute `ps`. -### Configuration: +**Supported Platforms**: Linux, FreeBSD, Darwin + +### Configuration ```toml # Get the number of processes and group them by status @@ -19,9 +21,10 @@ Using the environment variable `HOST_PROC` the plugin will retrieve process info `docker run -v /proc:/rootfs/proc:ro -e HOST_PROC=/rootfs/proc` -### Measurements & Fields: +### Metrics - processes + - fields: - blocked (aka disk sleep or uninterruptible sleep) - running - sleeping @@ -53,14 +56,8 @@ Linux FreeBSD Darwin meaning W W none paging (linux kernel < 2.6 only), wait (freebsd) ``` -### Tags: - -None - -### Example Output: +### Example Output ``` -$ telegraf --config ~/ws/telegraf.conf --input-filter processes --test -* Plugin: processes, Collection 1 -> processes blocked=8i,running=1i,sleeping=265i,stopped=0i,total=274i,zombie=0i,dead=0i,paging=0i,total_threads=687i 1457478636980905042 +processes blocked=8i,running=1i,sleeping=265i,stopped=0i,total=274i,zombie=0i,dead=0i,paging=0i,total_threads=687i 1457478636980905042 ``` diff --git a/plugins/inputs/processes/processes.go b/plugins/inputs/processes/processes.go index 379a9cb377929..9ee583dbacecf 100644 --- a/plugins/inputs/processes/processes.go +++ b/plugins/inputs/processes/processes.go @@ -1,242 +1,7 @@ -// +build !windows - package processes -import ( - "bytes" - "fmt" - "io/ioutil" - "log" - "os" - "os/exec" - "path/filepath" - "runtime" - "strconv" - "syscall" - - "github.com/influxdata/telegraf" - "github.com/influxdata/telegraf/plugins/inputs" - "github.com/influxdata/telegraf/plugins/inputs/linux_sysctl_fs" -) - -type Processes struct { - execPS func() ([]byte, error) - readProcFile func(filename string) ([]byte, error) - - forcePS bool - forceProc bool -} - func (p *Processes) Description() string { return "Get the number of processes and group them by status" } func (p *Processes) SampleConfig() string { return "" } - -func (p *Processes) Gather(acc telegraf.Accumulator) error { - // Get an empty map of metric fields - fields := getEmptyFields() - - // Decide if we will use 'ps' to get stats (use procfs otherwise) - usePS := true - if runtime.GOOS == "linux" { - usePS = false - } - if p.forcePS { - usePS = true - } else if p.forceProc { - usePS = false - } - - // Gather stats from 'ps' or procfs - if usePS { - if err := p.gatherFromPS(fields); err != nil { - return err - } - } else { - if err := p.gatherFromProc(fields); err != nil { - return err - } - } - - acc.AddGauge("processes", fields, nil) - return nil -} - -// Gets empty fields of metrics based on the OS -func getEmptyFields() map[string]interface{} { - fields := map[string]interface{}{ - "blocked": int64(0), - "zombies": int64(0), - "stopped": int64(0), - "running": int64(0), - "sleeping": int64(0), - "total": int64(0), - "unknown": int64(0), - } - switch runtime.GOOS { - case "freebsd": - fields["idle"] = int64(0) - fields["wait"] = int64(0) - case "darwin": - fields["idle"] = int64(0) - case "openbsd": - fields["idle"] = int64(0) - case "linux": - fields["dead"] = int64(0) - fields["paging"] = int64(0) - fields["total_threads"] = int64(0) - fields["idle"] = int64(0) - } - return fields -} - -// exec `ps` to get all process states -func (p *Processes) gatherFromPS(fields map[string]interface{}) error { - out, err := p.execPS() - if err != nil { - return err - } - - for i, status := range bytes.Fields(out) { - if i == 0 && string(status) == "STAT" { - // This is a header, skip it - continue - } - switch status[0] { - case 'W': - fields["wait"] = fields["wait"].(int64) + int64(1) - case 'U', 'D', 'L': - // Also known as uninterruptible sleep or disk sleep - fields["blocked"] = fields["blocked"].(int64) + int64(1) - case 'Z': - fields["zombies"] = fields["zombies"].(int64) + int64(1) - case 'X': - fields["dead"] = fields["dead"].(int64) + int64(1) - case 'T': - fields["stopped"] = fields["stopped"].(int64) + int64(1) - case 'R': - fields["running"] = fields["running"].(int64) + int64(1) - case 'S': - fields["sleeping"] = fields["sleeping"].(int64) + int64(1) - case 'I': - fields["idle"] = fields["idle"].(int64) + int64(1) - case '?': - fields["unknown"] = fields["unknown"].(int64) + int64(1) - default: - log.Printf("I! processes: Unknown state [ %s ] from ps", - string(status[0])) - } - fields["total"] = fields["total"].(int64) + int64(1) - } - return nil -} - -// get process states from /proc/(pid)/stat files -func (p *Processes) gatherFromProc(fields map[string]interface{}) error { - filenames, err := filepath.Glob(linux_sysctl_fs.GetHostProc() + "/[0-9]*/stat") - - if err != nil { - return err - } - - for _, filename := range filenames { - _, err := os.Stat(filename) - data, err := p.readProcFile(filename) - if err != nil { - return err - } - if data == nil { - continue - } - - // Parse out data after () - i := bytes.LastIndex(data, []byte(")")) - if i == -1 { - continue - } - data = data[i+2:] - - stats := bytes.Fields(data) - if len(stats) < 3 { - return fmt.Errorf("Something is terribly wrong with %s", filename) - } - switch stats[0][0] { - case 'R': - fields["running"] = fields["running"].(int64) + int64(1) - case 'S': - fields["sleeping"] = fields["sleeping"].(int64) + int64(1) - case 'D': - fields["blocked"] = fields["blocked"].(int64) + int64(1) - case 'Z': - fields["zombies"] = fields["zombies"].(int64) + int64(1) - case 'X': - fields["dead"] = fields["dead"].(int64) + int64(1) - case 'T', 't': - fields["stopped"] = fields["stopped"].(int64) + int64(1) - case 'W': - fields["paging"] = fields["paging"].(int64) + int64(1) - case 'I': - fields["idle"] = fields["idle"].(int64) + int64(1) - case 'P': - if _, ok := fields["parked"]; ok { - fields["parked"] = fields["parked"].(int64) + int64(1) - } - fields["parked"] = int64(1) - default: - log.Printf("I! processes: Unknown state [ %s ] in file %s", - string(stats[0][0]), filename) - } - fields["total"] = fields["total"].(int64) + int64(1) - - threads, err := strconv.Atoi(string(stats[17])) - if err != nil { - log.Printf("I! processes: Error parsing thread count: %s", err) - continue - } - fields["total_threads"] = fields["total_threads"].(int64) + int64(threads) - } - return nil -} - -func readProcFile(filename string) ([]byte, error) { - data, err := ioutil.ReadFile(filename) - if err != nil { - if os.IsNotExist(err) { - return nil, nil - } - - // Reading from /proc/ fails with ESRCH if the process has - // been terminated between open() and read(). - if perr, ok := err.(*os.PathError); ok && perr.Err == syscall.ESRCH { - return nil, nil - } - - return nil, err - } - - return data, nil -} - -func execPS() ([]byte, error) { - bin, err := exec.LookPath("ps") - if err != nil { - return nil, err - } - - out, err := exec.Command(bin, "axo", "state").Output() - if err != nil { - return nil, err - } - - return out, err -} - -func init() { - inputs.Add("processes", func() telegraf.Input { - return &Processes{ - execPS: execPS, - readProcFile: readProcFile, - } - }) -} diff --git a/plugins/inputs/processes/processes_notwindows.go b/plugins/inputs/processes/processes_notwindows.go new file mode 100644 index 0000000000000..445e7fb9f7255 --- /dev/null +++ b/plugins/inputs/processes/processes_notwindows.go @@ -0,0 +1,235 @@ +// +build !windows + +package processes + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "syscall" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" + "github.com/influxdata/telegraf/plugins/inputs/linux_sysctl_fs" +) + +type Processes struct { + execPS func() ([]byte, error) + readProcFile func(filename string) ([]byte, error) + + Log telegraf.Logger + + forcePS bool + forceProc bool +} + +func (p *Processes) Gather(acc telegraf.Accumulator) error { + // Get an empty map of metric fields + fields := getEmptyFields() + + // Decide if we will use 'ps' to get stats (use procfs otherwise) + usePS := true + if runtime.GOOS == "linux" { + usePS = false + } + if p.forcePS { + usePS = true + } else if p.forceProc { + usePS = false + } + + // Gather stats from 'ps' or procfs + if usePS { + if err := p.gatherFromPS(fields); err != nil { + return err + } + } else { + if err := p.gatherFromProc(fields); err != nil { + return err + } + } + + acc.AddGauge("processes", fields, nil) + return nil +} + +// Gets empty fields of metrics based on the OS +func getEmptyFields() map[string]interface{} { + fields := map[string]interface{}{ + "blocked": int64(0), + "zombies": int64(0), + "stopped": int64(0), + "running": int64(0), + "sleeping": int64(0), + "total": int64(0), + "unknown": int64(0), + } + switch runtime.GOOS { + case "freebsd": + fields["idle"] = int64(0) + fields["wait"] = int64(0) + case "darwin": + fields["idle"] = int64(0) + case "openbsd": + fields["idle"] = int64(0) + case "linux": + fields["dead"] = int64(0) + fields["paging"] = int64(0) + fields["total_threads"] = int64(0) + fields["idle"] = int64(0) + } + return fields +} + +// exec `ps` to get all process states +func (p *Processes) gatherFromPS(fields map[string]interface{}) error { + out, err := p.execPS() + if err != nil { + return err + } + + for i, status := range bytes.Fields(out) { + if i == 0 && string(status) == "STAT" { + // This is a header, skip it + continue + } + switch status[0] { + case 'W': + fields["wait"] = fields["wait"].(int64) + int64(1) + case 'U', 'D', 'L': + // Also known as uninterruptible sleep or disk sleep + fields["blocked"] = fields["blocked"].(int64) + int64(1) + case 'Z': + fields["zombies"] = fields["zombies"].(int64) + int64(1) + case 'X': + fields["dead"] = fields["dead"].(int64) + int64(1) + case 'T': + fields["stopped"] = fields["stopped"].(int64) + int64(1) + case 'R': + fields["running"] = fields["running"].(int64) + int64(1) + case 'S': + fields["sleeping"] = fields["sleeping"].(int64) + int64(1) + case 'I': + fields["idle"] = fields["idle"].(int64) + int64(1) + case '?': + fields["unknown"] = fields["unknown"].(int64) + int64(1) + default: + p.Log.Infof("Unknown state %q from ps", string(status[0])) + } + fields["total"] = fields["total"].(int64) + int64(1) + } + return nil +} + +// get process states from /proc/(pid)/stat files +func (p *Processes) gatherFromProc(fields map[string]interface{}) error { + filenames, err := filepath.Glob(linux_sysctl_fs.GetHostProc() + "/[0-9]*/stat") + + if err != nil { + return err + } + + for _, filename := range filenames { + _, err := os.Stat(filename) + data, err := p.readProcFile(filename) + if err != nil { + return err + } + if data == nil { + continue + } + + // Parse out data after () + i := bytes.LastIndex(data, []byte(")")) + if i == -1 { + continue + } + data = data[i+2:] + + stats := bytes.Fields(data) + if len(stats) < 3 { + return fmt.Errorf("Something is terribly wrong with %s", filename) + } + switch stats[0][0] { + case 'R': + fields["running"] = fields["running"].(int64) + int64(1) + case 'S': + fields["sleeping"] = fields["sleeping"].(int64) + int64(1) + case 'D': + fields["blocked"] = fields["blocked"].(int64) + int64(1) + case 'Z': + fields["zombies"] = fields["zombies"].(int64) + int64(1) + case 'X': + fields["dead"] = fields["dead"].(int64) + int64(1) + case 'T', 't': + fields["stopped"] = fields["stopped"].(int64) + int64(1) + case 'W': + fields["paging"] = fields["paging"].(int64) + int64(1) + case 'I': + fields["idle"] = fields["idle"].(int64) + int64(1) + case 'P': + if _, ok := fields["parked"]; ok { + fields["parked"] = fields["parked"].(int64) + int64(1) + } + fields["parked"] = int64(1) + default: + p.Log.Infof("Unknown state %q in file %q", string(stats[0][0]), filename) + } + fields["total"] = fields["total"].(int64) + int64(1) + + threads, err := strconv.Atoi(string(stats[17])) + if err != nil { + p.Log.Infof("Error parsing thread count: %s", err.Error()) + continue + } + fields["total_threads"] = fields["total_threads"].(int64) + int64(threads) + } + return nil +} + +func readProcFile(filename string) ([]byte, error) { + data, err := ioutil.ReadFile(filename) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + + // Reading from /proc/ fails with ESRCH if the process has + // been terminated between open() and read(). + if perr, ok := err.(*os.PathError); ok && perr.Err == syscall.ESRCH { + return nil, nil + } + + return nil, err + } + + return data, nil +} + +func execPS() ([]byte, error) { + bin, err := exec.LookPath("ps") + if err != nil { + return nil, err + } + + out, err := exec.Command(bin, "axo", "state").Output() + if err != nil { + return nil, err + } + + return out, err +} + +func init() { + inputs.Add("processes", func() telegraf.Input { + return &Processes{ + execPS: execPS, + readProcFile: readProcFile, + } + }) +} diff --git a/plugins/inputs/processes/processes_test.go b/plugins/inputs/processes/processes_test.go index f9bad4b6087bb..23359a85dc958 100644 --- a/plugins/inputs/processes/processes_test.go +++ b/plugins/inputs/processes/processes_test.go @@ -16,6 +16,7 @@ import ( func TestProcesses(t *testing.T) { processes := &Processes{ + Log: testutil.Logger{}, execPS: execPS, readProcFile: readProcFile, } @@ -35,6 +36,7 @@ func TestProcesses(t *testing.T) { func TestFromPS(t *testing.T) { processes := &Processes{ + Log: testutil.Logger{}, execPS: testExecPS, forcePS: true, } @@ -56,6 +58,7 @@ func TestFromPS(t *testing.T) { func TestFromPSError(t *testing.T) { processes := &Processes{ + Log: testutil.Logger{}, execPS: testExecPSError, forcePS: true, } @@ -71,6 +74,7 @@ func TestFromProcFiles(t *testing.T) { } tester := tester{} processes := &Processes{ + Log: testutil.Logger{}, readProcFile: tester.testProcFile, forceProc: true, } @@ -93,6 +97,7 @@ func TestFromProcFilesWithSpaceInCmd(t *testing.T) { } tester := tester{} processes := &Processes{ + Log: testutil.Logger{}, readProcFile: tester.testProcFile2, forceProc: true, } @@ -120,6 +125,7 @@ func TestParkedProcess(t *testing.T) { procstat := `88 (watchdog/13) P 2 0 0 0 -1 69238848 0 0 0 0 0 0 0 0 20 0 1 0 20 0 0 18446744073709551615 0 0 0 0 0 0 0 2147483647 0 1 0 0 17 0 0 0 0 0 0 0 0 0 0 0 0 0 0 ` plugin := &Processes{ + Log: testutil.Logger{}, readProcFile: func(string) ([]byte, error) { return []byte(procstat), nil }, @@ -147,7 +153,7 @@ func TestParkedProcess(t *testing.T) { "zombies": 0, }, time.Unix(0, 0), - telegraf.Untyped, + telegraf.Gauge, ), } actual := acc.GetTelegrafMetrics() diff --git a/plugins/inputs/processes/processes_windows.go b/plugins/inputs/processes/processes_windows.go index 32c73f918165d..567373c7c7260 100644 --- a/plugins/inputs/processes/processes_windows.go +++ b/plugins/inputs/processes/processes_windows.go @@ -1,3 +1,27 @@ // +build windows package processes + +import ( + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" +) + +type Processes struct { + Log telegraf.Logger +} + +func (e *Processes) Init() error { + e.Log.Warn("Current platform is not supported") + return nil +} + +func (e *Processes) Gather(acc telegraf.Accumulator) error { + return nil +} + +func init() { + inputs.Add("processes", func() telegraf.Input { + return &Processes{} + }) +} diff --git a/plugins/inputs/procstat/README.md b/plugins/inputs/procstat/README.md index 8ce834a7021f6..6163b8284818f 100644 --- a/plugins/inputs/procstat/README.md +++ b/plugins/inputs/procstat/README.md @@ -106,6 +106,7 @@ implemented as a WMI query. The pattern allows fuzzy matching using only - memory_rss (int) - memory_stack (int) - memory_swap (int) + - memory_usage (float) - memory_vms (int) - minor_faults (int) - nice_priority (int) diff --git a/plugins/inputs/prometheus/README.md b/plugins/inputs/prometheus/README.md index edc8a27d6db51..7b2e054a21049 100644 --- a/plugins/inputs/prometheus/README.md +++ b/plugins/inputs/prometheus/README.md @@ -11,6 +11,15 @@ in Prometheus format. ## An array of urls to scrape metrics from. urls = ["http://localhost:9100/metrics"] + ## Metric version controls the mapping from Prometheus metrics into + ## Telegraf metrics. When using the prometheus_client output, use the same + ## value in both plugins to ensure metrics are round-tripped without + ## modification. + ## + ## example: metric_version = 1; deprecated in 1.13 + ## metric_version = 2; recommended version + # metric_version = 1 + ## An array of Kubernetes services to scrape metrics from. # kubernetes_services = ["http://my-service-dns.my-namespace:9100/metrics"] @@ -140,3 +149,18 @@ cpu_usage_user,cpu=cpu1,url=http://example.org:9273/metrics gauge=5.829145728641 cpu_usage_user,cpu=cpu2,url=http://example.org:9273/metrics gauge=2.119071644805144 1505776751000000000 cpu_usage_user,cpu=cpu3,url=http://example.org:9273/metrics gauge=1.5228426395944945 1505776751000000000 ``` + +**Output (when metric_version = 2)** +``` +prometheus,quantile=1,url=http://example.org:9273/metrics go_gc_duration_seconds=0.005574303 1556075100000000000 +prometheus,quantile=0.75,url=http://example.org:9273/metrics go_gc_duration_seconds=0.0001046 1556075100000000000 +prometheus,quantile=0.5,url=http://example.org:9273/metrics go_gc_duration_seconds=0.0000719 1556075100000000000 +prometheus,quantile=0.25,url=http://example.org:9273/metrics go_gc_duration_seconds=0.0000579 1556075100000000000 +prometheus,quantile=0,url=http://example.org:9273/metrics go_gc_duration_seconds=0.0000349 1556075100000000000 +prometheus,url=http://example.org:9273/metrics go_gc_duration_seconds_count=324,go_gc_duration_seconds_sum=0.091340353 1556075100000000000 +prometheus,url=http://example.org:9273/metrics go_goroutines=15 1556075100000000000 +prometheus,cpu=cpu0,url=http://example.org:9273/metrics cpu_usage_user=1.513622603430151 1505776751000000000 +prometheus,cpu=cpu1,url=http://example.org:9273/metrics cpu_usage_user=5.829145728641773 1505776751000000000 +prometheus,cpu=cpu2,url=http://example.org:9273/metrics cpu_usage_user=2.119071644805144 1505776751000000000 +prometheus,cpu=cpu3,url=http://example.org:9273/metrics cpu_usage_user=1.5228426395944945 1505776751000000000 +``` diff --git a/plugins/inputs/prometheus/kubernetes.go b/plugins/inputs/prometheus/kubernetes.go index d92d90ead72fc..61750938403b0 100644 --- a/plugins/inputs/prometheus/kubernetes.go +++ b/plugins/inputs/prometheus/kubernetes.go @@ -68,7 +68,7 @@ func (p *Prometheus) start(ctx context.Context) error { case <-time.After(time.Second): err := p.watch(ctx, client) if err != nil { - log.Printf("E! [inputs.prometheus] unable to watch resources: %v", err) + p.Log.Errorf("Unable to watch resources: %s", err.Error()) } } } @@ -144,7 +144,7 @@ func registerPod(pod *corev1.Pod, p *Prometheus) { return } - log.Printf("D! [inputs.prometheus] will scrape metrics from %s", *targetURL) + log.Printf("D! [inputs.prometheus] will scrape metrics from %q", *targetURL) // add annotation as metrics tags tags := pod.GetMetadata().GetAnnotations() if tags == nil { @@ -158,7 +158,7 @@ func registerPod(pod *corev1.Pod, p *Prometheus) { } URL, err := url.Parse(*targetURL) if err != nil { - log.Printf("E! [inputs.prometheus] could not parse URL %s: %v", *targetURL, err) + log.Printf("E! [inputs.prometheus] could not parse URL %q: %s", *targetURL, err.Error()) return } podURL := p.AddressToURL(URL, URL.Hostname()) @@ -211,13 +211,13 @@ func unregisterPod(pod *corev1.Pod, p *Prometheus) { return } - log.Printf("D! [inputs.prometheus] registered a delete request for %s in namespace %s", + log.Printf("D! [inputs.prometheus] registered a delete request for %q in namespace %q", pod.GetMetadata().GetName(), pod.GetMetadata().GetNamespace()) p.lock.Lock() defer p.lock.Unlock() if _, ok := p.kubernetesPods[*url]; ok { delete(p.kubernetesPods, *url) - log.Printf("D! [inputs.prometheus] will stop scraping for %s", *url) + log.Printf("D! [inputs.prometheus] will stop scraping for %q", *url) } } diff --git a/plugins/inputs/prometheus/kubernetes_test.go b/plugins/inputs/prometheus/kubernetes_test.go index c1bbe0a1f8aaa..b926f7393bdd9 100644 --- a/plugins/inputs/prometheus/kubernetes_test.go +++ b/plugins/inputs/prometheus/kubernetes_test.go @@ -3,6 +3,7 @@ package prometheus import ( "testing" + "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/assert" v1 "github.com/ericchiang/k8s/apis/core/v1" @@ -53,7 +54,7 @@ func TestScrapeURLAnnotationsCustomPathWithSep(t *testing.T) { } func TestAddPod(t *testing.T) { - prom := &Prometheus{} + prom := &Prometheus{Log: testutil.Logger{}} p := pod() p.Metadata.Annotations = map[string]string{"prometheus.io/scrape": "true"} @@ -62,7 +63,7 @@ func TestAddPod(t *testing.T) { } func TestAddMultipleDuplicatePods(t *testing.T) { - prom := &Prometheus{} + prom := &Prometheus{Log: testutil.Logger{}} p := pod() p.Metadata.Annotations = map[string]string{"prometheus.io/scrape": "true"} @@ -73,7 +74,7 @@ func TestAddMultipleDuplicatePods(t *testing.T) { } func TestAddMultiplePods(t *testing.T) { - prom := &Prometheus{} + prom := &Prometheus{Log: testutil.Logger{}} p := pod() p.Metadata.Annotations = map[string]string{"prometheus.io/scrape": "true"} @@ -85,7 +86,7 @@ func TestAddMultiplePods(t *testing.T) { } func TestDeletePods(t *testing.T) { - prom := &Prometheus{} + prom := &Prometheus{Log: testutil.Logger{}} p := pod() p.Metadata.Annotations = map[string]string{"prometheus.io/scrape": "true"} diff --git a/plugins/inputs/prometheus/parser.go b/plugins/inputs/prometheus/parser.go index 6584fbc05e466..9e79249ec1b40 100644 --- a/plugins/inputs/prometheus/parser.go +++ b/plugins/inputs/prometheus/parser.go @@ -21,6 +21,145 @@ import ( "github.com/prometheus/common/expfmt" ) +// Parse returns a slice of Metrics from a text representation of a +// metrics +func ParseV2(buf []byte, header http.Header) ([]telegraf.Metric, error) { + var metrics []telegraf.Metric + var parser expfmt.TextParser + // parse even if the buffer begins with a newline + buf = bytes.TrimPrefix(buf, []byte("\n")) + // Read raw data + buffer := bytes.NewBuffer(buf) + reader := bufio.NewReader(buffer) + + mediatype, params, err := mime.ParseMediaType(header.Get("Content-Type")) + // Prepare output + metricFamilies := make(map[string]*dto.MetricFamily) + + if err == nil && mediatype == "application/vnd.google.protobuf" && + params["encoding"] == "delimited" && + params["proto"] == "io.prometheus.client.MetricFamily" { + for { + mf := &dto.MetricFamily{} + if _, ierr := pbutil.ReadDelimited(reader, mf); ierr != nil { + if ierr == io.EOF { + break + } + return nil, fmt.Errorf("reading metric family protocol buffer failed: %s", ierr) + } + metricFamilies[mf.GetName()] = mf + } + } else { + metricFamilies, err = parser.TextToMetricFamilies(reader) + if err != nil { + return nil, fmt.Errorf("reading text format failed: %s", err) + } + } + + // read metrics + for metricName, mf := range metricFamilies { + for _, m := range mf.Metric { + // reading tags + tags := makeLabels(m) + + if mf.GetType() == dto.MetricType_SUMMARY { + // summary metric + telegrafMetrics := makeQuantilesV2(m, tags, metricName, mf.GetType()) + metrics = append(metrics, telegrafMetrics...) + } else if mf.GetType() == dto.MetricType_HISTOGRAM { + // histogram metric + telegrafMetrics := makeBucketsV2(m, tags, metricName, mf.GetType()) + metrics = append(metrics, telegrafMetrics...) + } else { + // standard metric + // reading fields + fields := make(map[string]interface{}) + fields = getNameAndValueV2(m, metricName) + // converting to telegraf metric + if len(fields) > 0 { + var t time.Time + if m.TimestampMs != nil && *m.TimestampMs > 0 { + t = time.Unix(0, *m.TimestampMs*1000000) + } else { + t = time.Now() + } + metric, err := metric.New("prometheus", tags, fields, t, valueType(mf.GetType())) + if err == nil { + metrics = append(metrics, metric) + } + } + } + } + } + + return metrics, err +} + +// Get Quantiles for summary metric & Buckets for histogram +func makeQuantilesV2(m *dto.Metric, tags map[string]string, metricName string, metricType dto.MetricType) []telegraf.Metric { + var metrics []telegraf.Metric + fields := make(map[string]interface{}) + var t time.Time + if m.TimestampMs != nil && *m.TimestampMs > 0 { + t = time.Unix(0, *m.TimestampMs*1000000) + } else { + t = time.Now() + } + fields[metricName+"_count"] = float64(m.GetSummary().GetSampleCount()) + fields[metricName+"_sum"] = float64(m.GetSummary().GetSampleSum()) + met, err := metric.New("prometheus", tags, fields, t, valueType(metricType)) + if err == nil { + metrics = append(metrics, met) + } + + for _, q := range m.GetSummary().Quantile { + newTags := tags + fields = make(map[string]interface{}) + if !math.IsNaN(q.GetValue()) { + newTags["quantile"] = fmt.Sprint(q.GetQuantile()) + fields[metricName] = float64(q.GetValue()) + + quantileMetric, err := metric.New("prometheus", newTags, fields, t, valueType(metricType)) + if err == nil { + metrics = append(metrics, quantileMetric) + } + } + } + return metrics +} + +// Get Buckets from histogram metric +func makeBucketsV2(m *dto.Metric, tags map[string]string, metricName string, metricType dto.MetricType) []telegraf.Metric { + var metrics []telegraf.Metric + fields := make(map[string]interface{}) + var t time.Time + if m.TimestampMs != nil && *m.TimestampMs > 0 { + t = time.Unix(0, *m.TimestampMs*1000000) + } else { + t = time.Now() + } + fields[metricName+"_count"] = float64(m.GetHistogram().GetSampleCount()) + fields[metricName+"_sum"] = float64(m.GetHistogram().GetSampleSum()) + + met, err := metric.New("prometheus", tags, fields, t, valueType(metricType)) + if err == nil { + metrics = append(metrics, met) + } + + for _, b := range m.GetHistogram().Bucket { + newTags := tags + fields = make(map[string]interface{}) + newTags["le"] = fmt.Sprint(b.GetUpperBound()) + fields[metricName+"_bucket"] = float64(b.GetCumulativeCount()) + + histogramMetric, err := metric.New("prometheus", newTags, fields, t, valueType(metricType)) + if err == nil { + metrics = append(metrics, histogramMetric) + } + } + return metrics +} + // Parse returns a slice of Metrics from a text representation of a // metrics func Parse(buf []byte, header http.Header) ([]telegraf.Metric, error) { @@ -159,3 +298,22 @@ func getNameAndValue(m *dto.Metric) map[string]interface{} { } return fields } + +// Get name and value from metric +func getNameAndValueV2(m *dto.Metric, metricName string) map[string]interface{} { + fields := make(map[string]interface{}) + if m.Gauge != nil { + if !math.IsNaN(m.GetGauge().GetValue()) { + fields[metricName] = float64(m.GetGauge().GetValue()) + } + } else if m.Counter != nil { + if !math.IsNaN(m.GetCounter().GetValue()) { + fields[metricName] = float64(m.GetCounter().GetValue()) + } + } else if m.Untyped != nil { + if !math.IsNaN(m.GetUntyped().GetValue()) { + fields[metricName] = float64(m.GetUntyped().GetValue()) + } + } + return fields +} diff --git a/plugins/inputs/prometheus/prometheus.go b/plugins/inputs/prometheus/prometheus.go index 284114258852a..1f08627609d31 100644 --- a/plugins/inputs/prometheus/prometheus.go +++ b/plugins/inputs/prometheus/prometheus.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "io/ioutil" - "log" "net" "net/http" "net/url" @@ -18,7 +17,7 @@ import ( "github.com/influxdata/telegraf/plugins/inputs" ) -const acceptHeader = `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7,text/plain;version=0.0.4;q=0.3` +const acceptHeader = `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7,text/plain;version=0.0.4;q=0.3,*/*;q=0.1` type Prometheus struct { // An array of urls to scrape metrics from. @@ -40,8 +39,14 @@ type Prometheus struct { ResponseTimeout internal.Duration `toml:"response_timeout"` + MetricVersion int `toml:"metric_version"` + + URLTag string `toml:"url_tag"` + tls.ClientConfig + Log telegraf.Logger + client *http.Client // Should we scrape Kubernetes services for prometheus annotations @@ -57,6 +62,18 @@ var sampleConfig = ` ## An array of urls to scrape metrics from. urls = ["http://localhost:9100/metrics"] + ## Metric version controls the mapping from Prometheus metrics into + ## Telegraf metrics. When using the prometheus_client output, use the same + ## value in both plugins to ensure metrics are round-tripped without + ## modification. + ## + ## example: metric_version = 1; deprecated in 1.13 + ## metric_version = 2; recommended version + # metric_version = 1 + + ## Url tag name (tag containing scrapped url. optional, default is "url") + # url_tag = "scrapeUrl" + ## An array of Kubernetes services to scrape metrics from. # kubernetes_services = ["http://my-service-dns.my-namespace:9100/metrics"] @@ -84,7 +101,7 @@ var sampleConfig = ` # username = "" # password = "" - ## Specify timeout duration for slower prometheus clients (default is 3s) + ## Specify timeout duration for slower prometheus clients (default is 3s) # response_timeout = "3s" ## Optional TLS Config @@ -103,6 +120,13 @@ func (p *Prometheus) Description() string { return "Read metrics from one or many prometheus clients" } +func (p *Prometheus) Init() error { + if p.MetricVersion != 2 { + p.Log.Warnf("Use of deprecated configuration: 'metric_version = 1'; please update to 'metric_version = 2'") + } + return nil +} + var ErrProtocolError = errors.New("prometheus protocol error") func (p *Prometheus) AddressToURL(u *url.URL, address string) *url.URL { @@ -136,7 +160,7 @@ func (p *Prometheus) GetAllURLs() (map[string]URLAndAddress, error) { for _, u := range p.URLs { URL, err := url.Parse(u) if err != nil { - log.Printf("prometheus: Could not parse %s, skipping it. Error: %s", u, err.Error()) + p.Log.Errorf("Could not parse %q, skipping it. Error: %s", u, err.Error()) continue } allURLs[URL.String()] = URLAndAddress{URL: URL, OriginalURL: URL} @@ -157,7 +181,7 @@ func (p *Prometheus) GetAllURLs() (map[string]URLAndAddress, error) { resolvedAddresses, err := net.LookupHost(URL.Hostname()) if err != nil { - log.Printf("prometheus: Could not resolve %s, skipping it. Error: %s", URL.Host, err.Error()) + p.Log.Errorf("Could not resolve %q, skipping it. Error: %s", URL.Host, err.Error()) continue } for _, resolved := range resolvedAddresses { @@ -223,6 +247,7 @@ func (p *Prometheus) gatherURL(u URLAndAddress, acc telegraf.Accumulator) error var req *http.Request var err error var uClient *http.Client + var metrics []telegraf.Metric if u.URL.Scheme == "unix" { path := u.URL.Query().Get("path") if path == "" { @@ -284,7 +309,12 @@ func (p *Prometheus) gatherURL(u URLAndAddress, acc telegraf.Accumulator) error return fmt.Errorf("error reading body: %s", err) } - metrics, err := Parse(body, resp.Header) + if p.MetricVersion == 2 { + metrics, err = ParseV2(body, resp.Header) + } else { + metrics, err = Parse(body, resp.Header) + } + if err != nil { return fmt.Errorf("error reading metrics for %s: %s", u.URL, err) @@ -294,7 +324,9 @@ func (p *Prometheus) gatherURL(u URLAndAddress, acc telegraf.Accumulator) error tags := metric.Tags() // strip user and password from URL u.OriginalURL.User = nil - tags["url"] = u.OriginalURL.String() + if p.URLTag != "" { + tags[p.URLTag] = u.OriginalURL.String() + } if u.Address != "" { tags["address"] = u.Address } @@ -341,6 +373,7 @@ func init() { return &Prometheus{ ResponseTimeout: internal.Duration{Duration: time.Second * 3}, kubernetesPods: map[string]URLAndAddress{}, + URLTag: "url", } }) } diff --git a/plugins/inputs/prometheus/prometheus_test.go b/plugins/inputs/prometheus/prometheus_test.go index ef3902fc908cb..78629d3d7f240 100644 --- a/plugins/inputs/prometheus/prometheus_test.go +++ b/plugins/inputs/prometheus/prometheus_test.go @@ -29,6 +29,21 @@ go_goroutines 15 # TYPE test_metric untyped test_metric{label="value"} 1.0 1490802350000 ` +const sampleSummaryTextFormat = `# HELP go_gc_duration_seconds A summary of the GC invocation durations. +# TYPE go_gc_duration_seconds summary +go_gc_duration_seconds{quantile="0"} 0.00010425500000000001 +go_gc_duration_seconds{quantile="0.25"} 0.000139108 +go_gc_duration_seconds{quantile="0.5"} 0.00015749400000000002 +go_gc_duration_seconds{quantile="0.75"} 0.000331463 +go_gc_duration_seconds{quantile="1"} 0.000667154 +go_gc_duration_seconds_sum 0.0018183950000000002 +go_gc_duration_seconds_count 7 +` +const sampleGaugeTextFormat = ` +# HELP go_goroutines Number of goroutines that currently exist. +# TYPE go_goroutines gauge +go_goroutines 15 1490802350000 +` func TestPrometheusGeneratesMetrics(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -37,7 +52,9 @@ func TestPrometheusGeneratesMetrics(t *testing.T) { defer ts.Close() p := &Prometheus{ - URLs: []string{ts.URL}, + Log: testutil.Logger{}, + URLs: []string{ts.URL}, + URLTag: "url", } var acc testutil.Accumulator @@ -60,7 +77,9 @@ func TestPrometheusGeneratesMetricsWithHostNameTag(t *testing.T) { defer ts.Close() p := &Prometheus{ + Log: testutil.Logger{}, KubernetesServices: []string{ts.URL}, + URLTag: "url", } u, _ := url.Parse(ts.URL) tsAddress := u.Hostname() @@ -89,6 +108,7 @@ func TestPrometheusGeneratesMetricsAlthoughFirstDNSFails(t *testing.T) { defer ts.Close() p := &Prometheus{ + Log: testutil.Logger{}, URLs: []string{ts.URL}, KubernetesServices: []string{"http://random.telegraf.local:88/metrics"}, } @@ -103,3 +123,49 @@ func TestPrometheusGeneratesMetricsAlthoughFirstDNSFails(t *testing.T) { assert.True(t, acc.HasFloatField("test_metric", "value")) assert.True(t, acc.HasTimestamp("test_metric", time.Unix(1490802350, 0))) } + +func TestPrometheusGeneratesSummaryMetricsV2(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, sampleSummaryTextFormat) + })) + defer ts.Close() + + p := &Prometheus{ + URLs: []string{ts.URL}, + URLTag: "url", + MetricVersion: 2, + } + + var acc testutil.Accumulator + + err := acc.GatherError(p.Gather) + require.NoError(t, err) + + assert.True(t, acc.TagSetValue("prometheus", "quantile") == "0") + assert.True(t, acc.HasFloatField("prometheus", "go_gc_duration_seconds_sum")) + assert.True(t, acc.HasFloatField("prometheus", "go_gc_duration_seconds_count")) + assert.True(t, acc.TagValue("prometheus", "url") == ts.URL+"/metrics") + +} + +func TestPrometheusGeneratesGaugeMetricsV2(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, sampleGaugeTextFormat) + })) + defer ts.Close() + + p := &Prometheus{ + URLs: []string{ts.URL}, + URLTag: "url", + MetricVersion: 2, + } + + var acc testutil.Accumulator + + err := acc.GatherError(p.Gather) + require.NoError(t, err) + + assert.True(t, acc.HasFloatField("prometheus", "go_goroutines")) + assert.True(t, acc.TagValue("prometheus", "url") == ts.URL+"/metrics") + assert.True(t, acc.HasTimestamp("prometheus", time.Unix(1490802350, 0))) +} diff --git a/plugins/inputs/rabbitmq/README.md b/plugins/inputs/rabbitmq/README.md index d52a760f2f253..0e119b25e0231 100644 --- a/plugins/inputs/rabbitmq/README.md +++ b/plugins/inputs/rabbitmq/README.md @@ -7,7 +7,7 @@ For additional details reference the [RabbitMQ Management HTTP Stats][management [management]: https://www.rabbitmq.com/management.html [management-reference]: https://raw.githack.com/rabbitmq/rabbitmq-management/rabbitmq_v3_6_9/priv/www/api/index.html -### Configuration: +### Configuration ```toml [[inputs.rabbitmq]] @@ -52,151 +52,173 @@ For additional details reference the [RabbitMQ Management HTTP Stats][management ## Note that an empty array for both will include all queues # queue_name_include = [] # queue_name_exclude = [] -``` - -### Measurements & Fields: -- rabbitmq_overview - - channels (int, channels) - - connections (int, connections) - - consumers (int, consumers) - - exchanges (int, exchanges) - - messages (int, messages) - - messages_acked (int, messages) - - messages_delivered (int, messages) - - messages_delivered_get (int, messages) - - messages_published (int, messages) - - messages_ready (int, messages) - - messages_unacked (int, messages) - - queues (int, queues) - - clustering_listeners (int, cluster nodes) - - amqp_listeners (int, amqp nodes up) - - return_unroutable (int, number of unroutable messages) - - return_unroutable_rate (float, number of unroutable messages per second) - -- rabbitmq_node - - disk_free (int, bytes) - - disk_free_limit (int, bytes) - - disk_free_alarm (int, disk alarm) - - fd_total (int, file descriptors) - - fd_used (int, file descriptors) - - mem_limit (int, bytes) - - mem_used (int, bytes) - - mem_alarm (int, memory a) - - proc_total (int, erlang processes) - - proc_used (int, erlang processes) - - run_queue (int, erlang processes) - - sockets_total (int, sockets) - - sockets_used (int, sockets) - - running (int, node up) - - uptime (int, milliseconds) - - health_check_status (int, 1 or 0) - - mnesia_disk_tx_count (int, number of disk transaction) - - mnesia_ram_tx_count (int, number of ram transaction) - - mnesia_disk_tx_count_rate (float, number of disk transaction per second) - - mnesia_ram_tx_count_rate (float, number of ram transaction per second) - - gc_num (int, number of garbage collection) - - gc_bytes_reclaimed (int, bytes) - - gc_num_rate (float, number of garbage collection per second) - - gc_bytes_reclaimed_rate (float, bytes per second) - - io_read_avg_time (float, number of read operations) - - io_read_avg_time_rate (int, number of read operations per second) - - io_read_bytes (int, bytes) - - io_read_bytes_rate (float, bytes per second) - - io_write_avg_time (int, milliseconds) - - io_write_avg_time_rate (float, milliseconds per second) - - io_write_bytes (int, bytes) - - io_write_bytes_rate (float, bytes per second) - - mem_connection_readers (int, bytes) - - mem_connection_writers (int, bytes) - - mem_connection_channels (int, bytes) - - mem_connection_other (int, bytes) - - mem_queue_procs (int, bytes) - - mem_queue_slave_procs (int, bytes) - - mem_plugins (int, bytes) - - mem_other_proc (int, bytes) - - mem_metrics (int, bytes) - - mem_mgmt_db (int, bytes) - - mem_mnesia (int, bytes) - - mem_other_ets (int, bytes) - - mem_binary (int, bytes) - - mem_msg_index (int, bytes) - - mem_code (int, bytes) - - mem_atom (int, bytes) - - mem_other_system (int, bytes) - - mem_allocated_unused (int, bytes) - - mem_reserved_unallocated (int, bytes) - - mem_total (int, bytes) + ## Federation upstreams to include and exlude specified as an array of glob + ## pattern strings. Federation links can also be limited by the queue and + ## exchange filters. + # federation_upstream_include = [] + # federation_upstream_exclude = [] +``` -- rabbitmq_queue - - consumer_utilisation (float, percent) - - consumers (int, int) - - idle_since (string, time - e.g., "2006-01-02 15:04:05") - - memory (int, bytes) - - message_bytes (int, bytes) - - message_bytes_persist (int, bytes) - - message_bytes_ram (int, bytes) - - message_bytes_ready (int, bytes) - - message_bytes_unacked (int, bytes) - - messages (int, count) - - messages_ack (int, count) - - messages_ack_rate (float, messages per second) - - messages_deliver (int, count) - - messages_deliver_rate (float, messages per second) - - messages_deliver_get (int, count) - - messages_deliver_get_rate (float, messages per second) - - messages_publish (int, count) - - messages_publish_rate (float, messages per second) - - messages_ready (int, count) - - messages_redeliver (int, count) - - messages_redeliver_rate (float, messages per second) - - messages_unack (integer, count) - -- rabbitmq_exchange - - messages_publish_in (int, count) - - messages_publish_in_rate (int, messages per second) - - messages_publish_out (int, count) - - messages_publish_out_rate (int, messages per second) - -### Tags: - -- All measurements have the following tags: - - url +### Metrics - rabbitmq_overview - - name - -- rabbitmq_node - - node - - url + - tags: + - url + - name + - fields: + - channels (int, channels) + - connections (int, connections) + - consumers (int, consumers) + - exchanges (int, exchanges) + - messages (int, messages) + - messages_acked (int, messages) + - messages_delivered (int, messages) + - messages_delivered_get (int, messages) + - messages_published (int, messages) + - messages_ready (int, messages) + - messages_unacked (int, messages) + - queues (int, queues) + - clustering_listeners (int, cluster nodes) + - amqp_listeners (int, amqp nodes up) + - return_unroutable (int, number of unroutable messages) + - return_unroutable_rate (float, number of unroutable messages per second) + ++ rabbitmq_node + - tags: + - url + - node + - url + - fields: + - disk_free (int, bytes) + - disk_free_limit (int, bytes) + - disk_free_alarm (int, disk alarm) + - fd_total (int, file descriptors) + - fd_used (int, file descriptors) + - mem_limit (int, bytes) + - mem_used (int, bytes) + - mem_alarm (int, memory a) + - proc_total (int, erlang processes) + - proc_used (int, erlang processes) + - run_queue (int, erlang processes) + - sockets_total (int, sockets) + - sockets_used (int, sockets) + - running (int, node up) + - uptime (int, milliseconds) + - health_check_status (int, 1 or 0) + - mnesia_disk_tx_count (int, number of disk transaction) + - mnesia_ram_tx_count (int, number of ram transaction) + - mnesia_disk_tx_count_rate (float, number of disk transaction per second) + - mnesia_ram_tx_count_rate (float, number of ram transaction per second) + - gc_num (int, number of garbage collection) + - gc_bytes_reclaimed (int, bytes) + - gc_num_rate (float, number of garbage collection per second) + - gc_bytes_reclaimed_rate (float, bytes per second) + - io_read_avg_time (float, number of read operations) + - io_read_avg_time_rate (int, number of read operations per second) + - io_read_bytes (int, bytes) + - io_read_bytes_rate (float, bytes per second) + - io_write_avg_time (int, milliseconds) + - io_write_avg_time_rate (float, milliseconds per second) + - io_write_bytes (int, bytes) + - io_write_bytes_rate (float, bytes per second) + - mem_connection_readers (int, bytes) + - mem_connection_writers (int, bytes) + - mem_connection_channels (int, bytes) + - mem_connection_other (int, bytes) + - mem_queue_procs (int, bytes) + - mem_queue_slave_procs (int, bytes) + - mem_plugins (int, bytes) + - mem_other_proc (int, bytes) + - mem_metrics (int, bytes) + - mem_mgmt_db (int, bytes) + - mem_mnesia (int, bytes) + - mem_other_ets (int, bytes) + - mem_binary (int, bytes) + - mem_msg_index (int, bytes) + - mem_code (int, bytes) + - mem_atom (int, bytes) + - mem_other_system (int, bytes) + - mem_allocated_unused (int, bytes) + - mem_reserved_unallocated (int, bytes) + - mem_total (int, bytes) - rabbitmq_queue - - url - - queue - - vhost - - node - - durable - - auto_delete - -- rabbitmq_exchange - - url - - exchange - - type - - vhost - - internal - - durable - - auto_delete - -### Sample Queries: + - tags: + - url + - queue + - vhost + - node + - durable + - auto_delete + - fields: + - consumer_utilisation (float, percent) + - consumers (int, int) + - idle_since (string, time - e.g., "2006-01-02 15:04:05") + - memory (int, bytes) + - message_bytes (int, bytes) + - message_bytes_persist (int, bytes) + - message_bytes_ram (int, bytes) + - message_bytes_ready (int, bytes) + - message_bytes_unacked (int, bytes) + - messages (int, count) + - messages_ack (int, count) + - messages_ack_rate (float, messages per second) + - messages_deliver (int, count) + - messages_deliver_rate (float, messages per second) + - messages_deliver_get (int, count) + - messages_deliver_get_rate (float, messages per second) + - messages_publish (int, count) + - messages_publish_rate (float, messages per second) + - messages_ready (int, count) + - messages_redeliver (int, count) + - messages_redeliver_rate (float, messages per second) + - messages_unack (integer, count) + ++ rabbitmq_exchange + - tags: + - url + - exchange + - type + - vhost + - internal + - durable + - auto_delete + - fields: + - messages_publish_in (int, count) + - messages_publish_in_rate (int, messages per second) + - messages_publish_out (int, count) + - messages_publish_out_rate (int, messages per second) + +- rabbitmq_federation + - tags: + - url + - vhost + - type + - upstream + - exchange + - upstream_exchange + - queue + - upstream_queue + - fields: + - acks_uncommitted (int, count) + - consumers (int, count) + - messages_unacknowledged (int, count) + - messages_uncommitted (int, count) + - messages_unconfirmed (int, count) + - messages_confirm (int, count) + - messages_publish (int, count) + - messages_return_unroutable (int, count) + +### Sample Queries Message rates for the entire node can be calculated from total message counts. For instance, to get the rate of messages published per minute, use this query: ``` -SELECT NON_NEGATIVE_DERIVATIVE(LAST("messages_published"), 1m) AS messages_published_rate -FROM rabbitmq_overview WHERE time > now() - 10m GROUP BY time(1m) +SELECT NON_NEGATIVE_DERIVATIVE(LAST("messages_published"), 1m) AS messages_published_rate FROM rabbitmq_overview WHERE time > now() - 10m GROUP BY time(1m) ``` -### Example Output: +### Example Output ``` rabbitmq_queue,url=http://amqp.example.org:15672,queue=telegraf,vhost=influxdb,node=rabbit@amqp.example.org,durable=true,auto_delete=false,host=amqp.example.org messages_deliver_get=0i,messages_publish=329i,messages_publish_rate=0.2,messages_redeliver_rate=0,message_bytes_ready=0i,message_bytes_unacked=0i,messages_deliver=329i,messages_unack=0i,consumers=1i,idle_since="",messages=0i,messages_deliver_rate=0.2,messages_deliver_get_rate=0.2,messages_redeliver=0i,memory=43032i,message_bytes_ram=0i,messages_ack=329i,messages_ready=0i,messages_ack_rate=0.2,consumer_utilisation=1,message_bytes=0i,message_bytes_persist=0i 1493684035000000000 diff --git a/plugins/inputs/rabbitmq/rabbitmq.go b/plugins/inputs/rabbitmq/rabbitmq.go index 168a340b0b21f..d27c522bf2a48 100644 --- a/plugins/inputs/rabbitmq/rabbitmq.go +++ b/plugins/inputs/rabbitmq/rabbitmq.go @@ -34,27 +34,30 @@ const DefaultClientTimeout = 4 // RabbitMQ defines the configuration necessary for gathering metrics, // see the sample config for further details type RabbitMQ struct { - URL string - Name string - Username string - Password string + URL string `toml:"url"` + Name string `toml:"name"` + Username string `toml:"username"` + Password string `toml:"password"` tls.ClientConfig ResponseHeaderTimeout internal.Duration `toml:"header_timeout"` ClientTimeout internal.Duration `toml:"client_timeout"` - Nodes []string - Queues []string - Exchanges []string + Nodes []string `toml:"nodes"` + Queues []string `toml:"queues"` + Exchanges []string `toml:"exchanges"` - QueueInclude []string `toml:"queue_name_include"` - QueueExclude []string `toml:"queue_name_exclude"` + QueueInclude []string `toml:"queue_name_include"` + QueueExclude []string `toml:"queue_name_exclude"` + FederationUpstreamInclude []string `toml:"federation_upstream_include"` + FederationUpstreamExclude []string `toml:"federation_upstream_exclude"` - Client *http.Client + Client *http.Client `toml:"-"` filterCreated bool excludeEveryQueue bool queueFilter filter.Filter + upstreamFilter filter.Filter } // OverviewResponse ... @@ -178,6 +181,38 @@ type Exchange struct { AutoDelete bool `json:"auto_delete"` } +// FederationLinkChannelMessageStats ... +type FederationLinkChannelMessageStats struct { + Confirm int64 `json:"confirm"` + ConfirmDetails Details `json:"confirm_details"` + Publish int64 `json:"publish"` + PublishDetails Details `json:"publish_details"` + ReturnUnroutable int64 `json:"return_unroutable"` + ReturnUnroutableDetails Details `json:"return_unroutable_details"` +} + +// FederationLinkChannel ... +type FederationLinkChannel struct { + AcksUncommitted int64 `json:"acks_uncommitted"` + ConsumerCount int64 `json:"consumer_count"` + MessagesUnacknowledged int64 `json:"messages_unacknowledged"` + MessagesUncommitted int64 `json:"messages_uncommitted"` + MessagesUnconfirmed int64 `json:"messages_unconfirmed"` + MessageStats FederationLinkChannelMessageStats `json:"message_stats"` +} + +// FederationLink ... +type FederationLink struct { + Type string `json:"type"` + Queue string `json:"queue"` + UpstreamQueue string `json:"upstream_queue"` + Exchange string `json:"exchange"` + UpstreamExchange string `json:"upstream_exchange"` + Vhost string `json:"vhost"` + Upstream string `json:"upstream"` + LocalChannel FederationLinkChannel `json:"local_channel"` +} + type HealthCheck struct { Status string `json:"status"` } @@ -214,7 +249,7 @@ type Memory struct { // gatherFunc ... type gatherFunc func(r *RabbitMQ, acc telegraf.Accumulator) -var gatherFunctions = []gatherFunc{gatherOverview, gatherNodes, gatherQueues, gatherExchanges} +var gatherFunctions = []gatherFunc{gatherOverview, gatherNodes, gatherQueues, gatherExchanges, gatherFederationLinks} var sampleConfig = ` ## Management Plugin url. (default: http://localhost:15672) @@ -258,6 +293,15 @@ var sampleConfig = ` ## Note that an empty array for both will include all queues queue_name_include = [] queue_name_exclude = [] + + ## Federation upstreams include and exclude when gathering the rabbitmq_federation measurement. + ## If neither are specified, metrics for all federation upstreams are gathered. + ## Federation link metrics will only be gathered for queues and exchanges + ## whose non-federation metrics will be collected (e.g a queue excluded + ## by the 'queue_name_exclude' option will also be excluded from federation). + ## Globs accepted. + # federation_upstream_include = ["dataCentre-*"] + # federation_upstream_exclude = [] ` func boolToInt(b bool) int64 { @@ -294,12 +338,16 @@ func (r *RabbitMQ) Gather(acc telegraf.Accumulator) error { } } - // Create queue filter if not already created + // Create gather filters if not already created if !r.filterCreated { err := r.createQueueFilter() if err != nil { return err } + err = r.createUpstreamFilter() + if err != nil { + return err + } r.filterCreated = true } @@ -400,134 +448,112 @@ func gatherOverview(r *RabbitMQ, acc telegraf.Accumulator) { } func gatherNodes(r *RabbitMQ, acc telegraf.Accumulator) { - allNodes := make([]Node, 0) - // Gather information about nodes + allNodes := make([]*Node, 0) + err := r.requestJSON("/api/nodes", &allNodes) if err != nil { acc.AddError(err) return } - nodes := make(map[string]Node) + nodes := allNodes[:0] for _, node := range allNodes { if r.shouldGatherNode(node) { - nodes[node.Name] = node + nodes = append(nodes, node) } } - numberNodes := len(nodes) - if numberNodes == 0 { - return - } - - type NodeCheck struct { - NodeName string - HealthCheck HealthCheck - Memory *Memory - } - - nodeChecksChannel := make(chan NodeCheck, numberNodes) - + var wg sync.WaitGroup for _, node := range nodes { - go func(nodeName string, healthChecksChannel chan NodeCheck) { - var healthCheck HealthCheck - var memoryresponse MemoryResponse - - err := r.requestJSON("/api/healthchecks/node/"+nodeName, &healthCheck) - nodeCheck := NodeCheck{ - NodeName: nodeName, - HealthCheck: healthCheck, + wg.Add(1) + go func(node *Node) { + defer wg.Done() + + tags := map[string]string{"url": r.URL} + tags["node"] = node.Name + + fields := map[string]interface{}{ + "disk_free": node.DiskFree, + "disk_free_limit": node.DiskFreeLimit, + "disk_free_alarm": boolToInt(node.DiskFreeAlarm), + "fd_total": node.FdTotal, + "fd_used": node.FdUsed, + "mem_limit": node.MemLimit, + "mem_used": node.MemUsed, + "mem_alarm": boolToInt(node.MemAlarm), + "proc_total": node.ProcTotal, + "proc_used": node.ProcUsed, + "run_queue": node.RunQueue, + "sockets_total": node.SocketsTotal, + "sockets_used": node.SocketsUsed, + "uptime": node.Uptime, + "mnesia_disk_tx_count": node.MnesiaDiskTxCount, + "mnesia_disk_tx_count_rate": node.MnesiaDiskTxCountDetails.Rate, + "mnesia_ram_tx_count": node.MnesiaRamTxCount, + "mnesia_ram_tx_count_rate": node.MnesiaRamTxCountDetails.Rate, + "gc_num": node.GcNum, + "gc_num_rate": node.GcNumDetails.Rate, + "gc_bytes_reclaimed": node.GcBytesReclaimed, + "gc_bytes_reclaimed_rate": node.GcBytesReclaimedDetails.Rate, + "io_read_avg_time": node.IoReadAvgTime, + "io_read_avg_time_rate": node.IoReadAvgTimeDetails.Rate, + "io_read_bytes": node.IoReadBytes, + "io_read_bytes_rate": node.IoReadBytesDetails.Rate, + "io_write_avg_time": node.IoWriteAvgTime, + "io_write_avg_time_rate": node.IoWriteAvgTimeDetails.Rate, + "io_write_bytes": node.IoWriteBytes, + "io_write_bytes_rate": node.IoWriteBytesDetails.Rate, + "running": boolToInt(node.Running), } + + var health HealthCheck + err := r.requestJSON("/api/healthchecks/node/"+node.Name, &health) if err != nil { acc.AddError(err) return } - err = r.requestJSON("/api/nodes/"+nodeName+"/memory", &memoryresponse) - nodeCheck.Memory = memoryresponse.Memory + if health.Status == "ok" { + fields["health_check_status"] = int64(1) + } else { + fields["health_check_status"] = int64(0) + } + + var memory MemoryResponse + err = r.requestJSON("/api/nodes/"+node.Name+"/memory", &memory) if err != nil { acc.AddError(err) return } - nodeChecksChannel <- nodeCheck - }(node.Name, nodeChecksChannel) - } - - now := time.Now() - - for i := 0; i < len(nodes); i++ { - nodeCheck := <-nodeChecksChannel - - var healthCheckStatus int64 = 0 - - if nodeCheck.HealthCheck.Status == "ok" { - healthCheckStatus = 1 - } + if memory.Memory != nil { + fields["mem_connection_readers"] = memory.Memory.ConnectionReaders + fields["mem_connection_writers"] = memory.Memory.ConnectionWriters + fields["mem_connection_channels"] = memory.Memory.ConnectionChannels + fields["mem_connection_other"] = memory.Memory.ConnectionOther + fields["mem_queue_procs"] = memory.Memory.QueueProcs + fields["mem_queue_slave_procs"] = memory.Memory.QueueSlaveProcs + fields["mem_plugins"] = memory.Memory.Plugins + fields["mem_other_proc"] = memory.Memory.OtherProc + fields["mem_metrics"] = memory.Memory.Metrics + fields["mem_mgmt_db"] = memory.Memory.MgmtDb + fields["mem_mnesia"] = memory.Memory.Mnesia + fields["mem_other_ets"] = memory.Memory.OtherEts + fields["mem_binary"] = memory.Memory.Binary + fields["mem_msg_index"] = memory.Memory.MsgIndex + fields["mem_code"] = memory.Memory.Code + fields["mem_atom"] = memory.Memory.Atom + fields["mem_other_system"] = memory.Memory.OtherSystem + fields["mem_allocated_unused"] = memory.Memory.AllocatedUnused + fields["mem_reserved_unallocated"] = memory.Memory.ReservedUnallocated + fields["mem_total"] = memory.Memory.Total + } - node := nodes[nodeCheck.NodeName] - - tags := map[string]string{"url": r.URL} - tags["node"] = node.Name - - fields := map[string]interface{}{ - "disk_free": node.DiskFree, - "disk_free_limit": node.DiskFreeLimit, - "disk_free_alarm": boolToInt(node.DiskFreeAlarm), - "fd_total": node.FdTotal, - "fd_used": node.FdUsed, - "mem_limit": node.MemLimit, - "mem_used": node.MemUsed, - "mem_alarm": boolToInt(node.MemAlarm), - "proc_total": node.ProcTotal, - "proc_used": node.ProcUsed, - "run_queue": node.RunQueue, - "sockets_total": node.SocketsTotal, - "sockets_used": node.SocketsUsed, - "uptime": node.Uptime, - "mnesia_disk_tx_count": node.MnesiaDiskTxCount, - "mnesia_disk_tx_count_rate": node.MnesiaDiskTxCountDetails.Rate, - "mnesia_ram_tx_count": node.MnesiaRamTxCount, - "mnesia_ram_tx_count_rate": node.MnesiaRamTxCountDetails.Rate, - "gc_num": node.GcNum, - "gc_num_rate": node.GcNumDetails.Rate, - "gc_bytes_reclaimed": node.GcBytesReclaimed, - "gc_bytes_reclaimed_rate": node.GcBytesReclaimedDetails.Rate, - "io_read_avg_time": node.IoReadAvgTime, - "io_read_avg_time_rate": node.IoReadAvgTimeDetails.Rate, - "io_read_bytes": node.IoReadBytes, - "io_read_bytes_rate": node.IoReadBytesDetails.Rate, - "io_write_avg_time": node.IoWriteAvgTime, - "io_write_avg_time_rate": node.IoWriteAvgTimeDetails.Rate, - "io_write_bytes": node.IoWriteBytes, - "io_write_bytes_rate": node.IoWriteBytesDetails.Rate, - "running": boolToInt(node.Running), - "health_check_status": healthCheckStatus, - } - if nodeCheck.Memory != nil { - fields["mem_connection_readers"] = nodeCheck.Memory.ConnectionReaders - fields["mem_connection_writers"] = nodeCheck.Memory.ConnectionWriters - fields["mem_connection_channels"] = nodeCheck.Memory.ConnectionChannels - fields["mem_connection_other"] = nodeCheck.Memory.ConnectionOther - fields["mem_queue_procs"] = nodeCheck.Memory.QueueProcs - fields["mem_queue_slave_procs"] = nodeCheck.Memory.QueueSlaveProcs - fields["mem_plugins"] = nodeCheck.Memory.Plugins - fields["mem_other_proc"] = nodeCheck.Memory.OtherProc - fields["mem_metrics"] = nodeCheck.Memory.Metrics - fields["mem_mgmt_db"] = nodeCheck.Memory.MgmtDb - fields["mem_mnesia"] = nodeCheck.Memory.Mnesia - fields["mem_other_ets"] = nodeCheck.Memory.OtherEts - fields["mem_binary"] = nodeCheck.Memory.Binary - fields["mem_msg_index"] = nodeCheck.Memory.MsgIndex - fields["mem_code"] = nodeCheck.Memory.Code - fields["mem_atom"] = nodeCheck.Memory.Atom - fields["mem_other_system"] = nodeCheck.Memory.OtherSystem - fields["mem_allocated_unused"] = nodeCheck.Memory.AllocatedUnused - fields["mem_reserved_unallocated"] = nodeCheck.Memory.ReservedUnallocated - fields["mem_total"] = nodeCheck.Memory.Total - } - acc.AddFields("rabbitmq_node", fields, tags, now) + acc.AddFields("rabbitmq_node", fields, tags) + }(node) } + + wg.Wait() } func gatherQueues(r *RabbitMQ, acc telegraf.Accumulator) { @@ -598,7 +624,7 @@ func gatherExchanges(r *RabbitMQ, acc telegraf.Accumulator) { } for _, exchange := range exchanges { - if !r.shouldGatherExchange(exchange) { + if !r.shouldGatherExchange(exchange.Name) { continue } tags := map[string]string{ @@ -624,7 +650,53 @@ func gatherExchanges(r *RabbitMQ, acc telegraf.Accumulator) { } } -func (r *RabbitMQ) shouldGatherNode(node Node) bool { +func gatherFederationLinks(r *RabbitMQ, acc telegraf.Accumulator) { + // Gather information about federation links + federationLinks := make([]FederationLink, 0) + err := r.requestJSON("/api/federation-links", &federationLinks) + if err != nil { + acc.AddError(err) + return + } + + for _, link := range federationLinks { + if !r.shouldGatherFederationLink(link) { + continue + } + + tags := map[string]string{ + "url": r.URL, + "type": link.Type, + "vhost": link.Vhost, + "upstream": link.Upstream, + } + + if link.Type == "exchange" { + tags["exchange"] = link.Exchange + tags["upstream_exchange"] = link.UpstreamExchange + } else { + tags["queue"] = link.Queue + tags["upstream_queue"] = link.UpstreamQueue + } + + acc.AddFields( + "rabbitmq_federation", + map[string]interface{}{ + "acks_uncommitted": link.LocalChannel.AcksUncommitted, + "consumers": link.LocalChannel.ConsumerCount, + "messages_unacknowledged": link.LocalChannel.MessagesUnacknowledged, + "messages_uncommitted": link.LocalChannel.MessagesUncommitted, + "messages_unconfirmed": link.LocalChannel.MessagesUnconfirmed, + "messages_confirm": link.LocalChannel.MessageStats.Confirm, + "messages_publish": link.LocalChannel.MessageStats.Publish, + "messages_return_unroutable": link.LocalChannel.MessageStats.ReturnUnroutable, + }, + tags, + ) + } +} + +func (r *RabbitMQ) shouldGatherNode(node *Node) bool { if len(r.Nodes) == 0 { return true } @@ -659,13 +731,23 @@ func (r *RabbitMQ) createQueueFilter() error { return nil } -func (r *RabbitMQ) shouldGatherExchange(exchange Exchange) bool { +func (r *RabbitMQ) createUpstreamFilter() error { + upstreamFilter, err := filter.NewIncludeExcludeFilter(r.FederationUpstreamInclude, r.FederationUpstreamExclude) + if err != nil { + return err + } + r.upstreamFilter = upstreamFilter + + return nil +} + +func (r *RabbitMQ) shouldGatherExchange(exchangeName string) bool { if len(r.Exchanges) == 0 { return true } for _, name := range r.Exchanges { - if name == exchange.Name { + if name == exchangeName { return true } } @@ -673,6 +755,21 @@ func (r *RabbitMQ) shouldGatherExchange(exchange Exchange) bool { return false } +func (r *RabbitMQ) shouldGatherFederationLink(link FederationLink) bool { + if !r.upstreamFilter.Match(link.Upstream) { + return false + } + + switch link.Type { + case "exchange": + return r.shouldGatherExchange(link.Exchange) + case "queue": + return r.queueFilter.Match(link.Queue) + default: + return false + } +} + func init() { inputs.Add("rabbitmq", func() telegraf.Input { return &RabbitMQ{ diff --git a/plugins/inputs/rabbitmq/rabbitmq_test.go b/plugins/inputs/rabbitmq/rabbitmq_test.go index 9d35718d909d5..0991dd0c06c99 100644 --- a/plugins/inputs/rabbitmq/rabbitmq_test.go +++ b/plugins/inputs/rabbitmq/rabbitmq_test.go @@ -28,6 +28,8 @@ func TestRabbitMQGeneratesMetrics(t *testing.T) { jsonFilePath = "testdata/exchanges.json" case "/api/healthchecks/node/rabbit@vagrant-ubuntu-trusty-64": jsonFilePath = "testdata/healthchecks.json" + case "/api/federation-links": + jsonFilePath = "testdata/federation-links.json" case "/api/nodes/rabbit@vagrant-ubuntu-trusty-64/memory": jsonFilePath = "testdata/memory.json" default: @@ -162,6 +164,18 @@ func TestRabbitMQGeneratesMetrics(t *testing.T) { "messages_publish_out_rate": 5.1, } compareMetrics(t, exchangeMetrics, acc, "rabbitmq_exchange") + + federationLinkMetrics := map[string]interface{}{ + "acks_uncommitted": 1, + "consumers": 2, + "messages_unacknowledged": 3, + "messages_uncommitted": 4, + "messages_unconfirmed": 5, + "messages_confirm": 67, + "messages_publish": 890, + "messages_return_unroutable": 1, + } + compareMetrics(t, federationLinkMetrics, acc, "rabbitmq_federation") } func compareMetrics(t *testing.T, expectedMetrics map[string]interface{}, diff --git a/plugins/inputs/rabbitmq/testdata/federation-links.json b/plugins/inputs/rabbitmq/testdata/federation-links.json new file mode 100644 index 0000000000000..4cf5148705371 --- /dev/null +++ b/plugins/inputs/rabbitmq/testdata/federation-links.json @@ -0,0 +1,63 @@ +[ + { + "node": "rabbit@rmqlocal", + "queue": "exampleLocalQueue", + "upstream_queue": "exampleUpstreamQueue", + "type": "queue", + "vhost": "/", + "upstream": "ExampleFederationUpstream", + "id": "8ba5218f", + "status": "running", + "local_connection": "", + "uri": "amqp://appsv03", + "timestamp": "2019-08-19 15:34:15", + "local_channel": { + "acks_uncommitted": 1, + "confirm": true, + "connection_details": { + "name": "", + "peer_host": "undefined", + "peer_port": "undefined" + }, + "consumer_count": 2, + "garbage_collection": { + "fullsweep_after": 65535, + "max_heap_size": 0, + "min_bin_vheap_size": 46422, + "min_heap_size": 233, + "minor_gcs": 203 + }, + "global_prefetch_count": 0, + "message_stats": { + "confirm": 67, + "confirm_details": { + "rate": 2 + }, + "publish": 890, + "publish_details": { + "rate": 2 + }, + "return_unroutable": 1, + "return_unroutable_details": { + "rate": 0.1 + } + }, + "messages_unacknowledged": 3, + "messages_uncommitted": 4, + "messages_unconfirmed": 5, + "name": "", + "node": "rabbit@rmqlocal", + "number": 1, + "prefetch_count": 0, + "reductions": 1926653, + "reductions_details": { + "rate": 1068 + }, + "state": "running", + "transactional": false, + "user": "none", + "user_who_performed_action": "none", + "vhost": "sorandomsorandom" + } + } +] diff --git a/plugins/inputs/redis/redis.go b/plugins/inputs/redis/redis.go index 715b553c93d26..598c6c4f8f1d7 100644 --- a/plugins/inputs/redis/redis.go +++ b/plugins/inputs/redis/redis.go @@ -4,7 +4,6 @@ import ( "bufio" "fmt" "io" - "log" "net/url" "regexp" "strconv" @@ -23,6 +22,8 @@ type Redis struct { Password string tls.ClientConfig + Log telegraf.Logger + clients []Client initialized bool } @@ -101,13 +102,13 @@ func (r *Redis) init(acc telegraf.Accumulator) error { for i, serv := range r.Servers { if !strings.HasPrefix(serv, "tcp://") && !strings.HasPrefix(serv, "unix://") { - log.Printf("W! [inputs.redis]: server URL found without scheme; please update your configuration file") + r.Log.Warn("Server URL found without scheme; please update your configuration file") serv = "tcp://" + serv } u, err := url.Parse(serv) if err != nil { - return fmt.Errorf("Unable to parse to address %q: %v", serv, err) + return fmt.Errorf("unable to parse to address %q: %s", serv, err.Error()) } password := "" diff --git a/plugins/inputs/redis/redis_test.go b/plugins/inputs/redis/redis_test.go index e684225af42db..637b464f95e99 100644 --- a/plugins/inputs/redis/redis_test.go +++ b/plugins/inputs/redis/redis_test.go @@ -20,6 +20,7 @@ func TestRedisConnect(t *testing.T) { addr := fmt.Sprintf(testutil.GetLocalHost() + ":6379") r := &Redis{ + Log: testutil.Logger{}, Servers: []string{addr}, } diff --git a/plugins/inputs/smart/smart.go b/plugins/inputs/smart/smart.go index b17f979d3fdb5..6c83e98901ec6 100644 --- a/plugins/inputs/smart/smart.go +++ b/plugins/inputs/smart/smart.go @@ -3,7 +3,6 @@ package smart import ( "bufio" "fmt" - "log" "os/exec" "path" "regexp" @@ -24,7 +23,7 @@ var ( // Model Number: TS128GMTE850 modelInfo = regexp.MustCompile("^(Device Model|Product|Model Number):\\s+(.*)$") // Serial Number: S0X5NZBC422720 - serialInfo = regexp.MustCompile("^Serial Number:\\s+(.*)$") + serialInfo = regexp.MustCompile("(?i)^Serial Number:\\s+(.*)$") // LU WWN Device Id: 5 002538 655584d30 wwnInfo = regexp.MustCompile("^LU WWN Device Id:\\s+(.*)$") // User Capacity: 251,000,193,024 bytes [251 GB] @@ -209,10 +208,7 @@ func (m *Smart) scan() ([]string, error) { for _, line := range strings.Split(string(out), "\n") { dev := strings.Split(line, " ") if len(dev) > 1 && !excludedDev(m.Excludes, strings.TrimSpace(dev[0])) { - log.Printf("D! [inputs.smart] adding device: %+#v", dev) devices = append(devices, strings.TrimSpace(dev[0])) - } else { - log.Printf("D! [inputs.smart] skipping device: %+#v", dev) } } return devices, nil @@ -323,6 +319,7 @@ func gatherDisk(acc telegraf.Accumulator, timeout internal.Duration, usesudo, co attr := attribute.FindStringSubmatch(line) if len(attr) > 1 { + // attribute has been found, add it only if collectAttributes is true if collectAttributes { tags["id"] = attr[1] tags["name"] = attr[2] @@ -355,23 +352,25 @@ func gatherDisk(acc telegraf.Accumulator, timeout internal.Duration, usesudo, co } } } else { - if collectAttributes { - if matches := sasNvmeAttr.FindStringSubmatch(line); len(matches) > 2 { - if attr, ok := sasNvmeAttributes[matches[1]]; ok { - tags["name"] = attr.Name - if attr.ID != "" { - tags["id"] = attr.ID - } - - parse := parseCommaSeperatedInt - if attr.Parse != nil { - parse = attr.Parse - } - - if err := parse(fields, deviceFields, matches[2]); err != nil { - continue - } + // what was found is not a vendor attribute + if matches := sasNvmeAttr.FindStringSubmatch(line); len(matches) > 2 { + if attr, ok := sasNvmeAttributes[matches[1]]; ok { + tags["name"] = attr.Name + if attr.ID != "" { + tags["id"] = attr.ID + } + parse := parseCommaSeperatedInt + if attr.Parse != nil { + parse = attr.Parse + } + + if err := parse(fields, deviceFields, matches[2]); err != nil { + continue + } + // if the field is classified as an attribute, only add it + // if collectAttributes is true + if collectAttributes { acc.AddFields("smart_attribute", fields, tags) } } diff --git a/plugins/inputs/smart/smart_test.go b/plugins/inputs/smart/smart_test.go index d66a31fea0797..615ea9ba619c5 100644 --- a/plugins/inputs/smart/smart_test.go +++ b/plugins/inputs/smart/smart_test.go @@ -438,8 +438,56 @@ func TestGatherHtSAS(t *testing.T) { wg.Add(1) gatherDisk(acc, internal.Duration{Duration: time.Second * 30}, true, true, "", "", "", wg) - assert.Equal(t, 5, acc.NFields(), "Wrong number of fields gathered") - assert.Equal(t, uint64(3), acc.NMetrics(), "Wrong number of metrics gathered") + + expected := []telegraf.Metric{ + testutil.MustMetric( + "smart_attribute", + map[string]string{ + "device": ".", + "serial_no": "PDWAR9GE", + "enabled": "Enabled", + "id": "194", + "model": "HUC103030CSS600", + "name": "Temperature_Celsius", + }, + map[string]interface{}{ + "raw_value": 36, + }, + time.Unix(0, 0), + ), + testutil.MustMetric( + "smart_attribute", + map[string]string{ + "device": ".", + "serial_no": "PDWAR9GE", + "enabled": "Enabled", + "id": "4", + "model": "HUC103030CSS600", + "name": "Start_Stop_Count", + }, + map[string]interface{}{ + "raw_value": 47, + }, + time.Unix(0, 0), + ), + testutil.MustMetric( + "smart_device", + map[string]string{ + "device": ".", + "serial_no": "PDWAR9GE", + "enabled": "Enabled", + "model": "HUC103030CSS600", + }, + map[string]interface{}{ + "exit_status": 0, + "health_ok": true, + "temp_c": 36, + }, + time.Unix(0, 0), + ), + } + + testutil.RequireMetricsEqual(t, expected, acc.GetTelegrafMetrics(), testutil.SortMetrics(), testutil.IgnoreTime()) } func TestGatherSSD(t *testing.T) { diff --git a/plugins/inputs/snmp/CONFIG-EXAMPLES.md b/plugins/inputs/snmp/CONFIG-EXAMPLES.md deleted file mode 100644 index a0a52eeb327ef..0000000000000 --- a/plugins/inputs/snmp/CONFIG-EXAMPLES.md +++ /dev/null @@ -1,65 +0,0 @@ -Here are a few configuration examples for different use cases. - -### Switch/router interface metrics - -This setup will collect data on all interfaces from three different tables, `IF-MIB::ifTable`, `IF-MIB::ifXTable` and `EtherLike-MIB::dot3StatsTable`. It will also add the name from `IF-MIB::ifDescr` and use that as a tag. Depending on your needs and preferences you can easily use `IF-MIB::ifName` or `IF-MIB::ifAlias` instead or in addition. The values of these are typically: - - IF-MIB::ifName = Gi0/0/0 - IF-MIB::ifDescr = GigabitEthernet0/0/0 - IF-MIB::ifAlias = ### LAN ### - -This configuration also collects the hostname from the device (`RFC1213-MIB::sysName.0`) and adds as a tag. So each metric will both have the configured host/IP as `agent_host` as well as the device self-reported hostname as `hostname` and the name of the host that has collected these metrics as `host`. - -Here is the configuration that you add to your `telegraf.conf`: - -``` -[[inputs.snmp]] - agents = [ "host.example.com" ] - version = 2 - community = "public" - - [[inputs.snmp.field]] - name = "hostname" - oid = "RFC1213-MIB::sysName.0" - is_tag = true - - [[inputs.snmp.field]] - name = "uptime" - oid = "DISMAN-EXPRESSION-MIB::sysUpTimeInstance" - - # IF-MIB::ifTable contains counters on input and output traffic as well as errors and discards. - [[inputs.snmp.table]] - name = "interface" - inherit_tags = [ "hostname" ] - oid = "IF-MIB::ifTable" - - # Interface tag - used to identify interface in metrics database - [[inputs.snmp.table.field]] - name = "ifDescr" - oid = "IF-MIB::ifDescr" - is_tag = true - - # IF-MIB::ifXTable contains newer High Capacity (HC) counters that do not overflow as fast for a few of the ifTable counters - [[inputs.snmp.table]] - name = "interface" - inherit_tags = [ "hostname" ] - oid = "IF-MIB::ifXTable" - - # Interface tag - used to identify interface in metrics database - [[inputs.snmp.table.field]] - name = "ifDescr" - oid = "IF-MIB::ifDescr" - is_tag = true - - # EtherLike-MIB::dot3StatsTable contains detailed ethernet-level information about what kind of errors have been logged on an interface (such as FCS error, frame too long, etc) - [[inputs.snmp.table]] - name = "interface" - inherit_tags = [ "hostname" ] - oid = "EtherLike-MIB::dot3StatsTable" - - # Interface tag - used to identify interface in metrics database - [[inputs.snmp.table.field]] - name = "ifDescr" - oid = "IF-MIB::ifDescr" - is_tag = true -``` diff --git a/plugins/inputs/snmp/DEBUGGING.md b/plugins/inputs/snmp/DEBUGGING.md deleted file mode 100644 index f357c58b51c52..0000000000000 --- a/plugins/inputs/snmp/DEBUGGING.md +++ /dev/null @@ -1,53 +0,0 @@ -# Debugging & Testing SNMP Issues - -### Install net-snmp on your system: - -Mac: - -``` -brew install net-snmp -``` - -### Run an SNMP simulator docker image to get a full MIB on port 161: - -``` -docker run -d -p 161:161/udp xeemetric/snmp-simulator -``` - -### snmpget: - -snmpget corresponds to the inputs.snmp.field configuration. - -```bash -$ # get an snmp field with fully-qualified MIB name. -$ snmpget -v2c -c public localhost:161 system.sysUpTime.0 -DISMAN-EVENT-MIB::sysUpTimeInstance = Timeticks: (1643) 0:00:16.43 - -$ # get an snmp field, outputting the numeric OID. -$ snmpget -On -v2c -c public localhost:161 system.sysUpTime.0 -.1.3.6.1.2.1.1.3.0 = Timeticks: (1638) 0:00:16.38 -``` - -### snmptranslate: - -snmptranslate can be used to translate an OID to a MIB name: - -```bash -$ snmptranslate .1.3.6.1.2.1.1.3.0 -DISMAN-EVENT-MIB::sysUpTimeInstance -``` - -And to convert a partial MIB name to a fully qualified one: - -```bash -$ snmptranslate -IR sysUpTime.0 -DISMAN-EVENT-MIB::sysUpTimeInstance -``` - -And to convert a MIB name to an OID: - -```bash -$ snmptranslate -On -IR system.sysUpTime.0 -.1.3.6.1.2.1.1.3.0 -``` - diff --git a/plugins/inputs/snmp/README.md b/plugins/inputs/snmp/README.md index dab28e9b0a976..68760968a2a42 100644 --- a/plugins/inputs/snmp/README.md +++ b/plugins/inputs/snmp/README.md @@ -1,180 +1,221 @@ -# SNMP Plugin +# SNMP Input Plugin -The SNMP input plugin gathers metrics from SNMP agents. +The `snmp` input plugin uses polling to gather metrics from SNMP agents. +Support for gathering individual OIDs as well as complete SNMP tables is +included. -## Configuration: +### Prerequisites -See additional SNMP plugin configuration examples [here](./CONFIG-EXAMPLES.md). +This plugin uses the `snmptable` and `snmptranslate` programs from the +[net-snmp][] project. These tools will need to be installed into the `PATH` in +order to be located. Other utilities from the net-snmp project may be useful +for troubleshooting, but are not directly used by the plugin. -### Example: +These programs will load available MIBs on the system. Typically the default +directory for MIBs is `/usr/share/snmp/mibs`, but if your MIBs are in a +different location you may need to make the paths known to net-snmp. The +location of these files can be configured in the `snmp.conf` or via the +`MIBDIRS` environment variable. See [`man 1 snmpcmd`][man snmpcmd] for more +information. -SNMP data: -``` -.1.0.0.0.1.1.0 octet_str "foo" -.1.0.0.0.1.1.1 octet_str "bar" -.1.0.0.0.1.102 octet_str "bad" -.1.0.0.0.1.2.0 integer 1 -.1.0.0.0.1.2.1 integer 2 -.1.0.0.0.1.3.0 octet_str "0.123" -.1.0.0.0.1.3.1 octet_str "0.456" -.1.0.0.0.1.3.2 octet_str "9.999" -.1.0.0.1.1 octet_str "baz" -.1.0.0.1.2 uinteger 54321 -.1.0.0.1.3 uinteger 234 -``` - -Telegraf config: +### Configuration ```toml [[inputs.snmp]] - agents = [ "127.0.0.1:161" ] - version = 2 - community = "public" - - name = "system" - [[inputs.snmp.field]] - name = "hostname" - oid = ".1.0.0.1.1" - is_tag = true + ## Agent addresses to retrieve values from. + ## example: agents = ["udp://127.0.0.1:161"] + ## agents = ["tcp://127.0.0.1:161"] + agents = ["udp://127.0.0.1:161"] + + ## Timeout for each request. + # timeout = "5s" + + ## SNMP version; can be 1, 2, or 3. + # version = 2 + + ## SNMP community string. + # community = "public" + + ## Number of retries to attempt. + # retries = 3 + + ## The GETBULK max-repetitions parameter. + # max_repetitions = 10 + + ## SNMPv3 authentication and encryption options. + ## + ## Security Name. + # sec_name = "myuser" + ## Authentication protocol; one of "MD5", "SHA", or "". + # auth_protocol = "MD5" + ## Authentication password. + # auth_password = "pass" + ## Security Level; one of "noAuthNoPriv", "authNoPriv", or "authPriv". + # sec_level = "authNoPriv" + ## Context Name. + # context_name = "" + ## Privacy protocol used for encrypted messages; one of "DES", "AES" or "". + # priv_protocol = "" + ## Privacy password used for encrypted messages. + # priv_password = "" + + ## Add fields and tables defining the variables you wish to collect. This + ## example collects the system uptime and interface variables. Reference the + ## full plugin documentation for configuration details. [[inputs.snmp.field]] + oid = "RFC1213-MIB::sysUpTime.0" name = "uptime" - oid = ".1.0.0.1.2" + [[inputs.snmp.field]] - name = "loadavg" - oid = ".1.0.0.1.3" - conversion = "float(2)" + oid = "RFC1213-MIB::sysName.0" + name = "source" + is_tag = true [[inputs.snmp.table]] - name = "remote_servers" - inherit_tags = [ "hostname" ] + oid = "IF-MIB::ifTable" + name = "interface" + inherit_tags = ["source"] + [[inputs.snmp.table.field]] - name = "server" - oid = ".1.0.0.0.1.1" + oid = "IF-MIB::ifDescr" + name = "ifDescr" is_tag = true - [[inputs.snmp.table.field]] - name = "connections" - oid = ".1.0.0.0.1.2" - [[inputs.snmp.table.field]] - name = "latency" - oid = ".1.0.0.0.1.3" - conversion = "float" ``` -Resulting output: -``` -* Plugin: snmp, Collection 1 -> system,agent_host=127.0.0.1,host=mylocalhost,hostname=baz loadavg=2.34,uptime=54321i 1468953135000000000 -> remote_servers,agent_host=127.0.0.1,host=mylocalhost,hostname=baz,server=foo connections=1i,latency=0.123 1468953135000000000 -> remote_servers,agent_host=127.0.0.1,host=mylocalhost,hostname=baz,server=bar connections=2i,latency=0.456 1468953135000000000 -``` +#### Configure SNMP Requests -#### Configuration via MIB: +This plugin provides two methods for configuring the SNMP requests: `fields` +and `tables`. Use the `field` option to gather single ad-hoc variables. +To collect SNMP tables, use the `table` option. -This example uses the SNMP data above, but is configured via the MIB. -The example MIB file can be found in the `testdata` directory. See the [MIB lookups](#mib-lookups) section for more information. +##### Field + +Use a `field` to collect a variable by OID. Requests specified with this +option operate similar to the `snmpget` utility. -Telegraf config: ```toml [[inputs.snmp]] - agents = [ "127.0.0.1:161" ] - version = 2 - community = "public" + # ... snip ... [[inputs.snmp.field]] - oid = "TEST::hostname" - is_tag = true - - [[inputs.snmp.table]] - oid = "TEST::testTable" - inherit_tags = [ "hostname" ] -``` - -Resulting output: -``` -* Plugin: snmp, Collection 1 -> testTable,agent_host=127.0.0.1,host=mylocalhost,hostname=baz,server=foo connections=1i,latency="0.123" 1468953135000000000 -> testTable,agent_host=127.0.0.1,host=mylocalhost,hostname=baz,server=bar connections=2i,latency="0.456" 1468953135000000000 + ## Object identifier of the variable as a numeric or textual OID. + oid = "RFC1213-MIB::sysName.0" + + ## Name of the field or tag to create. If not specified, it defaults to + ## the value of 'oid'. If 'oid' is numeric, an attempt to translate the + ## numeric OID into a textual OID will be made. + # name = "" + + ## If true the variable will be added as a tag, otherwise a field will be + ## created. + # is_tag = false + + ## Apply one of the following conversions to the variable value: + ## float(X) Convert the input value into a float and divides by the + ## Xth power of 10. Efficively just moves the decimal left + ## X places. For example a value of `123` with `float(2)` + ## will result in `1.23`. + ## float: Convert the value into a float with no adjustment. Same + ## as `float(0)`. + ## int: Convert the value into an integer. + ## hwaddr: Convert the value to a MAC address. + ## ipaddr: Convert the value to an IP address. + # conversion = "" ``` -### Config parameters +##### Table -* `agents`: Default: `[]` -List of SNMP agents to connect to in the form of `IP[:PORT]`. If `:PORT` is unspecified, it defaults to `161`. +Use a `table` to configure the collection of a SNMP table. SNMP requests +formed with this option operate similarly way to the `snmptable` command. -* `version`: Default: `2` -SNMP protocol version to use. +Control the handling of specific table columns using a nested `field`. These +nested fields are specified similarly to a top-level `field`. -* `community`: Default: `"public"` -SNMP community to use. +All columns of the SNMP table will be collected, it is not required to add a +nested field for each column, only those which you wish to modify. To exclude +columns use [metric filtering][]. -* `max_repetitions`: Default: `50` -Maximum number of iterations for repeating variables. +One [metric][] is created for each row of the SNMP table. -* `sec_name`: -Security name for authenticated SNMPv3 requests. - -* `auth_protocol`: Values: `"MD5"`,`"SHA"`,`""`. Default: `""` -Authentication protocol for authenticated SNMPv3 requests. - -* `auth_password`: -Authentication password for authenticated SNMPv3 requests. - -* `sec_level`: Values: `"noAuthNoPriv"`,`"authNoPriv"`,`"authPriv"`. Default: `"noAuthNoPriv"` -Security level used for SNMPv3 messages. - -* `context_name`: -Context name used for SNMPv3 requests. +```toml +[[inputs.snmp]] + # ... snip ... -* `priv_protocol`: Values: `"DES"`,`"AES"`,`""`. Default: `""` -Privacy protocol used for encrypted SNMPv3 messages. + [[inputs.snmp.table]] + ## Object identifier of the SNMP table as a numeric or textual OID. + oid = "IF-MIB::ifTable" -* `priv_password`: -Privacy password used for encrypted SNMPv3 messages. + ## Name of the field or tag to create. If not specified, it defaults to + ## the value of 'oid'. If 'oid' is numeric an attempt to translate the + ## numeric OID into a textual OID will be made. + # name = "" + ## Which tags to inherit from the top-level config and to use in the output + ## of this table's measurement. + ## example: inherit_tags = ["source"] + # inherit_tags = [] -* `name`: -Output measurement name. + ## Add an 'index' tag with the table row number. Use this if the table has + ## no indexes or if you are excluding them. This option is normally not + ## required as any index columns are automatically added as tags. + # index_as_tag = false -#### Field parameters: -* `oid`: -OID to get. May be a numeric or textual OID. + [[inputs.snmp.table.field]] + ## OID to get. May be a numeric or textual module-qualified OID. + oid = "IF-MIB::ifDescr" -* `oid_index_suffix`: -The OID sub-identifier to strip off so that the index can be matched against other fields in the table. + ## Name of the field or tag to create. If not specified, it defaults to + ## the value of 'oid'. If 'oid' is numeric an attempt to translate the + ## numeric OID into a textual OID will be made. + # name = "" -* `oid_index_length`: -Specifies the length of the index after the supplied table OID (in OID path segments). Truncates the index after this point to remove non-fixed value or length index suffixes. + ## Output this field as a tag. + # is_tag = false -* `name`: -Output field/tag name. -If not specified, it defaults to the value of `oid`. If `oid` is numeric, an attempt to translate the numeric OID into a texual OID will be made. + ## The OID sub-identifier to strip off so that the index can be matched + ## against other fields in the table. + # oid_index_suffix = "" -* `is_tag`: -Output this field as a tag. + ## Specifies the length of the index after the supplied table OID (in OID + ## path segments). Truncates the index after this point to remove non-fixed + ## value or length index suffixes. + # oid_index_length = 0 +``` -* `conversion`: Values: `"float(X)"`,`"float"`,`"int"`,`""`. Default: `""` -Converts the value according to the given specification. +### Troubleshooting - - `float(X)`: Converts the input value into a float and divides by the Xth power of 10. Efficively just moves the decimal left X places. For example a value of `123` with `float(2)` will result in `1.23`. - - `float`: Converts the value into a float with no adjustment. Same as `float(0)`. - - `int`: Convertes the value into an integer. - - `hwaddr`: Converts the value to a MAC address. - - `ipaddr`: Converts the value to an IP address. +Check that a numeric field can be translated to a textual field: +``` +$ snmptranslate .1.3.6.1.2.1.1.3.0 +DISMAN-EVENT-MIB::sysUpTimeInstance +``` -#### Table parameters: -* `oid`: -Automatically populates the table's fields using data from the MIB. +Request a top-level field: +``` +$ snmpget -v2c -c public 127.0.0.1 sysUpTime.0 +``` -* `name`: -Output measurement name. -If not specified, it defaults to the value of `oid`. If `oid` is numeric, an attempt to translate the numeric OID into a texual OID will be made. +Request a table: +``` +$ snmptable -v2c -c public 127.0.0.1 ifTable +``` -* `inherit_tags`: -Which tags to inherit from the top-level config and to use in the output of this table's measurement. +To collect a packet capture, run this command in the background while running +Telegraf or one of the above commands. Adjust the interface, host and port as +needed: +``` +$ sudo tcpdump -s 0 -i eth0 -w telegraf-snmp.pcap host 127.0.0.1 and port 161 +``` -* `index_as_tag`: -Adds each row's index within the table as a tag. +### Example Output -### MIB lookups -If the plugin is configured such that it needs to perform lookups from the MIB, it will use the net-snmp utilities `snmptranslate` and `snmptable`. +``` +snmp,agent_host=127.0.0.1,source=loaner uptime=11331974i 1575509815000000000 +interface,agent_host=127.0.0.1,ifDescr=wlan0,ifIndex=3,source=example.org ifAdminStatus=1i,ifInDiscards=0i,ifInErrors=0i,ifInNUcastPkts=0i,ifInOctets=3436617431i,ifInUcastPkts=2717778i,ifInUnknownProtos=0i,ifLastChange=0i,ifMtu=1500i,ifOperStatus=1i,ifOutDiscards=0i,ifOutErrors=0i,ifOutNUcastPkts=0i,ifOutOctets=581368041i,ifOutQLen=0i,ifOutUcastPkts=1354338i,ifPhysAddress="c8:5b:76:c9:e6:8c",ifSpecific=".0.0",ifSpeed=0i,ifType=6i 1575509815000000000 +interface,agent_host=127.0.0.1,ifDescr=eth0,ifIndex=2,source=example.org ifAdminStatus=1i,ifInDiscards=0i,ifInErrors=0i,ifInNUcastPkts=21i,ifInOctets=3852386380i,ifInUcastPkts=3634004i,ifInUnknownProtos=0i,ifLastChange=9088763i,ifMtu=1500i,ifOperStatus=1i,ifOutDiscards=0i,ifOutErrors=0i,ifOutNUcastPkts=0i,ifOutOctets=434865441i,ifOutQLen=0i,ifOutUcastPkts=2110394i,ifPhysAddress="c8:5b:76:c9:e6:8c",ifSpecific=".0.0",ifSpeed=1000000000i,ifType=6i 1575509815000000000 +interface,agent_host=127.0.0.1,ifDescr=lo,ifIndex=1,source=example.org ifAdminStatus=1i,ifInDiscards=0i,ifInErrors=0i,ifInNUcastPkts=0i,ifInOctets=51555569i,ifInUcastPkts=339097i,ifInUnknownProtos=0i,ifLastChange=0i,ifMtu=65536i,ifOperStatus=1i,ifOutDiscards=0i,ifOutErrors=0i,ifOutNUcastPkts=0i,ifOutOctets=51555569i,ifOutQLen=0i,ifOutUcastPkts=339097i,ifSpecific=".0.0",ifSpeed=10000000i,ifType=24i 1575509815000000000 +``` -When performing the lookups, the plugin will load all available MIBs. If your MIB files are in a custom path, you may add the path using the `MIBDIRS` environment variable. See [`man 1 snmpcmd`](http://net-snmp.sourceforge.net/docs/man/snmpcmd.html#lbAK) for more information on the variable. +[net-snmp]: http://www.net-snmp.org/ +[man snmpcmd]: http://net-snmp.sourceforge.net/docs/man/snmpcmd.html#lbAK +[metric filtering]: /docs/CONFIGURATION.md#metric-filtering +[metric]: /docs/METRICS.md diff --git a/plugins/inputs/snmp/snmp.go b/plugins/inputs/snmp/snmp.go index 24250c22aac39..2fc56ff976d90 100644 --- a/plugins/inputs/snmp/snmp.go +++ b/plugins/inputs/snmp/snmp.go @@ -7,6 +7,7 @@ import ( "log" "math" "net" + "net/url" "os/exec" "strconv" "strings" @@ -22,61 +23,46 @@ import ( const description = `Retrieves SNMP values from remote agents` const sampleConfig = ` - agents = [ "127.0.0.1:161" ] - ## Timeout for each SNMP query. - timeout = "5s" - ## Number of retries to attempt within timeout. - retries = 3 - ## SNMP version, values can be 1, 2, or 3 - version = 2 + ## Agent addresses to retrieve values from. + ## example: agents = ["udp://127.0.0.1:161"] + ## agents = ["tcp://127.0.0.1:161"] + agents = ["udp://127.0.0.1:161"] + + ## Timeout for each request. + # timeout = "5s" + + ## SNMP version; can be 1, 2, or 3. + # version = 2 ## SNMP community string. - community = "public" - - ## The GETBULK max-repetitions parameter - max_repetitions = 10 - - ## SNMPv3 auth parameters - #sec_name = "myuser" - #auth_protocol = "md5" # Values: "MD5", "SHA", "" - #auth_password = "pass" - #sec_level = "authNoPriv" # Values: "noAuthNoPriv", "authNoPriv", "authPriv" - #context_name = "" - #priv_protocol = "" # Values: "DES", "AES", "" - #priv_password = "" - - ## measurement name - name = "system" - [[inputs.snmp.field]] - name = "hostname" - oid = ".1.0.0.1.1" - [[inputs.snmp.field]] - name = "uptime" - oid = ".1.0.0.1.2" - [[inputs.snmp.field]] - name = "load" - oid = ".1.0.0.1.3" - [[inputs.snmp.field]] - oid = "HOST-RESOURCES-MIB::hrMemorySize" - - [[inputs.snmp.table]] - ## measurement name - name = "remote_servers" - inherit_tags = [ "hostname" ] - [[inputs.snmp.table.field]] - name = "server" - oid = ".1.0.0.0.1.0" - is_tag = true - [[inputs.snmp.table.field]] - name = "connections" - oid = ".1.0.0.0.1.1" - [[inputs.snmp.table.field]] - name = "latency" - oid = ".1.0.0.0.1.2" - - [[inputs.snmp.table]] - ## auto populate table's fields using the MIB - oid = "HOST-RESOURCES-MIB::hrNetworkTable" + # community = "public" + + ## Number of retries to attempt. + # retries = 3 + + ## The GETBULK max-repetitions parameter. + # max_repetitions = 10 + + ## SNMPv3 authentication and encryption options. + ## + ## Security Name. + # sec_name = "myuser" + ## Authentication protocol; one of "MD5", "SHA", or "". + # auth_protocol = "MD5" + ## Authentication password. + # auth_password = "pass" + ## Security Level; one of "noAuthNoPriv", "authNoPriv", or "authPriv". + # sec_level = "authNoPriv" + ## Context Name. + # context_name = "" + ## Privacy protocol used for encrypted messages; one of "DES", "AES" or "". + # priv_protocol = "" + ## Privacy password used for encrypted messages. + # priv_password = "" + + ## Add fields and tables defining the variables you wish to collect. This + ## example collects the system uptime and interface variables. Reference the + ## full plugin documentation for configuration details. ` // execCommand is so tests can mock out exec.Command usage. @@ -90,7 +76,7 @@ func execCmd(arg0 string, args ...string) ([]byte, error) { for _, arg := range args { quoted = append(quoted, fmt.Sprintf("%q", arg)) } - log.Printf("D! [inputs.snmp] Executing %q %s", arg0, strings.Join(quoted, " ")) + log.Printf("D! [inputs.snmp] executing %q %s", arg0, strings.Join(quoted, " ")) } out, err := execCommand(arg0, args...).Output() @@ -108,41 +94,42 @@ func execCmd(arg0 string, args ...string) ([]byte, error) { // Snmp holds the configuration for the plugin. type Snmp struct { - // The SNMP agent to query. Format is ADDR[:PORT] (e.g. 1.2.3.4:161). - Agents []string + // The SNMP agent to query. Format is [SCHEME://]ADDR[:PORT] (e.g. + // udp://1.2.3.4:161). If the scheme is not specified then "udp" is used. + Agents []string `toml:"agents"` // Timeout to wait for a response. - Timeout internal.Duration - Retries int + Timeout internal.Duration `toml:"timeout"` + Retries int `toml:"retries"` // Values: 1, 2, 3 - Version uint8 + Version uint8 `toml:"version"` // Parameters for Version 1 & 2 - Community string + Community string `toml:"community"` // Parameters for Version 2 & 3 - MaxRepetitions uint8 + MaxRepetitions uint8 `toml:"max_repetitions"` // Parameters for Version 3 - ContextName string + ContextName string `toml:"context_name"` // Values: "noAuthNoPriv", "authNoPriv", "authPriv" - SecLevel string - SecName string + SecLevel string `toml:"sec_level"` + SecName string `toml:"sec_name"` // Values: "MD5", "SHA", "". Default: "" - AuthProtocol string - AuthPassword string + AuthProtocol string `toml:"auth_protocol"` + AuthPassword string `toml:"auth_password"` // Values: "DES", "AES", "". Default: "" - PrivProtocol string - PrivPassword string - EngineID string - EngineBoots uint32 - EngineTime uint32 + PrivProtocol string `toml:"priv_protocol"` + PrivPassword string `toml:"priv_password"` + EngineID string `toml:"-"` + EngineBoots uint32 `toml:"-"` + EngineTime uint32 `toml:"-"` Tables []Table `toml:"table"` // Name & Fields are the elements of a Table. // Telegraf chokes if we try to embed a Table. So instead we have to embed the // fields of a Table, and construct a Table during runtime. - Name string + Name string // deprecated in 1.14; use name_override Fields []Field `toml:"field"` connectionCache []snmpConnection @@ -277,7 +264,7 @@ func (f *Field) init() error { return nil } - _, oidNum, oidText, conversion, err := snmpTranslate(f.Oid) + _, oidNum, oidText, conversion, err := SnmpTranslate(f.Oid) if err != nil { return Errorf(err, "translating") } @@ -623,16 +610,30 @@ func (s *Snmp) getConnection(idx int) (snmpConnection, error) { gs := gosnmpWrapper{&gosnmp.GoSNMP{}} s.connectionCache[idx] = gs - host, portStr, err := net.SplitHostPort(agent) + if !strings.Contains(agent, "://") { + agent = "udp://" + agent + } + + u, err := url.Parse(agent) if err != nil { - if err, ok := err.(*net.AddrError); !ok || err.Err != "missing port in address" { - return nil, Errorf(err, "parsing host") - } - host = agent - portStr = "161" + return nil, err + } + + switch u.Scheme { + case "tcp": + gs.Transport = "tcp" + case "", "udp": + gs.Transport = "udp" + default: + return nil, fmt.Errorf("unsupported scheme: %v", u.Scheme) } - gs.Target = host + gs.Target = u.Hostname() + + portStr := u.Port() + if portStr == "" { + portStr = "161" + } port, err := strconv.ParseUint(portStr, 10, 16) if err != nil { return nil, Errorf(err, "parsing port") @@ -878,7 +879,7 @@ func snmpTable(oid string) (mibName string, oidNum string, oidText string, field } func snmpTableCall(oid string) (mibName string, oidNum string, oidText string, fields []Field, err error) { - mibName, oidNum, oidText, _, err = snmpTranslate(oid) + mibName, oidNum, oidText, _, err = SnmpTranslate(oid) if err != nil { return "", "", "", nil, Errorf(err, "translating") } @@ -948,7 +949,7 @@ var snmpTranslateCachesLock sync.Mutex var snmpTranslateCaches map[string]snmpTranslateCache // snmpTranslate resolves the given OID. -func snmpTranslate(oid string) (mibName string, oidNum string, oidText string, conversion string, err error) { +func SnmpTranslate(oid string) (mibName string, oidNum string, oidText string, conversion string, err error) { snmpTranslateCachesLock.Lock() if snmpTranslateCaches == nil { snmpTranslateCaches = map[string]snmpTranslateCache{} @@ -974,6 +975,28 @@ func snmpTranslate(oid string) (mibName string, oidNum string, oidText string, c return stc.mibName, stc.oidNum, stc.oidText, stc.conversion, stc.err } +func SnmpTranslateForce(oid string, mibName string, oidNum string, oidText string, conversion string) { + snmpTranslateCachesLock.Lock() + defer snmpTranslateCachesLock.Unlock() + if snmpTranslateCaches == nil { + snmpTranslateCaches = map[string]snmpTranslateCache{} + } + + var stc snmpTranslateCache + stc.mibName = mibName + stc.oidNum = oidNum + stc.oidText = oidText + stc.conversion = conversion + stc.err = nil + snmpTranslateCaches[oid] = stc +} + +func SnmpTranslateClear() { + snmpTranslateCachesLock.Lock() + defer snmpTranslateCachesLock.Unlock() + snmpTranslateCaches = map[string]snmpTranslateCache{} +} + func snmpTranslateCall(oid string) (mibName string, oidNum string, oidText string, conversion string, err error) { var out []byte if strings.ContainsAny(oid, ":abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") { diff --git a/plugins/inputs/snmp/snmp_test.go b/plugins/inputs/snmp/snmp_test.go index db1a49605df05..25382bd7ddebf 100644 --- a/plugins/inputs/snmp/snmp_test.go +++ b/plugins/inputs/snmp/snmp_test.go @@ -10,6 +10,7 @@ import ( "time" "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/plugins/inputs" "github.com/influxdata/telegraf/testutil" "github.com/influxdata/toml" "github.com/soniah/gosnmp" @@ -82,45 +83,20 @@ var tsc = &testSNMPConnection{ } func TestSampleConfig(t *testing.T) { - conf := struct { - Inputs struct { - Snmp []*Snmp - } - }{} - err := toml.Unmarshal([]byte("[[inputs.snmp]]\n"+(*Snmp)(nil).SampleConfig()), &conf) - assert.NoError(t, err) + conf := inputs.Inputs["snmp"]() + err := toml.Unmarshal([]byte(conf.SampleConfig()), conf) + require.NoError(t, err) - s := Snmp{ - Agents: []string{"127.0.0.1:161"}, + expected := &Snmp{ + Agents: []string{"udp://127.0.0.1:161"}, Timeout: internal.Duration{Duration: 5 * time.Second}, Version: 2, Community: "public", MaxRepetitions: 10, Retries: 3, - - Name: "system", - Fields: []Field{ - {Name: "hostname", Oid: ".1.0.0.1.1"}, - {Name: "uptime", Oid: ".1.0.0.1.2"}, - {Name: "load", Oid: ".1.0.0.1.3"}, - {Oid: "HOST-RESOURCES-MIB::hrMemorySize"}, - }, - Tables: []Table{ - { - Name: "remote_servers", - InheritTags: []string{"hostname"}, - Fields: []Field{ - {Name: "server", Oid: ".1.0.0.0.1.0", IsTag: true}, - {Name: "connections", Oid: ".1.0.0.0.1.1"}, - {Name: "latency", Oid: ".1.0.0.0.1.2"}, - }, - }, - { - Oid: "HOST-RESOURCES-MIB::hrNetworkTable", - }, - }, + Name: "snmp", } - assert.Equal(t, &s, conf.Inputs.Snmp[0]) + require.Equal(t, expected, conf) } func TestFieldInit(t *testing.T) { @@ -256,7 +232,7 @@ func TestSnmpInit_noTranslate(t *testing.T) { func TestGetSNMPConnection_v2(t *testing.T) { s := &Snmp{ - Agents: []string{"1.2.3.4:567", "1.2.3.4"}, + Agents: []string{"1.2.3.4:567", "1.2.3.4", "udp://127.0.0.1"}, Timeout: internal.Duration{Duration: 3 * time.Second}, Retries: 4, Version: 2, @@ -272,12 +248,53 @@ func TestGetSNMPConnection_v2(t *testing.T) { assert.EqualValues(t, 567, gs.Port) assert.Equal(t, gosnmp.Version2c, gs.Version) assert.Equal(t, "foo", gs.Community) + assert.Equal(t, "udp", gs.Transport) gsc, err = s.getConnection(1) require.NoError(t, err) gs = gsc.(gosnmpWrapper) assert.Equal(t, "1.2.3.4", gs.Target) assert.EqualValues(t, 161, gs.Port) + assert.Equal(t, "udp", gs.Transport) + + gsc, err = s.getConnection(2) + require.NoError(t, err) + gs = gsc.(gosnmpWrapper) + assert.Equal(t, "127.0.0.1", gs.Target) + assert.EqualValues(t, 161, gs.Port) + assert.Equal(t, "udp", gs.Transport) +} + +func TestGetSNMPConnectionTCP(t *testing.T) { + var wg sync.WaitGroup + wg.Add(1) + go stubTCPServer(&wg) + wg.Wait() + + s := &Snmp{ + Agents: []string{"tcp://127.0.0.1:56789"}, + } + err := s.init() + require.NoError(t, err) + + wg.Add(1) + gsc, err := s.getConnection(0) + require.NoError(t, err) + gs := gsc.(gosnmpWrapper) + assert.Equal(t, "127.0.0.1", gs.Target) + assert.EqualValues(t, 56789, gs.Port) + assert.Equal(t, "tcp", gs.Transport) + wg.Wait() +} + +func stubTCPServer(wg *sync.WaitGroup) { + defer wg.Done() + tcpAddr, _ := net.ResolveTCPAddr("tcp", "127.0.0.1:56789") + tcpServer, _ := net.ListenTCP("tcp", tcpAddr) + defer tcpServer.Close() + wg.Done() + conn, _ := tcpServer.AcceptTCP() + defer conn.Close() } func TestGetSNMPConnection_v3(t *testing.T) { @@ -708,7 +725,7 @@ func TestFieldConvert(t *testing.T) { func TestSnmpTranslateCache_miss(t *testing.T) { snmpTranslateCaches = nil oid := "IF-MIB::ifPhysAddress.1" - mibName, oidNum, oidText, conversion, err := snmpTranslate(oid) + mibName, oidNum, oidText, conversion, err := SnmpTranslate(oid) assert.Len(t, snmpTranslateCaches, 1) stc := snmpTranslateCaches[oid] require.NotNil(t, stc) @@ -729,7 +746,7 @@ func TestSnmpTranslateCache_hit(t *testing.T) { err: fmt.Errorf("e"), }, } - mibName, oidNum, oidText, conversion, err := snmpTranslate("foo") + mibName, oidNum, oidText, conversion, err := SnmpTranslate("foo") assert.Equal(t, "a", mibName) assert.Equal(t, "b", oidNum) assert.Equal(t, "c", oidText) diff --git a/plugins/inputs/snmp_legacy/snmp_legacy.go b/plugins/inputs/snmp_legacy/snmp_legacy.go index 57f9f4fe24738..8df9cff06fa2c 100644 --- a/plugins/inputs/snmp_legacy/snmp_legacy.go +++ b/plugins/inputs/snmp_legacy/snmp_legacy.go @@ -1,7 +1,6 @@ package snmp_legacy import ( - "fmt" "io/ioutil" "log" "net" @@ -24,6 +23,8 @@ type Snmp struct { Subtable []Subtable SnmptranslateFile string + Log telegraf.Logger + nameToOid map[string]string initNode Node subTableMap map[string]Subtable @@ -297,7 +298,7 @@ func (s *Snmp) Gather(acc telegraf.Accumulator) error { data, err := ioutil.ReadFile(s.SnmptranslateFile) if err != nil { - log.Printf("E! Reading SNMPtranslate file error: %s", err) + s.Log.Errorf("Reading SNMPtranslate file error: %s", err.Error()) return err } else { for _, line := range strings.Split(string(data), "\n") { @@ -395,16 +396,16 @@ func (s *Snmp) Gather(acc telegraf.Accumulator) error { // only if len(s.OidInstanceMapping) == 0 if len(host.OidInstanceMapping) >= 0 { if err := host.SNMPMap(acc, s.nameToOid, s.subTableMap); err != nil { - acc.AddError(fmt.Errorf("E! SNMP Mapping error for host '%s': %s", host.Address, err)) + s.Log.Errorf("Mapping error for host %q: %s", host.Address, err.Error()) continue } } // Launch Get requests if err := host.SNMPGet(acc, s.initNode); err != nil { - acc.AddError(fmt.Errorf("E! SNMP Error for host '%s': %s", host.Address, err)) + s.Log.Errorf("Error for host %q: %s", host.Address, err.Error()) } if err := host.SNMPBulk(acc, s.initNode); err != nil { - acc.AddError(fmt.Errorf("E! SNMP Error for host '%s': %s", host.Address, err)) + s.Log.Errorf("Error for host %q: %s", host.Address, err.Error()) } } return nil @@ -801,7 +802,7 @@ func (h *Host) HandleResponse( acc.AddFields(field_name, fields, tags) case gosnmp.NoSuchObject, gosnmp.NoSuchInstance: // Oid not found - log.Printf("E! [snmp input] Oid not found: %s", oid_key) + log.Printf("E! [inputs.snmp_legacy] oid %q not found", oid_key) default: // delete other data } diff --git a/plugins/inputs/snmp_trap/README.md b/plugins/inputs/snmp_trap/README.md new file mode 100644 index 0000000000000..ceb370d8f65e0 --- /dev/null +++ b/plugins/inputs/snmp_trap/README.md @@ -0,0 +1,84 @@ +# SNMP Trap Input Plugin + +The SNMP Trap plugin is a service input plugin that receives SNMP +notifications (traps and inform requests). + +Notifications are received on plain UDP. The port to listen is +configurable. + +### Prerequisites + +This plugin uses the `snmptranslate` programs from the +[net-snmp][] project. These tools will need to be installed into the `PATH` in +order to be located. Other utilities from the net-snmp project may be useful +for troubleshooting, but are not directly used by the plugin. + +These programs will load available MIBs on the system. Typically the default +directory for MIBs is `/usr/share/snmp/mibs`, but if your MIBs are in a +different location you may need to make the paths known to net-snmp. The +location of these files can be configured in the `snmp.conf` or via the +`MIBDIRS` environment variable. See [`man 1 snmpcmd`][man snmpcmd] for more +information. + +### Configuration +```toml +[[inputs.snmp_trap]] + ## Transport, local address, and port to listen on. Transport must + ## be "udp://". Omit local address to listen on all interfaces. + ## example: "udp://127.0.0.1:1234" + ## + ## Special permissions may be required to listen on a port less than + ## 1024. See README.md for details + ## + # service_address = "udp://:162" + ## Timeout running snmptranslate command + # timeout = "5s" +``` + +#### Using a Privileged Port + +On many operating systems, listening on a privileged port (a port +number less than 1024) requires extra permission. Since the default +SNMP trap port 162 is in this category, using telegraf to receive SNMP +traps may need extra permission. + +Instructions for listening on a privileged port vary by operating +system. It is not recommended to run telegraf as superuser in order to +use a privileged port. Instead follow the principle of least privilege +and use a more specific operating system mechanism to allow telegraf to +use the port. You may also be able to have telegraf use an +unprivileged port and then configure a firewall port forward rule from +the privileged port. + +To use a privileged port on Linux, you can use setcap to enable the +CAP_NET_BIND_SERVICE capability on the telegraf binary: + +``` +setcap cap_net_bind_service=+ep /usr/bin/telegraf +``` + +On Mac OS, listening on privileged ports is unrestricted on versions +10.14 and later. + +### Metrics + +- snmp_trap + - tags: + - source (string, IP address of trap source) + - name (string, value from SNMPv2-MIB::snmpTrapOID.0 PDU) + - mib (string, MIB from SNMPv2-MIB::snmpTrapOID.0 PDU) + - oid (string, OID string from SNMPv2-MIB::snmpTrapOID.0 PDU) + - version (string, "1" or "2c" or "3") + - fields: + - Fields are mapped from variables in the trap. Field names are + the trap variable names after MIB lookup. Field values are trap + variable values. + +### Example Output +``` +snmp_trap,mib=SNMPv2-MIB,name=coldStart,oid=.1.3.6.1.6.3.1.1.5.1,source=192.168.122.102,version=2c snmpTrapEnterprise.0="linux",sysUpTimeInstance=1i 1574109187723429814 +snmp_trap,mib=NET-SNMP-AGENT-MIB,name=nsNotifyShutdown,oid=.1.3.6.1.4.1.8072.4.0.2,source=192.168.122.102,version=2c sysUpTimeInstance=5803i,snmpTrapEnterprise.0="netSnmpNotificationPrefix" 1574109186555115459 +``` + +[net-snmp]: http://www.net-snmp.org/ +[man snmpcmd]: http://net-snmp.sourceforge.net/docs/man/snmpcmd.html#lbAK diff --git a/plugins/inputs/snmp_trap/snmp_trap.go b/plugins/inputs/snmp_trap/snmp_trap.go new file mode 100644 index 0000000000000..80fc28f7ccf6b --- /dev/null +++ b/plugins/inputs/snmp_trap/snmp_trap.go @@ -0,0 +1,302 @@ +package snmp_trap + +import ( + "bufio" + "bytes" + "fmt" + "net" + "os/exec" + "strconv" + "strings" + "sync" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/plugins/inputs" + + "github.com/soniah/gosnmp" +) + +var defaultTimeout = internal.Duration{Duration: time.Second * 5} + +type handler func(*gosnmp.SnmpPacket, *net.UDPAddr) +type execer func(internal.Duration, string, ...string) ([]byte, error) + +type mibEntry struct { + mibName string + oidText string +} + +type SnmpTrap struct { + ServiceAddress string `toml:"service_address"` + Timeout internal.Duration `toml:"timeout"` + + acc telegraf.Accumulator + listener *gosnmp.TrapListener + timeFunc func() time.Time + errCh chan error + + makeHandlerWrapper func(handler) handler + + Log telegraf.Logger `toml:"-"` + + cacheLock sync.Mutex + cache map[string]mibEntry + + execCmd execer +} + +var sampleConfig = ` + ## Transport, local address, and port to listen on. Transport must + ## be "udp://". Omit local address to listen on all interfaces. + ## example: "udp://127.0.0.1:1234" + ## + ## Special permissions may be required to listen on a port less than + ## 1024. See README.md for details + ## + # service_address = "udp://:162" + ## Timeout running snmptranslate command + # timeout = "5s" +` + +func (s *SnmpTrap) SampleConfig() string { + return sampleConfig +} + +func (s *SnmpTrap) Description() string { + return "Receive SNMP traps" +} + +func (s *SnmpTrap) Gather(_ telegraf.Accumulator) error { + return nil +} + +func init() { + inputs.Add("snmp_trap", func() telegraf.Input { + return &SnmpTrap{ + timeFunc: time.Now, + ServiceAddress: "udp://:162", + Timeout: defaultTimeout, + } + }) +} + +func realExecCmd(Timeout internal.Duration, arg0 string, args ...string) ([]byte, error) { + cmd := exec.Command(arg0, args...) + var out bytes.Buffer + cmd.Stdout = &out + err := internal.RunTimeout(cmd, Timeout.Duration) + if err != nil { + return nil, err + } + return out.Bytes(), nil +} + +func (s *SnmpTrap) Init() error { + s.cache = map[string]mibEntry{} + s.execCmd = realExecCmd + return nil +} + +func (s *SnmpTrap) Start(acc telegraf.Accumulator) error { + s.acc = acc + s.listener = gosnmp.NewTrapListener() + s.listener.OnNewTrap = makeTrapHandler(s) + s.listener.Params = gosnmp.Default + + // wrap the handler, used in unit tests + if nil != s.makeHandlerWrapper { + s.listener.OnNewTrap = s.makeHandlerWrapper(s.listener.OnNewTrap) + } + + split := strings.SplitN(s.ServiceAddress, "://", 2) + if len(split) != 2 { + return fmt.Errorf("invalid service address: %s", s.ServiceAddress) + } + + protocol := split[0] + addr := split[1] + + // gosnmp.TrapListener currently supports udp only. For forward + // compatibility, require udp in the service address + if protocol != "udp" { + return fmt.Errorf("unknown protocol '%s' in '%s'", protocol, s.ServiceAddress) + } + + // If (*TrapListener).Listen immediately returns an error we need + // to return it from this function. Use a channel to get it here + // from the goroutine. Buffer one in case Listen returns after + // Listening but before our Close is called. + s.errCh = make(chan error, 1) + go func() { + s.errCh <- s.listener.Listen(addr) + }() + + select { + case <-s.listener.Listening(): + s.Log.Infof("Listening on %s", s.ServiceAddress) + case err := <-s.errCh: + return err + } + + return nil +} + +func (s *SnmpTrap) Stop() { + s.listener.Close() + err := <-s.errCh + if nil != err { + s.Log.Errorf("Error stopping trap listener %v", err) + } +} + +func setTrapOid(tags map[string]string, oid string, e mibEntry) { + tags["oid"] = oid + tags["name"] = e.oidText + tags["mib"] = e.mibName +} + +func makeTrapHandler(s *SnmpTrap) handler { + return func(packet *gosnmp.SnmpPacket, addr *net.UDPAddr) { + tm := s.timeFunc() + fields := map[string]interface{}{} + tags := map[string]string{} + + tags["version"] = packet.Version.String() + tags["source"] = addr.IP.String() + + if packet.Version == gosnmp.Version1 { + // Follow the procedure described in RFC 2576 3.1 to + // translate a v1 trap to v2. + var trapOid string + + if packet.GenericTrap >= 0 && packet.GenericTrap < 6 { + trapOid = ".1.3.6.1.6.3.1.1.5." + strconv.Itoa(packet.GenericTrap+1) + } else if packet.GenericTrap == 6 { + trapOid = packet.Enterprise + ".0." + strconv.Itoa(packet.SpecificTrap) + } + + if trapOid != "" { + e, err := s.lookup(trapOid) + if err != nil { + s.Log.Errorf("Error resolving V1 OID: %v", err) + return + } + setTrapOid(tags, trapOid, e) + } + + if packet.AgentAddress != "" { + tags["agent_address"] = packet.AgentAddress + } + + fields["sysUpTimeInstance"] = packet.Timestamp + } + + for _, v := range packet.Variables { + // Use system mibs to resolve oids. Don't fall back to + // numeric oid because it's not useful enough to the end + // user and can be difficult to translate or remove from + // the database later. + + var value interface{} + + // todo: format the pdu value based on its snmp type and + // the mib's textual convention. The snmp input plugin + // only handles textual convention for ip and mac + // addresses + + switch v.Type { + case gosnmp.ObjectIdentifier: + val, ok := v.Value.(string) + if !ok { + s.Log.Errorf("Error getting value OID") + return + } + + var e mibEntry + var err error + e, err = s.lookup(val) + if nil != err { + s.Log.Errorf("Error resolving value OID: %v", err) + return + } + + value = e.oidText + + // 1.3.6.1.6.3.1.1.4.1.0 is SNMPv2-MIB::snmpTrapOID.0. + // If v.Name is this oid, set a tag of the trap name. + if v.Name == ".1.3.6.1.6.3.1.1.4.1.0" { + setTrapOid(tags, val, e) + continue + } + default: + value = v.Value + } + + e, err := s.lookup(v.Name) + if nil != err { + s.Log.Errorf("Error resolving OID: %v", err) + return + } + + name := e.oidText + + fields[name] = value + } + + s.acc.AddFields("snmp_trap", fields, tags, tm) + } +} + +func (s *SnmpTrap) lookup(oid string) (e mibEntry, err error) { + s.cacheLock.Lock() + defer s.cacheLock.Unlock() + var ok bool + if e, ok = s.cache[oid]; !ok { + // cache miss. exec snmptranlate + e, err = s.snmptranslate(oid) + if err == nil { + s.cache[oid] = e + } + return e, err + } + return e, nil +} + +func (s *SnmpTrap) clear() { + s.cacheLock.Lock() + defer s.cacheLock.Unlock() + s.cache = map[string]mibEntry{} +} + +func (s *SnmpTrap) load(oid string, e mibEntry) { + s.cacheLock.Lock() + defer s.cacheLock.Unlock() + s.cache[oid] = e +} + +func (s *SnmpTrap) snmptranslate(oid string) (e mibEntry, err error) { + var out []byte + out, err = s.execCmd(s.Timeout, "snmptranslate", "-Td", "-Ob", "-m", "all", oid) + + if err != nil { + return e, err + } + + scanner := bufio.NewScanner(bytes.NewBuffer(out)) + ok := scanner.Scan() + if err = scanner.Err(); !ok && err != nil { + return e, err + } + + e.oidText = scanner.Text() + + i := strings.Index(e.oidText, "::") + if i == -1 { + return e, fmt.Errorf("not found") + } + e.mibName = e.oidText[:i] + e.oidText = e.oidText[i+2:] + return e, nil +} diff --git a/plugins/inputs/snmp_trap/snmp_trap_test.go b/plugins/inputs/snmp_trap/snmp_trap_test.go new file mode 100644 index 0000000000000..34dd6cde0c6a3 --- /dev/null +++ b/plugins/inputs/snmp_trap/snmp_trap_test.go @@ -0,0 +1,339 @@ +package snmp_trap + +import ( + "fmt" + "net" + "strconv" + "strings" + "testing" + "time" + + "github.com/soniah/gosnmp" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/testutil" + + "github.com/stretchr/testify/require" +) + +func TestLoad(t *testing.T) { + s := &SnmpTrap{} + require.Nil(t, s.Init()) + + defer s.clear() + s.load( + ".1.3.6.1.6.3.1.1.5.1", + mibEntry{ + "SNMPv2-MIB", + "coldStart", + }, + ) + + e, err := s.lookup(".1.3.6.1.6.3.1.1.5.1") + require.NoError(t, err) + require.Equal(t, "SNMPv2-MIB", e.mibName) + require.Equal(t, "coldStart", e.oidText) +} + +func fakeExecCmd(_ internal.Duration, x string, y ...string) ([]byte, error) { + return nil, fmt.Errorf("mock " + x + " " + strings.Join(y, " ")) +} + +func sendTrap(t *testing.T, port uint16, now uint32, trap gosnmp.SnmpTrap, version gosnmp.SnmpVersion) { + s := &gosnmp.GoSNMP{ + Port: port, + Community: "public", + Version: version, + Timeout: time.Duration(2) * time.Second, + Retries: 3, + MaxOids: gosnmp.MaxOids, + Target: "127.0.0.1", + } + + err := s.Connect() + if err != nil { + t.Errorf("Connect() err: %v", err) + } + defer s.Conn.Close() + + _, err = s.SendTrap(trap) + if err != nil { + t.Errorf("SendTrap() err: %v", err) + } +} + +func TestReceiveTrap(t *testing.T) { + var now uint32 + now = 123123123 + + var fakeTime time.Time + fakeTime = time.Unix(456456456, 456) + + type entry struct { + oid string + e mibEntry + } + + // If the first pdu isn't type TimeTicks, gosnmp.SendTrap() will + // prepend one with time.Now() + var tests = []struct { + name string + + // send + version gosnmp.SnmpVersion + trap gosnmp.SnmpTrap // include pdus + + // recieve + entries []entry + metrics []telegraf.Metric + }{ + //ordinary v2c coldStart trap + { + name: "v2c coldStart", + version: gosnmp.Version2c, + trap: gosnmp.SnmpTrap{ + Variables: []gosnmp.SnmpPDU{ + { + Name: ".1.3.6.1.2.1.1.3.0", + Type: gosnmp.TimeTicks, + Value: now, + }, + { + Name: ".1.3.6.1.6.3.1.1.4.1.0", // SNMPv2-MIB::snmpTrapOID.0 + Type: gosnmp.ObjectIdentifier, + Value: ".1.3.6.1.6.3.1.1.5.1", // coldStart + }, + }, + }, + entries: []entry{ + { + oid: ".1.3.6.1.6.3.1.1.4.1.0", + e: mibEntry{ + "SNMPv2-MIB", + "snmpTrapOID.0", + }, + }, + { + oid: ".1.3.6.1.6.3.1.1.5.1", + e: mibEntry{ + "SNMPv2-MIB", + "coldStart", + }, + }, + { + oid: ".1.3.6.1.2.1.1.3.0", + e: mibEntry{ + "UNUSED_MIB_NAME", + "sysUpTimeInstance", + }, + }, + }, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "snmp_trap", // name + map[string]string{ // tags + "oid": ".1.3.6.1.6.3.1.1.5.1", + "name": "coldStart", + "mib": "SNMPv2-MIB", + "version": "2c", + "source": "127.0.0.1", + }, + map[string]interface{}{ // fields + "sysUpTimeInstance": now, + }, + fakeTime, + ), + }, + }, + //Check that we're not running snmptranslate to look up oids + //when we shouldn't be. This sends and receives a valid trap + //but metric production should fail because the oids aren't in + //the cache and oid lookup is intentionally mocked to fail. + { + name: "missing oid", + version: gosnmp.Version2c, + trap: gosnmp.SnmpTrap{ + Variables: []gosnmp.SnmpPDU{ + { + Name: ".1.3.6.1.2.1.1.3.0", + Type: gosnmp.TimeTicks, + Value: now, + }, + { + Name: ".1.3.6.1.6.3.1.1.4.1.0", // SNMPv2-MIB::snmpTrapOID.0 + Type: gosnmp.ObjectIdentifier, + Value: ".1.3.6.1.6.3.1.1.5.1", // coldStart + }, + }, + }, + entries: []entry{}, //nothing in cache + metrics: []telegraf.Metric{}, + }, + //v1 enterprise specific trap + { + name: "v1 trap enterprise", + version: gosnmp.Version1, + trap: gosnmp.SnmpTrap{ + Variables: []gosnmp.SnmpPDU{ + { + Name: ".1.2.3.4.5", + Type: gosnmp.OctetString, + Value: "payload", + }, + }, + Enterprise: ".1.2.3", + AgentAddress: "10.20.30.40", + GenericTrap: 6, // enterpriseSpecific + SpecificTrap: 55, + Timestamp: uint(now), + }, + entries: []entry{ + { + ".1.2.3.4.5", + mibEntry{ + "valueMIB", + "valueOID", + }, + }, + { + ".1.2.3.0.55", + mibEntry{ + "enterpriseMIB", + "enterpriseOID", + }, + }, + }, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "snmp_trap", // name + map[string]string{ // tags + "oid": ".1.2.3.0.55", + "name": "enterpriseOID", + "mib": "enterpriseMIB", + "version": "1", + "source": "127.0.0.1", + "agent_address": "10.20.30.40", + }, + map[string]interface{}{ // fields + "sysUpTimeInstance": uint(now), + "valueOID": "payload", + }, + fakeTime, + ), + }, + }, + //v1 generic trap + { + name: "v1 trap generic", + version: gosnmp.Version1, + trap: gosnmp.SnmpTrap{ + Variables: []gosnmp.SnmpPDU{ + { + Name: ".1.2.3.4.5", + Type: gosnmp.OctetString, + Value: "payload", + }, + }, + Enterprise: ".1.2.3", + AgentAddress: "10.20.30.40", + GenericTrap: 0, //coldStart + SpecificTrap: 0, + Timestamp: uint(now), + }, + entries: []entry{ + { + ".1.2.3.4.5", + mibEntry{ + "valueMIB", + "valueOID", + }, + }, + { + ".1.3.6.1.6.3.1.1.5.1", + mibEntry{ + "coldStartMIB", + "coldStartOID", + }, + }, + }, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "snmp_trap", // name + map[string]string{ // tags + "oid": ".1.3.6.1.6.3.1.1.5.1", + "name": "coldStartOID", + "mib": "coldStartMIB", + "version": "1", + "source": "127.0.0.1", + "agent_address": "10.20.30.40", + }, + map[string]interface{}{ // fields + "sysUpTimeInstance": uint(now), + "valueOID": "payload", + }, + fakeTime, + ), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // We would prefer to specify port 0 and let the network + // stack choose an unused port for us but TrapListener + // doesn't have a way to return the autoselected port. + // Instead, we'll use an unusual port and hope it's + // unused. + const port = 12399 + + // Hook into the trap handler so the test knows when the + // trap has been received + received := make(chan int) + wrap := func(f handler) handler { + return func(p *gosnmp.SnmpPacket, a *net.UDPAddr) { + f(p, a) + received <- 0 + } + } + + // Set up the service input plugin + s := &SnmpTrap{ + ServiceAddress: "udp://:" + strconv.Itoa(port), + makeHandlerWrapper: wrap, + timeFunc: func() time.Time { + return fakeTime + }, + Log: testutil.Logger{}, + } + require.Nil(t, s.Init()) + var acc testutil.Accumulator + require.Nil(t, s.Start(&acc)) + defer s.Stop() + + // Preload the cache with the oids we'll use in this test + // so snmptranslate and mibs don't need to be installed. + for _, entry := range tt.entries { + s.load(entry.oid, entry.e) + } + + // Don't look up oid with snmptranslate. + s.execCmd = fakeExecCmd + + // Send the trap + sendTrap(t, port, now, tt.trap, tt.version) + + // Wait for trap to be received + select { + case <-received: + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for trap to be received") + } + + // Verify plugin output + testutil.RequireMetricsEqual(t, + tt.metrics, acc.GetTelegrafMetrics(), + testutil.SortMetrics()) + }) + } + +} diff --git a/plugins/inputs/socket_listener/README.md b/plugins/inputs/socket_listener/README.md index 1740d8bcf15ff..ec1aa0befc4f0 100644 --- a/plugins/inputs/socket_listener/README.md +++ b/plugins/inputs/socket_listener/README.md @@ -66,6 +66,10 @@ This is a sample configuration for the plugin. ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md # data_format = "influx" + + ## Content encoding for message payloads, can be set to "gzip" to or + ## "identity" to apply no encoding. + # content_encoding = "identity" ``` ## A Note on UDP OS Buffer Sizes @@ -84,6 +88,7 @@ at least 8MB before trying to run large amounts of UDP traffic to your instance. 8MB is just a recommendation, and can be adjusted higher. ### Linux + Check the current UDP/IP receive buffer limit & default by typing the following commands: diff --git a/plugins/inputs/socket_listener/socket_listener.go b/plugins/inputs/socket_listener/socket_listener.go index a127a0738accd..b1e9338510d73 100644 --- a/plugins/inputs/socket_listener/socket_listener.go +++ b/plugins/inputs/socket_listener/socket_listener.go @@ -5,7 +5,6 @@ import ( "crypto/tls" "fmt" "io" - "log" "net" "os" "strconv" @@ -43,7 +42,7 @@ func (ssl *streamSocketListener) listen() { c, err := ssl.Accept() if err != nil { if !strings.HasSuffix(err.Error(), ": use of closed network connection") { - ssl.AddError(err) + ssl.Log.Error(err.Error()) } break } @@ -52,7 +51,7 @@ func (ssl *streamSocketListener) listen() { if srb, ok := c.(setReadBufferer); ok { srb.SetReadBuffer(int(ssl.ReadBufferSize.Size)) } else { - log.Printf("W! Unable to set read buffer on a %s socket", ssl.sockType) + ssl.Log.Warnf("Unable to set read buffer on a %s socket", ssl.sockType) } } @@ -66,7 +65,7 @@ func (ssl *streamSocketListener) listen() { ssl.connectionsMtx.Unlock() if err := ssl.setKeepAlive(c); err != nil { - ssl.AddError(fmt.Errorf("unable to configure keep alive (%s): %s", ssl.ServiceAddress, err)) + ssl.Log.Errorf("Unable to configure keep alive %q: %s", ssl.ServiceAddress, err.Error()) } wg.Add(1) @@ -120,9 +119,16 @@ func (ssl *streamSocketListener) read(c net.Conn) { if !scnr.Scan() { break } - metrics, err := ssl.Parse(scnr.Bytes()) + + body, err := ssl.decoder.Decode(scnr.Bytes()) if err != nil { - ssl.AddError(fmt.Errorf("unable to parse incoming line: %s", err)) + ssl.Log.Errorf("Unable to decode incoming line: %s", err.Error()) + continue + } + + metrics, err := ssl.Parse(body) + if err != nil { + ssl.Log.Errorf("Unable to parse incoming line: %s", err.Error()) // TODO rate limit continue } @@ -133,9 +139,9 @@ func (ssl *streamSocketListener) read(c net.Conn) { if err := scnr.Err(); err != nil { if netErr, ok := err.(net.Error); ok && netErr.Timeout() { - log.Printf("D! Timeout in plugin [input.socket_listener]: %s", err) + ssl.Log.Debugf("Timeout in plugin: %s", err.Error()) } else if netErr != nil && !strings.HasSuffix(err.Error(), ": use of closed network connection") { - ssl.AddError(err) + ssl.Log.Error(err.Error()) } } } @@ -151,14 +157,19 @@ func (psl *packetSocketListener) listen() { n, _, err := psl.ReadFrom(buf) if err != nil { if !strings.HasSuffix(err.Error(), ": use of closed network connection") { - psl.AddError(err) + psl.Log.Error(err.Error()) } break } - metrics, err := psl.Parse(buf[:n]) + body, err := psl.decoder.Decode(buf[:n]) if err != nil { - psl.AddError(fmt.Errorf("unable to parse incoming packet: %s", err)) + psl.Log.Errorf("Unable to decode incoming packet: %s", err.Error()) + } + + metrics, err := psl.Parse(body) + if err != nil { + psl.Log.Errorf("Unable to parse incoming packet: %s", err.Error()) // TODO rate limit continue } @@ -175,13 +186,17 @@ type SocketListener struct { ReadTimeout *internal.Duration `toml:"read_timeout"` KeepAlivePeriod *internal.Duration `toml:"keep_alive_period"` SocketMode string `toml:"socket_mode"` + ContentEncoding string `toml:"content_encoding"` tlsint.ServerConfig wg sync.WaitGroup + Log telegraf.Logger + parsers.Parser telegraf.Accumulator io.Closer + decoder internal.ContentDecoder } func (sl *SocketListener) Description() string { @@ -243,6 +258,10 @@ func (sl *SocketListener) SampleConfig() string { ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md # data_format = "influx" + + ## Content encoding for message payloads, can be set to "gzip" to or + ## "identity" to apply no encoding. + # content_encoding = "identity" ` } @@ -264,6 +283,12 @@ func (sl *SocketListener) Start(acc telegraf.Accumulator) error { protocol := spl[0] addr := spl[1] + var err error + sl.decoder, err = internal.NewContentDecoder(sl.ContentEncoding) + if err != nil { + return err + } + if protocol == "unix" || protocol == "unixpacket" || protocol == "unixgram" { // no good way of testing for "file does not exist". // Instead just ignore error and blow up when we try to listen, which will @@ -292,7 +317,7 @@ func (sl *SocketListener) Start(acc telegraf.Accumulator) error { return err } - log.Printf("I! [inputs.socket_listener] Listening on %s://%s", protocol, l.Addr()) + sl.Log.Infof("Listening on %s://%s", protocol, l.Addr()) // Set permissions on socket if (spl[0] == "unix" || spl[0] == "unixpacket") && sl.SocketMode != "" { @@ -339,11 +364,11 @@ func (sl *SocketListener) Start(acc telegraf.Accumulator) error { if srb, ok := pc.(setReadBufferer); ok { srb.SetReadBuffer(int(sl.ReadBufferSize.Size)) } else { - log.Printf("W! Unable to set read buffer on a %s socket", protocol) + sl.Log.Warnf("Unable to set read buffer on a %s socket", protocol) } } - log.Printf("I! [inputs.socket_listener] Listening on %s://%s", protocol, pc.LocalAddr()) + sl.Log.Infof("Listening on %s://%s", protocol, pc.LocalAddr()) psl := &packetSocketListener{ PacketConn: pc, diff --git a/plugins/inputs/socket_listener/socket_listener_test.go b/plugins/inputs/socket_listener/socket_listener_test.go index b4415e0922887..c6adf4cdebe7f 100644 --- a/plugins/inputs/socket_listener/socket_listener_test.go +++ b/plugins/inputs/socket_listener/socket_listener_test.go @@ -48,6 +48,7 @@ func TestSocketListener_tcp_tls(t *testing.T) { defer testEmptyLog(t)() sl := newSocketListener() + sl.Log = testutil.Logger{} sl.ServiceAddress = "tcp://127.0.0.1:0" sl.ServerConfig = *pki.TLSServerConfig() @@ -72,6 +73,7 @@ func TestSocketListener_unix_tls(t *testing.T) { sock := filepath.Join(tmpdir, "sl.TestSocketListener_unix_tls.sock") sl := newSocketListener() + sl.Log = testutil.Logger{} sl.ServiceAddress = "unix://" + sock sl.ServerConfig = *pki.TLSServerConfig() @@ -94,6 +96,7 @@ func TestSocketListener_tcp(t *testing.T) { defer testEmptyLog(t)() sl := newSocketListener() + sl.Log = testutil.Logger{} sl.ServiceAddress = "tcp://127.0.0.1:0" sl.ReadBufferSize = internal.Size{Size: 1024} @@ -112,6 +115,7 @@ func TestSocketListener_udp(t *testing.T) { defer testEmptyLog(t)() sl := newSocketListener() + sl.Log = testutil.Logger{} sl.ServiceAddress = "udp://127.0.0.1:0" sl.ReadBufferSize = internal.Size{Size: 1024} @@ -136,6 +140,7 @@ func TestSocketListener_unix(t *testing.T) { os.Create(sock) sl := newSocketListener() + sl.Log = testutil.Logger{} sl.ServiceAddress = "unix://" + sock sl.ReadBufferSize = internal.Size{Size: 1024} @@ -160,6 +165,7 @@ func TestSocketListener_unixgram(t *testing.T) { os.Create(sock) sl := newSocketListener() + sl.Log = testutil.Logger{} sl.ServiceAddress = "unixgram://" + sock sl.ReadBufferSize = internal.Size{Size: 1024} @@ -174,12 +180,65 @@ func TestSocketListener_unixgram(t *testing.T) { testSocketListener(t, sl, client) } +func TestSocketListenerDecode_tcp(t *testing.T) { + defer testEmptyLog(t)() + + sl := newSocketListener() + sl.Log = testutil.Logger{} + sl.ServiceAddress = "tcp://127.0.0.1:0" + sl.ReadBufferSize = internal.Size{Size: 1024} + sl.ContentEncoding = "gzip" + + acc := &testutil.Accumulator{} + err := sl.Start(acc) + require.NoError(t, err) + defer sl.Stop() + + client, err := net.Dial("tcp", sl.Closer.(net.Listener).Addr().String()) + require.NoError(t, err) + + testSocketListener(t, sl, client) +} + +func TestSocketListenerDecode_udp(t *testing.T) { + defer testEmptyLog(t)() + + sl := newSocketListener() + sl.Log = testutil.Logger{} + sl.ServiceAddress = "udp://127.0.0.1:0" + sl.ReadBufferSize = internal.Size{Size: 1024} + sl.ContentEncoding = "gzip" + + acc := &testutil.Accumulator{} + err := sl.Start(acc) + require.NoError(t, err) + defer sl.Stop() + + client, err := net.Dial("udp", sl.Closer.(net.PacketConn).LocalAddr().String()) + require.NoError(t, err) + + testSocketListener(t, sl, client) +} + func testSocketListener(t *testing.T, sl *SocketListener, client net.Conn) { - mstr12 := "test,foo=bar v=1i 123456789\ntest,foo=baz v=2i 123456790\n" - mstr3 := "test,foo=zab v=3i 123456791" - client.Write([]byte(mstr12)) - client.Write([]byte(mstr3)) - if _, ok := client.(net.Conn); ok { + mstr12 := []byte("test,foo=bar v=1i 123456789\ntest,foo=baz v=2i 123456790\n") + mstr3 := []byte("test,foo=zab v=3i 123456791") + + if sl.ContentEncoding == "gzip" { + encoder, err := internal.NewContentEncoder(sl.ContentEncoding) + require.NoError(t, err) + mstr12, err = encoder.Encode(mstr12) + require.NoError(t, err) + + encoder, err = internal.NewContentEncoder(sl.ContentEncoding) + require.NoError(t, err) + mstr3, err = encoder.Encode(mstr3) + require.NoError(t, err) + } + + client.Write(mstr12) + client.Write(mstr3) + if client.LocalAddr().Network() != "udp" { // stream connection. needs trailing newline to terminate mstr3 client.Write([]byte{'\n'}) } diff --git a/plugins/inputs/sqlserver/README.md b/plugins/inputs/sqlserver/README.md index 2d41c5dcca2c2..1b71165fbd78f 100644 --- a/plugins/inputs/sqlserver/README.md +++ b/plugins/inputs/sqlserver/README.md @@ -52,7 +52,7 @@ GO query_version = 2 ## If you are using AzureDB, setting this to true will gather resource utilization metrics - # azuredb = true + # azuredb = true ## If you would like to exclude some of the metrics queries, list them here ## Possible choices: @@ -67,8 +67,9 @@ GO ## - VolumeSpace ## - Schedulers ## - AzureDBResourceStats - ## - AzureDBResourceGovernance + ## - AzureDBResourceGovernance ## - SqlRequests + ## - ServerProperties exclude_query = [ 'Schedulers' , 'SqlRequests'] ``` @@ -108,7 +109,14 @@ The new (version 2) metrics provide: - *Server properties*: Number of databases in all possible states (online, offline, suspect, etc.), cpu count, physical memory, SQL Server service uptime, and SQL Server version. In the case of Azure SQL relevent properties such as Tier, #Vcores, Memory etc. - *Wait stats*: Wait time in ms, number of waiting tasks, resource wait time, signal wait time, max wait time in ms, wait type, and wait category. The waits are categorized using the same categories used in Query Store. - *Schedulers* - This captures sys.dm_os_schedulers. -- *SqlRequests* - This captures a snapshot of dm_exec_requests and dm_exec_sessions that gives you running requests as well as wait types and blocking sessions +- *SqlRequests* - This captures a snapshot of dm_exec_requests and + dm_exec_sessions that gives you running requests as well as wait types and + blocking sessions. + + In order to allow tracking on a per statement basis this query produces a + unique tag for each query. Depending on the database workload, this may + result in a high cardinality series. Reference the FAQ for tips on + [managing series cardinality][cardinality]. - *Azure Managed Instances* - Stats from `sys.server_resource_stats`: - cpu_count @@ -164,3 +172,5 @@ The following metrics can be used directly, with no delta calculations: Version 2 queries have the following tags: - `sql_instance`: Physical host and instance name (hostname:instance) - database_name: For Azure SQLDB, database_name denotes the name of the Azure SQL Database as server name is a logical construct. + +[cardinality]: /docs/FAQ.md#user-content-q-how-can-i-manage-series-cardinality diff --git a/plugins/inputs/sqlserver/sqlserver.go b/plugins/inputs/sqlserver/sqlserver.go index 51e729a31b17a..511bb5b49a82e 100644 --- a/plugins/inputs/sqlserver/sqlserver.go +++ b/plugins/inputs/sqlserver/sqlserver.go @@ -12,10 +12,12 @@ import ( // SQLServer struct type SQLServer struct { - Servers []string `toml:"servers"` - QueryVersion int `toml:"query_version"` - AzureDB bool `toml:"azuredb"` - ExcludeQuery []string `toml:"exclude_query"` + Servers []string `toml:"servers"` + QueryVersion int `toml:"query_version"` + AzureDB bool `toml:"azuredb"` + ExcludeQuery []string `toml:"exclude_query"` + queries MapQuery + isInitialized bool } // Query struct @@ -28,14 +30,9 @@ type Query struct { // MapQuery type type MapQuery map[string]Query -var queries MapQuery +const defaultServer = "Server=.;app name=telegraf;log=1;" -// Initialized flag -var isInitialized = false - -var defaultServer = "Server=.;app name=telegraf;log=1;" - -var sampleConfig = ` +const sampleConfig = ` ## Specify instances to monitor with a list of connection strings. ## All connection parameters are optional. ## By default, the host is localhost, listening on default port, TCP 1433. @@ -71,6 +68,7 @@ var sampleConfig = ` ## - AzureDBResourceStats ## - AzureDBResourceGovernance ## - SqlRequests + ## - ServerProperties exclude_query = [ 'Schedulers' ] ` @@ -89,8 +87,8 @@ type scanner interface { } func initQueries(s *SQLServer) { - queries = make(MapQuery) - + s.queries = make(MapQuery) + queries := s.queries // If this is an AzureDB instance, grab some extra metrics if s.AzureDB { queries["AzureDBResourceStats"] = Query{Script: sqlAzureDBResourceStats, ResultByRow: false} @@ -124,12 +122,12 @@ func initQueries(s *SQLServer) { } // Set a flag so we know that queries have already been initialized - isInitialized = true + s.isInitialized = true } // Gather collect data from SQL Server func (s *SQLServer) Gather(acc telegraf.Accumulator) error { - if !isInitialized { + if !s.isInitialized { initQueries(s) } @@ -140,7 +138,7 @@ func (s *SQLServer) Gather(acc telegraf.Accumulator) error { var wg sync.WaitGroup for _, serv := range s.Servers { - for _, query := range queries { + for _, query := range s.queries { wg.Add(1) go func(serv string, query Query) { defer wg.Done() @@ -352,25 +350,30 @@ EXEC(@SQL) // Conditional check based on Azure SQL DB OR On-prem SQL Server // EngineEdition=5 is Azure SQL DB -const sqlDatabaseIOV2 = `SET DEADLOCK_PRIORITY -10; +const sqlDatabaseIOV2 = ` +SET DEADLOCK_PRIORITY -10; IF SERVERPROPERTY('EngineEdition') = 5 BEGIN SELECT 'sqlserver_database_io' As [measurement], REPLACE(@@SERVERNAME,'\',':') AS [sql_instance], -DB_NAME([vfs].[database_id]) [database_name], +DB_NAME([vfs].[database_id]) AS [database_name], vfs.io_stall_read_ms AS read_latency_ms, vfs.num_of_reads AS reads, vfs.num_of_bytes_read AS read_bytes, vfs.io_stall_write_ms AS write_latency_ms, vfs.num_of_writes AS writes, vfs.num_of_bytes_written AS write_bytes, -b.name as logical_filename, -b.physical_name as physical_filename, -CASE WHEN vfs.file_id = 2 THEN 'LOG' ELSE 'DATA' END AS file_type +vfs.io_stall_queued_read_ms as rg_read_stall_ms, +vfs.io_stall_queued_write_ms as rg_write_stall_ms, +ISNULL(b.name ,'RBPEX') as logical_filename, +ISNULL(b.physical_name, 'RBPEX') as physical_filename, +CASE WHEN vfs.file_id = 2 THEN 'LOG'ELSE 'DATA' END AS file_type +,ISNULL(size,0)/128 AS current_size_mb +,ISNULL(FILEPROPERTY(b.name,'SpaceUsed')/128,0) as space_used_mb FROM [sys].[dm_io_virtual_file_stats](NULL,NULL) AS vfs -inner join sys.database_files b on b.file_id = vfs.file_id +LEFT OUTER join sys.database_files b on b.file_id = vfs.file_id END ELSE BEGIN @@ -384,12 +387,17 @@ vfs.num_of_bytes_read AS read_bytes, vfs.io_stall_write_ms AS write_latency_ms, vfs.num_of_writes AS writes, vfs.num_of_bytes_written AS write_bytes, -b.name as logical_filename, -b.physical_name as physical_filename, +vfs.io_stall_queued_read_ms as rg_read_stall_ms, +vfs.io_stall_queued_write_ms as rg_write_stall_ms, +ISNULL(b.name ,'RBPEX') as logical_filename, +ISNULL(b.physical_name, 'RBPEX') as physical_filename, CASE WHEN vfs.file_id = 2 THEN 'LOG' ELSE 'DATA' END AS file_type +,ISNULL(size,0)/128 AS current_size_mb +-- can't easily get space used without switching context to each DB for MI/On-prem making query expensive +, -1 as space_used_mb FROM [sys].[dm_io_virtual_file_stats](NULL,NULL) AS vfs -inner join sys.master_files b on b.database_id = vfs.database_id and b.file_id = vfs.file_id +LEFT OUTER join sys.master_files b on b.database_id = vfs.database_id and b.file_id = vfs.file_id END ` @@ -434,8 +442,9 @@ IF SERVERPROPERTY('EngineEdition') = 5 -- Azure SQL DB NULL AS available_storage_mb, -- Can we find out storage? NULL as uptime FROM sys.databases d - JOIN sys.database_service_objectives slo - ON d.database_id = slo.database_id + -- sys.databases.database_id may not match current DB_ID on Azure SQL DB + CROSS JOIN sys.database_service_objectives slo + WHERE d.name = DB_NAME() AND slo.database_id = DB_ID() ELSE BEGIN @@ -510,10 +519,32 @@ INSERT INTO @PCounters SELECT DISTINCT RTrim(spi.object_name) object_name, RTrim(spi.counter_name) counter_name, - RTrim(spi.instance_name) instance_name, + CASE WHEN ( + RTRIM(spi.object_name) LIKE '%:Databases' + OR RTRIM(spi.object_name) LIKE '%:Database Replica' + OR RTRIM(spi.object_name) LIKE '%:Catalog Metadata' + OR RTRIM(spi.object_name) LIKE '%:Query Store' + OR RTRIM(spi.object_name) LIKE '%:Columnstore' + OR RTRIM(spi.object_name) LIKE '%:Advanced Analytics') + AND SERVERPROPERTY ('EngineEdition') IN (5,8) + AND TRY_CONVERT(uniqueidentifier, spi.instance_name) IS NOT NULL -- for cloud only + THEN d.name + WHEN RTRIM(object_name) LIKE '%:Availability Replica' + AND SERVERPROPERTY ('EngineEdition') IN (5,8) + AND TRY_CONVERT(uniqueidentifier, spi.instance_name) IS NOT NULL -- for cloud only + THEN d.name + RTRIM(SUBSTRING(spi.instance_name, 37, LEN(spi.instance_name))) + ELSE spi.instance_name + END AS instance_name, CAST(spi.cntr_value AS BIGINT) AS cntr_value, spi.cntr_type FROM sys.dm_os_performance_counters AS spi +LEFT JOIN sys.databases AS d +ON LEFT(spi.instance_name, 36) -- some instance_name values have an additional identifier appended after the GUID + = CASE WHEN -- in SQL DB standalone, physical_database_name for master is the GUID of the user database + d.name = 'master' AND TRY_CONVERT(uniqueidentifier, d.physical_database_name) IS NOT NULL + THEN d.name + ELSE d.physical_database_name + END WHERE ( counter_name IN ( 'SQL Compilations/sec', @@ -527,12 +558,12 @@ WHERE ( 'Full Scans/sec', 'Index Searches/sec', 'Page Splits/sec', - 'Page Lookups/sec', - 'Page Reads/sec', - 'Page Writes/sec', - 'Readahead Pages/sec', - 'Lazy Writes/sec', - 'Checkpoint Pages/sec', + 'Page lookups/sec', + 'Page reads/sec', + 'Page writes/sec', + 'Readahead pages/sec', + 'Lazy writes/sec', + 'Checkpoint pages/sec', 'Page life expectancy', 'Log File(s) Size (KB)', 'Log File(s) Used Size (KB)', @@ -592,11 +623,16 @@ WHERE ( 'Background Writer pages/sec', 'Percent Log Used', 'Log Send Queue KB', - 'Redo Queue KB' + 'Redo Queue KB', + 'Mirrored Write Transactions/sec', + 'Group Commit Time', + 'Group Commits/Sec' ) ) OR ( object_name LIKE '%User Settable%' OR object_name LIKE '%SQL Errors%' + ) OR ( + object_name LIKE 'SQLServer:Batch Resp Statistics%' ) OR ( instance_name IN ('_Total') AND counter_name IN ( @@ -654,8 +690,7 @@ FROM @PCounters AS pc AND pc.object_name = pc1.object_name AND pc.instance_name = pc1.instance_name AND pc1.counter_name LIKE '%base' -WHERE pc.counter_name NOT LIKE '% base' -OPTION(RECOMPILE); +WHERE pc.counter_name NOT LIKE '% base'; ` // Conditional check based on Azure SQL DB v/s the rest aka (Azure SQL Managed instance OR On-prem SQL Server) @@ -1276,34 +1311,35 @@ ELSE const sqlAzureDBResourceStats string = `SET DEADLOCK_PRIORITY -10; IF SERVERPROPERTY('EngineEdition') = 5 -- Is this Azure SQL DB? BEGIN - SELECT TOP(1) - 'sqlserver_azurestats' AS [measurement], - REPLACE(@@SERVERNAME,'\',':') AS [sql_instance], - DB_NAME() as [database_name], - avg_cpu_percent, - avg_data_io_percent, - avg_log_write_percent, - avg_memory_usage_percent, - xtp_storage_percent, - max_worker_percent, - max_session_percent, - dtu_limit, - avg_login_rate_percent, - end_time, - avg_instance_memory_percent, - avg_instance_cpu_percent - FROM - sys.dm_db_resource_stats WITH (NOLOCK) - ORDER BY - end_time DESC -END` + SELECT TOP(1) + 'sqlserver_azure_db_resource_stats' AS [measurement], + REPLACE(@@SERVERNAME,'\',':') AS [sql_instance], + DB_NAME() as [database_name], + cast(avg_cpu_percent as float) as avg_cpu_percent, + cast(avg_data_io_percent as float) as avg_data_io_percent, + cast(avg_log_write_percent as float) as avg_log_write_percent, + cast(avg_memory_usage_percent as float) as avg_memory_usage_percent, + cast(xtp_storage_percent as float) as xtp_storage_percent, + cast(max_worker_percent as float) as max_worker_percent, + cast(max_session_percent as float) as max_session_percent, + dtu_limit, + cast(avg_login_rate_percent as float) as avg_login_rate_percent , + end_time, + cast(avg_instance_memory_percent as float) as avg_instance_memory_percent , + cast(avg_instance_cpu_percent as float) as avg_instance_cpu_percent + FROM + sys.dm_db_resource_stats WITH (NOLOCK) + ORDER BY + end_time DESC +END +` //Only executed if AzureDB Flag is set const sqlAzureDBResourceGovernance string = ` IF SERVERPROPERTY('EngineEdition') = 5 -- Is this Azure SQL DB? SELECT 'sqlserver_db_resource_governance' AS [measurement], - @@servername AS [sql_instance], + REPLACE(@@SERVERNAME,'\',':') AS [sql_instance], DB_NAME() as [database_name], slo_name, dtu_limit, @@ -1342,7 +1378,7 @@ BEGIN IF SERVERPROPERTY('EngineEdition') = 8 -- Is this Azure SQL Managed Instance? SELECT 'sqlserver_instance_resource_governance' AS [measurement], - @@SERVERNAME AS [sql_instance], + REPLACE(@@SERVERNAME,'\',':') AS [sql_instance], instance_cap_cpu, instance_max_log_rate, instance_max_worker_threads, @@ -1365,7 +1401,7 @@ SELECT blocking_session_id into #blockingSessions FROM sys.dm_exec_requests WHE create index ix_blockingSessions_1 on #blockingSessions (blocking_session_id) SELECT 'sqlserver_requests' AS [measurement], - @@servername AS [sql_instance], + REPLACE(@@SERVERNAME,'\',':') AS [sql_instance], DB_NAME() as [database_name], r.session_id , r.request_id @@ -1404,8 +1440,8 @@ SELECT , qt.objectid , QUOTENAME(OBJECT_SCHEMA_NAME(qt.objectid,qt.dbid)) + '.' + QUOTENAME(OBJECT_NAME(qt.objectid,qt.dbid)) as stmt_object_name , DB_NAME(qt.dbid) stmt_db_name - , r.query_hash - , r.query_plan_hash + ,CONVERT(varchar(20),[query_hash],1) as [query_hash] + ,CONVERT(varchar(20),[query_plan_hash],1) as [query_plan_hash] FROM sys.dm_exec_requests r LEFT OUTER JOIN sys.dm_exec_sessions s ON (s.session_id = r.session_id) OUTER APPLY sys.dm_exec_sql_text(sql_handle) AS qt diff --git a/plugins/inputs/sqlserver/sqlserver_test.go b/plugins/inputs/sqlserver/sqlserver_test.go index 063af75950a13..b493fb13cf257 100644 --- a/plugins/inputs/sqlserver/sqlserver_test.go +++ b/plugins/inputs/sqlserver/sqlserver_test.go @@ -1,6 +1,7 @@ package sqlserver import ( + "github.com/stretchr/testify/assert" "strconv" "strings" "testing" @@ -14,7 +15,7 @@ func TestSqlServer_ParseMetrics(t *testing.T) { var acc testutil.Accumulator - queries = make(MapQuery) + queries := make(MapQuery) queries["PerformanceCounters"] = Query{Script: mockPerformanceCounters, ResultByRow: true} queries["WaitStatsCategorized"] = Query{Script: mockWaitStatsCategorized, ResultByRow: false} queries["CPUHistory"] = Query{Script: mockCPUHistory, ResultByRow: false} @@ -81,6 +82,64 @@ func TestSqlServer_ParseMetrics(t *testing.T) { } } +func TestSqlServer_MultipleInstance(t *testing.T) { + // Invoke Gather() from two separate configurations and + // confirm they don't interfere with each other + if testing.Short() { + t.Skip("Skipping integration test in short mode") + } + testServer := "Server=127.0.0.1;Port=1433;User Id=SA;Password=ABCabc01;app name=telegraf;log=1" + s := &SQLServer{ + Servers: []string{testServer}, + ExcludeQuery: []string{"MemoryClerk"}, + } + s2 := &SQLServer{ + Servers: []string{testServer}, + ExcludeQuery: []string{"DatabaseSize"}, + } + + var acc, acc2 testutil.Accumulator + err := s.Gather(&acc) + require.NoError(t, err) + assert.Equal(t, s.isInitialized, true) + assert.Equal(t, s2.isInitialized, false) + + err = s2.Gather(&acc2) + require.NoError(t, err) + assert.Equal(t, s.isInitialized, true) + assert.Equal(t, s2.isInitialized, true) + + // acc includes size metrics, and excludes memory metrics + assert.False(t, acc.HasMeasurement("Memory breakdown (%)")) + assert.True(t, acc.HasMeasurement("Log size (bytes)")) + + // acc2 includes memory metrics, and excludes size metrics + assert.True(t, acc2.HasMeasurement("Memory breakdown (%)")) + assert.False(t, acc2.HasMeasurement("Log size (bytes)")) +} + +func TestSqlServer_MultipleInit(t *testing.T) { + + s := &SQLServer{} + s2 := &SQLServer{ + ExcludeQuery: []string{"DatabaseSize"}, + } + + initQueries(s) + _, ok := s.queries["DatabaseSize"] + // acc includes size metrics + assert.True(t, ok) + assert.Equal(t, s.isInitialized, true) + assert.Equal(t, s2.isInitialized, false) + + initQueries(s2) + _, ok = s2.queries["DatabaseSize"] + // acc2 excludes size metrics + assert.False(t, ok) + assert.Equal(t, s.isInitialized, true) + assert.Equal(t, s2.isInitialized, true) +} + const mockPerformanceMetrics = `measurement;servername;type;Point In Time Recovery;Available physical memory (bytes);Average pending disk IO;Average runnable tasks;Average tasks;Buffer pool rate (bytes/sec);Connection memory per connection (bytes);Memory grant pending;Page File Usage (%);Page lookup per batch request;Page split per batch request;Readahead per page read;Signal wait (%);Sql compilation per batch request;Sql recompilation per batch request;Total target memory ratio Performance metrics;WIN8-DEV;Performance metrics;0;6353158144;0;0;7;2773;415061;0;25;229371;130;10;18;188;52;14` diff --git a/plugins/inputs/stackdriver/stackdriver.go b/plugins/inputs/stackdriver/stackdriver.go index 4f4e35695fc21..f8b4294b77f49 100644 --- a/plugins/inputs/stackdriver/stackdriver.go +++ b/plugins/inputs/stackdriver/stackdriver.go @@ -3,7 +3,6 @@ package stackdriver import ( "context" "fmt" - "log" "math" "strconv" "strings" @@ -128,6 +127,8 @@ type ( DistributionAggregationAligners []string `toml:"distribution_aggregation_aligners"` Filter *ListTimeSeriesFilter `toml:"filter"` + Log telegraf.Logger + client metricClient timeSeriesConfCache *timeSeriesConfCache prevEnd time.Time @@ -167,6 +168,7 @@ type ( // stackdriverMetricClient is a metric client for stackdriver stackdriverMetricClient struct { + log telegraf.Logger conn *monitoring.MetricClient listMetricDescriptorsCalls selfstat.Stat @@ -206,7 +208,7 @@ func (c *stackdriverMetricClient) ListMetricDescriptors( mdChan := make(chan *metricpb.MetricDescriptor, 1000) go func() { - log.Printf("D! [inputs.stackdriver] ListMetricDescriptors: %s", req.Filter) + c.log.Debugf("List metric descriptor request filter: %s", req.Filter) defer close(mdChan) // Iterate over metric descriptors and send them to buffered channel @@ -216,7 +218,7 @@ func (c *stackdriverMetricClient) ListMetricDescriptors( mdDesc, mdErr := mdResp.Next() if mdErr != nil { if mdErr != iterator.Done { - log.Printf("E! [inputs.stackdriver] Received error response: %s: %v", req, mdErr) + c.log.Errorf("Failed iterating metric desciptor responses: %q: %v", req.String(), mdErr) } break } @@ -235,7 +237,7 @@ func (c *stackdriverMetricClient) ListTimeSeries( tsChan := make(chan *monitoringpb.TimeSeries, 1000) go func() { - log.Printf("D! [inputs.stackdriver] ListTimeSeries: %s", req.Filter) + c.log.Debugf("List time series request filter: %s", req.Filter) defer close(tsChan) // Iterate over timeseries and send them to buffered channel @@ -245,7 +247,7 @@ func (c *stackdriverMetricClient) ListTimeSeries( tsDesc, tsErr := tsResp.Next() if tsErr != nil { if tsErr != iterator.Done { - log.Printf("E! [inputs.stackdriver] Received error response: %s: %v", req, tsErr) + c.log.Errorf("Failed iterating time series responses: %q: %v", req.String(), tsErr) } break } @@ -458,6 +460,7 @@ func (s *Stackdriver) initializeStackdriverClient(ctx context.Context) error { "stackdriver", "list_timeseries_calls", tags) s.client = &stackdriverMetricClient{ + log: s.Log, conn: client, listMetricDescriptorsCalls: listMetricDescriptorsCalls, listTimeSeriesCalls: listTimeSeriesCalls, diff --git a/plugins/inputs/stackdriver/stackdriver_test.go b/plugins/inputs/stackdriver/stackdriver_test.go index 99e5deabdadf3..348cd497b09b1 100644 --- a/plugins/inputs/stackdriver/stackdriver_test.go +++ b/plugins/inputs/stackdriver/stackdriver_test.go @@ -640,6 +640,7 @@ func TestGather(t *testing.T) { t.Run(tt.name, func(t *testing.T) { var acc testutil.Accumulator s := &Stackdriver{ + Log: testutil.Logger{}, Project: "test", RateLimit: 10, GatherRawDistributionBuckets: true, @@ -775,6 +776,7 @@ func TestGatherAlign(t *testing.T) { } s := &Stackdriver{ + Log: testutil.Logger{}, Project: "test", RateLimit: 10, GatherRawDistributionBuckets: false, diff --git a/plugins/inputs/statsd/statsd.go b/plugins/inputs/statsd/statsd.go index 107a6e3887cd2..a0d3c9ee7fcbd 100644 --- a/plugins/inputs/statsd/statsd.go +++ b/plugins/inputs/statsd/statsd.go @@ -5,7 +5,6 @@ import ( "bytes" "errors" "fmt" - "log" "net" "sort" "strconv" @@ -34,13 +33,6 @@ const ( MaxTCPConnections = 250 ) -var dropwarn = "E! [inputs.statsd] Error: statsd message queue full. " + - "We have dropped %d messages so far. " + - "You may want to increase allowed_pending_messages in the config\n" - -var malformedwarn = "E! [inputs.statsd] Statsd over TCP has received %d malformed packets" + - " thus far." - // Statsd allows the importing of statsd and dogstatsd data. type Statsd struct { // Protocol used on listener - udp or tcp @@ -133,6 +125,8 @@ type Statsd struct { PacketsRecv selfstat.Stat BytesRecv selfstat.Stat + Log telegraf.Logger + // A pool of byte slices to handle parsing bufPool sync.Pool } @@ -312,7 +306,7 @@ func (s *Statsd) Gather(acc telegraf.Accumulator) error { func (s *Statsd) Start(ac telegraf.Accumulator) error { if s.ParseDataDogTags { s.DataDogExtensions = true - log.Printf("W! [inputs.statsd] The parse_data_dog_tags option is deprecated, use datadog_extensions instead.") + s.Log.Warn("'parse_data_dog_tags' config option is deprecated, please use 'datadog_extensions' instead") } s.acc = ac @@ -350,8 +344,7 @@ func (s *Statsd) Start(ac telegraf.Accumulator) error { } if s.ConvertNames { - log.Printf("W! [inputs.statsd] statsd: convert_names config option is deprecated," + - " please use metric_separator instead") + s.Log.Warn("'convert_names' config option is deprecated, please use 'metric_separator' instead") } if s.MetricSeparator == "" { @@ -369,7 +362,7 @@ func (s *Statsd) Start(ac telegraf.Accumulator) error { return err } - log.Println("I! [inputs.statsd] Statsd UDP listener listening on: ", conn.LocalAddr().String()) + s.Log.Infof("UDP listening on %q", conn.LocalAddr().String()) s.UDPlistener = conn s.wg.Add(1) @@ -387,7 +380,7 @@ func (s *Statsd) Start(ac telegraf.Accumulator) error { return err } - log.Println("I! [inputs.statsd] TCP Statsd listening on: ", listener.Addr().String()) + s.Log.Infof("TCP listening on %q", listener.Addr().String()) s.TCPlistener = listener s.wg.Add(1) @@ -403,7 +396,7 @@ func (s *Statsd) Start(ac telegraf.Accumulator) error { defer s.wg.Done() s.parser() }() - log.Printf("I! [inputs.statsd] Started the statsd service on %s\n", s.ServiceAddress) + s.Log.Infof("Started the statsd service on %q", s.ServiceAddress) return nil } @@ -463,7 +456,7 @@ func (s *Statsd) udpListen(conn *net.UDPConn) error { n, addr, err := conn.ReadFromUDP(buf) if err != nil { if !strings.Contains(err.Error(), "closed network") { - log.Printf("E! [inputs.statsd] Error READ: %s\n", err.Error()) + s.Log.Errorf("Error reading: %s", err.Error()) continue } return err @@ -479,7 +472,9 @@ func (s *Statsd) udpListen(conn *net.UDPConn) error { default: s.drops++ if s.drops == 1 || s.AllowedPendingMessages == 0 || s.drops%s.AllowedPendingMessages == 0 { - log.Printf(dropwarn, s.drops) + s.Log.Errorf("Statsd message queue full. "+ + "We have dropped %d messages so far. "+ + "You may want to increase allowed_pending_messages in the config", s.drops) } } } @@ -540,8 +535,8 @@ func (s *Statsd) parseStatsdLine(line string) error { // Validate splitting the line on ":" bits := strings.Split(line, ":") if len(bits) < 2 { - log.Printf("E! [inputs.statsd] Error: splitting ':', Unable to parse metric: %s\n", line) - return errors.New("Error Parsing statsd line") + s.Log.Errorf("Splitting ':', unable to parse metric: %s", line) + return errors.New("error Parsing statsd line") } // Extract bucket name from individual metric bits @@ -556,22 +551,22 @@ func (s *Statsd) parseStatsdLine(line string) error { // Validate splitting the bit on "|" pipesplit := strings.Split(bit, "|") if len(pipesplit) < 2 { - log.Printf("E! [inputs.statsd] Error: splitting '|', Unable to parse metric: %s\n", line) - return errors.New("Error Parsing statsd line") + s.Log.Errorf("Splitting '|', unable to parse metric: %s", line) + return errors.New("error parsing statsd line") } else if len(pipesplit) > 2 { sr := pipesplit[2] - errmsg := "E! [inputs.statsd] parsing sample rate, %s, it must be in format like: " + - "@0.1, @0.5, etc. Ignoring sample rate for line: %s\n" + if strings.Contains(sr, "@") && len(sr) > 1 { samplerate, err := strconv.ParseFloat(sr[1:], 64) if err != nil { - log.Printf(errmsg, err.Error(), line) + s.Log.Errorf("Parsing sample rate: %s", err.Error()) } else { // sample rate successfully parsed m.samplerate = samplerate } } else { - log.Printf(errmsg, "", line) + s.Log.Debugf("Sample rate must be in format like: "+ + "@0.1, @0.5, etc. Ignoring sample rate for line: %s", line) } } @@ -580,15 +575,15 @@ func (s *Statsd) parseStatsdLine(line string) error { case "g", "c", "s", "ms", "h": m.mtype = pipesplit[1] default: - log.Printf("E! [inputs.statsd] Error: Statsd Metric type %s unsupported", pipesplit[1]) - return errors.New("Error Parsing statsd line") + s.Log.Errorf("Metric type %q unsupported", pipesplit[1]) + return errors.New("error parsing statsd line") } // Parse the value if strings.HasPrefix(pipesplit[0], "-") || strings.HasPrefix(pipesplit[0], "+") { if m.mtype != "g" && m.mtype != "c" { - log.Printf("E! [inputs.statsd] Error: +- values are only supported for gauges & counters: %s\n", line) - return errors.New("Error Parsing statsd line") + s.Log.Errorf("+- values are only supported for gauges & counters, unable to parse metric: %s", line) + return errors.New("error parsing statsd line") } m.additive = true } @@ -597,8 +592,8 @@ func (s *Statsd) parseStatsdLine(line string) error { case "g", "ms", "h": v, err := strconv.ParseFloat(pipesplit[0], 64) if err != nil { - log.Printf("E! [inputs.statsd] Error: parsing value to float64: %s\n", line) - return errors.New("Error Parsing statsd line") + s.Log.Errorf("Parsing value to float64, unable to parse metric: %s", line) + return errors.New("error parsing statsd line") } m.floatvalue = v case "c": @@ -607,8 +602,8 @@ func (s *Statsd) parseStatsdLine(line string) error { if err != nil { v2, err2 := strconv.ParseFloat(pipesplit[0], 64) if err2 != nil { - log.Printf("E! [inputs.statsd] Error: parsing value to int64: %s\n", line) - return errors.New("Error Parsing statsd line") + s.Log.Errorf("Parsing value to int64, unable to parse metric: %s", line) + return errors.New("error parsing statsd line") } v = int64(v2) } @@ -852,7 +847,9 @@ func (s *Statsd) handler(conn *net.TCPConn, id string) { default: s.drops++ if s.drops == 1 || s.drops%s.AllowedPendingMessages == 0 { - log.Printf(dropwarn, s.drops) + s.Log.Errorf("Statsd message queue full. "+ + "We have dropped %d messages so far. "+ + "You may want to increase allowed_pending_messages in the config", s.drops) } } } @@ -862,9 +859,8 @@ func (s *Statsd) handler(conn *net.TCPConn, id string) { // refuser refuses a TCP connection func (s *Statsd) refuser(conn *net.TCPConn) { conn.Close() - log.Printf("I! [inputs.statsd] Refused TCP Connection from %s", conn.RemoteAddr()) - log.Printf("I! [inputs.statsd] WARNING: Maximum TCP Connections reached, you may want to" + - " adjust max_tcp_connections") + s.Log.Infof("Refused TCP Connection from %s", conn.RemoteAddr()) + s.Log.Warn("Maximum TCP Connections reached, you may want to adjust max_tcp_connections") } // forget a TCP connection @@ -883,7 +879,7 @@ func (s *Statsd) remember(id string, conn *net.TCPConn) { func (s *Statsd) Stop() { s.Lock() - log.Println("I! [inputs.statsd] Stopping the statsd service") + s.Log.Infof("Stopping the statsd service") close(s.done) if s.isUDP() { s.UDPlistener.Close() @@ -909,7 +905,7 @@ func (s *Statsd) Stop() { s.Lock() close(s.in) - log.Println("I! Stopped Statsd listener service on ", s.ServiceAddress) + s.Log.Infof("Stopped listener service on %q", s.ServiceAddress) s.Unlock() } diff --git a/plugins/inputs/statsd/statsd_test.go b/plugins/inputs/statsd/statsd_test.go index e629f164f856d..1215eeb2d0975 100644 --- a/plugins/inputs/statsd/statsd_test.go +++ b/plugins/inputs/statsd/statsd_test.go @@ -18,7 +18,7 @@ const ( ) func NewTestStatsd() *Statsd { - s := Statsd{} + s := Statsd{Log: testutil.Logger{}} // Make data structures s.done = make(chan struct{}) @@ -36,6 +36,7 @@ func NewTestStatsd() *Statsd { // Test that MaxTCPConections is respected func TestConcurrentConns(t *testing.T) { listener := Statsd{ + Log: testutil.Logger{}, Protocol: "tcp", ServiceAddress: "localhost:8125", AllowedPendingMessages: 10000, @@ -66,6 +67,7 @@ func TestConcurrentConns(t *testing.T) { // Test that MaxTCPConections is respected when max==1 func TestConcurrentConns1(t *testing.T) { listener := Statsd{ + Log: testutil.Logger{}, Protocol: "tcp", ServiceAddress: "localhost:8125", AllowedPendingMessages: 10000, @@ -94,6 +96,7 @@ func TestConcurrentConns1(t *testing.T) { // Test that MaxTCPConections is respected func TestCloseConcurrentConns(t *testing.T) { listener := Statsd{ + Log: testutil.Logger{}, Protocol: "tcp", ServiceAddress: "localhost:8125", AllowedPendingMessages: 10000, @@ -115,6 +118,7 @@ func TestCloseConcurrentConns(t *testing.T) { // benchmark how long it takes to accept & process 100,000 metrics: func BenchmarkUDP(b *testing.B) { listener := Statsd{ + Log: testutil.Logger{}, Protocol: "udp", ServiceAddress: "localhost:8125", AllowedPendingMessages: 250000, @@ -145,6 +149,7 @@ func BenchmarkUDP(b *testing.B) { // benchmark how long it takes to accept & process 100,000 metrics: func BenchmarkTCP(b *testing.B) { listener := Statsd{ + Log: testutil.Logger{}, Protocol: "tcp", ServiceAddress: "localhost:8125", AllowedPendingMessages: 250000, @@ -870,6 +875,7 @@ func TestParse_DataDogTags(t *testing.T) { "value": 1, }, time.Now(), + telegraf.Counter, ), }, }, @@ -887,6 +893,7 @@ func TestParse_DataDogTags(t *testing.T) { "value": 10.1, }, time.Now(), + telegraf.Gauge, ), }, }, @@ -943,6 +950,7 @@ func TestParse_DataDogTags(t *testing.T) { "value": 42, }, time.Now(), + telegraf.Counter, ), }, }, @@ -1625,6 +1633,7 @@ func testValidateGauge( func TestTCP(t *testing.T) { statsd := Statsd{ + Log: testutil.Logger{}, Protocol: "tcp", ServiceAddress: "localhost:0", AllowedPendingMessages: 10000, @@ -1662,6 +1671,7 @@ func TestTCP(t *testing.T) { "value": 42, }, time.Now(), + telegraf.Counter, ), }, acc.GetTelegrafMetrics(), diff --git a/plugins/inputs/suricata/suricata_test.go b/plugins/inputs/suricata/suricata_test.go index 093efd347e38f..02f298b977284 100644 --- a/plugins/inputs/suricata/suricata_test.go +++ b/plugins/inputs/suricata/suricata_test.go @@ -1,7 +1,6 @@ package suricata import ( - "bytes" "fmt" "io/ioutil" "log" @@ -9,35 +8,21 @@ import ( "net" "os" "path/filepath" - "regexp" "strings" "testing" "time" - "github.com/influxdata/telegraf/plugins/inputs" + "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/testutil" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var ex2 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats":{"capture":{"kernel_packets":905344474,"kernel_drops":78355440,"kernel_packets_delta":2376742,"kernel_drops_delta":82049}}}` -var ex3 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats":{"threads": { "foo": { "capture":{"kernel_packets":905344474,"kernel_drops":78355440}}}}}` -var ex4 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats":{"threads": { "W1#en..bar1": { "capture":{"kernel_packets":905344474,"kernel_drops":78355440}}}}}` -var brokenType1 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats":{"threads": { "W1#en..bar1": { "capture":{"kernel_packets":905344474,"kernel_drops": true}}}}}` -var brokenType2 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats":{"threads": { "W1#en..bar1": { "capture":{"kernel_packets":905344474,"kernel_drops": ["foo"]}}}}}` -var brokenType3 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats":{"threads": { "W1#en..bar1": { "capture":{"kernel_packets":905344474,"kernel_drops":"none this time"}}}}}` -var brokenType4 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats":{"threads": { "W1#en..bar1": { "capture":{"kernel_packets":905344474,"kernel_drops":null}}}}}` -var brokenType5 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats":{"foo": null}}` -var brokenStruct1 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats":{"threads": ["foo"]}}` -var brokenStruct2 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats"}` -var brokenStruct3 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats": "foobar"}` -var brokenStruct4 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats": null}` -var singleDotRegexp = regexp.MustCompilePOSIX(`[^.]\.[^.]`) +var ex3 = `{"timestamp":"2017-03-06T07:43:39.000397+0000","event_type":"stats","stats":{"threads": { "W#05-wlp4s0": { "capture":{"kernel_packets":905344474,"kernel_drops":78355440}}}}}` func TestSuricataLarge(t *testing.T) { dir, err := ioutil.TempDir("", "test") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(dir) tmpfn := filepath.Join(dir, fmt.Sprintf("t%d", rand.Int63())) @@ -49,32 +34,24 @@ func TestSuricataLarge(t *testing.T) { }, } acc := testutil.Accumulator{} - acc.SetDebug(true) - assert.NoError(t, s.Start(&acc)) + require.NoError(t, s.Start(&acc)) + defer s.Stop() data, err := ioutil.ReadFile("testdata/test1.json") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) c, err := net.Dial("unix", tmpfn) - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) c.Write([]byte(data)) c.Write([]byte("\n")) c.Close() acc.Wait(1) - - s.Stop() } func TestSuricata(t *testing.T) { dir, err := ioutil.TempDir("", "test") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(dir) tmpfn := filepath.Join(dir, fmt.Sprintf("t%d", rand.Int63())) @@ -86,20 +63,17 @@ func TestSuricata(t *testing.T) { }, } acc := testutil.Accumulator{} - acc.SetDebug(true) - assert.NoError(t, s.Start(&acc)) + require.NoError(t, s.Start(&acc)) + defer s.Stop() c, err := net.Dial("unix", tmpfn) - if err != nil { - t.Fatalf("failed: %s", err.Error()) - } + require.NoError(t, err) c.Write([]byte(ex2)) c.Write([]byte("\n")) c.Close() acc.Wait(1) - s.Stop() s = Suricata{ Source: tmpfn, Delimiter: ".", @@ -108,105 +82,78 @@ func TestSuricata(t *testing.T) { }, } - acc.AssertContainsTaggedFields(t, "suricata", - map[string]interface{}{ - "capture.kernel_packets": float64(905344474), - "capture.kernel_drops": float64(78355440), - "capture.kernel_packets_delta": float64(2376742), - "capture.kernel_drops_delta": float64(82049), - }, - map[string]string{"thread": "total"}) - - acc = testutil.Accumulator{} - acc.SetDebug(true) - assert.NoError(t, s.Start(&acc)) - - c, err = net.Dial("unix", tmpfn) - if err != nil { - log.Println(err) + expected := []telegraf.Metric{ + testutil.MustMetric( + "suricata", + map[string]string{ + "thread": "total", + }, + map[string]interface{}{ + "capture.kernel_packets": float64(905344474), + "capture.kernel_drops": float64(78355440), + "capture.kernel_packets_delta": float64(2376742), + "capture.kernel_drops_delta": float64(82049), + }, + time.Unix(0, 0), + ), } - c.Write([]byte("")) - c.Write([]byte("\n")) - c.Write([]byte("foobard}\n")) - c.Write([]byte(ex3)) - c.Write([]byte("\n")) - c.Close() - acc.Wait(1) - s.Stop() - - acc.AssertContainsTaggedFields(t, "suricata", - map[string]interface{}{ - "capture.kernel_packets": float64(905344474), - "capture.kernel_drops": float64(78355440), - }, - map[string]string{"thread": "foo"}) + testutil.RequireMetricsEqual(t, expected, acc.GetTelegrafMetrics(), testutil.IgnoreTime()) } -func TestSuricataInvalid(t *testing.T) { +func TestThreadStats(t *testing.T) { dir, err := ioutil.TempDir("", "test") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(dir) tmpfn := filepath.Join(dir, fmt.Sprintf("t%d", rand.Int63())) s := Suricata{ - Source: tmpfn, + Source: tmpfn, + Delimiter: ".", Log: testutil.Logger{ Name: "inputs.suricata", }, } - acc := testutil.Accumulator{} - acc.SetDebug(true) - assert.NoError(t, s.Start(&acc)) + acc := testutil.Accumulator{} + require.NoError(t, s.Start(&acc)) + defer s.Stop() c, err := net.Dial("unix", tmpfn) - if err != nil { - log.Println(err) - } - c.Write([]byte("sfjiowef")) + require.NoError(t, err) + c.Write([]byte("")) + c.Write([]byte("\n")) + c.Write([]byte("foobard}\n")) + c.Write([]byte(ex3)) c.Write([]byte("\n")) c.Close() + acc.Wait(1) - acc.WaitError(1) - s.Stop() -} - -func splitAtSingleDot(in string) []string { - res := singleDotRegexp.FindAllStringIndex(in, -1) - if res == nil { - return []string{in} - } - ret := make([]string, 0) - startpos := 0 - for _, v := range res { - ret = append(ret, in[startpos:v[0]+1]) - startpos = v[1] - 1 + expected := []telegraf.Metric{ + testutil.MustMetric( + "suricata", + map[string]string{ + "thread": "W#05-wlp4s0", + }, + map[string]interface{}{ + "capture.kernel_packets": float64(905344474), + "capture.kernel_drops": float64(78355440), + }, + time.Unix(0, 0), + ), } - return append(ret, in[startpos:]) + + testutil.RequireMetricsEqual(t, expected, acc.GetTelegrafMetrics(), testutil.IgnoreTime()) } -func TestSuricataSplitDots(t *testing.T) { +func TestSuricataInvalid(t *testing.T) { dir, err := ioutil.TempDir("", "test") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(dir) tmpfn := filepath.Join(dir, fmt.Sprintf("t%d", rand.Int63())) - out := splitAtSingleDot("foo") - if len(out) != 1 { - t.Fatalf("splitting 'foo' should yield one result") - } - if out[0] != "foo" { - t.Fatalf("splitting 'foo' should yield one result, 'foo'") - } - s := Suricata{ - Source: tmpfn, - Delimiter: ".", + Source: tmpfn, Log: testutil.Logger{ Name: "inputs.suricata", }, @@ -214,25 +161,16 @@ func TestSuricataSplitDots(t *testing.T) { acc := testutil.Accumulator{} acc.SetDebug(true) - assert.NoError(t, s.Start(&acc)) + require.NoError(t, s.Start(&acc)) + defer s.Stop() c, err := net.Dial("unix", tmpfn) - if err != nil { - log.Println(err) - } - c.Write([]byte(ex4)) + require.NoError(t, err) + c.Write([]byte("sfjiowef")) c.Write([]byte("\n")) c.Close() - acc.Wait(1) - acc.AssertContainsTaggedFields(t, "suricata", - map[string]interface{}{ - "capture.kernel_packets": float64(905344474), - "capture.kernel_drops": float64(78355440), - }, - map[string]string{"thread": "W1#en..bar1"}) - - s.Stop() + acc.WaitError(1) } func TestSuricataInvalidPath(t *testing.T) { @@ -245,16 +183,12 @@ func TestSuricataInvalidPath(t *testing.T) { } acc := testutil.Accumulator{} - acc.SetDebug(true) - - assert.Error(t, s.Start(&acc)) + require.Error(t, s.Start(&acc)) } func TestSuricataTooLongLine(t *testing.T) { dir, err := ioutil.TempDir("", "test") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(dir) tmpfn := filepath.Join(dir, fmt.Sprintf("t%d", rand.Int63())) @@ -265,28 +199,23 @@ func TestSuricataTooLongLine(t *testing.T) { }, } acc := testutil.Accumulator{} - acc.SetDebug(true) - assert.NoError(t, s.Start(&acc)) + require.NoError(t, s.Start(&acc)) + defer s.Stop() c, err := net.Dial("unix", tmpfn) - if err != nil { - log.Println(err) - } + require.NoError(t, err) c.Write([]byte(strings.Repeat("X", 20000000))) c.Write([]byte("\n")) c.Close() acc.WaitError(1) - s.Stop() } func TestSuricataEmptyJSON(t *testing.T) { dir, err := ioutil.TempDir("", "test") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(dir) tmpfn := filepath.Join(dir, fmt.Sprintf("t%d", rand.Int63())) @@ -297,85 +226,23 @@ func TestSuricataEmptyJSON(t *testing.T) { }, } acc := testutil.Accumulator{} - acc.SetDebug(true) - - assert.NoError(t, s.Start(&acc)) + require.NoError(t, s.Start(&acc)) + defer s.Stop() c, err := net.Dial("unix", tmpfn) if err != nil { log.Println(err) + } c.Write([]byte("\n")) c.Close() acc.WaitError(1) - - s.Stop() -} - -func TestSuricataInvalidInputs(t *testing.T) { - dir, err := ioutil.TempDir("", "test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - defer func() { - log.SetOutput(os.Stderr) - }() - tmpfn := filepath.Join(dir, fmt.Sprintf("t%d", rand.Int63())) - - for input, errmsg := range map[string]string{ - brokenType1: `Unsupported type bool encountered`, - brokenType2: `Unsupported type []interface {} encountered`, - brokenType3: `Unsupported type string encountered`, - brokenType4: `Unsupported type encountered`, - brokenType5: `Unsupported type encountered`, - brokenStruct1: `The 'threads' sub-object does not have required structure`, - brokenStruct2: `Input does not contain necessary 'stats' sub-object`, - brokenStruct3: `The 'stats' sub-object does not have required structure`, - brokenStruct4: `The 'stats' sub-object does not have required structure`, - } { - var logBuf buffer - logBuf.Reset() - log.SetOutput(&logBuf) - - acc := testutil.Accumulator{} - acc.SetDebug(true) - - s := Suricata{ - Source: tmpfn, - Delimiter: ".", - Log: testutil.Logger{ - Name: "inputs.suricata", - }, - } - assert.NoError(t, s.Start(&acc)) - - c, err := net.Dial("unix", tmpfn) - if err != nil { - t.Fatal(err) - } - c.Write([]byte(input)) - c.Write([]byte("\n")) - c.Close() - - for { - if bytes.Count(logBuf.Bytes(), []byte{'\n'}) > 0 { - break - } - time.Sleep(50 * time.Millisecond) - } - - assert.Contains(t, logBuf.String(), errmsg) - s.Stop() - } } func TestSuricataDisconnectSocket(t *testing.T) { dir, err := ioutil.TempDir("", "test") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(dir) tmpfn := filepath.Join(dir, fmt.Sprintf("t%d", rand.Int63())) @@ -386,47 +253,28 @@ func TestSuricataDisconnectSocket(t *testing.T) { }, } acc := testutil.Accumulator{} - acc.SetDebug(true) - assert.NoError(t, s.Start(&acc)) + require.NoError(t, s.Start(&acc)) + defer s.Stop() c, err := net.Dial("unix", tmpfn) - if err != nil { - log.Println(err) - } + require.NoError(t, err) c.Write([]byte(ex2)) c.Write([]byte("\n")) c.Close() c, err = net.Dial("unix", tmpfn) - if err != nil { - log.Println(err) - } + require.NoError(t, err) c.Write([]byte(ex3)) c.Write([]byte("\n")) c.Close() acc.Wait(2) - - s.Stop() -} - -func TestSuricataPluginDesc(t *testing.T) { - v, ok := inputs.Inputs["suricata"] - if !ok { - t.Fatal("suricata plugin not registered") - } - desc := v().Description() - if desc != "Suricata stats plugin" { - t.Fatal("invalid description ", desc) - } } func TestSuricataStartStop(t *testing.T) { dir, err := ioutil.TempDir("", "test") - if err != nil { - t.Fatal(err) - } + require.NoError(t, err) defer os.RemoveAll(dir) tmpfn := filepath.Join(dir, fmt.Sprintf("t%d", rand.Int63())) @@ -437,36 +285,6 @@ func TestSuricataStartStop(t *testing.T) { }, } acc := testutil.Accumulator{} - acc.SetDebug(true) - assert.NoError(t, s.Start(&acc)) + require.NoError(t, s.Start(&acc)) s.Stop() } - -func TestSuricataGather(t *testing.T) { - dir, err := ioutil.TempDir("", "test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) - tmpfn := filepath.Join(dir, fmt.Sprintf("t%d", rand.Int63())) - - s := Suricata{ - Source: tmpfn, - Log: testutil.Logger{ - Name: "inputs.suricata", - }, - } - acc := testutil.Accumulator{} - acc.SetDebug(true) - assert.NoError(t, s.Gather(&acc)) -} - -func TestSuricataSampleConfig(t *testing.T) { - v, ok := inputs.Inputs["suricata"] - if !ok { - t.Fatal("suricata plugin not registered") - } - if v().SampleConfig() != sampleConfig { - t.Fatal("wrong sampleconfig") - } -} diff --git a/plugins/inputs/synproxy/README.md b/plugins/inputs/synproxy/README.md new file mode 100644 index 0000000000000..4e275886fbfa9 --- /dev/null +++ b/plugins/inputs/synproxy/README.md @@ -0,0 +1,49 @@ +# Synproxy Input Plugin + +The synproxy plugin gathers the synproxy counters. Synproxy is a Linux netfilter module used for SYN attack mitigation. +The use of synproxy is documented in `man iptables-extensions` under the SYNPROXY section. + + +### Configuration + +The synproxy plugin does not need any configuration + +```toml +[[inputs.synproxy]] + # no configuration +``` + +### Metrics + +The following synproxy counters are gathered + +- synproxy + - fields: + - cookie_invalid (uint32, packets, counter) - Invalid cookies + - cookie_retrans (uint32, packets, counter) - Cookies retransmitted + - cookie_valid (uint32, packets, counter) - Valid cookies + - entries (uint32, packets, counter) - Entries + - syn_received (uint32, packets, counter) - SYN received + - conn_reopened (uint32, packets, counter) - Connections reopened + +### Sample Queries + +Get the number of packets per 5 minutes for the measurement in the last hour from InfluxDB: +``` +SELECT difference(last("cookie_invalid")) AS "cookie_invalid", difference(last("cookie_retrans")) AS "cookie_retrans", difference(last("cookie_valid")) AS "cookie_valid", difference(last("entries")) AS "entries", difference(last("syn_received")) AS "syn_received", difference(last("conn_reopened")) AS "conn_reopened" FROM synproxy WHERE time > NOW() - 1h GROUP BY time(5m) FILL(null); +``` + +### Troubleshooting + +Execute the following CLI command in Linux to test the synproxy counters: +``` +cat /proc/net/stat/synproxy +``` + +### Example Output + +This section shows example output in Line Protocol format. + +``` +synproxy,host=Filter-GW01,rack=filter-node1 conn_reopened=0i,cookie_invalid=235i,cookie_retrans=0i,cookie_valid=8814i,entries=0i,syn_received=8742i 1549550634000000000 +``` diff --git a/plugins/inputs/synproxy/synproxy.go b/plugins/inputs/synproxy/synproxy.go new file mode 100644 index 0000000000000..6a5b2b3239ed9 --- /dev/null +++ b/plugins/inputs/synproxy/synproxy.go @@ -0,0 +1,40 @@ +package synproxy + +import ( + "os" + "path" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" +) + +type Synproxy struct { + Log telegraf.Logger `toml:"-"` + + // Synproxy stats filename (proc filesystem) + statFile string +} + +func (k *Synproxy) Description() string { + return "Get synproxy counter statistics from procfs" +} + +func (k *Synproxy) SampleConfig() string { + return "" +} + +func getHostProc() string { + procPath := "/proc" + if os.Getenv("HOST_PROC") != "" { + procPath = os.Getenv("HOST_PROC") + } + return procPath +} + +func init() { + inputs.Add("synproxy", func() telegraf.Input { + return &Synproxy{ + statFile: path.Join(getHostProc(), "/net/stat/synproxy"), + } + }) +} diff --git a/plugins/inputs/synproxy/synproxy_linux.go b/plugins/inputs/synproxy/synproxy_linux.go new file mode 100644 index 0000000000000..bcc9729384282 --- /dev/null +++ b/plugins/inputs/synproxy/synproxy_linux.go @@ -0,0 +1,90 @@ +// +build linux + +package synproxy + +import ( + "bufio" + "fmt" + "os" + "strconv" + "strings" + + "github.com/influxdata/telegraf" +) + +func (k *Synproxy) Gather(acc telegraf.Accumulator) error { + data, err := k.getSynproxyStat() + if err != nil { + return err + } + + acc.AddCounter("synproxy", data, map[string]string{}) + return nil +} + +func inSlice(haystack []string, needle string) bool { + for _, val := range haystack { + if needle == val { + return true + } + } + return false +} + +func (k *Synproxy) getSynproxyStat() (map[string]interface{}, error) { + var hname []string + counters := []string{"entries", "syn_received", "cookie_invalid", "cookie_valid", "cookie_retrans", "conn_reopened"} + fields := make(map[string]interface{}) + + // Open synproxy file in proc filesystem + file, err := os.Open(k.statFile) + if err != nil { + return nil, err + } + defer file.Close() + + // Initialise expected fields + for _, val := range counters { + fields[val] = uint32(0) + } + + scanner := bufio.NewScanner(file) + // Read header row + if scanner.Scan() { + line := scanner.Text() + // Parse fields separated by whitespace + dataFields := strings.Fields(line) + for _, val := range dataFields { + if !inSlice(counters, val) { + val = "" + } + hname = append(hname, val) + } + } + if len(hname) == 0 { + return nil, fmt.Errorf("invalid data") + } + // Read data rows + for scanner.Scan() { + line := scanner.Text() + // Parse fields separated by whitespace + dataFields := strings.Fields(line) + // If number of data fields do not match number of header fields + if len(dataFields) != len(hname) { + return nil, fmt.Errorf("invalid number of columns in data, expected %d found %d", len(hname), + len(dataFields)) + } + for i, val := range dataFields { + // Convert from hexstring to int32 + x, err := strconv.ParseUint(val, 16, 32) + // If field is not a valid hexstring + if err != nil { + return nil, fmt.Errorf("invalid value '%s' found", val) + } + if hname[i] != "" { + fields[hname[i]] = fields[hname[i]].(uint32) + uint32(x) + } + } + } + return fields, nil +} diff --git a/plugins/inputs/synproxy/synproxy_notlinux.go b/plugins/inputs/synproxy/synproxy_notlinux.go new file mode 100644 index 0000000000000..71a223644d8ed --- /dev/null +++ b/plugins/inputs/synproxy/synproxy_notlinux.go @@ -0,0 +1,23 @@ +// +build !linux + +package synproxy + +import ( + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/inputs" +) + +func (k *Synproxy) Init() error { + k.Log.Warn("Current platform is not supported") + return nil +} + +func (k *Synproxy) Gather(acc telegraf.Accumulator) error { + return nil +} + +func init() { + inputs.Add("synproxy", func() telegraf.Input { + return &Synproxy{} + }) +} diff --git a/plugins/inputs/synproxy/synproxy_test.go b/plugins/inputs/synproxy/synproxy_test.go new file mode 100644 index 0000000000000..83d752ff16f8c --- /dev/null +++ b/plugins/inputs/synproxy/synproxy_test.go @@ -0,0 +1,169 @@ +// +build linux + +package synproxy + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/influxdata/telegraf/testutil" + + "github.com/stretchr/testify/assert" +) + +func TestSynproxyFileNormal(t *testing.T) { + testSynproxyFileData(t, synproxyFileNormal, synproxyResultNormal) +} + +func TestSynproxyFileOverflow(t *testing.T) { + testSynproxyFileData(t, synproxyFileOverflow, synproxyResultOverflow) +} + +func TestSynproxyFileExtended(t *testing.T) { + testSynproxyFileData(t, synproxyFileExtended, synproxyResultNormal) +} + +func TestSynproxyFileAltered(t *testing.T) { + testSynproxyFileData(t, synproxyFileAltered, synproxyResultNormal) +} + +func TestSynproxyFileHeaderMismatch(t *testing.T) { + tmpfile := makeFakeSynproxyFile([]byte(synproxyFileHeaderMismatch)) + defer os.Remove(tmpfile) + + k := Synproxy{ + statFile: tmpfile, + } + + acc := testutil.Accumulator{} + err := k.Gather(&acc) + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid number of columns in data") +} + +func TestSynproxyFileInvalidHex(t *testing.T) { + tmpfile := makeFakeSynproxyFile([]byte(synproxyFileInvalidHex)) + defer os.Remove(tmpfile) + + k := Synproxy{ + statFile: tmpfile, + } + + acc := testutil.Accumulator{} + err := k.Gather(&acc) + assert.Error(t, err) + assert.Contains(t, err.Error(), "invalid value") +} + +func TestNoSynproxyFile(t *testing.T) { + tmpfile := makeFakeSynproxyFile([]byte(synproxyFileNormal)) + // Remove file to generate "no such file" error + os.Remove(tmpfile) + + k := Synproxy{ + statFile: tmpfile, + } + + acc := testutil.Accumulator{} + err := k.Gather(&acc) + assert.Error(t, err) +} + +// Valid Synproxy file +const synproxyFileNormal = `entries syn_received cookie_invalid cookie_valid cookie_retrans conn_reopened +00000000 00007a88 00002af7 00007995 00000000 00000000 +00000000 0000892c 000015e3 00008852 00000000 00000000 +00000000 00007a80 00002ccc 0000796a 00000000 00000000 +00000000 000079f7 00002bf5 0000790a 00000000 00000000 +00000000 00007a08 00002c9a 00007901 00000000 00000000 +00000000 00007cfc 00002b36 000078fd 00000000 00000000 +00000000 000079c2 00002c2b 000078d6 00000000 00000000 +00000000 0000798a 00002ba8 000078a0 00000000 00000000` + +const synproxyFileOverflow = `entries syn_received cookie_invalid cookie_valid cookie_retrans conn_reopened +00000000 80000001 e0000000 80000001 00000000 00000000 +00000000 80000003 f0000009 80000003 00000000 00000000` + +const synproxyFileHeaderMismatch = `entries syn_received cookie_invalid cookie_valid cookie_retrans +00000000 00000002 00000000 00000002 00000000 00000000 +00000000 00000004 00000015 00000004 00000000 00000000 +00000000 00000003 00000000 00000003 00000000 00000000 +00000000 00000002 00000000 00000002 00000000 00000000 +00000000 00000003 00000009 00000003 00000000 00000000 +00000000 00000003 00000009 00000003 00000000 00000000 +00000000 00000001 00000000 00000001 00000000 00000000 +00000000 00000003 00000009 00000003 00000000 00000000` + +const synproxyFileInvalidHex = `entries syn_received cookie_invalid cookie_valid cookie_retrans conn_reopened +entries 00000002 00000000 00000002 00000000 00000000 +00000000 00000003 00000009 00000003 00000000 00000000` + +const synproxyFileExtended = `entries syn_received cookie_invalid cookie_valid cookie_retrans conn_reopened new_counter +00000000 00007a88 00002af7 00007995 00000000 00000000 00000000 +00000000 0000892c 000015e3 00008852 00000000 00000000 00000000 +00000000 00007a80 00002ccc 0000796a 00000000 00000000 00000000 +00000000 000079f7 00002bf5 0000790a 00000000 00000000 00000000 +00000000 00007a08 00002c9a 00007901 00000000 00000000 00000000 +00000000 00007cfc 00002b36 000078fd 00000000 00000000 00000000 +00000000 000079c2 00002c2b 000078d6 00000000 00000000 00000000 +00000000 0000798a 00002ba8 000078a0 00000000 00000000 00000000` + +const synproxyFileAltered = `entries cookie_invalid cookie_valid syn_received conn_reopened +00000000 00002af7 00007995 00007a88 00000000 +00000000 000015e3 00008852 0000892c 00000000 +00000000 00002ccc 0000796a 00007a80 00000000 +00000000 00002bf5 0000790a 000079f7 00000000 +00000000 00002c9a 00007901 00007a08 00000000 +00000000 00002b36 000078fd 00007cfc 00000000 +00000000 00002c2b 000078d6 000079c2 00000000 +00000000 00002ba8 000078a0 0000798a 00000000` + +var synproxyResultNormal = map[string]interface{}{ + "entries": uint32(0x00000000), + "syn_received": uint32(0x0003e27b), + "cookie_invalid": uint32(0x0001493e), + "cookie_valid": uint32(0x0003d7cf), + "cookie_retrans": uint32(0x00000000), + "conn_reopened": uint32(0x00000000), +} + +var synproxyResultOverflow = map[string]interface{}{ + "entries": uint32(0x00000000), + "syn_received": uint32(0x00000004), + "cookie_invalid": uint32(0xd0000009), + "cookie_valid": uint32(0x00000004), + "cookie_retrans": uint32(0x00000000), + "conn_reopened": uint32(0x00000000), +} + +func testSynproxyFileData(t *testing.T, fileData string, telegrafData map[string]interface{}) { + tmpfile := makeFakeSynproxyFile([]byte(fileData)) + defer os.Remove(tmpfile) + + k := Synproxy{ + statFile: tmpfile, + } + + acc := testutil.Accumulator{} + err := k.Gather(&acc) + assert.NoError(t, err) + + acc.AssertContainsFields(t, "synproxy", telegrafData) +} + +func makeFakeSynproxyFile(content []byte) string { + tmpfile, err := ioutil.TempFile("", "synproxy_test") + if err != nil { + panic(err) + } + + if _, err := tmpfile.Write(content); err != nil { + panic(err) + } + if err := tmpfile.Close(); err != nil { + panic(err) + } + + return tmpfile.Name() +} diff --git a/plugins/inputs/syslog/commons_test.go b/plugins/inputs/syslog/commons_test.go index 5d5562fc74c12..10f2ddf511d22 100644 --- a/plugins/inputs/syslog/commons_test.go +++ b/plugins/inputs/syslog/commons_test.go @@ -1,10 +1,12 @@ package syslog import ( + "time" + + "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal" framing "github.com/influxdata/telegraf/internal/syslog" "github.com/influxdata/telegraf/testutil" - "time" ) var ( @@ -14,16 +16,16 @@ var ( type testCasePacket struct { name string data []byte - wantBestEffort *testutil.Metric - wantStrict *testutil.Metric + wantBestEffort telegraf.Metric + wantStrict telegraf.Metric werr bool } type testCaseStream struct { name string data []byte - wantBestEffort []testutil.Metric - wantStrict []testutil.Metric + wantBestEffort []telegraf.Metric + wantStrict []telegraf.Metric werr int // how many errors we expect in the strict mode? } diff --git a/plugins/inputs/syslog/nontransparent_test.go b/plugins/inputs/syslog/nontransparent_test.go index 2bf6aa4efe0dd..d0352c6ae1c7f 100644 --- a/plugins/inputs/syslog/nontransparent_test.go +++ b/plugins/inputs/syslog/nontransparent_test.go @@ -9,7 +9,7 @@ import ( "testing" "time" - "github.com/google/go-cmp/cmp" + "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal" framing "github.com/influxdata/telegraf/internal/syslog" "github.com/influxdata/telegraf/testutil" @@ -21,10 +21,16 @@ func getTestCasesForNonTransparent() []testCaseStream { { name: "1st/avg/ok", data: []byte(`<29>1 2016-02-21T04:32:57+00:00 web1 someservice 2341 2 [origin][meta sequence="14125553" service="someservice"] "GET /v1/ok HTTP/1.1" 200 145 "-" "hacheck 0.9.0" 24306 127.0.0.1:40124 575`), - wantStrict: []testutil.Metric{ - { - Measurement: "syslog", - Fields: map[string]interface{}{ + wantStrict: []telegraf.Metric{ + testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "notice", + "facility": "daemon", + "hostname": "web1", + "appname": "someservice", + }, + map[string]interface{}{ "version": uint16(1), "timestamp": time.Unix(1456029177, 0).UnixNano(), "procid": "2341", @@ -36,19 +42,19 @@ func getTestCasesForNonTransparent() []testCaseStream { "severity_code": 5, "facility_code": 3, }, - Tags: map[string]string{ + defaultTime, + ), + }, + wantBestEffort: []telegraf.Metric{ + testutil.MustMetric( + "syslog", + map[string]string{ "severity": "notice", "facility": "daemon", "hostname": "web1", "appname": "someservice", }, - Time: defaultTime, - }, - }, - wantBestEffort: []testutil.Metric{ - { - Measurement: "syslog", - Fields: map[string]interface{}{ + map[string]interface{}{ "version": uint16(1), "timestamp": time.Unix(1456029177, 0).UnixNano(), "procid": "2341", @@ -60,75 +66,69 @@ func getTestCasesForNonTransparent() []testCaseStream { "severity_code": 5, "facility_code": 3, }, - Tags: map[string]string{ - "severity": "notice", - "facility": "daemon", - "hostname": "web1", - "appname": "someservice", - }, - Time: defaultTime, - }, + defaultTime, + ), }, werr: 1, }, { name: "1st/min/ok//2nd/min/ok", data: []byte("<1>2 - - - - - -\n<4>11 - - - - - -\n"), - wantStrict: []testutil.Metric{ - { - Measurement: "syslog", - Fields: map[string]interface{}{ + wantStrict: []telegraf.Metric{ + testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "alert", + "facility": "kern", + }, + map[string]interface{}{ "version": uint16(2), "severity_code": 1, "facility_code": 0, }, - Tags: map[string]string{ - "severity": "alert", + defaultTime, + ), + testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "warning", "facility": "kern", }, - Time: defaultTime, - }, - { - Measurement: "syslog", - Fields: map[string]interface{}{ + map[string]interface{}{ "version": uint16(11), "severity_code": 4, "facility_code": 0, }, - Tags: map[string]string{ - "severity": "warning", + defaultTime.Add(time.Nanosecond), + ), + }, + wantBestEffort: []telegraf.Metric{ + testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "alert", "facility": "kern", }, - Time: defaultTime.Add(time.Nanosecond), - }, - }, - wantBestEffort: []testutil.Metric{ - { - Measurement: "syslog", - Fields: map[string]interface{}{ + map[string]interface{}{ "version": uint16(2), "severity_code": 1, "facility_code": 0, }, - Tags: map[string]string{ - "severity": "alert", + defaultTime, + ), + testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "warning", "facility": "kern", }, - Time: defaultTime, - }, - { - Measurement: "syslog", - Fields: map[string]interface{}{ + map[string]interface{}{ "version": uint16(11), "severity_code": 4, "facility_code": 0, }, - Tags: map[string]string{ - "severity": "warning", - "facility": "kern", - }, - Time: defaultTime.Add(time.Nanosecond), - }, + defaultTime.Add(time.Nanosecond), + ), }, }, } @@ -186,13 +186,7 @@ func testStrictNonTransparent(t *testing.T, protocol string, address string, wan if len(acc.Errors) != tc.werr { t.Fatalf("Got unexpected errors. want error = %v, errors = %v\n", tc.werr, acc.Errors) } - var got []testutil.Metric - for _, metric := range acc.Metrics { - got = append(got, *metric) - } - if !cmp.Equal(tc.wantStrict, got) { - t.Fatalf("Got (+) / Want (-)\n %s", cmp.Diff(tc.wantStrict, got)) - } + testutil.RequireMetricsEqual(t, tc.wantStrict, acc.GetTelegrafMetrics()) }) } } @@ -240,14 +234,7 @@ func testBestEffortNonTransparent(t *testing.T, protocol string, address string, acc.Wait(len(tc.wantBestEffort)) } - // Verify - var got []testutil.Metric - for _, metric := range acc.Metrics { - got = append(got, *metric) - } - if !cmp.Equal(tc.wantBestEffort, got) { - t.Fatalf("Got (+) / Want (-)\n %s", cmp.Diff(tc.wantBestEffort, got)) - } + testutil.RequireMetricsEqual(t, tc.wantStrict, acc.GetTelegrafMetrics()) }) } } diff --git a/plugins/inputs/syslog/octetcounting_test.go b/plugins/inputs/syslog/octetcounting_test.go index 4f8f2d2783450..ea86b808dc02a 100644 --- a/plugins/inputs/syslog/octetcounting_test.go +++ b/plugins/inputs/syslog/octetcounting_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/google/go-cmp/cmp" + "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal" framing "github.com/influxdata/telegraf/internal/syslog" "github.com/influxdata/telegraf/testutil" @@ -22,10 +22,16 @@ func getTestCasesForOctetCounting() []testCaseStream { { name: "1st/avg/ok", data: []byte(`188 <29>1 2016-02-21T04:32:57+00:00 web1 someservice 2341 2 [origin][meta sequence="14125553" service="someservice"] "GET /v1/ok HTTP/1.1" 200 145 "-" "hacheck 0.9.0" 24306 127.0.0.1:40124 575`), - wantStrict: []testutil.Metric{ - { - Measurement: "syslog", - Fields: map[string]interface{}{ + wantStrict: []telegraf.Metric{ + testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "notice", + "facility": "daemon", + "hostname": "web1", + "appname": "someservice", + }, + map[string]interface{}{ "version": uint16(1), "timestamp": time.Unix(1456029177, 0).UnixNano(), "procid": "2341", @@ -37,19 +43,19 @@ func getTestCasesForOctetCounting() []testCaseStream { "severity_code": 5, "facility_code": 3, }, - Tags: map[string]string{ + defaultTime, + ), + }, + wantBestEffort: []telegraf.Metric{ + testutil.MustMetric( + "syslog", + map[string]string{ "severity": "notice", "facility": "daemon", "hostname": "web1", "appname": "someservice", }, - Time: defaultTime, - }, - }, - wantBestEffort: []testutil.Metric{ - { - Measurement: "syslog", - Fields: map[string]interface{}{ + map[string]interface{}{ "version": uint16(1), "timestamp": time.Unix(1456029177, 0).UnixNano(), "procid": "2341", @@ -61,221 +67,215 @@ func getTestCasesForOctetCounting() []testCaseStream { "severity_code": 5, "facility_code": 3, }, - Tags: map[string]string{ - "severity": "notice", - "facility": "daemon", - "hostname": "web1", - "appname": "someservice", - }, - Time: defaultTime, - }, + defaultTime, + ), }, }, { name: "1st/min/ok//2nd/min/ok", data: []byte("16 <1>2 - - - - - -17 <4>11 - - - - - -"), - wantStrict: []testutil.Metric{ - { - Measurement: "syslog", - Fields: map[string]interface{}{ + wantStrict: []telegraf.Metric{ + testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "alert", + "facility": "kern", + }, + map[string]interface{}{ "version": uint16(2), "severity_code": 1, "facility_code": 0, }, - Tags: map[string]string{ - "severity": "alert", + defaultTime, + ), + testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "warning", "facility": "kern", }, - Time: defaultTime, - }, - { - Measurement: "syslog", - Fields: map[string]interface{}{ + map[string]interface{}{ "version": uint16(11), "severity_code": 4, "facility_code": 0, }, - Tags: map[string]string{ - "severity": "warning", + defaultTime.Add(time.Nanosecond), + ), + }, + wantBestEffort: []telegraf.Metric{ + testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "alert", "facility": "kern", }, - Time: defaultTime.Add(time.Nanosecond), - }, - }, - wantBestEffort: []testutil.Metric{ - { - Measurement: "syslog", - Fields: map[string]interface{}{ + map[string]interface{}{ "version": uint16(2), "severity_code": 1, "facility_code": 0, }, - Tags: map[string]string{ - "severity": "alert", + defaultTime, + ), + testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "warning", "facility": "kern", }, - Time: defaultTime, - }, - { - Measurement: "syslog", - Fields: map[string]interface{}{ + map[string]interface{}{ "version": uint16(11), "severity_code": 4, "facility_code": 0, }, - Tags: map[string]string{ - "severity": "warning", - "facility": "kern", - }, - Time: defaultTime.Add(time.Nanosecond), - }, + defaultTime.Add(time.Nanosecond), + ), }, }, { name: "1st/utf8/ok", data: []byte("23 <1>1 - - - - - - hellø"), - wantStrict: []testutil.Metric{ - { - Measurement: "syslog", - Fields: map[string]interface{}{ + wantStrict: []telegraf.Metric{ + testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "alert", + "facility": "kern", + }, + map[string]interface{}{ "version": uint16(1), "message": "hellø", "severity_code": 1, "facility_code": 0, }, - Tags: map[string]string{ + defaultTime, + ), + }, + wantBestEffort: []telegraf.Metric{ + testutil.MustMetric( + "syslog", + map[string]string{ "severity": "alert", "facility": "kern", }, - Time: defaultTime, - }, - }, - wantBestEffort: []testutil.Metric{ - { - Measurement: "syslog", - Fields: map[string]interface{}{ + map[string]interface{}{ "version": uint16(1), "message": "hellø", "severity_code": 1, "facility_code": 0, }, - Tags: map[string]string{ - "severity": "alert", - "facility": "kern", - }, - Time: defaultTime, - }, + defaultTime, + ), }, }, { name: "1st/nl/ok", // newline data: []byte("28 <1>3 - - - - - - hello\nworld"), - wantStrict: []testutil.Metric{ - { - Measurement: "syslog", - Fields: map[string]interface{}{ + wantStrict: []telegraf.Metric{ + testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "alert", + "facility": "kern", + }, + map[string]interface{}{ "version": uint16(3), "message": "hello\nworld", "severity_code": 1, "facility_code": 0, }, - Tags: map[string]string{ + defaultTime, + ), + }, + wantBestEffort: []telegraf.Metric{ + testutil.MustMetric( + "syslog", + map[string]string{ "severity": "alert", "facility": "kern", }, - Time: defaultTime, - }, - }, - wantBestEffort: []testutil.Metric{ - { - Measurement: "syslog", - Fields: map[string]interface{}{ + map[string]interface{}{ "version": uint16(3), "message": "hello\nworld", "severity_code": 1, "facility_code": 0, }, - Tags: map[string]string{ - "severity": "alert", - "facility": "kern", - }, - Time: defaultTime, - }, + defaultTime, + ), }, }, { name: "1st/uf/ko", // underflow (msglen less than provided octets) data: []byte("16 <1>2"), wantStrict: nil, - wantBestEffort: []testutil.Metric{ - { - Measurement: "syslog", - Fields: map[string]interface{}{ + wantBestEffort: []telegraf.Metric{ + testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "alert", + "facility": "kern", + }, + map[string]interface{}{ "version": uint16(2), "severity_code": 1, "facility_code": 0, }, - Tags: map[string]string{ - "severity": "alert", - "facility": "kern", - }, - Time: defaultTime, - }, + defaultTime, + ), }, werr: 1, }, { name: "1st/min/ok", data: []byte("16 <1>1 - - - - - -"), - wantStrict: []testutil.Metric{ - { - Measurement: "syslog", - Fields: map[string]interface{}{ + wantStrict: []telegraf.Metric{ + testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "alert", + "facility": "kern", + }, + map[string]interface{}{ "version": uint16(1), "severity_code": 1, "facility_code": 0, }, - Tags: map[string]string{ + defaultTime, + ), + }, + wantBestEffort: []telegraf.Metric{ + testutil.MustMetric( + "syslog", + map[string]string{ "severity": "alert", "facility": "kern", }, - Time: defaultTime, - }, - }, - wantBestEffort: []testutil.Metric{ - { - Measurement: "syslog", - Fields: map[string]interface{}{ + map[string]interface{}{ "version": uint16(1), "severity_code": 1, "facility_code": 0, }, - Tags: map[string]string{ - "severity": "alert", - "facility": "kern", - }, - Time: defaultTime, - }, + defaultTime, + ), }, }, { name: "1st/uf/mf", // The first "underflow" message breaks also the second one data: []byte("16 <1>217 <11>1 - - - - - -"), wantStrict: nil, - wantBestEffort: []testutil.Metric{ - { - Measurement: "syslog", - Fields: map[string]interface{}{ + wantBestEffort: []telegraf.Metric{ + testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "alert", + "facility": "kern", + }, + map[string]interface{}{ "version": uint16(217), "severity_code": 1, "facility_code": 0, }, - Tags: map[string]string{ - "severity": "alert", - "facility": "kern", - }, - Time: defaultTime, - }, + defaultTime, + ), }, werr: 1, }, @@ -287,10 +287,16 @@ func getTestCasesForOctetCounting() []testCaseStream { { name: "1st/max/ok", data: []byte(fmt.Sprintf("8192 <%d>%d %s %s %s %s %s - %s", maxP, maxV, maxTS, maxH, maxA, maxPID, maxMID, message7681)), - wantStrict: []testutil.Metric{ - { - Measurement: "syslog", - Fields: map[string]interface{}{ + wantStrict: []telegraf.Metric{ + testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "debug", + "facility": "local7", + "hostname": maxH, + "appname": maxA, + }, + map[string]interface{}{ "version": maxV, "timestamp": time.Unix(1514764799, 999999000).UnixNano(), "message": message7681, @@ -299,19 +305,19 @@ func getTestCasesForOctetCounting() []testCaseStream { "facility_code": 23, "severity_code": 7, }, - Tags: map[string]string{ + defaultTime, + ), + }, + wantBestEffort: []telegraf.Metric{ + testutil.MustMetric( + "syslog", + map[string]string{ "severity": "debug", "facility": "local7", "hostname": maxH, "appname": maxA, }, - Time: defaultTime, - }, - }, - wantBestEffort: []testutil.Metric{ - { - Measurement: "syslog", - Fields: map[string]interface{}{ + map[string]interface{}{ "version": maxV, "timestamp": time.Unix(1514764799, 999999000).UnixNano(), "message": message7681, @@ -320,14 +326,8 @@ func getTestCasesForOctetCounting() []testCaseStream { "facility_code": 23, "severity_code": 7, }, - Tags: map[string]string{ - "severity": "debug", - "facility": "local7", - "hostname": maxH, - "appname": maxA, - }, - Time: defaultTime, - }, + defaultTime, + ), }, }, } @@ -386,13 +386,7 @@ func testStrictOctetCounting(t *testing.T, protocol string, address string, want if len(acc.Errors) != tc.werr { t.Fatalf("Got unexpected errors. want error = %v, errors = %v\n", tc.werr, acc.Errors) } - var got []testutil.Metric - for _, metric := range acc.Metrics { - got = append(got, *metric) - } - if !cmp.Equal(tc.wantStrict, got) { - t.Fatalf("Got (+) / Want (-)\n %s", cmp.Diff(tc.wantStrict, got)) - } + testutil.RequireMetricsEqual(t, tc.wantStrict, acc.GetTelegrafMetrics()) }) } } @@ -440,14 +434,7 @@ func testBestEffortOctetCounting(t *testing.T, protocol string, address string, acc.Wait(len(tc.wantBestEffort)) } - // Verify - var got []testutil.Metric - for _, metric := range acc.Metrics { - got = append(got, *metric) - } - if !cmp.Equal(tc.wantBestEffort, got) { - t.Fatalf("Got (+) / Want (-)\n %s", cmp.Diff(tc.wantBestEffort, got)) - } + testutil.RequireMetricsEqual(t, tc.wantBestEffort, acc.GetTelegrafMetrics()) }) } } diff --git a/plugins/inputs/syslog/rfc5426_test.go b/plugins/inputs/syslog/rfc5426_test.go index ba856b0ac2bc8..00efb9479da32 100644 --- a/plugins/inputs/syslog/rfc5426_test.go +++ b/plugins/inputs/syslog/rfc5426_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/google/go-cmp/cmp" + "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/require" ) @@ -25,73 +25,79 @@ func getTestCasesForRFC5426() []testCasePacket { { name: "complete", data: []byte("<1>1 - - - - - - A"), - wantBestEffort: &testutil.Metric{ - Measurement: "syslog", - Fields: map[string]interface{}{ + wantBestEffort: testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "alert", + "facility": "kern", + }, + map[string]interface{}{ "version": uint16(1), "message": "A", "facility_code": 0, "severity_code": 1, }, - Tags: map[string]string{ + defaultTime, + ), + wantStrict: testutil.MustMetric( + "syslog", + map[string]string{ "severity": "alert", "facility": "kern", }, - Time: defaultTime, - }, - wantStrict: &testutil.Metric{ - Measurement: "syslog", - Fields: map[string]interface{}{ + map[string]interface{}{ "version": uint16(1), "message": "A", "facility_code": 0, "severity_code": 1, }, - Tags: map[string]string{ - "severity": "alert", - "facility": "kern", - }, - Time: defaultTime, - }, + defaultTime, + ), }, { name: "one/per/packet", data: []byte("<1>3 - - - - - - A<1>4 - - - - - - B"), - wantBestEffort: &testutil.Metric{ - Measurement: "syslog", - Fields: map[string]interface{}{ + wantBestEffort: testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "alert", + "facility": "kern", + }, + map[string]interface{}{ "version": uint16(3), "message": "A<1>4 - - - - - - B", "severity_code": 1, "facility_code": 0, }, - Tags: map[string]string{ + defaultTime, + ), + wantStrict: testutil.MustMetric( + "syslog", + map[string]string{ "severity": "alert", "facility": "kern", }, - Time: defaultTime, - }, - wantStrict: &testutil.Metric{ - Measurement: "syslog", - Fields: map[string]interface{}{ + map[string]interface{}{ "version": uint16(3), "message": "A<1>4 - - - - - - B", "severity_code": 1, "facility_code": 0, }, - Tags: map[string]string{ - "severity": "alert", - "facility": "kern", - }, - Time: defaultTime, - }, + defaultTime, + ), }, { name: "average", data: []byte(`<29>1 2016-02-21T04:32:57+00:00 web1 someservice 2341 2 [origin][meta sequence="14125553" service="someservice"] "GET /v1/ok HTTP/1.1" 200 145 "-" "hacheck 0.9.0" 24306 127.0.0.1:40124 575`), - wantBestEffort: &testutil.Metric{ - Measurement: "syslog", - Fields: map[string]interface{}{ + wantBestEffort: testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "notice", + "facility": "daemon", + "hostname": "web1", + "appname": "someservice", + }, + map[string]interface{}{ "version": uint16(1), "timestamp": time.Unix(1456029177, 0).UnixNano(), "procid": "2341", @@ -103,17 +109,17 @@ func getTestCasesForRFC5426() []testCasePacket { "severity_code": 5, "facility_code": 3, }, - Tags: map[string]string{ + defaultTime, + ), + wantStrict: testutil.MustMetric( + "syslog", + map[string]string{ "severity": "notice", "facility": "daemon", "hostname": "web1", "appname": "someservice", }, - Time: defaultTime, - }, - wantStrict: &testutil.Metric{ - Measurement: "syslog", - Fields: map[string]interface{}{ + map[string]interface{}{ "version": uint16(1), "timestamp": time.Unix(1456029177, 0).UnixNano(), "procid": "2341", @@ -125,21 +131,21 @@ func getTestCasesForRFC5426() []testCasePacket { "severity_code": 5, "facility_code": 3, }, - Tags: map[string]string{ - "severity": "notice", - "facility": "daemon", - "hostname": "web1", - "appname": "someservice", - }, - Time: defaultTime, - }, + defaultTime, + ), }, { name: "max", data: []byte(fmt.Sprintf("<%d>%d %s %s %s %s %s - %s", maxP, maxV, maxTS, maxH, maxA, maxPID, maxMID, message7681)), - wantBestEffort: &testutil.Metric{ - Measurement: "syslog", - Fields: map[string]interface{}{ + wantBestEffort: testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "debug", + "facility": "local7", + "hostname": maxH, + "appname": maxA, + }, + map[string]interface{}{ "version": maxV, "timestamp": time.Unix(1514764799, 999999000).UnixNano(), "message": message7681, @@ -148,17 +154,17 @@ func getTestCasesForRFC5426() []testCasePacket { "severity_code": 7, "facility_code": 23, }, - Tags: map[string]string{ + defaultTime, + ), + wantStrict: testutil.MustMetric( + "syslog", + map[string]string{ "severity": "debug", "facility": "local7", "hostname": maxH, "appname": maxA, }, - Time: defaultTime, - }, - wantStrict: &testutil.Metric{ - Measurement: "syslog", - Fields: map[string]interface{}{ + map[string]interface{}{ "version": maxV, "timestamp": time.Unix(1514764799, 999999000).UnixNano(), "message": message7681, @@ -167,64 +173,58 @@ func getTestCasesForRFC5426() []testCasePacket { "severity_code": 7, "facility_code": 23, }, - Tags: map[string]string{ - "severity": "debug", - "facility": "local7", - "hostname": maxH, - "appname": maxA, - }, - Time: defaultTime, - }, + defaultTime, + ), }, { name: "minimal/incomplete", data: []byte("<1>2"), - wantBestEffort: &testutil.Metric{ - Measurement: "syslog", - Fields: map[string]interface{}{ + wantBestEffort: testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "alert", + "facility": "kern", + }, + map[string]interface{}{ "version": uint16(2), "facility_code": 0, "severity_code": 1, }, - Tags: map[string]string{ - "severity": "alert", - "facility": "kern", - }, - Time: defaultTime, - }, + defaultTime, + ), werr: true, }, { name: "trim message", data: []byte("<1>1 - - - - - - \tA\n"), - wantBestEffort: &testutil.Metric{ - Measurement: "syslog", - Fields: map[string]interface{}{ + wantBestEffort: testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "alert", + "facility": "kern", + }, + map[string]interface{}{ "version": uint16(1), "message": "\tA", "facility_code": 0, "severity_code": 1, }, - Tags: map[string]string{ + defaultTime, + ), + wantStrict: testutil.MustMetric( + "syslog", + map[string]string{ "severity": "alert", "facility": "kern", }, - Time: defaultTime, - }, - wantStrict: &testutil.Metric{ - Measurement: "syslog", - Fields: map[string]interface{}{ + map[string]interface{}{ "version": uint16(1), "message": "\tA", "facility_code": 0, "severity_code": 1, }, - Tags: map[string]string{ - "severity": "alert", - "facility": "kern", - }, - Time: defaultTime, - }, + defaultTime, + ), }, } @@ -269,19 +269,17 @@ func testRFC5426(t *testing.T, protocol string, address string, bestEffort bool) } // Compare - var got *testutil.Metric - var want *testutil.Metric + var got telegraf.Metric + var want telegraf.Metric if len(acc.Metrics) > 0 { - got = acc.Metrics[0] + got = acc.GetTelegrafMetrics()[0] } if bestEffort { want = tc.wantBestEffort } else { want = tc.wantStrict } - if !cmp.Equal(want, got) { - t.Fatalf("Got (+) / Want (-)\n %s", cmp.Diff(want, got)) - } + testutil.RequireMetricEqual(t, want, got) }) } } @@ -346,23 +344,22 @@ func TestTimeIncrement_udp(t *testing.T) { // Wait acc.Wait(1) - want := &testutil.Metric{ - Measurement: "syslog", - Fields: map[string]interface{}{ - "version": uint16(1), - "facility_code": 0, - "severity_code": 1, - }, - Tags: map[string]string{ - "severity": "alert", - "facility": "kern", - }, - Time: getNow(), - } - - if !cmp.Equal(want, acc.Metrics[0]) { - t.Fatalf("Got (+) / Want (-)\n %s", cmp.Diff(want, acc.Metrics[0])) + want := []telegraf.Metric{ + testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "alert", + "facility": "kern", + }, + map[string]interface{}{ + "version": uint16(1), + "facility_code": 0, + "severity_code": 1, + }, + getNow(), + ), } + testutil.RequireMetricsEqual(t, want, acc.GetTelegrafMetrics()) // New one with different time atomic.StoreInt64(&i, atomic.LoadInt64(&i)+1) @@ -377,23 +374,22 @@ func TestTimeIncrement_udp(t *testing.T) { // Wait acc.Wait(1) - want = &testutil.Metric{ - Measurement: "syslog", - Fields: map[string]interface{}{ - "version": uint16(1), - "facility_code": 0, - "severity_code": 1, - }, - Tags: map[string]string{ - "severity": "alert", - "facility": "kern", - }, - Time: getNow(), - } - - if !cmp.Equal(want, acc.Metrics[0]) { - t.Fatalf("Got (+) / Want (-)\n %s", cmp.Diff(want, acc.Metrics[0])) + want = []telegraf.Metric{ + testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "alert", + "facility": "kern", + }, + map[string]interface{}{ + "version": uint16(1), + "facility_code": 0, + "severity_code": 1, + }, + getNow(), + ), } + testutil.RequireMetricsEqual(t, want, acc.GetTelegrafMetrics()) // New one with same time as previous one @@ -407,21 +403,20 @@ func TestTimeIncrement_udp(t *testing.T) { // Wait acc.Wait(1) - want = &testutil.Metric{ - Measurement: "syslog", - Fields: map[string]interface{}{ - "version": uint16(1), - "facility_code": 0, - "severity_code": 1, - }, - Tags: map[string]string{ - "severity": "alert", - "facility": "kern", - }, - Time: getNow().Add(time.Nanosecond), - } - - if !cmp.Equal(want, acc.Metrics[0]) { - t.Fatalf("Got (+) / Want (-)\n %s", cmp.Diff(want, acc.Metrics[0])) + want = []telegraf.Metric{ + testutil.MustMetric( + "syslog", + map[string]string{ + "severity": "alert", + "facility": "kern", + }, + map[string]interface{}{ + "version": uint16(1), + "facility_code": 0, + "severity_code": 1, + }, + getNow().Add(time.Nanosecond), + ), } + testutil.RequireMetricsEqual(t, want, acc.GetTelegrafMetrics()) } diff --git a/plugins/inputs/syslog/syslog.go b/plugins/inputs/syslog/syslog.go index 43d02de5e85c4..92d1340920e2c 100644 --- a/plugins/inputs/syslog/syslog.go +++ b/plugins/inputs/syslog/syslog.go @@ -12,10 +12,10 @@ import ( "time" "unicode" - "github.com/influxdata/go-syslog" - "github.com/influxdata/go-syslog/nontransparent" - "github.com/influxdata/go-syslog/octetcounting" - "github.com/influxdata/go-syslog/rfc5424" + "github.com/influxdata/go-syslog/v2" + "github.com/influxdata/go-syslog/v2/nontransparent" + "github.com/influxdata/go-syslog/v2/octetcounting" + "github.com/influxdata/go-syslog/v2/rfc5424" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal" framing "github.com/influxdata/telegraf/internal/syslog" diff --git a/plugins/inputs/sysstat/README.md b/plugins/inputs/sysstat/README.md index d8e0e95d84f9f..9775c1a305c95 100644 --- a/plugins/inputs/sysstat/README.md +++ b/plugins/inputs/sysstat/README.md @@ -16,18 +16,15 @@ the created binary data file with the `sadf` utility. ## On Debian and Arch Linux the default path is /usr/lib/sa/sadc whereas ## on RHEL and CentOS the default path is /usr/lib64/sa/sadc sadc_path = "/usr/lib/sa/sadc" # required - # - # + ## Path to the sadf command, if it is not in PATH # sadf_path = "/usr/bin/sadf" - # - # + ## Activities is a list of activities, that are passed as argument to the ## sadc collector utility (e.g: DISK, SNMP etc...) ## The more activities that are added, the more data is collected. # activities = ["DISK"] - # - # + ## Group metrics to measurements. ## ## If group is false each metric will be prefixed with a description @@ -35,8 +32,7 @@ the created binary data file with the `sadf` utility. ## ## If Group is true, corresponding metrics are grouped to a single measurement. # group = true - # - # + ## Options for the sadf command. The values on the left represent the sadf options and ## the values on the right their description (wich are used for grouping and prefixing metrics). ## @@ -58,8 +54,7 @@ the created binary data file with the `sadf` utility. -w = "task" # -H = "hugepages" # only available for newer linux distributions # "-I ALL" = "interrupts" # requires INT activity - # - # + ## Device tags can be used to add additional tags for devices. For example the configuration below ## adds a tag vg with value rootvg for all metrics with sda devices. # [[inputs.sysstat.device_tags.sda]] diff --git a/plugins/inputs/sysstat/sysstat.go b/plugins/inputs/sysstat/sysstat.go index f1778fd6a2150..9f530024b52d8 100644 --- a/plugins/inputs/sysstat/sysstat.go +++ b/plugins/inputs/sysstat/sysstat.go @@ -7,7 +7,6 @@ import ( "encoding/csv" "fmt" "io" - "log" "os" "os/exec" "path" @@ -67,6 +66,8 @@ type Sysstat struct { DeviceTags map[string][]map[string]string `toml:"device_tags"` tmpFile string interval int + + Log telegraf.Logger } func (*Sysstat) Description() string { @@ -81,18 +82,15 @@ var sampleConfig = ` ## Arch: /usr/lib/sa/sadc ## RHEL/CentOS: /usr/lib64/sa/sadc sadc_path = "/usr/lib/sa/sadc" # required - # - # + ## Path to the sadf command, if it is not in PATH # sadf_path = "/usr/bin/sadf" - # - # + ## Activities is a list of activities, that are passed as argument to the ## sadc collector utility (e.g: DISK, SNMP etc...) ## The more activities that are added, the more data is collected. # activities = ["DISK"] - # - # + ## Group metrics to measurements. ## ## If group is false each metric will be prefixed with a description @@ -100,8 +98,7 @@ var sampleConfig = ` ## ## If Group is true, corresponding metrics are grouped to a single measurement. # group = true - # - # + ## Options for the sadf command. The values on the left represent the sadf ## options and the values on the right their description (which are used for ## grouping and prefixing metrics). @@ -125,8 +122,7 @@ var sampleConfig = ` -w = "task" # -H = "hugepages" # only available for newer linux distributions # "-I ALL" = "interrupts" # requires INT activity - # - # + ## Device tags can be used to add additional tags for devices. ## For example the configuration below adds a tag vg with value rootvg for ## all metrics with sda devices. @@ -196,7 +192,7 @@ func (s *Sysstat) collect() error { out, err := internal.CombinedOutputTimeout(cmd, time.Second*time.Duration(collectInterval+parseInterval)) if err != nil { if err := os.Remove(s.tmpFile); err != nil { - log.Printf("E! failed to remove tmp file after %s command: %s", strings.Join(cmd.Args, " "), err) + s.Log.Errorf("Failed to remove tmp file after %q command: %s", strings.Join(cmd.Args, " "), err.Error()) } return fmt.Errorf("failed to run command %s: %s - %s", strings.Join(cmd.Args, " "), err, string(out)) } diff --git a/plugins/inputs/sysstat/sysstat_test.go b/plugins/inputs/sysstat/sysstat_test.go index 1674f2747fdda..4aecfaacc2a15 100644 --- a/plugins/inputs/sysstat/sysstat_test.go +++ b/plugins/inputs/sysstat/sysstat_test.go @@ -13,6 +13,7 @@ import ( ) var s = Sysstat{ + Log: testutil.Logger{}, interval: 10, Sadc: "/usr/lib/sa/sadc", Sadf: "/usr/bin/sadf", diff --git a/plugins/inputs/system/system.go b/plugins/inputs/system/system.go index 82e6b6db074d7..32747cca20314 100644 --- a/plugins/inputs/system/system.go +++ b/plugins/inputs/system/system.go @@ -4,7 +4,6 @@ import ( "bufio" "bytes" "fmt" - "log" "os" "strings" "time" @@ -16,20 +15,22 @@ import ( "github.com/shirou/gopsutil/load" ) -type SystemStats struct{} +type SystemStats struct { + Log telegraf.Logger +} -func (_ *SystemStats) Description() string { +func (*SystemStats) Description() string { return "Read metrics about system load & uptime" } -func (_ *SystemStats) SampleConfig() string { +func (*SystemStats) SampleConfig() string { return ` ## Uncomment to remove deprecated metrics. # fielddrop = ["uptime_format"] ` } -func (_ *SystemStats) Gather(acc telegraf.Accumulator) error { +func (s *SystemStats) Gather(acc telegraf.Accumulator) error { loadavg, err := load.Avg() if err != nil && !strings.Contains(err.Error(), "not implemented") { return err @@ -51,9 +52,9 @@ func (_ *SystemStats) Gather(acc telegraf.Accumulator) error { if err == nil { fields["n_users"] = len(users) } else if os.IsNotExist(err) { - log.Printf("D! [inputs.system] Error reading users: %v", err) + s.Log.Debugf("Reading users: %s", err.Error()) } else if os.IsPermission(err) { - log.Printf("D! [inputs.system] %v", err) + s.Log.Debug(err.Error()) } now := time.Now() diff --git a/plugins/inputs/systemd_units/README.md b/plugins/inputs/systemd_units/README.md new file mode 100644 index 0000000000000..f6b8796f968b1 --- /dev/null +++ b/plugins/inputs/systemd_units/README.md @@ -0,0 +1,135 @@ +# Systemd Units Plugin + +The systemd_units plugin gathers systemd unit status on Linux. It relies on +`systemctl list-units --all --type=service` to collect data on service status. + +The results are tagged with the unit name and provide enumerated fields for +loaded, active and running fields, indicating the unit health. + +This plugin is related to the [win_services module](/plugins/inputs/win_services/), which +fulfills the same purpose on windows. + +In addition to services, this plugin can gather other unit types as well, +see `systemctl list-units --all --type help` for possible options. + +### Configuration +```toml +[[inputs.systemd_units]] + ## Set timeout for systemctl execution + # timeout = "1s" + # + ## Filter for a specific unit type, default is "service", other possible + ## values are "socket", "target", "device", "mount", "automount", "swap", + ## "timer", "path", "slice" and "scope ": + # unittype = "service" +``` + +### Metrics +- systemd_units: + - tags: + - name (string, unit name) + - load (string, load state) + - active (string, active state) + - sub (string, sub state) + - fields: + - load_code (int, see below) + - active_code (int, see below) + - sub_code (int, see below) + +#### Load + +enumeration of [unit_load_state_table](https://github.com/systemd/systemd/blob/c87700a1335f489be31cd3549927da68b5638819/src/basic/unit-def.c#L87) + +| Value | Meaning | Description | +| ----- | ------- | ----------- | +| 0 | loaded | unit is ~ | +| 1 | stub | unit is ~ | +| 2 | not-found | unit is ~ | +| 3 | bad-setting | unit is ~ | +| 4 | error | unit is ~ | +| 5 | merged | unit is ~ | +| 6 | masked | unit is ~ | + +#### Active + +enumeration of [unit_active_state_table](https://github.com/systemd/systemd/blob/c87700a1335f489be31cd3549927da68b5638819/src/basic/unit-def.c#L99) + +| Value | Meaning | Description | +| ----- | ------- | ----------- | +| 0 | active | unit is ~ | +| 1 | reloading | unit is ~ | +| 2 | inactive | unit is ~ | +| 3 | failed | unit is ~ | +| 4 | activating | unit is ~ | +| 5 | deactivating | unit is ~ | + +#### Sub + +enumeration of sub states, see various [unittype_state_tables](https://github.com/systemd/systemd/blob/c87700a1335f489be31cd3549927da68b5638819/src/basic/unit-def.c#L163); +duplicates were removed, tables are hex aligned to keep some space for future +values + +| Value | Meaning | Description | +| ----- | ------- | ----------- | +| | | service_state_table start at 0x0000 | +| 0x0000 | running | unit is ~ | +| 0x0001 | dead | unit is ~ | +| 0x0002 | start-pre | unit is ~ | +| 0x0003 | start | unit is ~ | +| 0x0004 | exited | unit is ~ | +| 0x0005 | reload | unit is ~ | +| 0x0006 | stop | unit is ~ | +| 0x0007 | stop-watchdog | unit is ~ | +| 0x0008 | stop-sigterm | unit is ~ | +| 0x0009 | stop-sigkill | unit is ~ | +| 0x000a | stop-post | unit is ~ | +| 0x000b | final-sigterm | unit is ~ | +| 0x000c | failed | unit is ~ | +| 0x000d | auto-restart | unit is ~ | +| | | service_state_table start at 0x0010 | +| 0x0010 | waiting | unit is ~ | +| | | service_state_table start at 0x0020 | +| 0x0020 | tentative | unit is ~ | +| 0x0021 | plugged | unit is ~ | +| | | service_state_table start at 0x0030 | +| 0x0030 | mounting | unit is ~ | +| 0x0031 | mounting-done | unit is ~ | +| 0x0032 | mounted | unit is ~ | +| 0x0033 | remounting | unit is ~ | +| 0x0034 | unmounting | unit is ~ | +| 0x0035 | remounting-sigterm | unit is ~ | +| 0x0036 | remounting-sigkill | unit is ~ | +| 0x0037 | unmounting-sigterm | unit is ~ | +| 0x0038 | unmounting-sigkill | unit is ~ | +| | | service_state_table start at 0x0040 | +| | | service_state_table start at 0x0050 | +| 0x0050 | abandoned | unit is ~ | +| | | service_state_table start at 0x0060 | +| 0x0060 | active | unit is ~ | +| | | service_state_table start at 0x0070 | +| 0x0070 | start-chown | unit is ~ | +| 0x0071 | start-post | unit is ~ | +| 0x0072 | listening | unit is ~ | +| 0x0073 | stop-pre | unit is ~ | +| 0x0074 | stop-pre-sigterm | unit is ~ | +| 0x0075 | stop-pre-sigkill | unit is ~ | +| 0x0076 | final-sigkill | unit is ~ | +| | | service_state_table start at 0x0080 | +| 0x0080 | activating | unit is ~ | +| 0x0081 | activating-done | unit is ~ | +| 0x0082 | deactivating | unit is ~ | +| 0x0083 | deactivating-sigterm | unit is ~ | +| 0x0084 | deactivating-sigkill | unit is ~ | +| | | service_state_table start at 0x0090 | +| | | service_state_table start at 0x00a0 | +| 0x00a0 | elapsed | unit is ~ | +| | | | + +### Example Output + +``` +systemd_units,host=host1.example.com,name=dbus.service,load=loaded,active=active,sub=running load_code=0i,active_code=0i,sub_code=0i 1533730725000000000 +systemd_units,host=host1.example.com,name=networking.service,load=loaded,active=failed,sub=failed load_code=0i,active_code=3i,sub_code=12i 1533730725000000000 +systemd_units,host=host1.example.com,name=ssh.service,load=loaded,active=active,sub=running load_code=0i,active_code=0i,sub_code=0i 1533730725000000000 +... +``` diff --git a/plugins/inputs/systemd_units/systemd_units_linux.go b/plugins/inputs/systemd_units/systemd_units_linux.go new file mode 100644 index 0000000000000..64caf03d007f3 --- /dev/null +++ b/plugins/inputs/systemd_units/systemd_units_linux.go @@ -0,0 +1,221 @@ +package systemd_units + +import ( + "bufio" + "bytes" + "fmt" + "os/exec" + "strings" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/plugins/inputs" +) + +// SystemdUnits is a telegraf plugin to gather systemd unit status +type SystemdUnits struct { + Timeout internal.Duration + UnitType string `toml:"unittype"` + systemctl systemctl +} + +type systemctl func(Timeout internal.Duration, UnitType string) (*bytes.Buffer, error) + +const measurement = "systemd_units" + +// Below are mappings of systemd state tables as defined in +// https://github.com/systemd/systemd/blob/c87700a1335f489be31cd3549927da68b5638819/src/basic/unit-def.c +// Duplicate strings are removed from this list. +var load_map = map[string]int{ + "loaded": 0, + "stub": 1, + "not-found": 2, + "bad-setting": 3, + "error": 4, + "merged": 5, + "masked": 6, +} + +var active_map = map[string]int{ + "active": 0, + "reloading": 1, + "inactive": 2, + "failed": 3, + "activating": 4, + "deactivating": 5, +} + +var sub_map = map[string]int{ + // service_state_table, offset 0x0000 + "running": 0x0000, + "dead": 0x0001, + "start-pre": 0x0002, + "start": 0x0003, + "exited": 0x0004, + "reload": 0x0005, + "stop": 0x0006, + "stop-watchdog": 0x0007, + "stop-sigterm": 0x0008, + "stop-sigkill": 0x0009, + "stop-post": 0x000a, + "final-sigterm": 0x000b, + "failed": 0x000c, + "auto-restart": 0x000d, + + // automount_state_table, offset 0x0010 + "waiting": 0x0010, + + // device_state_table, offset 0x0020 + "tentative": 0x0020, + "plugged": 0x0021, + + // mount_state_table, offset 0x0030 + "mounting": 0x0030, + "mounting-done": 0x0031, + "mounted": 0x0032, + "remounting": 0x0033, + "unmounting": 0x0034, + "remounting-sigterm": 0x0035, + "remounting-sigkill": 0x0036, + "unmounting-sigterm": 0x0037, + "unmounting-sigkill": 0x0038, + + // path_state_table, offset 0x0040 + + // scope_state_table, offset 0x0050 + "abandoned": 0x0050, + + // slice_state_table, offset 0x0060 + "active": 0x0060, + + // socket_state_table, offset 0x0070 + "start-chown": 0x0070, + "start-post": 0x0071, + "listening": 0x0072, + "stop-pre": 0x0073, + "stop-pre-sigterm": 0x0074, + "stop-pre-sigkill": 0x0075, + "final-sigkill": 0x0076, + + // swap_state_table, offset 0x0080 + "activating": 0x0080, + "activating-done": 0x0081, + "deactivating": 0x0082, + "deactivating-sigterm": 0x0083, + "deactivating-sigkill": 0x0084, + + // target_state_table, offset 0x0090 + + // timer_state_table, offset 0x00a0 + "elapsed": 0x00a0, +} + +var ( + defaultTimeout = internal.Duration{Duration: time.Second} + defaultUnitType = "service" +) + +// Description returns a short description of the plugin +func (s *SystemdUnits) Description() string { + return "Gather systemd units state" +} + +// SampleConfig returns sample configuration options. +func (s *SystemdUnits) SampleConfig() string { + return ` + ## Set timeout for systemctl execution + # timeout = "1s" + # + ## Filter for a specific unit type, default is "service", other possible + ## values are "socket", "target", "device", "mount", "automount", "swap", + ## "timer", "path", "slice" and "scope ": + # unittype = "service" +` +} + +// Gather parses systemctl outputs and adds counters to the Accumulator +func (s *SystemdUnits) Gather(acc telegraf.Accumulator) error { + out, err := s.systemctl(s.Timeout, s.UnitType) + if err != nil { + return err + } + + scanner := bufio.NewScanner(out) + for scanner.Scan() { + line := scanner.Text() + + data := strings.Fields(line) + if len(data) < 4 { + acc.AddError(fmt.Errorf("Error parsing line (expected at least 4 fields): %s", line)) + continue + } + name := data[0] + load := data[1] + active := data[2] + sub := data[3] + tags := map[string]string{ + "name": name, + "load": load, + "active": active, + "sub": sub, + } + + var ( + load_code int + active_code int + sub_code int + ok bool + ) + if load_code, ok = load_map[load]; !ok { + acc.AddError(fmt.Errorf("Error parsing field 'load', value not in map: %s", load)) + continue + } + if active_code, ok = active_map[active]; !ok { + acc.AddError(fmt.Errorf("Error parsing field 'active', value not in map: %s", active)) + continue + } + if sub_code, ok = sub_map[sub]; !ok { + acc.AddError(fmt.Errorf("Error parsing field 'sub', value not in map: %s", sub)) + continue + } + fields := map[string]interface{}{ + "load_code": load_code, + "active_code": active_code, + "sub_code": sub_code, + } + + acc.AddFields(measurement, fields, tags) + } + + return nil +} + +func setSystemctl(Timeout internal.Duration, UnitType string) (*bytes.Buffer, error) { + // is systemctl available ? + systemctlPath, err := exec.LookPath("systemctl") + if err != nil { + return nil, err + } + + cmd := exec.Command(systemctlPath, "list-units", "--all", fmt.Sprintf("--type=%s", UnitType), "--no-legend") + + var out bytes.Buffer + cmd.Stdout = &out + err = internal.RunTimeout(cmd, Timeout.Duration) + if err != nil { + return &out, fmt.Errorf("error running systemctl list-units --all --type=%s --no-legend: %s", UnitType, err) + } + + return &out, nil +} + +func init() { + inputs.Add("systemd_units", func() telegraf.Input { + return &SystemdUnits{ + systemctl: setSystemctl, + Timeout: defaultTimeout, + UnitType: defaultUnitType, + } + }) +} diff --git a/plugins/inputs/systemd_units/systemd_units_linux_test.go b/plugins/inputs/systemd_units/systemd_units_linux_test.go new file mode 100644 index 0000000000000..f45922bb91af0 --- /dev/null +++ b/plugins/inputs/systemd_units/systemd_units_linux_test.go @@ -0,0 +1,100 @@ +package systemd_units + +import ( + "bytes" + "fmt" + "reflect" + "testing" + + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/testutil" +) + +func TestSystemdUnits(t *testing.T) { + tests := []struct { + name string + line string + tags map[string]string + fields map[string]interface{} + status int + err error + }{ + { + name: "example loaded active running", + line: "example.service loaded active running example service description", + tags: map[string]string{"name": "example.service", "load": "loaded", "active": "active", "sub": "running"}, + fields: map[string]interface{}{ + "load_code": 0, + "active_code": 0, + "sub_code": 0, + }, + }, + { + name: "example loaded active exited", + line: "example.service loaded active exited example service description", + tags: map[string]string{"name": "example.service", "load": "loaded", "active": "active", "sub": "exited"}, + fields: map[string]interface{}{ + "load_code": 0, + "active_code": 0, + "sub_code": 4, + }, + }, + { + name: "example loaded failed failed", + line: "example.service loaded failed failed example service description", + tags: map[string]string{"name": "example.service", "load": "loaded", "active": "failed", "sub": "failed"}, + fields: map[string]interface{}{ + "load_code": 0, + "active_code": 3, + "sub_code": 12, + }, + }, + { + name: "example not-found inactive dead", + line: "example.service not-found inactive dead example service description", + tags: map[string]string{"name": "example.service", "load": "not-found", "active": "inactive", "sub": "dead"}, + fields: map[string]interface{}{ + "load_code": 2, + "active_code": 2, + "sub_code": 1, + }, + }, + { + name: "example unknown unknown unknown", + line: "example.service unknown unknown unknown example service description", + err: fmt.Errorf("Error parsing field 'load', value not in map: %s", "unknown"), + }, + { + name: "example too few fields", + line: "example.service loaded fai", + err: fmt.Errorf("Error parsing line (expected at least 4 fields): %s", "example.service loaded fai"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + systemd_units := &SystemdUnits{ + systemctl: func(Timeout internal.Duration, UnitType string) (*bytes.Buffer, error) { + return bytes.NewBufferString(tt.line), nil + }, + } + acc := new(testutil.Accumulator) + err := acc.GatherError(systemd_units.Gather) + if !reflect.DeepEqual(tt.err, err) { + t.Errorf("%s: expected error '%#v' got '%#v'", tt.name, tt.err, err) + } + if len(acc.Metrics) > 0 { + m := acc.Metrics[0] + if !reflect.DeepEqual(m.Measurement, measurement) { + t.Errorf("%s: expected measurement '%#v' got '%#v'\n", tt.name, measurement, m.Measurement) + } + if !reflect.DeepEqual(m.Tags, tt.tags) { + t.Errorf("%s: expected tags\n%#v got\n%#v\n", tt.name, tt.tags, m.Tags) + } + if !reflect.DeepEqual(m.Fields, tt.fields) { + t.Errorf("%s: expected fields\n%#v got\n%#v\n", tt.name, tt.fields, m.Fields) + } + } + }) + } +} diff --git a/plugins/inputs/systemd_units/systemd_units_notlinux.go b/plugins/inputs/systemd_units/systemd_units_notlinux.go new file mode 100644 index 0000000000000..f53cea3de6eba --- /dev/null +++ b/plugins/inputs/systemd_units/systemd_units_notlinux.go @@ -0,0 +1,3 @@ +// +build !linux + +package systemd_units diff --git a/plugins/inputs/tail/tail.go b/plugins/inputs/tail/tail.go index da5b81a60eaea..db4d564249295 100644 --- a/plugins/inputs/tail/tail.go +++ b/plugins/inputs/tail/tail.go @@ -3,8 +3,6 @@ package tail import ( - "fmt" - "log" "strings" "sync" @@ -31,6 +29,8 @@ type Tail struct { Pipe bool WatchMethod string + Log telegraf.Logger + tailers map[string]*tail.Tail offsets map[string]int64 parserFunc parsers.ParserFunc @@ -124,7 +124,7 @@ func (t *Tail) tailNewFiles(fromBeginning bool) error { for _, filepath := range t.Files { g, err := globpath.Compile(filepath) if err != nil { - t.acc.AddError(fmt.Errorf("glob %s failed to compile, %s", filepath, err)) + t.Log.Errorf("Glob %q failed to compile: %s", filepath, err.Error()) } for _, file := range g.Match() { if _, ok := t.tailers[file]; ok { @@ -135,7 +135,7 @@ func (t *Tail) tailNewFiles(fromBeginning bool) error { var seek *tail.SeekInfo if !t.Pipe && !fromBeginning { if offset, ok := t.offsets[file]; ok { - log.Printf("D! [inputs.tail] using offset %d for file: %v", offset, file) + t.Log.Debugf("Using offset %d for %q", offset, file) seek = &tail.SeekInfo{ Whence: 0, Offset: offset, @@ -159,15 +159,15 @@ func (t *Tail) tailNewFiles(fromBeginning bool) error { Logger: tail.DiscardingLogger, }) if err != nil { - t.acc.AddError(err) + t.Log.Debugf("Failed to open file (%s): %v", file, err) continue } - log.Printf("D! [inputs.tail] tail added for file: %v", file) + t.Log.Debugf("Tail added for %q", file) parser, err := t.parserFunc() if err != nil { - t.acc.AddError(fmt.Errorf("error creating parser: %v", err)) + t.Log.Errorf("Creating parser: %s", err.Error()) } // create a goroutine for each "tailer" @@ -213,7 +213,7 @@ func (t *Tail) receiver(parser parsers.Parser, tailer *tail.Tail) { var firstLine = true for line := range tailer.Lines { if line.Err != nil { - t.acc.AddError(fmt.Errorf("error tailing file %s, Error: %s", tailer.Filename, line.Err)) + t.Log.Errorf("Tailing %q: %s", tailer.Filename, line.Err.Error()) continue } // Fix up files with Windows line endings. @@ -221,8 +221,8 @@ func (t *Tail) receiver(parser parsers.Parser, tailer *tail.Tail) { metrics, err := parseLine(parser, text, firstLine) if err != nil { - t.acc.AddError(fmt.Errorf("malformed log line in %s: [%s], Error: %s", - tailer.Filename, line.Text, err)) + t.Log.Errorf("Malformed log line in %q: [%q]: %s", + tailer.Filename, line.Text, err.Error()) continue } firstLine = false @@ -233,10 +233,10 @@ func (t *Tail) receiver(parser parsers.Parser, tailer *tail.Tail) { } } - log.Printf("D! [inputs.tail] tail removed for file: %v", tailer.Filename) + t.Log.Debugf("Tail removed for %q", tailer.Filename) if err := tailer.Err(); err != nil { - t.acc.AddError(fmt.Errorf("error tailing file %s, Error: %s", tailer.Filename, err)) + t.Log.Errorf("Tailing %q: %s", tailer.Filename, err.Error()) } } @@ -249,14 +249,14 @@ func (t *Tail) Stop() { // store offset for resume offset, err := tailer.Tell() if err == nil { - log.Printf("D! [inputs.tail] recording offset %d for file: %v", offset, tailer.Filename) + t.Log.Debugf("Recording offset %d for %q", offset, tailer.Filename) } else { - t.acc.AddError(fmt.Errorf("error recording offset for file %s", tailer.Filename)) + t.Log.Errorf("Recording offset for %q: %s", tailer.Filename, err.Error()) } } err := tailer.Stop() if err != nil { - t.acc.AddError(fmt.Errorf("error stopping tail on file %s", tailer.Filename)) + t.Log.Errorf("Stopping tail on %q: %s", tailer.Filename, err.Error()) } } diff --git a/plugins/inputs/tail/tail_test.go b/plugins/inputs/tail/tail_test.go index 41db76cacf8e4..4b96e092ff98d 100644 --- a/plugins/inputs/tail/tail_test.go +++ b/plugins/inputs/tail/tail_test.go @@ -1,7 +1,9 @@ package tail import ( + "bytes" "io/ioutil" + "log" "os" "runtime" "testing" @@ -28,6 +30,7 @@ func TestTailFromBeginning(t *testing.T) { require.NoError(t, err) tt := NewTail() + tt.Log = testutil.Logger{} tt.FromBeginning = true tt.Files = []string{tmpfile.Name()} tt.SetParserFunc(parsers.NewInfluxParser) @@ -61,6 +64,7 @@ func TestTailFromEnd(t *testing.T) { require.NoError(t, err) tt := NewTail() + tt.Log = testutil.Logger{} tt.Files = []string{tmpfile.Name()} tt.SetParserFunc(parsers.NewInfluxParser) defer tt.Stop() @@ -97,6 +101,7 @@ func TestTailBadLine(t *testing.T) { defer os.Remove(tmpfile.Name()) tt := NewTail() + tt.Log = testutil.Logger{} tt.FromBeginning = true tt.Files = []string{tmpfile.Name()} tt.SetParserFunc(parsers.NewInfluxParser) @@ -105,13 +110,17 @@ func TestTailBadLine(t *testing.T) { acc := testutil.Accumulator{} require.NoError(t, tt.Start(&acc)) + + buf := &bytes.Buffer{} + log.SetOutput(buf) + require.NoError(t, acc.GatherError(tt.Gather)) _, err = tmpfile.WriteString("cpu mytag= foo usage_idle= 100\n") require.NoError(t, err) - acc.WaitError(1) - assert.Contains(t, acc.Errors[0].Error(), "malformed log line") + time.Sleep(500 * time.Millisecond) + assert.Contains(t, buf.String(), "Malformed log line") } func TestTailDosLineendings(t *testing.T) { @@ -122,6 +131,7 @@ func TestTailDosLineendings(t *testing.T) { require.NoError(t, err) tt := NewTail() + tt.Log = testutil.Logger{} tt.FromBeginning = true tt.Files = []string{tmpfile.Name()} tt.SetParserFunc(parsers.NewInfluxParser) @@ -160,6 +170,7 @@ cpu,42 require.NoError(t, err) plugin := NewTail() + plugin.Log = testutil.Logger{} plugin.FromBeginning = true plugin.Files = []string{tmpfile.Name()} plugin.SetParserFunc(func() (parsers.Parser, error) { @@ -217,6 +228,7 @@ func TestMultipleMetricsOnFirstLine(t *testing.T) { require.NoError(t, err) plugin := NewTail() + plugin.Log = testutil.Logger{} plugin.FromBeginning = true plugin.Files = []string{tmpfile.Name()} plugin.SetParserFunc(func() (parsers.Parser, error) { diff --git a/plugins/inputs/tcp_listener/tcp_listener.go b/plugins/inputs/tcp_listener/tcp_listener.go index 544f36bd61246..41b8e463766ba 100644 --- a/plugins/inputs/tcp_listener/tcp_listener.go +++ b/plugins/inputs/tcp_listener/tcp_listener.go @@ -48,13 +48,15 @@ type TcpListener struct { TotalConnections selfstat.Stat PacketsRecv selfstat.Stat BytesRecv selfstat.Stat + + Log telegraf.Logger } -var dropwarn = "E! Error: tcp_listener message queue full. " + +var dropwarn = "tcp_listener message queue full. " + "We have dropped %d messages so far. " + - "You may want to increase allowed_pending_messages in the config\n" + "You may want to increase allowed_pending_messages in the config" -var malformedwarn = "E! tcp_listener has received %d malformed packets" + +var malformedwarn = "tcp_listener has received %d malformed packets" + " thus far." const sampleConfig = ` @@ -114,16 +116,15 @@ func (t *TcpListener) Start(acc telegraf.Accumulator) error { address, _ := net.ResolveTCPAddr("tcp", t.ServiceAddress) t.listener, err = net.ListenTCP("tcp", address) if err != nil { - log.Fatalf("ERROR: ListenUDP - %s", err) + t.Log.Errorf("Failed to listen: %s", err.Error()) return err } - log.Println("I! TCP server listening on: ", t.listener.Addr().String()) t.wg.Add(2) go t.tcpListen() go t.tcpParser() - log.Printf("I! Started TCP listener service on %s\n", t.ServiceAddress) + t.Log.Infof("Started TCP listener service on %q", t.ServiceAddress) return nil } @@ -150,7 +151,7 @@ func (t *TcpListener) Stop() { t.wg.Wait() close(t.in) - log.Println("I! Stopped TCP listener service on ", t.ServiceAddress) + t.Log.Infof("Stopped TCP listener service on %q", t.ServiceAddress) } // tcpListen listens for incoming TCP connections. @@ -191,9 +192,8 @@ func (t *TcpListener) refuser(conn *net.TCPConn) { " reached, closing.\nYou may want to increase max_tcp_connections in"+ " the Telegraf tcp listener configuration.\n", t.MaxTCPConnections) conn.Close() - log.Printf("I! Refused TCP Connection from %s", conn.RemoteAddr()) - log.Printf("I! WARNING: Maximum TCP Connections reached, you may want to" + - " adjust max_tcp_connections") + t.Log.Infof("Refused TCP Connection from %s", conn.RemoteAddr()) + t.Log.Warn("Maximum TCP Connections reached, you may want to adjust max_tcp_connections") } // handler handles a single TCP Connection @@ -235,7 +235,7 @@ func (t *TcpListener) handler(conn *net.TCPConn, id string) { default: t.drops++ if t.drops == 1 || t.drops%t.AllowedPendingMessages == 0 { - log.Printf(dropwarn, t.drops) + t.Log.Errorf(dropwarn, t.drops) } } } @@ -268,7 +268,7 @@ func (t *TcpListener) tcpParser() error { } else { t.malformed++ if t.malformed == 1 || t.malformed%1000 == 0 { - log.Printf(malformedwarn, t.malformed) + t.Log.Errorf(malformedwarn, t.malformed) } } } diff --git a/plugins/inputs/tcp_listener/tcp_listener_test.go b/plugins/inputs/tcp_listener/tcp_listener_test.go index 6ff40ad87d6ab..7c04ecaba9280 100644 --- a/plugins/inputs/tcp_listener/tcp_listener_test.go +++ b/plugins/inputs/tcp_listener/tcp_listener_test.go @@ -33,6 +33,7 @@ cpu_load_short,host=server06 value=12.0 1422568543702900257 func newTestTcpListener() (*TcpListener, chan []byte) { in := make(chan []byte, 1500) listener := &TcpListener{ + Log: testutil.Logger{}, ServiceAddress: "localhost:8194", AllowedPendingMessages: 10000, MaxTCPConnections: 250, @@ -45,6 +46,7 @@ func newTestTcpListener() (*TcpListener, chan []byte) { // benchmark how long it takes to accept & process 100,000 metrics: func BenchmarkTCP(b *testing.B) { listener := TcpListener{ + Log: testutil.Logger{}, ServiceAddress: "localhost:8198", AllowedPendingMessages: 100000, MaxTCPConnections: 250, @@ -76,6 +78,7 @@ func BenchmarkTCP(b *testing.B) { func TestHighTrafficTCP(t *testing.T) { listener := TcpListener{ + Log: testutil.Logger{}, ServiceAddress: "localhost:8199", AllowedPendingMessages: 100000, MaxTCPConnections: 250, @@ -103,6 +106,7 @@ func TestHighTrafficTCP(t *testing.T) { func TestConnectTCP(t *testing.T) { listener := TcpListener{ + Log: testutil.Logger{}, ServiceAddress: "localhost:8194", AllowedPendingMessages: 10000, MaxTCPConnections: 250, @@ -140,6 +144,7 @@ func TestConnectTCP(t *testing.T) { // Test that MaxTCPConections is respected func TestConcurrentConns(t *testing.T) { listener := TcpListener{ + Log: testutil.Logger{}, ServiceAddress: "localhost:8195", AllowedPendingMessages: 10000, MaxTCPConnections: 2, @@ -175,6 +180,7 @@ func TestConcurrentConns(t *testing.T) { // Test that MaxTCPConections is respected when max==1 func TestConcurrentConns1(t *testing.T) { listener := TcpListener{ + Log: testutil.Logger{}, ServiceAddress: "localhost:8196", AllowedPendingMessages: 10000, MaxTCPConnections: 1, @@ -208,6 +214,7 @@ func TestConcurrentConns1(t *testing.T) { // Test that MaxTCPConections is respected func TestCloseConcurrentConns(t *testing.T) { listener := TcpListener{ + Log: testutil.Logger{}, ServiceAddress: "localhost:8195", AllowedPendingMessages: 10000, MaxTCPConnections: 2, diff --git a/plugins/inputs/temp/README.md b/plugins/inputs/temp/README.md index 873a732855719..8398d25ca2d7c 100644 --- a/plugins/inputs/temp/README.md +++ b/plugins/inputs/temp/README.md @@ -5,13 +5,14 @@ meant to be multi platform and uses platform specific collection methods. Currently supports Linux and Windows. -### Configuration: +### Configuration -``` +```toml [[inputs.temp]] + # no configuration ``` -### Metrics: +### Metrics - temp - tags: @@ -19,7 +20,16 @@ Currently supports Linux and Windows. - fields: - temp (float, celcius) -### Example Output: + +### Troubleshooting + +On **Windows**, the plugin uses a WMI call that is can be replicated with the +following command: +``` +wmic /namespace:\\root\wmi PATH MSAcpi_ThermalZoneTemperature +``` + +### Example Output ``` temp,sensor=coretemp_physicalid0_crit temp=100 1531298763000000000 diff --git a/plugins/inputs/udp_listener/udp_listener.go b/plugins/inputs/udp_listener/udp_listener.go index d0a728b3c8484..7fa59fdb121bc 100644 --- a/plugins/inputs/udp_listener/udp_listener.go +++ b/plugins/inputs/udp_listener/udp_listener.go @@ -53,17 +53,19 @@ type UdpListener struct { PacketsRecv selfstat.Stat BytesRecv selfstat.Stat + + Log telegraf.Logger } // UDP_MAX_PACKET_SIZE is packet limit, see // https://en.wikipedia.org/wiki/User_Datagram_Protocol#Packet_structure const UDP_MAX_PACKET_SIZE int = 64 * 1024 -var dropwarn = "E! Error: udp_listener message queue full. " + +var dropwarn = "udp_listener message queue full. " + "We have dropped %d messages so far. " + - "You may want to increase allowed_pending_messages in the config\n" + "You may want to increase allowed_pending_messages in the config" -var malformedwarn = "E! udp_listener has received %d malformed packets" + +var malformedwarn = "udp_listener has received %d malformed packets" + " thus far." const sampleConfig = ` @@ -113,7 +115,7 @@ func (u *UdpListener) Start(acc telegraf.Accumulator) error { u.wg.Add(1) go u.udpParser() - log.Printf("I! Started UDP listener service on %s (ReadBuffer: %d)\n", u.ServiceAddress, u.UDPBufferSize) + u.Log.Infof("Started service on %q (ReadBuffer: %d)", u.ServiceAddress, u.UDPBufferSize) return nil } @@ -124,7 +126,7 @@ func (u *UdpListener) Stop() { u.wg.Wait() u.listener.Close() close(u.in) - log.Println("I! Stopped UDP listener service on ", u.ServiceAddress) + u.Log.Infof("Stopped service on %q", u.ServiceAddress) } func (u *UdpListener) udpListen() error { @@ -134,15 +136,15 @@ func (u *UdpListener) udpListen() error { u.listener, err = net.ListenUDP("udp", address) if err != nil { - return fmt.Errorf("E! Error: ListenUDP - %s", err) + return err } - log.Println("I! UDP server listening on: ", u.listener.LocalAddr().String()) + u.Log.Infof("Server listening on %q", u.listener.LocalAddr().String()) if u.UDPBufferSize > 0 { err = u.listener.SetReadBuffer(u.UDPBufferSize) // if we want to move away from OS default if err != nil { - return fmt.Errorf("E! Failed to set UDP read buffer to %d: %s", u.UDPBufferSize, err) + return fmt.Errorf("failed to set UDP read buffer to %d: %s", u.UDPBufferSize, err) } } @@ -166,7 +168,7 @@ func (u *UdpListener) udpListenLoop() { if err != nil { if err, ok := err.(net.Error); ok && err.Timeout() { } else { - log.Printf("E! Error: %s\n", err.Error()) + u.Log.Error(err.Error()) } continue } @@ -180,7 +182,7 @@ func (u *UdpListener) udpListenLoop() { default: u.drops++ if u.drops == 1 || u.drops%u.AllowedPendingMessages == 0 { - log.Printf(dropwarn, u.drops) + u.Log.Errorf(dropwarn, u.drops) } } } @@ -208,7 +210,7 @@ func (u *UdpListener) udpParser() error { } else { u.malformed++ if u.malformed == 1 || u.malformed%1000 == 0 { - log.Printf(malformedwarn, u.malformed) + u.Log.Errorf(malformedwarn, u.malformed) } } } diff --git a/plugins/inputs/udp_listener/udp_listener_test.go b/plugins/inputs/udp_listener/udp_listener_test.go index 345db62a44e91..b241235e4d61d 100644 --- a/plugins/inputs/udp_listener/udp_listener_test.go +++ b/plugins/inputs/udp_listener/udp_listener_test.go @@ -31,6 +31,7 @@ cpu_load_short,host=server06 value=12.0 1422568543702900257 func newTestUdpListener() (*UdpListener, chan []byte) { in := make(chan []byte, 1500) listener := &UdpListener{ + Log: testutil.Logger{}, ServiceAddress: ":8125", AllowedPendingMessages: 10000, in: in, @@ -78,6 +79,7 @@ func newTestUdpListener() (*UdpListener, chan []byte) { func TestConnectUDP(t *testing.T) { listener := UdpListener{ + Log: testutil.Logger{}, ServiceAddress: ":8127", AllowedPendingMessages: 10000, } diff --git a/plugins/inputs/unbound/README.md b/plugins/inputs/unbound/README.md index 36c9aa47de850..d7d5c8ba9f1b1 100644 --- a/plugins/inputs/unbound/README.md +++ b/plugins/inputs/unbound/README.md @@ -18,6 +18,9 @@ a validating, recursive, and caching DNS resolver. ## The default location of the unbound-control binary can be overridden with: # binary = "/usr/sbin/unbound-control" + ## The default location of the unbound config file can be overridden with: + # config_file = "/etc/unbound/unbound.conf" + ## The default timeout of 1s can be overriden with: # timeout = "1s" diff --git a/plugins/inputs/unbound/unbound.go b/plugins/inputs/unbound/unbound.go index 02067c739c572..c8247d0cf04e2 100644 --- a/plugins/inputs/unbound/unbound.go +++ b/plugins/inputs/unbound/unbound.go @@ -17,7 +17,7 @@ import ( "github.com/influxdata/telegraf/plugins/inputs" ) -type runner func(cmdName string, Timeout internal.Duration, UseSudo bool, Server string, ThreadAsTag bool) (*bytes.Buffer, error) +type runner func(cmdName string, Timeout internal.Duration, UseSudo bool, Server string, ThreadAsTag bool, ConfigFile string) (*bytes.Buffer, error) // Unbound is used to store configuration values type Unbound struct { @@ -26,6 +26,7 @@ type Unbound struct { UseSudo bool Server string ThreadAsTag bool + ConfigFile string filter filter.Filter run runner @@ -45,6 +46,9 @@ var sampleConfig = ` ## The default location of the unbound-control binary can be overridden with: # binary = "/usr/sbin/unbound-control" + ## The default location of the unbound config file can be overridden with: + # config_file = "/etc/unbound/unbound.conf" + ## The default timeout of 1s can be overriden with: # timeout = "1s" @@ -67,7 +71,7 @@ func (s *Unbound) SampleConfig() string { } // Shell out to unbound_stat and return the output -func unboundRunner(cmdName string, Timeout internal.Duration, UseSudo bool, Server string, ThreadAsTag bool) (*bytes.Buffer, error) { +func unboundRunner(cmdName string, Timeout internal.Duration, UseSudo bool, Server string, ThreadAsTag bool, ConfigFile string) (*bytes.Buffer, error) { cmdArgs := []string{"stats_noreset"} if Server != "" { @@ -96,6 +100,10 @@ func unboundRunner(cmdName string, Timeout internal.Duration, UseSudo bool, Serv cmdArgs = append([]string{"-s", server}, cmdArgs...) } + if ConfigFile != "" { + cmdArgs = append([]string{"-c", ConfigFile}, cmdArgs...) + } + cmd := exec.Command(cmdName, cmdArgs...) if UseSudo { @@ -125,7 +133,7 @@ func (s *Unbound) Gather(acc telegraf.Accumulator) error { return err } - out, err := s.run(s.Binary, s.Timeout, s.UseSudo, s.Server, s.ThreadAsTag) + out, err := s.run(s.Binary, s.Timeout, s.UseSudo, s.Server, s.ThreadAsTag, s.ConfigFile) if err != nil { return fmt.Errorf("error gathering metrics: %s", err) } @@ -207,6 +215,7 @@ func init() { UseSudo: false, Server: "", ThreadAsTag: false, + ConfigFile: "", } }) } diff --git a/plugins/inputs/unbound/unbound_test.go b/plugins/inputs/unbound/unbound_test.go index b1d6206c39900..cc4b99daecc59 100644 --- a/plugins/inputs/unbound/unbound_test.go +++ b/plugins/inputs/unbound/unbound_test.go @@ -12,8 +12,8 @@ import ( var TestTimeout = internal.Duration{Duration: time.Second} -func UnboundControl(output string, Timeout internal.Duration, useSudo bool, Server string, ThreadAsTag bool) func(string, internal.Duration, bool, string, bool) (*bytes.Buffer, error) { - return func(string, internal.Duration, bool, string, bool) (*bytes.Buffer, error) { +func UnboundControl(output string, Timeout internal.Duration, useSudo bool, Server string, ThreadAsTag bool, ConfigFile string) func(string, internal.Duration, bool, string, bool, string) (*bytes.Buffer, error) { + return func(string, internal.Duration, bool, string, bool, string) (*bytes.Buffer, error) { return bytes.NewBuffer([]byte(output)), nil } } @@ -21,7 +21,7 @@ func UnboundControl(output string, Timeout internal.Duration, useSudo bool, Serv func TestParseFullOutput(t *testing.T) { acc := &testutil.Accumulator{} v := &Unbound{ - run: UnboundControl(fullOutput, TestTimeout, true, "", false), + run: UnboundControl(fullOutput, TestTimeout, true, "", false, ""), } err := v.Gather(acc) @@ -38,7 +38,7 @@ func TestParseFullOutput(t *testing.T) { func TestParseFullOutputThreadAsTag(t *testing.T) { acc := &testutil.Accumulator{} v := &Unbound{ - run: UnboundControl(fullOutput, TestTimeout, true, "", true), + run: UnboundControl(fullOutput, TestTimeout, true, "", true, ""), ThreadAsTag: true, } err := v.Gather(acc) diff --git a/plugins/inputs/uwsgi/uwsgi.go b/plugins/inputs/uwsgi/uwsgi.go index 15a9bbe2261fb..a20f3b2bfcaf9 100644 --- a/plugins/inputs/uwsgi/uwsgi.go +++ b/plugins/inputs/uwsgi/uwsgi.go @@ -91,13 +91,13 @@ func (u *Uwsgi) gatherServer(acc telegraf.Accumulator, url *url.URL) error { } s.source = url.Host case "unix": - r, err = net.DialTimeout(url.Scheme, url.Host, u.Timeout.Duration) + r, err = net.DialTimeout(url.Scheme, url.Path, u.Timeout.Duration) if err != nil { return err } s.source, err = os.Hostname() if err != nil { - s.source = url.Host + s.source = "" } case "http": resp, err := u.client.Get(url.String()) diff --git a/plugins/inputs/vsphere/METRICS.md b/plugins/inputs/vsphere/METRICS.md index 0b9e0482fd8f8..d1a34bb26c4f9 100644 --- a/plugins/inputs/vsphere/METRICS.md +++ b/plugins/inputs/vsphere/METRICS.md @@ -4,6 +4,8 @@ and the set of available metrics may vary depending hardware, as well as what pl are installed. Therefore, providing a definitive list of available metrics is difficult. The metrics listed below are the most commonly available as of vSphere 6.5. +For a complete list of metrics available from vSphere and the units they measure in, please reference the [VMWare vCenter Converter API Reference](https://www.vmware.com/support/developer/converter-sdk/conv60_apireference/vim.PerformanceManager.html). + To list the exact set in your environment, please use the govc tool available [here](https://github.com/vmware/govmomi/tree/master/govc) To obtain the set of metrics for e.g. a VM, you may use the following command: @@ -284,4 +286,4 @@ disk.capacity.latest disk.capacity.contention.average disk.capacity.provisioned.average disk.capacity.usage.average -``` \ No newline at end of file +``` diff --git a/plugins/inputs/vsphere/README.md b/plugins/inputs/vsphere/README.md index 4009c8cdeb59d..f69bd28624166 100644 --- a/plugins/inputs/vsphere/README.md +++ b/plugins/inputs/vsphere/README.md @@ -31,6 +31,7 @@ vm_metric_exclude = [ "*" ] ## VMs ## Typical VM metrics (if omitted or empty, all metrics are collected) # vm_include = [ "/*/vm/**"] # Inventory path to VMs to collect (by default all are collected) + # vm_exclude = [] # Inventory paths to exclude vm_metric_include = [ "cpu.demand.average", "cpu.idle.summation", @@ -73,6 +74,7 @@ vm_metric_exclude = [ "*" ] ## Hosts ## Typical host metrics (if omitted or empty, all metrics are collected) # host_include = [ "/*/host/**"] # Inventory path to hosts to collect (by default all are collected) + # host_exclude [] # Inventory paths to exclude host_metric_include = [ "cpu.coreUtilization.average", "cpu.costop.summation", @@ -130,18 +132,21 @@ vm_metric_exclude = [ "*" ] ## Clusters # cluster_include = [ "/*/host/**"] # Inventory path to clusters to collect (by default all are collected) + # cluster_exclude = [] # Inventory paths to exclude # cluster_metric_include = [] ## if omitted or empty, all metrics are collected # cluster_metric_exclude = [] ## Nothing excluded by default # cluster_instances = false ## false by default ## Datastores # cluster_include = [ "/*/datastore/**"] # Inventory path to datastores to collect (by default all are collected) + # cluster_exclude = [] # Inventory paths to exclude # datastore_metric_include = [] ## if omitted or empty, all metrics are collected # datastore_metric_exclude = [] ## Nothing excluded by default # datastore_instances = false ## false by default ## Datacenters # datacenter_include = [ "/*/host/**"] # Inventory path to clusters to collect (by default all are collected) + # datacenter_exclude = [] # Inventory paths to exclude datacenter_metric_include = [] ## if omitted or empty, all metrics are collected datacenter_metric_exclude = [ "*" ] ## Datacenters are not collected by default. # datacenter_instances = false ## false by default diff --git a/plugins/inputs/vsphere/client.go b/plugins/inputs/vsphere/client.go index b514813ab03bf..176f483231904 100644 --- a/plugins/inputs/vsphere/client.go +++ b/plugins/inputs/vsphere/client.go @@ -4,13 +4,13 @@ import ( "context" "crypto/tls" "fmt" - "log" "net/url" "strconv" "strings" "sync" "time" + "github.com/influxdata/telegraf" "github.com/vmware/govmomi" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/performance" @@ -45,6 +45,7 @@ type Client struct { Valid bool Timeout time.Duration closeGate sync.Once + log telegraf.Logger } // NewClientFactory creates a new ClientFactory and prepares it for use. @@ -76,7 +77,7 @@ func (cf *ClientFactory) GetClient(ctx context.Context) (*Client, error) { ctx1, cancel1 := context.WithTimeout(ctx, cf.parent.Timeout.Duration) defer cancel1() if _, err := methods.GetCurrentTime(ctx1, cf.client.Client); err != nil { - log.Printf("I! [inputs.vsphere]: Client session seems to have time out. Reauthenticating!") + cf.parent.Log.Info("Client session seems to have time out. Reauthenticating!") ctx2, cancel2 := context.WithTimeout(ctx, cf.parent.Timeout.Duration) defer cancel2() if err := cf.client.Client.SessionManager.Login(ctx2, url.UserPassword(cf.parent.Username, cf.parent.Password)); err != nil { @@ -88,7 +89,7 @@ func (cf *ClientFactory) GetClient(ctx context.Context) (*Client, error) { cf.client = nil continue } - return nil, fmt.Errorf("Renewing authentication failed: %v", err) + return nil, fmt.Errorf("renewing authentication failed: %s", err.Error()) } } @@ -113,7 +114,7 @@ func NewClient(ctx context.Context, u *url.URL, vs *VSphere) (*Client, error) { u.User = url.UserPassword(vs.Username, vs.Password) } - log.Printf("D! [inputs.vsphere]: Creating client: %s", u.Host) + vs.Log.Debugf("Creating client: %s", u.Host) soapClient := soap.NewClient(u, tlsCfg.InsecureSkipVerify) // Add certificate if we have it. Use it to log us in. @@ -170,6 +171,7 @@ func NewClient(ctx context.Context, u *url.URL, vs *VSphere) (*Client, error) { p := performance.NewManager(c.Client) client := &Client{ + log: vs.Log, Client: c, Views: m, Root: v, @@ -184,9 +186,9 @@ func NewClient(ctx context.Context, u *url.URL, vs *VSphere) (*Client, error) { if err != nil { return nil, err } - log.Printf("D! [inputs.vsphere] vCenter says max_query_metrics should be %d", n) + vs.Log.Debugf("vCenter says max_query_metrics should be %d", n) if n < vs.MaxQueryMetrics { - log.Printf("W! [inputs.vsphere] Configured max_query_metrics is %d, but server limits it to %d. Reducing.", vs.MaxQueryMetrics, n) + vs.Log.Warnf("Configured max_query_metrics is %d, but server limits it to %d. Reducing.", vs.MaxQueryMetrics, n) vs.MaxQueryMetrics = n } return client, nil @@ -202,7 +204,6 @@ func (cf *ClientFactory) Close() { } func (c *Client) close() { - // Use a Once to prevent us from panics stemming from trying // to close it multiple times. c.closeGate.Do(func() { @@ -210,7 +211,7 @@ func (c *Client) close() { defer cancel() if c.Client != nil { if err := c.Client.Logout(ctx); err != nil { - log.Printf("E! [inputs.vsphere]: Error during logout: %s", err) + c.log.Errorf("Logout: %s", err.Error()) } } }) @@ -239,7 +240,7 @@ func (c *Client) GetMaxQueryMetrics(ctx context.Context) (int, error) { if s, ok := res[0].GetOptionValue().Value.(string); ok { v, err := strconv.Atoi(s) if err == nil { - log.Printf("D! [inputs.vsphere] vCenter maxQueryMetrics is defined: %d", v) + c.log.Debugf("vCenter maxQueryMetrics is defined: %d", v) if v == -1 { // Whatever the server says, we never ask for more metrics than this. return absoluteMaxMetrics, nil @@ -250,17 +251,17 @@ func (c *Client) GetMaxQueryMetrics(ctx context.Context) (int, error) { // Fall through version-based inference if value isn't usable } } else { - log.Println("D! [inputs.vsphere] Option query for maxQueryMetrics failed. Using default") + c.log.Debug("Option query for maxQueryMetrics failed. Using default") } // No usable maxQueryMetrics setting. Infer based on version ver := c.Client.Client.ServiceContent.About.Version parts := strings.Split(ver, ".") if len(parts) < 2 { - log.Printf("W! [inputs.vsphere] vCenter returned an invalid version string: %s. Using default query size=64", ver) + c.log.Warnf("vCenter returned an invalid version string: %s. Using default query size=64", ver) return 64, nil } - log.Printf("D! [inputs.vsphere] vCenter version is: %s", ver) + c.log.Debugf("vCenter version is: %s", ver) major, err := strconv.Atoi(parts[0]) if err != nil { return 0, err diff --git a/plugins/inputs/vsphere/endpoint.go b/plugins/inputs/vsphere/endpoint.go index c361754ab0cae..c049e495fa3ff 100644 --- a/plugins/inputs/vsphere/endpoint.go +++ b/plugins/inputs/vsphere/endpoint.go @@ -32,8 +32,6 @@ var isIPv6 = regexp.MustCompile("^(?:[A-Fa-f0-9]{0,4}:){1,7}[A-Fa-f0-9]{1,4}$") const metricLookback = 3 // Number of time periods to look back at for non-realtime metrics -const rtMetricLookback = 3 // Number of time periods to look back at for realtime metrics - const maxSampleConst = 10 // Absolute maximim number of samples regardless of period const maxMetadataSamples = 100 // Number of resources to sample for metric metadata @@ -67,6 +65,7 @@ type resourceKind struct { objects objectMap filters filter.Filter paths []string + excludePaths []string collectInstances bool getObjects func(context.Context, *Endpoint, *ResourceFilter) (objectMap, error) include []string @@ -84,7 +83,7 @@ type metricEntry struct { fields map[string]interface{} } -type objectMap map[string]objectRef +type objectMap map[string]*objectRef type objectRef struct { name string @@ -100,7 +99,7 @@ type objectRef struct { func (e *Endpoint) getParent(obj *objectRef, res *resourceKind) (*objectRef, bool) { if pKind, ok := e.resourceKinds[res.parent]; ok { if p, ok := pKind.objects[obj.parentRef.Value]; ok { - return &p, true + return p, true } } return nil, false @@ -132,6 +131,7 @@ func NewEndpoint(ctx context.Context, parent *VSphere, url *url.URL) (*Endpoint, objects: make(objectMap), filters: newFilterOrPanic(parent.DatacenterMetricInclude, parent.DatacenterMetricExclude), paths: parent.DatacenterInclude, + excludePaths: parent.DatacenterExclude, simple: isSimple(parent.DatacenterMetricInclude, parent.DatacenterMetricExclude), include: parent.DatacenterMetricInclude, collectInstances: parent.DatacenterInstances, @@ -149,6 +149,7 @@ func NewEndpoint(ctx context.Context, parent *VSphere, url *url.URL) (*Endpoint, objects: make(objectMap), filters: newFilterOrPanic(parent.ClusterMetricInclude, parent.ClusterMetricExclude), paths: parent.ClusterInclude, + excludePaths: parent.ClusterExclude, simple: isSimple(parent.ClusterMetricInclude, parent.ClusterMetricExclude), include: parent.ClusterMetricInclude, collectInstances: parent.ClusterInstances, @@ -166,6 +167,7 @@ func NewEndpoint(ctx context.Context, parent *VSphere, url *url.URL) (*Endpoint, objects: make(objectMap), filters: newFilterOrPanic(parent.HostMetricInclude, parent.HostMetricExclude), paths: parent.HostInclude, + excludePaths: parent.HostExclude, simple: isSimple(parent.HostMetricInclude, parent.HostMetricExclude), include: parent.HostMetricInclude, collectInstances: parent.HostInstances, @@ -183,6 +185,7 @@ func NewEndpoint(ctx context.Context, parent *VSphere, url *url.URL) (*Endpoint, objects: make(objectMap), filters: newFilterOrPanic(parent.VMMetricInclude, parent.VMMetricExclude), paths: parent.VMInclude, + excludePaths: parent.VMExclude, simple: isSimple(parent.VMMetricInclude, parent.VMMetricExclude), include: parent.VMMetricInclude, collectInstances: parent.VMInstances, @@ -199,6 +202,7 @@ func NewEndpoint(ctx context.Context, parent *VSphere, url *url.URL) (*Endpoint, objects: make(objectMap), filters: newFilterOrPanic(parent.DatastoreMetricInclude, parent.DatastoreMetricExclude), paths: parent.DatastoreInclude, + excludePaths: parent.DatastoreExclude, simple: isSimple(parent.DatastoreMetricInclude, parent.DatastoreMetricExclude), include: parent.DatastoreMetricInclude, collectInstances: parent.DatastoreInstances, @@ -250,10 +254,10 @@ func (e *Endpoint) startDiscovery(ctx context.Context) { case <-e.discoveryTicker.C: err := e.discover(ctx) if err != nil && err != context.Canceled { - log.Printf("E! [inputs.vsphere]: Error in discovery for %s: %v", e.URL.Host, err) + e.Parent.Log.Errorf("Discovery for %s: %s", e.URL.Host, err.Error()) } case <-ctx.Done(): - log.Printf("D! [inputs.vsphere]: Exiting discovery goroutine for %s", e.URL.Host) + e.Parent.Log.Debugf("Exiting discovery goroutine for %s", e.URL.Host) e.discoveryTicker.Stop() return } @@ -264,7 +268,7 @@ func (e *Endpoint) startDiscovery(ctx context.Context) { func (e *Endpoint) initalDiscovery(ctx context.Context) { err := e.discover(ctx) if err != nil && err != context.Canceled { - log.Printf("E! [inputs.vsphere]: Error in discovery for %s: %v", e.URL.Host, err) + e.Parent.Log.Errorf("Discovery for %s: %s", e.URL.Host, err.Error()) } e.startDiscovery(ctx) } @@ -279,27 +283,15 @@ func (e *Endpoint) init(ctx context.Context) error { if e.customAttrEnabled { fields, err := client.GetCustomFields(ctx) if err != nil { - log.Println("W! [inputs.vsphere] Could not load custom field metadata") + e.Parent.Log.Warn("Could not load custom field metadata") } else { e.customFields = fields } } if e.Parent.ObjectDiscoveryInterval.Duration > 0 { - - // Run an initial discovery. If force_discovery_on_init isn't set, we kick it off as a - // goroutine without waiting for it. This will probably cause us to report an empty - // dataset on the first collection, but it solves the issue of the first collection timing out. - if e.Parent.ForceDiscoverOnInit { - log.Printf("D! [inputs.vsphere]: Running initial discovery and waiting for it to finish") - e.initalDiscovery(ctx) - } else { - // Otherwise, just run it in the background. We'll probably have an incomplete first metric - // collection this way. - go func() { - e.initalDiscovery(ctx) - }() - } + e.Parent.Log.Debug("Running initial discovery") + e.initalDiscovery(ctx) } e.initialized = true return nil @@ -322,7 +314,7 @@ func (e *Endpoint) getMetricNameMap(ctx context.Context) (map[int32]string, erro return names, nil } -func (e *Endpoint) getMetadata(ctx context.Context, obj objectRef, sampling int32) (performance.MetricList, error) { +func (e *Endpoint) getMetadata(ctx context.Context, obj *objectRef, sampling int32) (performance.MetricList, error) { client, err := e.clientFactory.GetClient(ctx) if err != nil { return nil, err @@ -341,32 +333,36 @@ func (e *Endpoint) getDatacenterName(ctx context.Context, client *Client, cache path := make([]string, 0) returnVal := "" here := r - for { - if name, ok := cache[here.Reference().String()]; ok { - // Populate cache for the entire chain of objects leading here. - returnVal = name - break - } - path = append(path, here.Reference().String()) - o := object.NewCommon(client.Client.Client, r) - var result mo.ManagedEntity - ctx1, cancel1 := context.WithTimeout(ctx, e.Parent.Timeout.Duration) - defer cancel1() - err := o.Properties(ctx1, here, []string{"parent", "name"}, &result) - if err != nil { - log.Printf("W! [inputs.vsphere]: Error while resolving parent. Assuming no parent exists. Error: %s", err) - break - } - if result.Reference().Type == "Datacenter" { - // Populate cache for the entire chain of objects leading here. - returnVal = result.Name - break - } - if result.Parent == nil { - log.Printf("D! [inputs.vsphere]: No parent found for %s (ascending from %s)", here.Reference(), r.Reference()) - break - } - here = result.Parent.Reference() + done := false + for !done { + done = func() bool { + if name, ok := cache[here.Reference().String()]; ok { + // Populate cache for the entire chain of objects leading here. + returnVal = name + return true + } + path = append(path, here.Reference().String()) + o := object.NewCommon(client.Client.Client, r) + var result mo.ManagedEntity + ctx1, cancel1 := context.WithTimeout(ctx, e.Parent.Timeout.Duration) + defer cancel1() + err := o.Properties(ctx1, here, []string{"parent", "name"}, &result) + if err != nil { + e.Parent.Log.Warnf("Error while resolving parent. Assuming no parent exists. Error: %s", err.Error()) + return true + } + if result.Reference().Type == "Datacenter" { + // Populate cache for the entire chain of objects leading here. + returnVal = result.Name + return true + } + if result.Parent == nil { + e.Parent.Log.Debugf("No parent found for %s (ascending from %s)", here.Reference(), r.Reference()) + return true + } + here = result.Parent.Reference() + return false + }() } for _, s := range path { cache[s] = returnVal @@ -393,7 +389,7 @@ func (e *Endpoint) discover(ctx context.Context) error { return err } - log.Printf("D! [inputs.vsphere]: Discover new objects for %s", e.URL.Host) + e.Parent.Log.Debugf("Discover new objects for %s", e.URL.Host) dcNameCache := make(map[string]string) numRes := int64(0) @@ -401,43 +397,51 @@ func (e *Endpoint) discover(ctx context.Context) error { // Populate resource objects, and endpoint instance info. newObjects := make(map[string]objectMap) for k, res := range e.resourceKinds { - log.Printf("D! [inputs.vsphere] Discovering resources for %s", res.name) - // Need to do this for all resource types even if they are not enabled - if res.enabled || k != "vm" { - rf := ResourceFilter{ - finder: &Finder{client}, - resType: res.vcName, - paths: res.paths} + err := func() error { + e.Parent.Log.Debugf("Discovering resources for %s", res.name) + // Need to do this for all resource types even if they are not enabled + if res.enabled || k != "vm" { + rf := ResourceFilter{ + finder: &Finder{client}, + resType: res.vcName, + paths: res.paths, + excludePaths: res.excludePaths, + } - ctx1, cancel1 := context.WithTimeout(ctx, e.Parent.Timeout.Duration) - defer cancel1() - objects, err := res.getObjects(ctx1, e, &rf) - if err != nil { - return err - } + ctx1, cancel1 := context.WithTimeout(ctx, e.Parent.Timeout.Duration) + defer cancel1() + objects, err := res.getObjects(ctx1, e, &rf) + if err != nil { + return err + } - // Fill in datacenter names where available (no need to do it for Datacenters) - if res.name != "Datacenter" { - for k, obj := range objects { - if obj.parentRef != nil { - obj.dcname = e.getDatacenterName(ctx, client, dcNameCache, *obj.parentRef) - objects[k] = obj + // Fill in datacenter names where available (no need to do it for Datacenters) + if res.name != "Datacenter" { + for k, obj := range objects { + if obj.parentRef != nil { + obj.dcname = e.getDatacenterName(ctx, client, dcNameCache, *obj.parentRef) + objects[k] = obj + } } } - } - // No need to collect metric metadata if resource type is not enabled - if res.enabled { - if res.simple { - e.simpleMetadataSelect(ctx, client, res) - } else { - e.complexMetadataSelect(ctx, res, objects, metricNames) + // No need to collect metric metadata if resource type is not enabled + if res.enabled { + if res.simple { + e.simpleMetadataSelect(ctx, client, res) + } else { + e.complexMetadataSelect(ctx, res, objects, metricNames) + } } - } - newObjects[k] = objects + newObjects[k] = objects - SendInternalCounterWithTags("discovered_objects", e.URL.Host, map[string]string{"type": res.name}, int64(len(objects))) - numRes += int64(len(objects)) + SendInternalCounterWithTags("discovered_objects", e.URL.Host, map[string]string{"type": res.name}, int64(len(objects))) + numRes += int64(len(objects)) + } + return nil + }() + if err != nil { + return err } } @@ -445,8 +449,8 @@ func (e *Endpoint) discover(ctx context.Context) error { dss := newObjects["datastore"] l2d := make(map[string]string) for _, ds := range dss { - url := ds.altID - m := isolateLUN.FindStringSubmatch(url) + lunId := ds.altID + m := isolateLUN.FindStringSubmatch(lunId) if m != nil { l2d[m[1]] = ds.name } @@ -457,7 +461,7 @@ func (e *Endpoint) discover(ctx context.Context) error { if e.customAttrEnabled { fields, err = client.GetCustomFields(ctx) if err != nil { - log.Println("W! [inputs.vsphere] Could not load custom field metadata") + e.Parent.Log.Warn("Could not load custom field metadata") fields = nil } } @@ -481,10 +485,10 @@ func (e *Endpoint) discover(ctx context.Context) error { } func (e *Endpoint) simpleMetadataSelect(ctx context.Context, client *Client, res *resourceKind) { - log.Printf("D! [inputs.vsphere] Using fast metric metadata selection for %s", res.name) + e.Parent.Log.Debugf("Using fast metric metadata selection for %s", res.name) m, err := client.CounterInfoByName(ctx) if err != nil { - log.Printf("E! [inputs.vsphere]: Error while getting metric metadata. Discovery will be incomplete. Error: %s", err) + e.Parent.Log.Errorf("Getting metric metadata. Discovery will be incomplete. Error: %s", err.Error()) return } res.metrics = make(performance.MetricList, 0, len(res.include)) @@ -500,7 +504,7 @@ func (e *Endpoint) simpleMetadataSelect(ctx context.Context, client *Client, res } res.metrics = append(res.metrics, cnt) } else { - log.Printf("W! [inputs.vsphere] Metric name %s is unknown. Will not be collected", s) + e.Parent.Log.Warnf("Metric name %s is unknown. Will not be collected", s) } } } @@ -508,7 +512,7 @@ func (e *Endpoint) simpleMetadataSelect(ctx context.Context, client *Client, res func (e *Endpoint) complexMetadataSelect(ctx context.Context, res *resourceKind, objects objectMap, metricNames map[int32]string) { // We're only going to get metadata from maxMetadataSamples resources. If we have // more resources than that, we pick maxMetadataSamples samples at random. - sampledObjects := make([]objectRef, len(objects)) + sampledObjects := make([]*objectRef, len(objects)) i := 0 for _, obj := range objects { sampledObjects[i] = obj @@ -529,11 +533,11 @@ func (e *Endpoint) complexMetadataSelect(ctx context.Context, res *resourceKind, instInfoMux := sync.Mutex{} te := NewThrottledExecutor(e.Parent.DiscoverConcurrency) for _, obj := range sampledObjects { - func(obj objectRef) { + func(obj *objectRef) { te.Run(ctx, func() { metrics, err := e.getMetadata(ctx, obj, res.sampling) if err != nil { - log.Printf("E! [inputs.vsphere]: Error while getting metric metadata. Discovery will be incomplete. Error: %s", err) + e.Parent.Log.Errorf("Getting metric metadata. Discovery will be incomplete. Error: %s", err.Error()) } mMap := make(map[string]types.PerfMetricId) for _, m := range metrics { @@ -546,7 +550,7 @@ func (e *Endpoint) complexMetadataSelect(ctx context.Context, res *resourceKind, mMap[strconv.Itoa(int(m.CounterId))+"|"+m.Instance] = m } } - log.Printf("D! [inputs.vsphere] Found %d metrics for %s", len(mMap), obj.name) + e.Parent.Log.Debugf("Found %d metrics for %s", len(mMap), obj.name) instInfoMux.Lock() defer instInfoMux.Unlock() if len(mMap) > len(res.metrics) { @@ -573,8 +577,13 @@ func getDatacenters(ctx context.Context, e *Endpoint, filter *ResourceFilter) (o } m := make(objectMap, len(resources)) for _, r := range resources { - m[r.ExtensibleManagedObject.Reference().Value] = objectRef{ - name: r.Name, ref: r.ExtensibleManagedObject.Reference(), parentRef: r.Parent, dcname: r.Name} + m[r.ExtensibleManagedObject.Reference().Value] = &objectRef{ + name: r.Name, + ref: r.ExtensibleManagedObject.Reference(), + parentRef: r.Parent, + dcname: r.Name, + customValues: e.loadCustomAttributes(&r.ManagedEntity), + } } return m, nil } @@ -590,35 +599,47 @@ func getClusters(ctx context.Context, e *Endpoint, filter *ResourceFilter) (obje cache := make(map[string]*types.ManagedObjectReference) m := make(objectMap, len(resources)) for _, r := range resources { - // We're not interested in the immediate parent (a folder), but the data center. - p, ok := cache[r.Parent.Value] - if !ok { - ctx2, cancel2 := context.WithTimeout(ctx, e.Parent.Timeout.Duration) - defer cancel2() - client, err := e.clientFactory.GetClient(ctx2) - if err != nil { - return nil, err + // Wrap in a function to make defer work correctly. + err := func() error { + // We're not interested in the immediate parent (a folder), but the data center. + p, ok := cache[r.Parent.Value] + if !ok { + ctx2, cancel2 := context.WithTimeout(ctx, e.Parent.Timeout.Duration) + defer cancel2() + client, err := e.clientFactory.GetClient(ctx2) + if err != nil { + return err + } + o := object.NewFolder(client.Client.Client, *r.Parent) + var folder mo.Folder + ctx3, cancel3 := context.WithTimeout(ctx, e.Parent.Timeout.Duration) + defer cancel3() + err = o.Properties(ctx3, *r.Parent, []string{"parent"}, &folder) + if err != nil { + e.Parent.Log.Warnf("Error while getting folder parent: %s", err.Error()) + p = nil + } else { + pp := folder.Parent.Reference() + p = &pp + cache[r.Parent.Value] = p + } } - o := object.NewFolder(client.Client.Client, *r.Parent) - var folder mo.Folder - ctx3, cancel3 := context.WithTimeout(ctx, e.Parent.Timeout.Duration) - defer cancel3() - err = o.Properties(ctx3, *r.Parent, []string{"parent"}, &folder) - if err != nil { - log.Printf("W! [inputs.vsphere] Error while getting folder parent: %e", err) - p = nil - } else { - pp := folder.Parent.Reference() - p = &pp - cache[r.Parent.Value] = p + m[r.ExtensibleManagedObject.Reference().Value] = &objectRef{ + name: r.Name, + ref: r.ExtensibleManagedObject.Reference(), + parentRef: p, + customValues: e.loadCustomAttributes(&r.ManagedEntity), } + return nil + }() + if err != nil { + return nil, err } - m[r.ExtensibleManagedObject.Reference().Value] = objectRef{ - name: r.Name, ref: r.ExtensibleManagedObject.Reference(), parentRef: p} } return m, nil } +//noinspection GoUnusedParameter func getHosts(ctx context.Context, e *Endpoint, filter *ResourceFilter) (objectMap, error) { var resources []mo.HostSystem err := filter.FindAll(ctx, &resources) @@ -627,8 +648,12 @@ func getHosts(ctx context.Context, e *Endpoint, filter *ResourceFilter) (objectM } m := make(objectMap) for _, r := range resources { - m[r.ExtensibleManagedObject.Reference().Value] = objectRef{ - name: r.Name, ref: r.ExtensibleManagedObject.Reference(), parentRef: r.Parent} + m[r.ExtensibleManagedObject.Reference().Value] = &objectRef{ + name: r.Name, + ref: r.ExtensibleManagedObject.Reference(), + parentRef: r.Parent, + customValues: e.loadCustomAttributes(&r.ManagedEntity), + } } return m, nil } @@ -693,30 +718,13 @@ func getVMs(ctx context.Context, e *Endpoint, filter *ResourceFilter) (objectMap guest = cleanGuestID(r.Config.GuestId) uuid = r.Config.Uuid } - cvs := make(map[string]string) - if e.customAttrEnabled { - for _, cv := range r.Summary.CustomValue { - val := cv.(*types.CustomFieldStringValue) - if val.Value == "" { - continue - } - key, ok := e.customFields[val.Key] - if !ok { - log.Printf("W! [inputs.vsphere] Metadata for custom field %d not found. Skipping", val.Key) - continue - } - if e.customAttrFilter.Match(key) { - cvs[key] = val.Value - } - } - } - m[r.ExtensibleManagedObject.Reference().Value] = objectRef{ + m[r.ExtensibleManagedObject.Reference().Value] = &objectRef{ name: r.Name, ref: r.ExtensibleManagedObject.Reference(), parentRef: r.Runtime.Host, guest: guest, altID: uuid, - customValues: cvs, + customValues: e.loadCustomAttributes(&r.ManagedEntity), lookup: lookup, } } @@ -733,19 +741,47 @@ func getDatastores(ctx context.Context, e *Endpoint, filter *ResourceFilter) (ob } m := make(objectMap) for _, r := range resources { - url := "" + lunId := "" if r.Info != nil { info := r.Info.GetDatastoreInfo() if info != nil { - url = info.Url + lunId = info.Url } } - m[r.ExtensibleManagedObject.Reference().Value] = objectRef{ - name: r.Name, ref: r.ExtensibleManagedObject.Reference(), parentRef: r.Parent, altID: url} + m[r.ExtensibleManagedObject.Reference().Value] = &objectRef{ + name: r.Name, + ref: r.ExtensibleManagedObject.Reference(), + parentRef: r.Parent, + altID: lunId, + customValues: e.loadCustomAttributes(&r.ManagedEntity), + } } return m, nil } +func (e *Endpoint) loadCustomAttributes(entity *mo.ManagedEntity) map[string]string { + if !e.customAttrEnabled { + return map[string]string{} + } + cvs := make(map[string]string) + for _, v := range entity.CustomValue { + cv, ok := v.(*types.CustomFieldStringValue) + if !ok { + e.Parent.Log.Warnf("Metadata for custom field %d not of string type. Skipping", cv.Key) + continue + } + key, ok := e.customFields[cv.Key] + if !ok { + e.Parent.Log.Warnf("Metadata for custom field %d not found. Skipping", cv.Key) + continue + } + if e.customAttrFilter.Match(key) { + cvs[key] = cv.Value + } + } + return cvs +} + // Close shuts down an Endpoint and releases any resources associated with it. func (e *Endpoint) Close() { e.clientFactory.Close() @@ -802,7 +838,7 @@ func submitChunkJob(ctx context.Context, te *ThrottledExecutor, job func([]types }) } -func (e *Endpoint) chunkify(ctx context.Context, res *resourceKind, now time.Time, latest time.Time, acc telegraf.Accumulator, job func([]types.PerfQuerySpec)) { +func (e *Endpoint) chunkify(ctx context.Context, res *resourceKind, now time.Time, latest time.Time, job func([]types.PerfQuerySpec)) { te := NewThrottledExecutor(e.Parent.CollectConcurrency) maxMetrics := e.Parent.MaxQueryMetrics if maxMetrics < 1 { @@ -819,7 +855,7 @@ func (e *Endpoint) chunkify(ctx context.Context, res *resourceKind, now time.Tim metrics := 0 total := 0 nRes := 0 - for _, object := range res.objects { + for _, resource := range res.objects { mr := len(res.metrics) for mr > 0 { mc := mr @@ -829,14 +865,14 @@ func (e *Endpoint) chunkify(ctx context.Context, res *resourceKind, now time.Tim } fm := len(res.metrics) - mr pq := types.PerfQuerySpec{ - Entity: object.ref, + Entity: resource.ref, MaxSample: maxSampleConst, MetricId: res.metrics[fm : fm+mc], IntervalId: res.sampling, Format: "normal", } - start, ok := e.hwMarks.Get(object.ref.Value) + start, ok := e.hwMarks.Get(resource.ref.Value) if !ok { // Look back 3 sampling periods by default start = latest.Add(time.Duration(-res.sampling) * time.Second * (metricLookback - 1)) @@ -847,7 +883,7 @@ func (e *Endpoint) chunkify(ctx context.Context, res *resourceKind, now time.Tim // Make sure endtime is always after start time. We may occasionally see samples from the future // returned from vCenter. This is presumably due to time drift between vCenter and EXSi nodes. if pq.StartTime.After(*pq.EndTime) { - log.Printf("D! [inputs.vsphere] Future sample. Res: %s, StartTime: %s, EndTime: %s, Now: %s", pq.Entity, *pq.StartTime, *pq.EndTime, now) + e.Parent.Log.Debugf("Future sample. Res: %s, StartTime: %s, EndTime: %s, Now: %s", pq.Entity, *pq.StartTime, *pq.EndTime, now) end := start.Add(time.Second) pq.EndTime = &end } @@ -861,7 +897,7 @@ func (e *Endpoint) chunkify(ctx context.Context, res *resourceKind, now time.Tim // 2) We are at the last resource and have no more data to process. // 3) The query contains more than 100,000 individual metrics if mr > 0 || nRes >= e.Parent.MaxQueryObjects || len(pqs) > 100000 { - log.Printf("D! [inputs.vsphere]: Queueing query: %d objects, %d metrics (%d remaining) of type %s for %s. Processed objects: %d. Total objects %d", + e.Parent.Log.Debugf("Queueing query: %d objects, %d metrics (%d remaining) of type %s for %s. Processed objects: %d. Total objects %d", len(pqs), metrics, mr, res.name, e.URL.Host, total+1, len(res.objects)) // Don't send work items if the context has been cancelled. @@ -882,7 +918,7 @@ func (e *Endpoint) chunkify(ctx context.Context, res *resourceKind, now time.Tim // Handle final partially filled chunk if len(pqs) > 0 { // Run collection job - log.Printf("D! [inputs.vsphere]: Queuing query: %d objects, %d metrics (0 remaining) of type %s for %s. Total objects %d (final chunk)", + e.Parent.Log.Debugf("Queuing query: %d objects, %d metrics (0 remaining) of type %s for %s. Total objects %d (final chunk)", len(pqs), metrics, res.name, e.URL.Host, len(res.objects)) submitChunkJob(ctx, te, job, pqs) } @@ -905,7 +941,7 @@ func (e *Endpoint) collectResource(ctx context.Context, resourceType string, acc // Estimate the interval at which we're invoked. Use local time (not server time) // since this is about how we got invoked locally. localNow := time.Now() - estInterval := time.Duration(time.Minute) + estInterval := time.Minute if !res.lastColl.IsZero() { s := time.Duration(res.sampling) * time.Second rawInterval := localNow.Sub(res.lastColl) @@ -914,18 +950,18 @@ func (e *Endpoint) collectResource(ctx context.Context, resourceType string, acc if estInterval < s { estInterval = s } - log.Printf("D! [inputs.vsphere] Raw interval %s, padded: %s, estimated: %s", rawInterval, paddedInterval, estInterval) + e.Parent.Log.Debugf("Raw interval %s, padded: %s, estimated: %s", rawInterval, paddedInterval, estInterval) } - log.Printf("D! [inputs.vsphere] Interval estimated to %s", estInterval) + e.Parent.Log.Debugf("Interval estimated to %s", estInterval) res.lastColl = localNow latest := res.latestSample if !latest.IsZero() { elapsed := now.Sub(latest).Seconds() + 5.0 // Allow 5 second jitter. - log.Printf("D! [inputs.vsphere]: Latest: %s, elapsed: %f, resource: %s", latest, elapsed, resourceType) + e.Parent.Log.Debugf("Latest: %s, elapsed: %f, resource: %s", latest, elapsed, resourceType) if !res.realTime && elapsed < float64(res.sampling) { // No new data would be available. We're outta here! - log.Printf("D! [inputs.vsphere]: Sampling period for %s of %d has not elapsed on %s", + e.Parent.Log.Debugf("Sampling period for %s of %d has not elapsed on %s", resourceType, res.sampling, e.URL.Host) return nil } @@ -936,7 +972,7 @@ func (e *Endpoint) collectResource(ctx context.Context, resourceType string, acc internalTags := map[string]string{"resourcetype": resourceType} sw := NewStopwatchWithTags("gather_duration", e.URL.Host, internalTags) - log.Printf("D! [inputs.vsphere]: Collecting metrics for %d objects of type %s for %s", + e.Parent.Log.Debugf("Collecting metrics for %d objects of type %s for %s", len(res.objects), resourceType, e.URL.Host) count := int64(0) @@ -945,13 +981,14 @@ func (e *Endpoint) collectResource(ctx context.Context, resourceType string, acc latestSample := time.Time{} // Divide workload into chunks and process them concurrently - e.chunkify(ctx, res, now, latest, acc, + e.chunkify(ctx, res, now, latest, func(chunk []types.PerfQuerySpec) { - n, localLatest, err := e.collectChunk(ctx, chunk, res, acc, now, estInterval) - log.Printf("D! [inputs.vsphere] CollectChunk for %s returned %d metrics", resourceType, n) + n, localLatest, err := e.collectChunk(ctx, chunk, res, acc, estInterval) if err != nil { - acc.AddError(errors.New("While collecting " + res.name + ": " + err.Error())) + acc.AddError(errors.New("while collecting " + res.name + ": " + err.Error())) + return } + e.Parent.Log.Debugf("CollectChunk for %s returned %d metrics", resourceType, n) atomic.AddInt64(&count, int64(n)) tsMux.Lock() defer tsMux.Unlock() @@ -960,7 +997,7 @@ func (e *Endpoint) collectResource(ctx context.Context, resourceType string, acc } }) - log.Printf("D! [inputs.vsphere] Latest sample for %s set to %s", resourceType, latestSample) + e.Parent.Log.Debugf("Latest sample for %s set to %s", resourceType, latestSample) if !latestSample.IsZero() { res.latestSample = latestSample } @@ -992,7 +1029,7 @@ func alignSamples(info []types.PerfSampleInfo, values []int64, interval time.Dur if roundedTs == lastBucket { bi++ p := len(rValues) - 1 - rValues[p] = ((bi-1)/bi)*float64(rValues[p]) + v/bi + rValues[p] = ((bi-1)/bi)*rValues[p] + v/bi } else { rValues = append(rValues, v) roundedInfo := types.PerfSampleInfo{ @@ -1004,12 +1041,11 @@ func alignSamples(info []types.PerfSampleInfo, values []int64, interval time.Dur lastBucket = roundedTs } } - //log.Printf("D! [inputs.vsphere] Aligned samples: %d collapsed into %d", len(info), len(rInfo)) return rInfo, rValues } -func (e *Endpoint) collectChunk(ctx context.Context, pqs []types.PerfQuerySpec, res *resourceKind, acc telegraf.Accumulator, now time.Time, interval time.Duration) (int, time.Time, error) { - log.Printf("D! [inputs.vsphere] Query for %s has %d QuerySpecs", res.name, len(pqs)) +func (e *Endpoint) collectChunk(ctx context.Context, pqs []types.PerfQuerySpec, res *resourceKind, acc telegraf.Accumulator, interval time.Duration) (int, time.Time, error) { + e.Parent.Log.Debugf("Query for %s has %d QuerySpecs", res.name, len(pqs)) latestSample := time.Time{} count := 0 resourceType := res.name @@ -1030,14 +1066,14 @@ func (e *Endpoint) collectChunk(ctx context.Context, pqs []types.PerfQuerySpec, return count, latestSample, err } - log.Printf("D! [inputs.vsphere] Query for %s returned metrics for %d objects", resourceType, len(ems)) + e.Parent.Log.Debugf("Query for %s returned metrics for %d objects", resourceType, len(ems)) // Iterate through results for _, em := range ems { moid := em.Entity.Reference().Value instInfo, found := res.objects[moid] if !found { - log.Printf("E! [inputs.vsphere]: MOID %s not found in cache. Skipping! (This should not happen!)", moid) + e.Parent.Log.Errorf("MOID %s not found in cache. Skipping! (This should not happen!)", moid) continue } buckets := make(map[string]metricEntry) @@ -1052,10 +1088,10 @@ func (e *Endpoint) collectChunk(ctx context.Context, pqs []types.PerfQuerySpec, // Populate tags objectRef, ok := res.objects[moid] if !ok { - log.Printf("E! [inputs.vsphere]: MOID %s not found in cache. Skipping", moid) + e.Parent.Log.Errorf("MOID %s not found in cache. Skipping", moid) continue } - e.populateTags(&objectRef, resourceType, res, t, &v) + e.populateTags(objectRef, resourceType, res, t, &v) nValues := 0 alignedInfo, alignedValues := alignSamples(em.SampleInfo, v.Value, interval) @@ -1064,7 +1100,7 @@ func (e *Endpoint) collectChunk(ctx context.Context, pqs []types.PerfQuerySpec, // According to the docs, SampleInfo and Value should have the same length, but we've seen corrupted // data coming back with missing values. Take care of that gracefully! if idx >= len(alignedValues) { - log.Printf("D! [inputs.vsphere] len(SampleInfo)>len(Value) %d > %d", len(alignedInfo), len(alignedValues)) + e.Parent.Log.Debugf("Len(SampleInfo)>len(Value) %d > %d", len(alignedInfo), len(alignedValues)) break } ts := sample.Timestamp @@ -1085,11 +1121,11 @@ func (e *Endpoint) collectChunk(ctx context.Context, pqs []types.PerfQuerySpec, // Percentage values must be scaled down by 100. info, ok := metricInfo[name] if !ok { - log.Printf("E! [inputs.vsphere]: Could not determine unit for %s. Skipping", name) + e.Parent.Log.Errorf("Could not determine unit for %s. Skipping", name) } v := alignedValues[idx] if info.UnitInfo.GetElementDescription().Key == "percent" { - bucket.fields[fn] = float64(v) / 100.0 + bucket.fields[fn] = v / 100.0 } else { if e.Parent.UseIntSamples { bucket.fields[fn] = int64(round(v)) @@ -1103,7 +1139,7 @@ func (e *Endpoint) collectChunk(ctx context.Context, pqs []types.PerfQuerySpec, e.hwMarks.Put(moid, ts) } if nValues == 0 { - log.Printf("D! [inputs.vsphere]: Missing value for: %s, %s", name, objectRef.name) + e.Parent.Log.Debugf("Missing value for: %s, %s", name, objectRef.name) continue } } diff --git a/plugins/inputs/vsphere/finder.go b/plugins/inputs/vsphere/finder.go index 24427b205309a..e49bf80f33fe5 100644 --- a/plugins/inputs/vsphere/finder.go +++ b/plugins/inputs/vsphere/finder.go @@ -2,7 +2,6 @@ package vsphere import ( "context" - "log" "reflect" "strings" @@ -26,35 +25,55 @@ type Finder struct { // ResourceFilter is a convenience class holding a finder and a set of paths. It is useful when you need a // self contained object capable of returning a certain set of resources. type ResourceFilter struct { - finder *Finder - resType string - paths []string + finder *Finder + resType string + paths []string + excludePaths []string } // FindAll returns the union of resources found given the supplied resource type and paths. -func (f *Finder) FindAll(ctx context.Context, resType string, paths []string, dst interface{}) error { +func (f *Finder) FindAll(ctx context.Context, resType string, paths, excludePaths []string, dst interface{}) error { + objs := make(map[string]types.ObjectContent) for _, p := range paths { - if err := f.Find(ctx, resType, p, dst); err != nil { + if err := f.find(ctx, resType, p, objs); err != nil { return err } } - return nil + if len(excludePaths) > 0 { + excludes := make(map[string]types.ObjectContent) + for _, p := range excludePaths { + if err := f.find(ctx, resType, p, excludes); err != nil { + return err + } + } + for k := range excludes { + delete(objs, k) + } + } + return objectContentToTypedArray(objs, dst) } // Find returns the resources matching the specified path. func (f *Finder) Find(ctx context.Context, resType, path string, dst interface{}) error { + objs := make(map[string]types.ObjectContent) + err := f.find(ctx, resType, path, objs) + if err != nil { + return err + } + return objectContentToTypedArray(objs, dst) +} + +func (f *Finder) find(ctx context.Context, resType, path string, objs map[string]types.ObjectContent) error { p := strings.Split(path, "/") flt := make([]property.Filter, len(p)-1) for i := 1; i < len(p); i++ { flt[i-1] = property.Filter{"name": p[i]} } - objs := make(map[string]types.ObjectContent) err := f.descend(ctx, f.client.Client.ServiceContent.RootFolder, resType, flt, 0, objs) if err != nil { return err } - objectContentToTypedArray(objs, dst) - log.Printf("D! [inputs.vsphere] Find(%s, %s) returned %d objects", resType, path, len(objs)) + f.client.log.Debugf("Find(%s, %s) returned %d objects", resType, path, len(objs)) return nil } @@ -95,6 +114,9 @@ func (f *Finder) descend(ctx context.Context, root types.ManagedObjectReference, // Special case: The last token is a recursive wildcard, so we can grab everything // recursively in a single call. v2, err := m.CreateContainerView(ctx, root, []string{resType}, true) + if err != nil { + return err + } defer v2.Destroy(ctx) err = v2.Retrieve(ctx, []string{resType}, fields, &content) if err != nil { @@ -205,7 +227,7 @@ func objectContentToTypedArray(objs map[string]types.ObjectContent, dst interfac // FindAll finds all resources matching the paths that were specified upon creation of // the ResourceFilter. func (r *ResourceFilter) FindAll(ctx context.Context, dst interface{}) error { - return r.finder.FindAll(ctx, r.resType, r.paths, dst) + return r.finder.FindAll(ctx, r.resType, r.paths, r.excludePaths, dst) } func matchName(f property.Filter, props []types.DynamicProperty) bool { @@ -234,12 +256,12 @@ func init() { } addFields = map[string][]string{ - "HostSystem": {"parent"}, + "HostSystem": {"parent", "summary.customValue", "customValue"}, "VirtualMachine": {"runtime.host", "config.guestId", "config.uuid", "runtime.powerState", - "summary.customValue", "guest.net", "guest.hostName"}, - "Datastore": {"parent", "info"}, - "ClusterComputeResource": {"parent"}, - "Datacenter": {"parent"}, + "summary.customValue", "guest.net", "guest.hostName", "customValue"}, + "Datastore": {"parent", "info", "customValue"}, + "ClusterComputeResource": {"parent", "customValue"}, + "Datacenter": {"parent", "customValue"}, } containers = map[string]interface{}{ diff --git a/plugins/inputs/vsphere/tscache.go b/plugins/inputs/vsphere/tscache.go index 4f73c4fe89155..6e7d00c8b4b80 100644 --- a/plugins/inputs/vsphere/tscache.go +++ b/plugins/inputs/vsphere/tscache.go @@ -34,7 +34,7 @@ func (t *TSCache) Purge() { n++ } } - log.Printf("D! [inputs.vsphere] Purged timestamp cache. %d deleted with %d remaining", n, len(t.table)) + log.Printf("D! [inputs.vsphere] purged timestamp cache. %d deleted with %d remaining", n, len(t.table)) } // IsNew returns true if the supplied timestamp for the supplied key is more recent than the diff --git a/plugins/inputs/vsphere/vsphere.go b/plugins/inputs/vsphere/vsphere.go index 2f9f08cc685b3..bc4042980a887 100644 --- a/plugins/inputs/vsphere/vsphere.go +++ b/plugins/inputs/vsphere/vsphere.go @@ -2,7 +2,6 @@ package vsphere import ( "context" - "log" "sync" "time" @@ -23,22 +22,27 @@ type VSphere struct { DatacenterMetricInclude []string DatacenterMetricExclude []string DatacenterInclude []string + DatacenterExclude []string ClusterInstances bool ClusterMetricInclude []string ClusterMetricExclude []string ClusterInclude []string + ClusterExclude []string HostInstances bool HostMetricInclude []string HostMetricExclude []string HostInclude []string + HostExclude []string VMInstances bool `toml:"vm_instances"` VMMetricInclude []string `toml:"vm_metric_include"` VMMetricExclude []string `toml:"vm_metric_exclude"` VMInclude []string `toml:"vm_include"` + VMExclude []string `toml:"vm_exclude"` DatastoreInstances bool DatastoreMetricInclude []string DatastoreMetricExclude []string DatastoreInclude []string + DatastoreExclude []string Separator string CustomAttributeInclude []string CustomAttributeExclude []string @@ -58,6 +62,8 @@ type VSphere struct { // Mix in the TLS/SSL goodness from core tls.ClientConfig + + Log telegraf.Logger } var sampleConfig = ` @@ -194,11 +200,6 @@ var sampleConfig = ` # collect_concurrency = 1 # discover_concurrency = 1 - ## whether or not to force discovery of new objects on initial gather call before collecting metrics - ## when true for large environments this may cause errors for time elapsed while collecting metrics - ## when false (default) the first collection cycle may result in no or limited metrics while objects are discovered - # force_discover_on_init = false - ## the interval before (re)discovering objects subject to metrics collection (default: 300s) # object_discovery_interval = "300s" @@ -243,10 +244,15 @@ func (v *VSphere) Description() string { // Start is called from telegraf core when a plugin is started and allows it to // perform initialization tasks. func (v *VSphere) Start(acc telegraf.Accumulator) error { - log.Println("D! [inputs.vsphere]: Starting plugin") + v.Log.Info("Starting plugin") ctx, cancel := context.WithCancel(context.Background()) v.cancel = cancel + // Check for deprecated settings + if !v.ForceDiscoverOnInit { + v.Log.Warn("The 'force_discover_on_init' configuration parameter has been deprecated. Setting it to 'false' has no effect") + } + // Create endpoints, one for each vCenter we're monitoring v.endpoints = make([]*Endpoint, len(v.Vcenters)) for i, rawURL := range v.Vcenters { @@ -266,7 +272,7 @@ func (v *VSphere) Start(acc telegraf.Accumulator) error { // Stop is called from telegraf core when a plugin is stopped and allows it to // perform shutdown tasks. func (v *VSphere) Stop() { - log.Println("D! [inputs.vsphere]: Stopping plugin") + v.Log.Info("Stopping plugin") v.cancel() // Wait for all endpoints to finish. No need to wait for @@ -275,7 +281,7 @@ func (v *VSphere) Stop() { // wait for any discovery to complete by trying to grab the // "busy" mutex. for _, ep := range v.endpoints { - log.Printf("D! [inputs.vsphere]: Waiting for endpoint %s to finish", ep.URL.Host) + v.Log.Debugf("Waiting for endpoint %q to finish", ep.URL.Host) func() { ep.busy.Lock() // Wait until discovery is finished defer ep.busy.Unlock() @@ -343,7 +349,7 @@ func init() { MaxQueryMetrics: 256, CollectConcurrency: 1, DiscoverConcurrency: 1, - ForceDiscoverOnInit: false, + ForceDiscoverOnInit: true, ObjectDiscoveryInterval: internal.Duration{Duration: time.Second * 300}, Timeout: internal.Duration{Duration: time.Second * 60}, } diff --git a/plugins/inputs/vsphere/vsphere_test.go b/plugins/inputs/vsphere/vsphere_test.go index 28c2c7934b335..b66fa45eb942e 100644 --- a/plugins/inputs/vsphere/vsphere_test.go +++ b/plugins/inputs/vsphere/vsphere_test.go @@ -42,6 +42,7 @@ var configHeader = ` func defaultVSphere() *VSphere { return &VSphere{ + Log: testutil.Logger{}, ClusterMetricInclude: []string{ "cpu.usage.*", "cpu.usagemhz.*", @@ -376,10 +377,59 @@ func TestFinder(t *testing.T) { testLookupVM(ctx, t, &f, "/*/host/**/*DC*/*/*DC*", 4, "") vm = []mo.VirtualMachine{} - err = f.FindAll(ctx, "VirtualMachine", []string{"/DC0/vm/DC0_H0*", "/DC0/vm/DC0_C0*"}, &vm) + err = f.FindAll(ctx, "VirtualMachine", []string{"/DC0/vm/DC0_H0*", "/DC0/vm/DC0_C0*"}, []string{}, &vm) require.NoError(t, err) require.Equal(t, 4, len(vm)) + rf := ResourceFilter{ + finder: &f, + paths: []string{"/DC0/vm/DC0_H0*", "/DC0/vm/DC0_C0*"}, + excludePaths: []string{"/DC0/vm/DC0_H0_VM0"}, + resType: "VirtualMachine", + } + vm = []mo.VirtualMachine{} + require.NoError(t, rf.FindAll(ctx, &vm)) + require.Equal(t, 3, len(vm)) + + rf = ResourceFilter{ + finder: &f, + paths: []string{"/DC0/vm/DC0_H0*", "/DC0/vm/DC0_C0*"}, + excludePaths: []string{"/**"}, + resType: "VirtualMachine", + } + vm = []mo.VirtualMachine{} + require.NoError(t, rf.FindAll(ctx, &vm)) + require.Equal(t, 0, len(vm)) + + rf = ResourceFilter{ + finder: &f, + paths: []string{"/**"}, + excludePaths: []string{"/**"}, + resType: "VirtualMachine", + } + vm = []mo.VirtualMachine{} + require.NoError(t, rf.FindAll(ctx, &vm)) + require.Equal(t, 0, len(vm)) + + rf = ResourceFilter{ + finder: &f, + paths: []string{"/**"}, + excludePaths: []string{"/this won't match anything"}, + resType: "VirtualMachine", + } + vm = []mo.VirtualMachine{} + require.NoError(t, rf.FindAll(ctx, &vm)) + require.Equal(t, 8, len(vm)) + + rf = ResourceFilter{ + finder: &f, + paths: []string{"/**"}, + excludePaths: []string{"/**/*VM0"}, + resType: "VirtualMachine", + } + vm = []mo.VirtualMachine{} + require.NoError(t, rf.FindAll(ctx, &vm)) + require.Equal(t, 4, len(vm)) } func TestFolders(t *testing.T) { diff --git a/plugins/inputs/win_perf_counters/win_perf_counters.go b/plugins/inputs/win_perf_counters/win_perf_counters.go index f858ba6e7364d..bd130a3fd79e9 100644 --- a/plugins/inputs/win_perf_counters/win_perf_counters.go +++ b/plugins/inputs/win_perf_counters/win_perf_counters.go @@ -5,7 +5,6 @@ package win_perf_counters import ( "errors" "fmt" - "log" "strings" "time" @@ -147,6 +146,8 @@ type Win_PerfCounters struct { CountersRefreshInterval internal.Duration UseWildcardsExpansion bool + Log telegraf.Logger + lastRefreshed time.Time counters []*counter query PerformanceQuery @@ -289,7 +290,7 @@ func (m *Win_PerfCounters) AddItem(counterPath string, objectName string, instan m.counters = append(m.counters, newItem) if m.PrintValid { - log.Printf("Valid: %s\n", counterPath) + m.Log.Infof("Valid: %s", counterPath) } } } else { @@ -297,7 +298,7 @@ func (m *Win_PerfCounters) AddItem(counterPath string, objectName string, instan includeTotal, counterHandle} m.counters = append(m.counters, newItem) if m.PrintValid { - log.Printf("Valid: %s\n", counterPath) + m.Log.Infof("Valid: %s", counterPath) } } @@ -323,7 +324,7 @@ func (m *Win_PerfCounters) ParseConfig() error { if err != nil { if PerfObject.FailOnMissing || PerfObject.WarnOnMissing { - log.Printf("Invalid counterPath: '%s'. Error: %s\n", counterPath, err.Error()) + m.Log.Errorf("Invalid counterPath: '%s'. Error: %s\n", counterPath, err.Error()) } if PerfObject.FailOnMissing { return err diff --git a/plugins/inputs/win_perf_counters/win_perf_counters_test.go b/plugins/inputs/win_perf_counters/win_perf_counters_test.go index 5052fb7a263a5..13eebdc95e2c3 100644 --- a/plugins/inputs/win_perf_counters/win_perf_counters_test.go +++ b/plugins/inputs/win_perf_counters/win_perf_counters_test.go @@ -247,13 +247,17 @@ func TestCounterPathParsing(t *testing.T) { func TestAddItemSimple(t *testing.T) { var err error cps1 := []string{"\\O(I)\\C"} - m := Win_PerfCounters{PrintValid: false, Object: nil, query: &FakePerformanceQuery{ - counters: createCounterMap(cps1, []float64{1.1}, []uint32{0}), - expandPaths: map[string][]string{ - cps1[0]: cps1, - }, - vistaAndNewer: true, - }} + m := Win_PerfCounters{ + Log: testutil.Logger{}, + PrintValid: false, + Object: nil, + query: &FakePerformanceQuery{ + counters: createCounterMap(cps1, []float64{1.1}, []uint32{0}), + expandPaths: map[string][]string{ + cps1[0]: cps1, + }, + vistaAndNewer: true, + }} err = m.query.Open() require.NoError(t, err) err = m.AddItem(cps1[0], "O", "I", "c", "test", false) @@ -265,13 +269,18 @@ func TestAddItemSimple(t *testing.T) { func TestAddItemInvalidCountPath(t *testing.T) { var err error cps1 := []string{"\\O\\C"} - m := Win_PerfCounters{PrintValid: false, Object: nil, UseWildcardsExpansion: true, query: &FakePerformanceQuery{ - counters: createCounterMap(cps1, []float64{1.1}, []uint32{0}), - expandPaths: map[string][]string{ - cps1[0]: {"\\O/C"}, - }, - vistaAndNewer: true, - }} + m := Win_PerfCounters{ + Log: testutil.Logger{}, + PrintValid: false, + Object: nil, + UseWildcardsExpansion: true, + query: &FakePerformanceQuery{ + counters: createCounterMap(cps1, []float64{1.1}, []uint32{0}), + expandPaths: map[string][]string{ + cps1[0]: {"\\O/C"}, + }, + vistaAndNewer: true, + }} err = m.query.Open() require.NoError(t, err) err = m.AddItem("\\O\\C", "O", "------", "C", "test", false) @@ -284,16 +293,20 @@ func TestParseConfigBasic(t *testing.T) { var err error perfObjects := createPerfObject("m", "O", []string{"I1", "I2"}, []string{"C1", "C2"}, false, false) cps1 := []string{"\\O(I1)\\C1", "\\O(I1)\\C2", "\\O(I2)\\C1", "\\O(I2)\\C2"} - m := Win_PerfCounters{PrintValid: false, Object: perfObjects, query: &FakePerformanceQuery{ - counters: createCounterMap(cps1, []float64{1.1, 1.2, 1.3, 1.4}, []uint32{0, 0, 0, 0}), - expandPaths: map[string][]string{ - cps1[0]: {cps1[0]}, - cps1[1]: {cps1[1]}, - cps1[2]: {cps1[2]}, - cps1[3]: {cps1[3]}, - }, - vistaAndNewer: true, - }} + m := Win_PerfCounters{ + Log: testutil.Logger{}, + PrintValid: false, + Object: perfObjects, + query: &FakePerformanceQuery{ + counters: createCounterMap(cps1, []float64{1.1, 1.2, 1.3, 1.4}, []uint32{0, 0, 0, 0}), + expandPaths: map[string][]string{ + cps1[0]: {cps1[0]}, + cps1[1]: {cps1[1]}, + cps1[2]: {cps1[2]}, + cps1[3]: {cps1[3]}, + }, + vistaAndNewer: true, + }} err = m.query.Open() require.NoError(t, err) err = m.ParseConfig() @@ -318,14 +331,19 @@ func TestParseConfigNoInstance(t *testing.T) { var err error perfObjects := createPerfObject("m", "O", []string{"------"}, []string{"C1", "C2"}, false, false) cps1 := []string{"\\O\\C1", "\\O\\C2"} - m := Win_PerfCounters{PrintValid: false, Object: perfObjects, UseWildcardsExpansion: false, query: &FakePerformanceQuery{ - counters: createCounterMap(cps1, []float64{1.1, 1.2}, []uint32{0, 0}), - expandPaths: map[string][]string{ - cps1[0]: {cps1[0]}, - cps1[1]: {cps1[1]}, - }, - vistaAndNewer: true, - }} + m := Win_PerfCounters{ + Log: testutil.Logger{}, + PrintValid: false, + Object: perfObjects, + UseWildcardsExpansion: false, + query: &FakePerformanceQuery{ + counters: createCounterMap(cps1, []float64{1.1, 1.2}, []uint32{0, 0}), + expandPaths: map[string][]string{ + cps1[0]: {cps1[0]}, + cps1[1]: {cps1[1]}, + }, + vistaAndNewer: true, + }} err = m.query.Open() require.NoError(t, err) err = m.ParseConfig() @@ -350,15 +368,19 @@ func TestParseConfigInvalidCounterError(t *testing.T) { var err error perfObjects := createPerfObject("m", "O", []string{"I1", "I2"}, []string{"C1", "C2"}, true, false) cps1 := []string{"\\O(I1)\\C2", "\\O(I2)\\C1", "\\O(I2)\\C2"} - m := Win_PerfCounters{PrintValid: false, Object: perfObjects, query: &FakePerformanceQuery{ - counters: createCounterMap(cps1, []float64{1.1, 1.2, 1.3}, []uint32{0, 0, 0}), - expandPaths: map[string][]string{ - cps1[0]: {cps1[0]}, - cps1[1]: {cps1[1]}, - cps1[2]: {cps1[2]}, - }, - vistaAndNewer: true, - }} + m := Win_PerfCounters{ + Log: testutil.Logger{}, + PrintValid: false, + Object: perfObjects, + query: &FakePerformanceQuery{ + counters: createCounterMap(cps1, []float64{1.1, 1.2, 1.3}, []uint32{0, 0, 0}), + expandPaths: map[string][]string{ + cps1[0]: {cps1[0]}, + cps1[1]: {cps1[1]}, + cps1[2]: {cps1[2]}, + }, + vistaAndNewer: true, + }} err = m.query.Open() require.NoError(t, err) err = m.ParseConfig() @@ -381,15 +403,19 @@ func TestParseConfigInvalidCounterNoError(t *testing.T) { var err error perfObjects := createPerfObject("m", "O", []string{"I1", "I2"}, []string{"C1", "C2"}, false, false) cps1 := []string{"\\O(I1)\\C2", "\\O(I2)\\C1", "\\O(I2)\\C2"} - m := Win_PerfCounters{PrintValid: false, Object: perfObjects, query: &FakePerformanceQuery{ - counters: createCounterMap(cps1, []float64{1.1, 1.2, 1.3}, []uint32{0, 0, 0}), - expandPaths: map[string][]string{ - cps1[0]: {cps1[0]}, - cps1[1]: {cps1[1]}, - cps1[2]: {cps1[2]}, - }, - vistaAndNewer: true, - }} + m := Win_PerfCounters{ + Log: testutil.Logger{}, + PrintValid: false, + Object: perfObjects, + query: &FakePerformanceQuery{ + counters: createCounterMap(cps1, []float64{1.1, 1.2, 1.3}, []uint32{0, 0, 0}), + expandPaths: map[string][]string{ + cps1[0]: {cps1[0]}, + cps1[1]: {cps1[1]}, + cps1[2]: {cps1[2]}, + }, + vistaAndNewer: true, + }} err = m.query.Open() require.NoError(t, err) err = m.ParseConfig() @@ -413,13 +439,18 @@ func TestParseConfigTotalExpansion(t *testing.T) { var err error perfObjects := createPerfObject("m", "O", []string{"*"}, []string{"*"}, true, true) cps1 := []string{"\\O(I1)\\C1", "\\O(I1)\\C2", "\\O(_Total)\\C1", "\\O(_Total)\\C2"} - m := Win_PerfCounters{PrintValid: false, UseWildcardsExpansion: true, Object: perfObjects, query: &FakePerformanceQuery{ - counters: createCounterMap(append(cps1, "\\O(*)\\*"), []float64{1.1, 1.2, 1.3, 1.4, 0}, []uint32{0, 0, 0, 0, 0}), - expandPaths: map[string][]string{ - "\\O(*)\\*": cps1, - }, - vistaAndNewer: true, - }} + m := Win_PerfCounters{ + Log: testutil.Logger{}, + PrintValid: false, + UseWildcardsExpansion: true, + Object: perfObjects, + query: &FakePerformanceQuery{ + counters: createCounterMap(append(cps1, "\\O(*)\\*"), []float64{1.1, 1.2, 1.3, 1.4, 0}, []uint32{0, 0, 0, 0, 0}), + expandPaths: map[string][]string{ + "\\O(*)\\*": cps1, + }, + vistaAndNewer: true, + }} err = m.query.Open() require.NoError(t, err) err = m.ParseConfig() @@ -430,13 +461,18 @@ func TestParseConfigTotalExpansion(t *testing.T) { perfObjects[0].IncludeTotal = false - m = Win_PerfCounters{PrintValid: false, UseWildcardsExpansion: true, Object: perfObjects, query: &FakePerformanceQuery{ - counters: createCounterMap(append(cps1, "\\O(*)\\*"), []float64{1.1, 1.2, 1.3, 1.4, 0}, []uint32{0, 0, 0, 0, 0}), - expandPaths: map[string][]string{ - "\\O(*)\\*": cps1, - }, - vistaAndNewer: true, - }} + m = Win_PerfCounters{ + Log: testutil.Logger{}, + PrintValid: false, + UseWildcardsExpansion: true, + Object: perfObjects, + query: &FakePerformanceQuery{ + counters: createCounterMap(append(cps1, "\\O(*)\\*"), []float64{1.1, 1.2, 1.3, 1.4, 0}, []uint32{0, 0, 0, 0, 0}), + expandPaths: map[string][]string{ + "\\O(*)\\*": cps1, + }, + vistaAndNewer: true, + }} err = m.query.Open() require.NoError(t, err) err = m.ParseConfig() @@ -450,13 +486,18 @@ func TestParseConfigExpand(t *testing.T) { var err error perfObjects := createPerfObject("m", "O", []string{"*"}, []string{"*"}, false, false) cps1 := []string{"\\O(I1)\\C1", "\\O(I1)\\C2", "\\O(I2)\\C1", "\\O(I2)\\C2"} - m := Win_PerfCounters{PrintValid: false, UseWildcardsExpansion: true, Object: perfObjects, query: &FakePerformanceQuery{ - counters: createCounterMap(append(cps1, "\\O(*)\\*"), []float64{1.1, 1.2, 1.3, 1.4, 0}, []uint32{0, 0, 0, 0, 0}), - expandPaths: map[string][]string{ - "\\O(*)\\*": cps1, - }, - vistaAndNewer: true, - }} + m := Win_PerfCounters{ + Log: testutil.Logger{}, + PrintValid: false, + UseWildcardsExpansion: true, + Object: perfObjects, + query: &FakePerformanceQuery{ + counters: createCounterMap(append(cps1, "\\O(*)\\*"), []float64{1.1, 1.2, 1.3, 1.4, 0}, []uint32{0, 0, 0, 0, 0}), + expandPaths: map[string][]string{ + "\\O(*)\\*": cps1, + }, + vistaAndNewer: true, + }} err = m.query.Open() require.NoError(t, err) err = m.ParseConfig() @@ -474,13 +515,17 @@ func TestSimpleGather(t *testing.T) { measurement := "test" perfObjects := createPerfObject(measurement, "O", []string{"I"}, []string{"C"}, false, false) cp1 := "\\O(I)\\C" - m := Win_PerfCounters{PrintValid: false, Object: perfObjects, query: &FakePerformanceQuery{ - counters: createCounterMap([]string{cp1}, []float64{1.2}, []uint32{0}), - expandPaths: map[string][]string{ - cp1: {cp1}, - }, - vistaAndNewer: false, - }} + m := Win_PerfCounters{ + Log: testutil.Logger{}, + PrintValid: false, + Object: perfObjects, + query: &FakePerformanceQuery{ + counters: createCounterMap([]string{cp1}, []float64{1.2}, []uint32{0}), + expandPaths: map[string][]string{ + cp1: {cp1}, + }, + vistaAndNewer: false, + }} var acc1 testutil.Accumulator err = m.Gather(&acc1) require.NoError(t, err) @@ -513,13 +558,17 @@ func TestSimpleGatherNoData(t *testing.T) { measurement := "test" perfObjects := createPerfObject(measurement, "O", []string{"I"}, []string{"C"}, false, false) cp1 := "\\O(I)\\C" - m := Win_PerfCounters{PrintValid: false, Object: perfObjects, query: &FakePerformanceQuery{ - counters: createCounterMap([]string{cp1}, []float64{1.2}, []uint32{PDH_NO_DATA}), - expandPaths: map[string][]string{ - cp1: {cp1}, - }, - vistaAndNewer: false, - }} + m := Win_PerfCounters{ + Log: testutil.Logger{}, + PrintValid: false, + Object: perfObjects, + query: &FakePerformanceQuery{ + counters: createCounterMap([]string{cp1}, []float64{1.2}, []uint32{PDH_NO_DATA}), + expandPaths: map[string][]string{ + cp1: {cp1}, + }, + vistaAndNewer: false, + }} var acc1 testutil.Accumulator err = m.Gather(&acc1) // this "PDH_NO_DATA" error should not be returned to caller, but checked, and handled @@ -555,13 +604,18 @@ func TestSimpleGatherWithTimestamp(t *testing.T) { measurement := "test" perfObjects := createPerfObject(measurement, "O", []string{"I"}, []string{"C"}, false, false) cp1 := "\\O(I)\\C" - m := Win_PerfCounters{PrintValid: false, UsePerfCounterTime: true, Object: perfObjects, query: &FakePerformanceQuery{ - counters: createCounterMap([]string{cp1}, []float64{1.2}, []uint32{0}), - expandPaths: map[string][]string{ - cp1: {cp1}, - }, - vistaAndNewer: true, - }} + m := Win_PerfCounters{ + Log: testutil.Logger{}, + PrintValid: false, + UsePerfCounterTime: true, + Object: perfObjects, + query: &FakePerformanceQuery{ + counters: createCounterMap([]string{cp1}, []float64{1.2}, []uint32{0}), + expandPaths: map[string][]string{ + cp1: {cp1}, + }, + vistaAndNewer: true, + }} var acc1 testutil.Accumulator err = m.Gather(&acc1) require.NoError(t, err) @@ -586,13 +640,17 @@ func TestGatherError(t *testing.T) { measurement := "test" perfObjects := createPerfObject(measurement, "O", []string{"I"}, []string{"C"}, false, false) cp1 := "\\O(I)\\C" - m := Win_PerfCounters{PrintValid: false, Object: perfObjects, query: &FakePerformanceQuery{ - counters: createCounterMap([]string{cp1}, []float64{-2}, []uint32{PDH_PLA_VALIDATION_WARNING}), - expandPaths: map[string][]string{ - cp1: {cp1}, - }, - vistaAndNewer: false, - }} + m := Win_PerfCounters{ + Log: testutil.Logger{}, + PrintValid: false, + Object: perfObjects, + query: &FakePerformanceQuery{ + counters: createCounterMap([]string{cp1}, []float64{-2}, []uint32{PDH_PLA_VALIDATION_WARNING}), + expandPaths: map[string][]string{ + cp1: {cp1}, + }, + vistaAndNewer: false, + }} var acc1 testutil.Accumulator err = m.Gather(&acc1) require.Error(t, err) @@ -617,15 +675,19 @@ func TestGatherInvalidDataIgnore(t *testing.T) { measurement := "test" perfObjects := createPerfObject(measurement, "O", []string{"I"}, []string{"C1", "C2", "C3"}, false, false) cps1 := []string{"\\O(I)\\C1", "\\O(I)\\C2", "\\O(I)\\C3"} - m := Win_PerfCounters{PrintValid: false, Object: perfObjects, query: &FakePerformanceQuery{ - counters: createCounterMap(cps1, []float64{1.2, 1, 0}, []uint32{0, PDH_INVALID_DATA, 0}), - expandPaths: map[string][]string{ - cps1[0]: {cps1[0]}, - cps1[1]: {cps1[1]}, - cps1[2]: {cps1[2]}, - }, - vistaAndNewer: false, - }} + m := Win_PerfCounters{ + Log: testutil.Logger{}, + PrintValid: false, + Object: perfObjects, + query: &FakePerformanceQuery{ + counters: createCounterMap(cps1, []float64{1.2, 1, 0}, []uint32{0, PDH_INVALID_DATA, 0}), + expandPaths: map[string][]string{ + cps1[0]: {cps1[0]}, + cps1[1]: {cps1[1]}, + cps1[2]: {cps1[2]}, + }, + vistaAndNewer: false, + }} var acc1 testutil.Accumulator err = m.Gather(&acc1) require.NoError(t, err) @@ -666,7 +728,14 @@ func TestGatherRefreshingWithExpansion(t *testing.T) { }, vistaAndNewer: true, } - m := Win_PerfCounters{PrintValid: false, Object: perfObjects, UseWildcardsExpansion: true, query: fpm, CountersRefreshInterval: internal.Duration{Duration: time.Second * 10}} + m := Win_PerfCounters{ + Log: testutil.Logger{}, + PrintValid: false, + Object: perfObjects, + UseWildcardsExpansion: true, + query: fpm, + CountersRefreshInterval: internal.Duration{Duration: time.Second * 10}, + } var acc1 testutil.Accumulator err = m.Gather(&acc1) assert.Len(t, m.counters, 4) @@ -752,7 +821,13 @@ func TestGatherRefreshingWithoutExpansion(t *testing.T) { }, vistaAndNewer: true, } - m := Win_PerfCounters{PrintValid: false, Object: perfObjects, UseWildcardsExpansion: false, query: fpm, CountersRefreshInterval: internal.Duration{Duration: time.Second * 10}} + m := Win_PerfCounters{ + Log: testutil.Logger{}, + PrintValid: false, + Object: perfObjects, + UseWildcardsExpansion: false, + query: fpm, + CountersRefreshInterval: internal.Duration{Duration: time.Second * 10}} var acc1 testutil.Accumulator err = m.Gather(&acc1) assert.Len(t, m.counters, 2) @@ -862,14 +937,19 @@ func TestGatherTotalNoExpansion(t *testing.T) { measurement := "m" perfObjects := createPerfObject(measurement, "O", []string{"*"}, []string{"C1", "C2"}, true, true) cps1 := []string{"\\O(I1)\\C1", "\\O(I1)\\C2", "\\O(_Total)\\C1", "\\O(_Total)\\C2"} - m := Win_PerfCounters{PrintValid: false, UseWildcardsExpansion: false, Object: perfObjects, query: &FakePerformanceQuery{ - counters: createCounterMap(append([]string{"\\O(*)\\C1", "\\O(*)\\C2"}, cps1...), []float64{0, 0, 1.1, 1.2, 1.3, 1.4}, []uint32{0, 0, 0, 0, 0, 0}), - expandPaths: map[string][]string{ - "\\O(*)\\C1": {cps1[0], cps1[2]}, - "\\O(*)\\C2": {cps1[1], cps1[3]}, - }, - vistaAndNewer: true, - }} + m := Win_PerfCounters{ + Log: testutil.Logger{}, + PrintValid: false, + UseWildcardsExpansion: false, + Object: perfObjects, + query: &FakePerformanceQuery{ + counters: createCounterMap(append([]string{"\\O(*)\\C1", "\\O(*)\\C2"}, cps1...), []float64{0, 0, 1.1, 1.2, 1.3, 1.4}, []uint32{0, 0, 0, 0, 0, 0}), + expandPaths: map[string][]string{ + "\\O(*)\\C1": {cps1[0], cps1[2]}, + "\\O(*)\\C2": {cps1[1], cps1[3]}, + }, + vistaAndNewer: true, + }} var acc1 testutil.Accumulator err = m.Gather(&acc1) require.NoError(t, err) diff --git a/plugins/inputs/win_services/win_services.go b/plugins/inputs/win_services/win_services.go index 1befc4a601602..6ac1bde68ca20 100644 --- a/plugins/inputs/win_services/win_services.go +++ b/plugins/inputs/win_services/win_services.go @@ -4,7 +4,6 @@ package win_services import ( "fmt" - "log" "os" "github.com/influxdata/telegraf" @@ -90,6 +89,8 @@ var description = "Input plugin to report Windows services info." //WinServices is an implementation if telegraf.Input interface, providing info about Windows Services type WinServices struct { + Log telegraf.Logger + ServiceNames []string `toml:"service_names"` mgrProvider ManagerProvider } @@ -125,9 +126,9 @@ func (m *WinServices) Gather(acc telegraf.Accumulator) error { service, err := collectServiceInfo(scmgr, srvName) if err != nil { if IsPermission(err) { - log.Printf("D! Error in plugin [inputs.win_services]: %v", err) + m.Log.Debug(err.Error()) } else { - acc.AddError(err) + m.Log.Error(err.Error()) } continue } diff --git a/plugins/inputs/win_services/win_services_integration_test.go b/plugins/inputs/win_services/win_services_integration_test.go index a39df49c7371d..0c375c3dd2e65 100644 --- a/plugins/inputs/win_services/win_services_integration_test.go +++ b/plugins/inputs/win_services/win_services_integration_test.go @@ -47,7 +47,11 @@ func TestGatherErrors(t *testing.T) { if testing.Short() { t.Skip("Skipping integration test in short mode") } - ws := &WinServices{InvalidServices, &MgProvider{}} + ws := &WinServices{ + Log: testutil.Logger{}, + ServiceNames: InvalidServices, + mgrProvider: &MgProvider{}, + } require.Len(t, ws.ServiceNames, 3, "Different number of services") var acc testutil.Accumulator require.NoError(t, ws.Gather(&acc)) diff --git a/plugins/inputs/win_services/win_services_test.go b/plugins/inputs/win_services/win_services_test.go index 37dc3f08c7a95..e33ab2ddce622 100644 --- a/plugins/inputs/win_services/win_services_test.go +++ b/plugins/inputs/win_services/win_services_test.go @@ -3,8 +3,10 @@ package win_services import ( + "bytes" "errors" "fmt" + "log" "testing" "github.com/influxdata/telegraf/testutil" @@ -128,47 +130,51 @@ var testErrors = []testData{ func TestBasicInfo(t *testing.T) { - winServices := &WinServices{nil, &FakeMgProvider{testErrors[0]}} + winServices := &WinServices{testutil.Logger{}, nil, &FakeMgProvider{testErrors[0]}} assert.NotEmpty(t, winServices.SampleConfig()) assert.NotEmpty(t, winServices.Description()) } func TestMgrErrors(t *testing.T) { //mgr.connect error - winServices := &WinServices{nil, &FakeMgProvider{testErrors[0]}} + winServices := &WinServices{testutil.Logger{}, nil, &FakeMgProvider{testErrors[0]}} var acc1 testutil.Accumulator err := winServices.Gather(&acc1) require.Error(t, err) assert.Contains(t, err.Error(), testErrors[0].mgrConnectError.Error()) ////mgr.listServices error - winServices = &WinServices{nil, &FakeMgProvider{testErrors[1]}} + winServices = &WinServices{testutil.Logger{}, nil, &FakeMgProvider{testErrors[1]}} var acc2 testutil.Accumulator err = winServices.Gather(&acc2) require.Error(t, err) assert.Contains(t, err.Error(), testErrors[1].mgrListServicesError.Error()) ////mgr.listServices error 2 - winServices = &WinServices{[]string{"Fake service 1"}, &FakeMgProvider{testErrors[3]}} + winServices = &WinServices{testutil.Logger{}, []string{"Fake service 1"}, &FakeMgProvider{testErrors[3]}} var acc3 testutil.Accumulator - err = winServices.Gather(&acc3) - require.NoError(t, err) - assert.Len(t, acc3.Errors, 1) + buf := &bytes.Buffer{} + log.SetOutput(buf) + require.NoError(t, winServices.Gather(&acc3)) + + require.Contains(t, buf.String(), testErrors[2].services[0].serviceOpenError.Error()) } func TestServiceErrors(t *testing.T) { - winServices := &WinServices{nil, &FakeMgProvider{testErrors[2]}} + winServices := &WinServices{testutil.Logger{}, nil, &FakeMgProvider{testErrors[2]}} var acc1 testutil.Accumulator + + buf := &bytes.Buffer{} + log.SetOutput(buf) require.NoError(t, winServices.Gather(&acc1)) - assert.Len(t, acc1.Errors, 3) + //open service error - assert.Contains(t, acc1.Errors[0].Error(), testErrors[2].services[0].serviceOpenError.Error()) + require.Contains(t, buf.String(), testErrors[2].services[0].serviceOpenError.Error()) //query service error - assert.Contains(t, acc1.Errors[1].Error(), testErrors[2].services[1].serviceQueryError.Error()) + require.Contains(t, buf.String(), testErrors[2].services[1].serviceQueryError.Error()) //config service error - assert.Contains(t, acc1.Errors[2].Error(), testErrors[2].services[2].serviceConfigError.Error()) - + require.Contains(t, buf.String(), testErrors[2].services[2].serviceConfigError.Error()) } var testSimpleData = []testData{ @@ -179,7 +185,7 @@ var testSimpleData = []testData{ } func TestGather2(t *testing.T) { - winServices := &WinServices{nil, &FakeMgProvider{testSimpleData[0]}} + winServices := &WinServices{testutil.Logger{}, nil, &FakeMgProvider{testSimpleData[0]}} var acc1 testutil.Accumulator require.NoError(t, winServices.Gather(&acc1)) assert.Len(t, acc1.Errors, 0, "There should be no errors after gather") @@ -193,5 +199,4 @@ func TestGather2(t *testing.T) { tags["display_name"] = s.displayName acc1.AssertContainsTaggedFields(t, "win_services", fields, tags) } - } diff --git a/plugins/inputs/wireless/wireless.go b/plugins/inputs/wireless/wireless.go index eb488ef594693..911d7fb097d17 100644 --- a/plugins/inputs/wireless/wireless.go +++ b/plugins/inputs/wireless/wireless.go @@ -7,7 +7,8 @@ import ( // Wireless is used to store configuration values. type Wireless struct { - HostProc string `toml:"host_proc"` + HostProc string `toml:"host_proc"` + Log telegraf.Logger `toml:"-"` } var sampleConfig = ` diff --git a/plugins/inputs/wireless/wireless_nonlinux.go b/plugins/inputs/wireless/wireless_notlinux.go similarity index 75% rename from plugins/inputs/wireless/wireless_nonlinux.go rename to plugins/inputs/wireless/wireless_notlinux.go index 0fbe5eb062bb8..4769acc970e42 100644 --- a/plugins/inputs/wireless/wireless_nonlinux.go +++ b/plugins/inputs/wireless/wireless_notlinux.go @@ -3,19 +3,21 @@ package wireless import ( - "log" - "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/inputs" ) +func (w *Wireless) Init() error { + w.Log.Warn("Current platform is not supported") + return nil +} + func (w *Wireless) Gather(acc telegraf.Accumulator) error { return nil } func init() { inputs.Add("wireless", func() telegraf.Input { - log.Print("W! [inputs.wireless] Current platform is not supported") return &Wireless{} }) } diff --git a/plugins/inputs/x509_cert/README.md b/plugins/inputs/x509_cert/README.md index 450dd3d1039e0..b302d4992eeb3 100644 --- a/plugins/inputs/x509_cert/README.md +++ b/plugins/inputs/x509_cert/README.md @@ -33,6 +33,12 @@ file or network connection. - province - locality - verification + - serial_number + - signature_algorithm + - public_key_algorithm + - issuer_common_name + - issuer_serial_number + - san - fields: - verification_code (int) - verification_error (string) diff --git a/plugins/inputs/x509_cert/x509_cert.go b/plugins/inputs/x509_cert/x509_cert.go index 825fd5eeb18d1..21e64fcbb6e48 100644 --- a/plugins/inputs/x509_cert/x509_cert.go +++ b/plugins/inputs/x509_cert/x509_cert.go @@ -2,9 +2,9 @@ package x509_cert import ( + "bytes" "crypto/tls" "crypto/x509" - "crypto/x509/pkix" "encoding/pem" "fmt" "io/ioutil" @@ -96,18 +96,26 @@ func (c *X509Cert) getCert(u *url.URL, timeout time.Duration) ([]*x509.Certifica if err != nil { return nil, err } + var certs []*x509.Certificate + for { + block, rest := pem.Decode(bytes.TrimSpace(content)) + if block == nil { + return nil, fmt.Errorf("failed to parse certificate PEM") + } - block, _ := pem.Decode(content) - if block == nil { - return nil, fmt.Errorf("failed to parse certificate PEM") - } - - cert, err := x509.ParseCertificate(block.Bytes) - if err != nil { - return nil, err + if block.Type == "CERTIFICATE" { + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, err + } + certs = append(certs, cert) + } + if rest == nil || len(rest) == 0 { + break + } + content = rest } - - return []*x509.Certificate{cert}, nil + return certs, nil default: return nil, fmt.Errorf("unsuported scheme '%s' in location %s", u.Scheme, u.String()) } @@ -129,27 +137,42 @@ func getFields(cert *x509.Certificate, now time.Time) map[string]interface{} { return fields } -func getTags(subject pkix.Name, location string) map[string]string { +func getTags(cert *x509.Certificate, location string) map[string]string { tags := map[string]string{ - "source": location, - "common_name": subject.CommonName, + "source": location, + "common_name": cert.Subject.CommonName, + "serial_number": cert.SerialNumber.Text(16), + "signature_algorithm": cert.SignatureAlgorithm.String(), + "public_key_algorithm": cert.PublicKeyAlgorithm.String(), } - if len(subject.Organization) > 0 { - tags["organization"] = subject.Organization[0] + if len(cert.Subject.Organization) > 0 { + tags["organization"] = cert.Subject.Organization[0] + } + if len(cert.Subject.OrganizationalUnit) > 0 { + tags["organizational_unit"] = cert.Subject.OrganizationalUnit[0] } - if len(subject.OrganizationalUnit) > 0 { - tags["organizational_unit"] = subject.OrganizationalUnit[0] + if len(cert.Subject.Country) > 0 { + tags["country"] = cert.Subject.Country[0] } - if len(subject.Country) > 0 { - tags["country"] = subject.Country[0] + if len(cert.Subject.Province) > 0 { + tags["province"] = cert.Subject.Province[0] } - if len(subject.Province) > 0 { - tags["province"] = subject.Province[0] + if len(cert.Subject.Locality) > 0 { + tags["locality"] = cert.Subject.Locality[0] + } + + tags["issuer_common_name"] = cert.Issuer.CommonName + tags["issuer_serial_number"] = cert.Issuer.SerialNumber + + san := append(cert.DNSNames, cert.EmailAddresses...) + for _, ip := range cert.IPAddresses { + san = append(san, ip.String()) } - if len(subject.Locality) > 0 { - tags["locality"] = subject.Locality[0] + for _, uri := range cert.URIs { + san = append(san, uri.String()) } + tags["san"] = strings.Join(san, ",") return tags } @@ -172,7 +195,7 @@ func (c *X509Cert) Gather(acc telegraf.Accumulator) error { for i, cert := range certs { fields := getFields(cert, now) - tags := getTags(cert.Subject, location) + tags := getTags(cert, location) // The first certificate is the leaf/end-entity certificate which needs DNS // name validation against the URL hostname. diff --git a/plugins/inputs/x509_cert/x509_cert_test.go b/plugins/inputs/x509_cert/x509_cert_test.go index 188b510d263d9..fa90a90eb8b47 100644 --- a/plugins/inputs/x509_cert/x509_cert_test.go +++ b/plugins/inputs/x509_cert/x509_cert_test.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "fmt" "io/ioutil" + "math/big" "os" "testing" "time" @@ -141,6 +142,15 @@ func TestGatherLocal(t *testing.T) { {name: "not a certificate", mode: 0640, content: "test", error: true}, {name: "wrong certificate", mode: 0640, content: wrongCert, error: true}, {name: "correct certificate", mode: 0640, content: pki.ReadServerCert()}, + {name: "correct certificate and extra trailing space", mode: 0640, content: pki.ReadServerCert() + " "}, + {name: "correct certificate and extra leading space", mode: 0640, content: " " + pki.ReadServerCert()}, + {name: "correct multiple certificates", mode: 0640, content: pki.ReadServerCert() + pki.ReadCACert()}, + {name: "correct multiple certificates and key", mode: 0640, content: pki.ReadServerCert() + pki.ReadCACert() + pki.ReadServerKey()}, + {name: "correct certificate and wrong certificate", mode: 0640, content: pki.ReadServerCert() + "\n" + wrongCert, error: true}, + {name: "correct certificate and not a certificate", mode: 0640, content: pki.ReadServerCert() + "\ntest", error: true}, + {name: "correct multiple certificates and extra trailing space", mode: 0640, content: pki.ReadServerCert() + pki.ReadServerCert() + " "}, + {name: "correct multiple certificates and extra leading space", mode: 0640, content: " " + pki.ReadServerCert() + pki.ReadServerCert()}, + {name: "correct multiple certificates and extra middle space", mode: 0640, content: pki.ReadServerCert() + " " + pki.ReadServerCert()}, } for _, test := range tests { @@ -187,6 +197,61 @@ func TestGatherLocal(t *testing.T) { } } +func TestTags(t *testing.T) { + cert := fmt.Sprintf("%s\n%s", pki.ReadServerCert(), pki.ReadCACert()) + + f, err := ioutil.TempFile("", "x509_cert") + if err != nil { + t.Fatal(err) + } + + _, err = f.Write([]byte(cert)) + if err != nil { + t.Fatal(err) + } + + err = f.Close() + if err != nil { + t.Fatal(err) + } + + defer os.Remove(f.Name()) + + sc := X509Cert{ + Sources: []string{f.Name()}, + } + sc.Init() + + acc := testutil.Accumulator{} + err = sc.Gather(&acc) + require.NoError(t, err) + + assert.True(t, acc.HasMeasurement("x509_cert")) + + assert.True(t, acc.HasTag("x509_cert", "common_name")) + assert.Equal(t, "server.localdomain", acc.TagValue("x509_cert", "common_name")) + + assert.True(t, acc.HasTag("x509_cert", "signature_algorithm")) + assert.Equal(t, "SHA256-RSA", acc.TagValue("x509_cert", "signature_algorithm")) + + assert.True(t, acc.HasTag("x509_cert", "public_key_algorithm")) + assert.Equal(t, "RSA", acc.TagValue("x509_cert", "public_key_algorithm")) + + assert.True(t, acc.HasTag("x509_cert", "issuer_common_name")) + assert.Equal(t, "Telegraf Test CA", acc.TagValue("x509_cert", "issuer_common_name")) + + assert.True(t, acc.HasTag("x509_cert", "san")) + assert.Equal(t, "localhost,127.0.0.1", acc.TagValue("x509_cert", "san")) + + assert.True(t, acc.HasTag("x509_cert", "serial_number")) + serialNumber := new(big.Int) + _, validSerialNumber := serialNumber.SetString(acc.TagValue("x509_cert", "serial_number"), 16) + if !validSerialNumber { + t.Errorf("Expected a valid Hex serial number but got %s", acc.TagValue("x509_cert", "serial_number")) + } + assert.Equal(t, big.NewInt(1), serialNumber) +} + func TestGatherChain(t *testing.T) { cert := fmt.Sprintf("%s\n%s", pki.ReadServerCert(), pki.ReadCACert()) diff --git a/plugins/inputs/zipkin/convert_test.go b/plugins/inputs/zipkin/convert_test.go index 92c1ba3ff33db..23a951594da1a 100644 --- a/plugins/inputs/zipkin/convert_test.go +++ b/plugins/inputs/zipkin/convert_test.go @@ -121,6 +121,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "duration_ns": (time.Duration(53106) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851331000).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -138,6 +139,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "duration_ns": (time.Duration(53106) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851331000).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -152,6 +154,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "duration_ns": (time.Duration(50410) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360904552000).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -169,6 +172,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "duration_ns": (time.Duration(50410) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360904552000).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -183,6 +187,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851318000).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -199,6 +204,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851318000).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -215,6 +221,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851318000).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -231,6 +238,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851318000).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -248,6 +256,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851318000).UTC(), + Type: telegraf.Untyped, }, }, wantErr: false, @@ -296,6 +305,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "duration_ns": (time.Duration(1) * time.Nanosecond).Nanoseconds(), }, Time: time.Unix(1, 0).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -312,6 +322,7 @@ func TestLineProtocolConverter_Record(t *testing.T) { "duration_ns": (time.Duration(1) * time.Nanosecond).Nanoseconds(), }, Time: time.Unix(1, 0).UTC(), + Type: telegraf.Untyped, }, }, }, diff --git a/plugins/inputs/zipkin/zipkin.go b/plugins/inputs/zipkin/zipkin.go index 18a63dccd4cb1..4224fea3d2928 100644 --- a/plugins/inputs/zipkin/zipkin.go +++ b/plugins/inputs/zipkin/zipkin.go @@ -3,7 +3,6 @@ package zipkin import ( "context" "fmt" - "log" "net" "net/http" "strconv" @@ -60,6 +59,8 @@ type Zipkin struct { Port int Path string + Log telegraf.Logger + address string handler Handler server *http.Server @@ -105,7 +106,7 @@ func (z *Zipkin) Start(acc telegraf.Accumulator) error { } z.address = ln.Addr().String() - log.Printf("I! Started the zipkin listener on %s", z.address) + z.Log.Infof("Started the zipkin listener on %s", z.address) go func() { wg.Add(1) diff --git a/plugins/inputs/zipkin/zipkin_test.go b/plugins/inputs/zipkin/zipkin_test.go index 2ac269db19693..77bef853b7e52 100644 --- a/plugins/inputs/zipkin/zipkin_test.go +++ b/plugins/inputs/zipkin/zipkin_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/google/go-cmp/cmp" + "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/testutil" ) @@ -40,6 +41,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": (time.Duration(53106) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851331000).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -57,6 +59,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": (time.Duration(53106) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851331000).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -71,6 +74,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": (time.Duration(50410) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360904552000).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -88,6 +92,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": (time.Duration(50410) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360904552000).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -102,6 +107,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851318000).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -118,6 +124,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851318000).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -134,6 +141,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851318000).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -150,6 +158,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851318000).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -167,6 +176,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": (time.Duration(103680) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1498688360851318000).UTC(), + Type: telegraf.Untyped, }, }, wantErr: false, @@ -189,6 +199,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": (time.Duration(1) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1433330263415871*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -205,6 +216,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": (time.Duration(1) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1433330263415871*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -221,6 +233,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": (time.Duration(1) * time.Microsecond).Nanoseconds(), }, Time: time.Unix(0, 1433330263415871*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, }, }, @@ -240,7 +253,9 @@ func TestZipkinPlugin(t *testing.T) { }, Fields: map[string]interface{}{ "duration_ns": int64(3000000), - }, Time: time.Unix(0, 1503031538791000*int64(time.Microsecond)).UTC(), + }, + Time: time.Unix(0, 1503031538791000*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -257,6 +272,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": int64(3000000), }, Time: time.Unix(0, 1503031538791000*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -273,6 +289,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": int64(3000000), }, Time: time.Unix(0, 1503031538791000*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -290,6 +307,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": int64(3000000), }, Time: time.Unix(0, 1503031538791000*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -307,6 +325,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": int64(3000000), }, Time: time.Unix(0, 1503031538791000*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -324,6 +343,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": int64(3000000), }, Time: time.Unix(0, 1503031538791000*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -338,6 +358,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": int64(10000000), }, Time: time.Unix(0, 1503031538786000*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -354,6 +375,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": int64(10000000), }, Time: time.Unix(0, 1503031538786000*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -370,6 +392,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": int64(10000000), }, Time: time.Unix(0, 1503031538786000*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -387,6 +410,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": int64(10000000), }, Time: time.Unix(0, 1503031538786000*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -404,6 +428,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": int64(10000000), }, Time: time.Unix(0, 1503031538786000*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -421,6 +446,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": int64(10000000), }, Time: time.Unix(0, 1503031538786000*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -438,6 +464,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": int64(10000000), }, Time: time.Unix(0, 1503031538786000*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -455,6 +482,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": int64(10000000), }, Time: time.Unix(0, 1503031538786000*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -469,6 +497,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": int64(23393000), }, Time: time.Unix(0, 1503031538778000*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -485,6 +514,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": int64(23393000), }, Time: time.Unix(0, 1503031538778000*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -501,6 +531,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": int64(23393000), }, Time: time.Unix(0, 1503031538778000*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -518,6 +549,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": int64(23393000), }, Time: time.Unix(0, 1503031538778000*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -535,6 +567,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": int64(23393000), }, Time: time.Unix(0, 1503031538778000*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, { Measurement: "zipkin", @@ -552,6 +585,7 @@ func TestZipkinPlugin(t *testing.T) { "duration_ns": int64(23393000), }, Time: time.Unix(0, 1503031538778000*int64(time.Microsecond)).UTC(), + Type: telegraf.Untyped, }, }, }, @@ -562,6 +596,7 @@ func TestZipkinPlugin(t *testing.T) { DefaultNetwork = "tcp4" z := &Zipkin{ + Log: testutil.Logger{}, Path: "/api/v1/spans", Port: 0, } diff --git a/plugins/inputs/zookeeper/README.md b/plugins/inputs/zookeeper/README.md index d54caae44471b..23009c519b64a 100644 --- a/plugins/inputs/zookeeper/README.md +++ b/plugins/inputs/zookeeper/README.md @@ -1,7 +1,7 @@ ## Zookeeper Input Plugin The zookeeper plugin collects variables outputted from the 'mntr' command -[Zookeeper Admin](https://zookeeper.apache.org/doc/trunk/zookeeperAdmin.html). +[Zookeeper Admin](https://zookeeper.apache.org/doc/current/zookeeperAdmin.html). ### Configuration @@ -19,7 +19,7 @@ The zookeeper plugin collects variables outputted from the 'mntr' command # timeout = "5s" ## Optional TLS Config - # enable_ssl = true + # enable_tls = true # tls_ca = "/etc/telegraf/ca.pem" # tls_cert = "/etc/telegraf/cert.pem" # tls_key = "/etc/telegraf/key.pem" diff --git a/plugins/outputs/all/all.go b/plugins/outputs/all/all.go index e40230993b51a..35e0393de1cad 100644 --- a/plugins/outputs/all/all.go +++ b/plugins/outputs/all/all.go @@ -33,5 +33,6 @@ import ( _ "github.com/influxdata/telegraf/plugins/outputs/socket_writer" _ "github.com/influxdata/telegraf/plugins/outputs/stackdriver" _ "github.com/influxdata/telegraf/plugins/outputs/syslog" + _ "github.com/influxdata/telegraf/plugins/outputs/warp10" _ "github.com/influxdata/telegraf/plugins/outputs/wavefront" ) diff --git a/plugins/outputs/application_insights/application_insights_test.go b/plugins/outputs/application_insights/application_insights_test.go index 561e6c9f93fea..7255ad0689608 100644 --- a/plugins/outputs/application_insights/application_insights_test.go +++ b/plugins/outputs/application_insights/application_insights_test.go @@ -184,7 +184,7 @@ func TestSimpleMetricCreated(t *testing.T) { {"neither value nor count", map[string]interface{}{"v1": "alpha", "v2": 45.8}, "", []string{"v2"}}, {"value is of wrong type", map[string]interface{}{"value": "alpha", "count": 15}, "", []string{"count"}}, {"count is of wrong type", map[string]interface{}{"value": 23.77, "count": 7.5}, "", []string{"count", "value"}}, - {"count is out of range", map[string]interface{}{"value": -98.45E4, "count": math.MaxUint64 - uint64(20)}, "", []string{"value", "count"}}, + {"count is out of range", map[string]interface{}{"value": -98.45e4, "count": math.MaxUint64 - uint64(20)}, "", []string{"value", "count"}}, {"several additional fields", map[string]interface{}{"alpha": 10, "bravo": "bravo", "charlie": 30, "delta": 40.7}, "", []string{"alpha", "charlie", "delta"}}, } diff --git a/plugins/outputs/cloudwatch/README.md b/plugins/outputs/cloudwatch/README.md index 31619263f26f9..418fe86ffa489 100644 --- a/plugins/outputs/cloudwatch/README.md +++ b/plugins/outputs/cloudwatch/README.md @@ -45,4 +45,7 @@ also save AWS API cost. If enable this flag, this plugin would parse the require [CloudWatch statistic fields](https://docs.aws.amazon.com/sdk-for-go/api/service/cloudwatch/#StatisticSet) (count, min, max, and sum) and send them to CloudWatch. You could use `basicstats` aggregator to calculate those fields. If not all statistic fields are available, -all fields would still be sent as raw metrics. \ No newline at end of file +all fields would still be sent as raw metrics. + +### high_resolution_metrics +Enable high resolution metrics (1 second precision) instead of standard ones (60 seconds precision) \ No newline at end of file diff --git a/plugins/outputs/cloudwatch/cloudwatch.go b/plugins/outputs/cloudwatch/cloudwatch.go index aaefa89ec2383..1ae8bd4f83217 100644 --- a/plugins/outputs/cloudwatch/cloudwatch.go +++ b/plugins/outputs/cloudwatch/cloudwatch.go @@ -25,8 +25,9 @@ type CloudWatch struct { Token string `toml:"token"` EndpointURL string `toml:"endpoint_url"` - Namespace string `toml:"namespace"` // CloudWatch Metrics Namespace - svc *cloudwatch.CloudWatch + Namespace string `toml:"namespace"` // CloudWatch Metrics Namespace + HighResolutionMetrics bool `toml:"high_resolution_metrics"` + svc *cloudwatch.CloudWatch WriteStatistics bool `toml:"write_statistics"` } @@ -47,11 +48,12 @@ type cloudwatchField interface { } type statisticField struct { - metricName string - fieldName string - tags map[string]string - values map[statisticType]float64 - timestamp time.Time + metricName string + fieldName string + tags map[string]string + values map[statisticType]float64 + timestamp time.Time + storageResolution int64 } func (f *statisticField) addValue(sType statisticType, value float64) { @@ -81,6 +83,7 @@ func (f *statisticField) buildDatum() []*cloudwatch.MetricDatum { Sum: aws.Float64(sum), SampleCount: aws.Float64(count), }, + StorageResolution: aws.Int64(f.storageResolution), } datums = append(datums, datum) @@ -126,11 +129,12 @@ func (f *statisticField) hasAllFields() bool { } type valueField struct { - metricName string - fieldName string - tags map[string]string - value float64 - timestamp time.Time + metricName string + fieldName string + tags map[string]string + value float64 + timestamp time.Time + storageResolution int64 } func (f *valueField) addValue(sType statisticType, value float64) { @@ -143,10 +147,11 @@ func (f *valueField) buildDatum() []*cloudwatch.MetricDatum { return []*cloudwatch.MetricDatum{ { - MetricName: aws.String(strings.Join([]string{f.metricName, f.fieldName}, "_")), - Value: aws.Float64(f.value), - Dimensions: BuildDimensions(f.tags), - Timestamp: aws.Time(f.timestamp), + MetricName: aws.String(strings.Join([]string{f.metricName, f.fieldName}, "_")), + Value: aws.Float64(f.value), + Dimensions: BuildDimensions(f.tags), + Timestamp: aws.Time(f.timestamp), + StorageResolution: aws.Int64(f.storageResolution), }, } } @@ -186,6 +191,9 @@ var sampleConfig = ` ## You could use basicstats aggregator to calculate those fields. If not all statistic ## fields are available, all fields would still be sent as raw metrics. # write_statistics = false + + ## Enable high resolution metrics of 1 second (if not enabled, standard resolution are of 60 seconds precision) + # high_resolution_metrics = false ` func (c *CloudWatch) SampleConfig() string { @@ -220,7 +228,7 @@ func (c *CloudWatch) Write(metrics []telegraf.Metric) error { var datums []*cloudwatch.MetricDatum for _, m := range metrics { - d := BuildMetricDatum(c.WriteStatistics, m) + d := BuildMetricDatum(c.WriteStatistics, c.HighResolutionMetrics, m) datums = append(datums, d...) } @@ -278,10 +286,14 @@ func PartitionDatums(size int, datums []*cloudwatch.MetricDatum) [][]*cloudwatch // Make a MetricDatum from telegraf.Metric. It would check if all required fields of // cloudwatch.StatisticSet are available. If so, it would build MetricDatum from statistic values. // Otherwise, fields would still been built independently. -func BuildMetricDatum(buildStatistic bool, point telegraf.Metric) []*cloudwatch.MetricDatum { +func BuildMetricDatum(buildStatistic bool, highResolutionMetrics bool, point telegraf.Metric) []*cloudwatch.MetricDatum { fields := make(map[string]cloudwatchField) tags := point.Tags() + storageResolution := int64(60) + if highResolutionMetrics { + storageResolution = 1 + } for k, v := range point.Fields() { @@ -297,11 +309,12 @@ func BuildMetricDatum(buildStatistic bool, point telegraf.Metric) []*cloudwatch. // If statistic metric is not enabled or non-statistic type, just take current field as a value field. if !buildStatistic || sType == statisticTypeNone { fields[k] = &valueField{ - metricName: point.Name(), - fieldName: k, - tags: tags, - timestamp: point.Time(), - value: val, + metricName: point.Name(), + fieldName: k, + tags: tags, + timestamp: point.Time(), + value: val, + storageResolution: storageResolution, } continue } @@ -317,6 +330,7 @@ func BuildMetricDatum(buildStatistic bool, point telegraf.Metric) []*cloudwatch. values: map[statisticType]float64{ sType: val, }, + storageResolution: storageResolution, } } else { // Add new statistic value to this field diff --git a/plugins/outputs/cloudwatch/cloudwatch_test.go b/plugins/outputs/cloudwatch/cloudwatch_test.go index acadca8424f8b..b2466e4d046d4 100644 --- a/plugins/outputs/cloudwatch/cloudwatch_test.go +++ b/plugins/outputs/cloudwatch/cloudwatch_test.go @@ -75,11 +75,11 @@ func TestBuildMetricDatums(t *testing.T) { testutil.TestMetric(float64(1.174272e+108)), // largest should be 1.174271e+108 } for _, point := range validMetrics { - datums := BuildMetricDatum(false, point) + datums := BuildMetricDatum(false, false, point) assert.Equal(1, len(datums), fmt.Sprintf("Valid point should create a Datum {value: %v}", point)) } for _, point := range invalidMetrics { - datums := BuildMetricDatum(false, point) + datums := BuildMetricDatum(false, false, point) assert.Equal(0, len(datums), fmt.Sprintf("Valid point should not create a Datum {value: %v}", point)) } @@ -89,7 +89,7 @@ func TestBuildMetricDatums(t *testing.T) { map[string]interface{}{"value_max": float64(10), "value_min": float64(0), "value_sum": float64(100), "value_count": float64(20)}, time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC), ) - datums := BuildMetricDatum(true, statisticMetric) + datums := BuildMetricDatum(true, false, statisticMetric) assert.Equal(1, len(datums), fmt.Sprintf("Valid point should create a Datum {value: %v}", statisticMetric)) multiFieldsMetric, _ := metric.New( @@ -98,7 +98,7 @@ func TestBuildMetricDatums(t *testing.T) { map[string]interface{}{"valueA": float64(10), "valueB": float64(0), "valueC": float64(100), "valueD": float64(20)}, time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC), ) - datums = BuildMetricDatum(true, multiFieldsMetric) + datums = BuildMetricDatum(true, false, multiFieldsMetric) assert.Equal(4, len(datums), fmt.Sprintf("Each field should create a Datum {value: %v}", multiFieldsMetric)) multiStatisticMetric, _ := metric.New( @@ -112,10 +112,27 @@ func TestBuildMetricDatums(t *testing.T) { }, time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC), ) - datums = BuildMetricDatum(true, multiStatisticMetric) + datums = BuildMetricDatum(true, false, multiStatisticMetric) assert.Equal(7, len(datums), fmt.Sprintf("Valid point should create a Datum {value: %v}", multiStatisticMetric)) } +func TestMetricDatumResolution(t *testing.T) { + const expectedStandardResolutionValue = int64(60) + const expectedHighResolutionValue = int64(1) + + assert := assert.New(t) + + metric := testutil.TestMetric(1) + + standardResolutionDatum := BuildMetricDatum(false, false, metric) + actualStandardResolutionValue := *standardResolutionDatum[0].StorageResolution + assert.Equal(expectedStandardResolutionValue, actualStandardResolutionValue) + + highResolutionDatum := BuildMetricDatum(false, true, metric) + actualHighResolutionValue := *highResolutionDatum[0].StorageResolution + assert.Equal(expectedHighResolutionValue, actualHighResolutionValue) +} + func TestBuildMetricDatums_SkipEmptyTags(t *testing.T) { input := testutil.MustMetric( "cpu", @@ -129,7 +146,7 @@ func TestBuildMetricDatums_SkipEmptyTags(t *testing.T) { time.Unix(0, 0), ) - datums := BuildMetricDatum(true, input) + datums := BuildMetricDatum(true, false, input) require.Len(t, datums[0].Dimensions, 1) } diff --git a/plugins/outputs/exec/exec.go b/plugins/outputs/exec/exec.go index 583646bb5379f..474c967916660 100644 --- a/plugins/outputs/exec/exec.go +++ b/plugins/outputs/exec/exec.go @@ -67,13 +67,11 @@ func (e *Exec) SampleConfig() string { // Write writes the metrics to the configured command. func (e *Exec) Write(metrics []telegraf.Metric) error { var buffer bytes.Buffer - for _, metric := range metrics { - value, err := e.serializer.Serialize(metric) - if err != nil { - return err - } - buffer.Write(value) + serializedMetrics, err := e.serializer.SerializeBatch(metrics) + if err != nil { + return err } + buffer.Write(serializedMetrics) if buffer.Len() <= 0 { return nil diff --git a/plugins/outputs/file/README.md b/plugins/outputs/file/README.md index e34f80807eb33..350633c56c09b 100644 --- a/plugins/outputs/file/README.md +++ b/plugins/outputs/file/README.md @@ -9,6 +9,11 @@ This plugin writes telegraf metrics to files ## Files to write to, "stdout" is a specially handled file. files = ["stdout", "/tmp/metrics.out"] + ## Use batch serialization format instead of line based delimiting. The + ## batch format allows for the production of non line based output formats and + ## may more effiently encode and write metrics. + # use_batch_format = false + ## The file will be rotated after the time interval specified. When set ## to 0 no time based rotation is performed. # rotation_interval = "0h" diff --git a/plugins/outputs/file/file.go b/plugins/outputs/file/file.go index 11793bc4f204c..12d70d8f37bd4 100644 --- a/plugins/outputs/file/file.go +++ b/plugins/outputs/file/file.go @@ -3,7 +3,6 @@ package file import ( "fmt" "io" - "log" "os" "github.com/influxdata/telegraf" @@ -18,6 +17,8 @@ type File struct { RotationInterval internal.Duration `toml:"rotation_interval"` RotationMaxSize internal.Size `toml:"rotation_max_size"` RotationMaxArchives int `toml:"rotation_max_archives"` + UseBatchFormat bool `toml:"use_batch_format"` + Log telegraf.Logger `toml:"-"` writer io.Writer closers []io.Closer @@ -28,6 +29,11 @@ var sampleConfig = ` ## Files to write to, "stdout" is a specially handled file. files = ["stdout", "/tmp/metrics.out"] + ## Use batch serialization format instead of line based delimiting. The + ## batch format allows for the production of non line based output formats and + ## may more effiently encode metric groups. + # use_batch_format = false + ## The file will be rotated after the time interval specified. When set ## to 0 no time based rotation is performed. # rotation_interval = "0d" @@ -98,15 +104,27 @@ func (f *File) Description() string { func (f *File) Write(metrics []telegraf.Metric) error { var writeErr error = nil - for _, metric := range metrics { - b, err := f.serializer.Serialize(metric) + if f.UseBatchFormat { + octets, err := f.serializer.SerializeBatch(metrics) if err != nil { - log.Printf("D! [outputs.file] Could not serialize metric: %v", err) + f.Log.Errorf("Could not serialize metric: %v", err) } - _, err = f.writer.Write(b) + _, err = f.writer.Write(octets) if err != nil { - writeErr = fmt.Errorf("E! [outputs.file] failed to write message: %v", err) + f.Log.Errorf("Error writing to file: %v", err) + } + } else { + for _, metric := range metrics { + b, err := f.serializer.Serialize(metric) + if err != nil { + f.Log.Debugf("Could not serialize metric: %v", err) + } + + _, err = f.writer.Write(b) + if err != nil { + writeErr = fmt.Errorf("E! [outputs.file] failed to write message: %v", err) + } } } diff --git a/plugins/outputs/http/http.go b/plugins/outputs/http/http.go index 1967b6ef99dc4..746cba346b96d 100644 --- a/plugins/outputs/http/http.go +++ b/plugins/outputs/http/http.go @@ -176,10 +176,12 @@ func (h *HTTP) write(reqBody []byte) error { var err error if h.ContentEncoding == "gzip" { - reqBodyBuffer, err = internal.CompressWithGzip(reqBodyBuffer) + rc, err := internal.CompressWithGzip(reqBodyBuffer) if err != nil { return err } + defer rc.Close() + reqBodyBuffer = rc } req, err := http.NewRequest(h.Method, h.URL, reqBodyBuffer) diff --git a/plugins/outputs/influxdb/http.go b/plugins/outputs/influxdb/http.go index 9497cadcc970d..d449c94564e18 100644 --- a/plugins/outputs/influxdb/http.go +++ b/plugins/outputs/influxdb/http.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" "io" + "io/ioutil" "net" "net/http" "net/url" @@ -255,6 +256,9 @@ func (c *httpClient) Write(ctx context.Context, metrics []telegraf.Metric) error } if c.config.ExcludeDatabaseTag { + // Avoid modifying the metric in case we need to retry the request. + metric = metric.Copy() + metric.Accept() metric.RemoveTag(c.config.DatabaseTag) } @@ -265,7 +269,7 @@ func (c *httpClient) Write(ctx context.Context, metrics []telegraf.Metric) error if !c.config.SkipDatabaseCreation && !c.createdDatabases[db] { err := c.CreateDatabase(ctx, db) if err != nil { - c.log.Warnf("when writing to [%s]: database %q creation failed: %v", + c.log.Warnf("When writing to [%s]: database %q creation failed: %v", c.config.URL, db, err) } } @@ -285,7 +289,12 @@ func (c *httpClient) writeBatch(ctx context.Context, db string, metrics []telegr return err } - reader := influx.NewReader(metrics, c.config.Serializer) + reader, err := c.requestBodyReader(metrics) + if err != nil { + return err + } + defer reader.Close() + req, err := c.makeWriteRequest(url, reader) if err != nil { return err @@ -331,7 +340,7 @@ func (c *httpClient) writeBatch(ctx context.Context, db string, metrics []telegr // discarded for being older than the retention policy. Usually this not // a cause for concern and we don't want to retry. if strings.Contains(desc, errStringPointsBeyondRP) { - c.log.Warnf("when writing to [%s]: received error %v", + c.log.Warnf("When writing to [%s]: received error %v", c.URL(), desc) return nil } @@ -340,7 +349,7 @@ func (c *httpClient) writeBatch(ctx context.Context, db string, metrics []telegr // correctable at this point and so the point is dropped instead of // retrying. if strings.Contains(desc, errStringPartialWrite) { - c.log.Errorf("when writing to [%s]: received error %v; discarding points", + c.log.Errorf("When writing to [%s]: received error %v; discarding points", c.URL(), desc) return nil } @@ -348,7 +357,7 @@ func (c *httpClient) writeBatch(ctx context.Context, db string, metrics []telegr // This error indicates a bug in either Telegraf line protocol // serialization, retries would not be successful. if strings.Contains(desc, errStringUnableToParse) { - c.log.Errorf("when writing to [%s]: received error %v; discarding points", + c.log.Errorf("When writing to [%s]: received error %v; discarding points", c.URL(), desc) return nil } @@ -383,12 +392,6 @@ func (c *httpClient) makeQueryRequest(query string) (*http.Request, error) { func (c *httpClient) makeWriteRequest(url string, body io.Reader) (*http.Request, error) { var err error - if c.config.ContentEncoding == "gzip" { - body, err = internal.CompressWithGzip(body) - if err != nil { - return nil, err - } - } req, err := http.NewRequest("POST", url, body) if err != nil { @@ -405,6 +408,23 @@ func (c *httpClient) makeWriteRequest(url string, body io.Reader) (*http.Request return req, nil } +// requestBodyReader warp io.Reader from influx.NewReader to io.ReadCloser, which is usefully to fast close the write +// side of the connection in case of error +func (c *httpClient) requestBodyReader(metrics []telegraf.Metric) (io.ReadCloser, error) { + reader := influx.NewReader(metrics, c.config.Serializer) + + if c.config.ContentEncoding == "gzip" { + rc, err := internal.CompressWithGzip(reader) + if err != nil { + return nil, err + } + + return rc, nil + } + + return ioutil.NopCloser(reader), nil +} + func (c *httpClient) addHeaders(req *http.Request) { if c.config.Username != "" || c.config.Password != "" { req.SetBasicAuth(c.config.Username, c.config.Password) diff --git a/plugins/outputs/influxdb/http_test.go b/plugins/outputs/influxdb/http_test.go index e4acb16415d40..a09b02d43cd39 100644 --- a/plugins/outputs/influxdb/http_test.go +++ b/plugins/outputs/influxdb/http_test.go @@ -675,3 +675,61 @@ func TestHTTP_UnixSocket(t *testing.T) { }) } } + +func TestHTTP_WriteDatabaseTagWorksOnRetry(t *testing.T) { + ts := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/write": + r.ParseForm() + require.Equal(t, r.Form["db"], []string{"foo"}) + + body, err := ioutil.ReadAll(r.Body) + require.NoError(t, err) + require.Contains(t, string(body), "cpu value=42") + + w.WriteHeader(http.StatusNoContent) + return + default: + w.WriteHeader(http.StatusNotFound) + return + } + }), + ) + defer ts.Close() + + addr := &url.URL{ + Scheme: "http", + Host: ts.Listener.Addr().String(), + } + + config := influxdb.HTTPConfig{ + URL: addr, + Database: "telegraf", + DatabaseTag: "database", + ExcludeDatabaseTag: true, + Log: testutil.Logger{}, + } + + client, err := influxdb.NewHTTPClient(config) + require.NoError(t, err) + + metrics := []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{ + "database": "foo", + }, + map[string]interface{}{ + "value": 42.0, + }, + time.Unix(0, 0), + ), + } + + ctx := context.Background() + err = client.Write(ctx, metrics) + require.NoError(t, err) + err = client.Write(ctx, metrics) + require.NoError(t, err) +} diff --git a/plugins/outputs/influxdb/influxdb.go b/plugins/outputs/influxdb/influxdb.go index 6af6dc173e592..be462ba03157f 100644 --- a/plugins/outputs/influxdb/influxdb.go +++ b/plugins/outputs/influxdb/influxdb.go @@ -57,8 +57,7 @@ type InfluxDB struct { CreateHTTPClientF func(config *HTTPConfig) (Client, error) CreateUDPClientF func(config *UDPConfig) (Client, error) - serializer *influx.Serializer - Log telegraf.Logger + Log telegraf.Logger } var sampleConfig = ` @@ -145,11 +144,6 @@ func (i *InfluxDB) Connect() error { urls = append(urls, defaultURL) } - i.serializer = influx.NewSerializer() - if i.InfluxUintSupport { - i.serializer.SetFieldTypeSupport(influx.UintSupport) - } - for _, u := range urls { parts, err := url.Parse(u) if err != nil { @@ -221,13 +215,13 @@ func (i *InfluxDB) Write(metrics []telegraf.Metric) error { if !i.SkipDatabaseCreation { err := client.CreateDatabase(ctx, apiError.Database) if err != nil { - i.Log.Errorf("when writing to [%s]: database %q not found and failed to recreate", + i.Log.Errorf("When writing to [%s]: database %q not found and failed to recreate", client.URL(), apiError.Database) } } } - i.Log.Errorf("when writing to [%s]: %v", client.URL(), err) + i.Log.Errorf("When writing to [%s]: %v", client.URL(), err) } return errors.New("could not write any address") @@ -237,7 +231,7 @@ func (i *InfluxDB) udpClient(url *url.URL) (Client, error) { config := &UDPConfig{ URL: url, MaxPayloadSize: int(i.UDPPayload.Size), - Serializer: i.serializer, + Serializer: i.newSerializer(), Log: i.Log, } @@ -271,7 +265,7 @@ func (i *InfluxDB) httpClient(ctx context.Context, url *url.URL, proxy *url.URL) SkipDatabaseCreation: i.SkipDatabaseCreation, RetentionPolicy: i.RetentionPolicy, Consistency: i.WriteConsistency, - Serializer: i.serializer, + Serializer: i.newSerializer(), Log: i.Log, } @@ -283,14 +277,23 @@ func (i *InfluxDB) httpClient(ctx context.Context, url *url.URL, proxy *url.URL) if !i.SkipDatabaseCreation { err = c.CreateDatabase(ctx, c.Database()) if err != nil { - i.Log.Warnf("when writing to [%s]: database %q creation failed: %v", - c.URL(), i.Database, err) + i.Log.Warnf("When writing to [%s]: database %q creation failed: %v", + c.URL(), c.Database(), err) } } return c, nil } +func (i *InfluxDB) newSerializer() *influx.Serializer { + serializer := influx.NewSerializer() + if i.InfluxUintSupport { + serializer.SetFieldTypeSupport(influx.UintSupport) + } + + return serializer +} + func init() { outputs.Add("influxdb", func() telegraf.Output { return &InfluxDB{ diff --git a/plugins/outputs/influxdb/udp.go b/plugins/outputs/influxdb/udp.go index a50516c97da23..0add3c6c39de6 100644 --- a/plugins/outputs/influxdb/udp.go +++ b/plugins/outputs/influxdb/udp.go @@ -95,7 +95,7 @@ func (c *udpClient) Write(ctx context.Context, metrics []telegraf.Metric) error if err != nil { // Since we are serializing multiple metrics, don't fail the // entire batch just because of one unserializable metric. - c.log.Errorf("when writing to [%s] could not serialize metric: %v", + c.log.Errorf("When writing to [%s] could not serialize metric: %v", c.URL(), err) continue } diff --git a/plugins/outputs/influxdb_v2/http.go b/plugins/outputs/influxdb_v2/http.go index fbfdf6958df4a..b94df889bfc6e 100644 --- a/plugins/outputs/influxdb_v2/http.go +++ b/plugins/outputs/influxdb_v2/http.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "io" + "io/ioutil" "log" "net" "net/http" @@ -189,6 +190,9 @@ func (c *httpClient) Write(ctx context.Context, metrics []telegraf.Metric) error } if c.ExcludeBucketTag { + // Avoid modifying the metric in case we need to retry the request. + metric = metric.Copy() + metric.Accept() metric.RemoveTag(c.BucketTag) } @@ -211,7 +215,12 @@ func (c *httpClient) writeBatch(ctx context.Context, bucket string, metrics []te return err } - reader := influx.NewReader(metrics, c.serializer) + reader, err := c.requestBodyReader(metrics) + if err != nil { + return err + } + defer reader.Close() + req, err := c.makeWriteRequest(url, reader) if err != nil { return err @@ -279,12 +288,6 @@ func (c *httpClient) writeBatch(ctx context.Context, bucket string, metrics []te func (c *httpClient) makeWriteRequest(url string, body io.Reader) (*http.Request, error) { var err error - if c.ContentEncoding == "gzip" { - body, err = internal.CompressWithGzip(body) - if err != nil { - return nil, err - } - } req, err := http.NewRequest("POST", url, body) if err != nil { @@ -301,6 +304,23 @@ func (c *httpClient) makeWriteRequest(url string, body io.Reader) (*http.Request return req, nil } +// requestBodyReader warp io.Reader from influx.NewReader to io.ReadCloser, which is usefully to fast close the write +// side of the connection in case of error +func (c *httpClient) requestBodyReader(metrics []telegraf.Metric) (io.ReadCloser, error) { + reader := influx.NewReader(metrics, c.serializer) + + if c.ContentEncoding == "gzip" { + rc, err := internal.CompressWithGzip(reader) + if err != nil { + return nil, err + } + + return rc, nil + } + + return ioutil.NopCloser(reader), nil +} + func (c *httpClient) addHeaders(req *http.Request) { for header, value := range c.Headers { req.Header.Set(header, value) diff --git a/plugins/outputs/influxdb_v2/http_test.go b/plugins/outputs/influxdb_v2/http_test.go index 33ff9e24b90e3..23c3ff05e17b6 100644 --- a/plugins/outputs/influxdb_v2/http_test.go +++ b/plugins/outputs/influxdb_v2/http_test.go @@ -1,10 +1,17 @@ package influxdb_v2_test import ( + "context" + "io/ioutil" + "net/http" + "net/http/httptest" "net/url" "testing" + "time" + "github.com/influxdata/telegraf" influxdb "github.com/influxdata/telegraf/plugins/outputs/influxdb_v2" + "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/require" ) @@ -47,3 +54,60 @@ func TestNewHTTPClient(t *testing.T) { } } } + +func TestWriteBucketTagWorksOnRetry(t *testing.T) { + ts := httptest.NewServer( + http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/v2/write": + r.ParseForm() + require.Equal(t, r.Form["bucket"], []string{"foo"}) + + body, err := ioutil.ReadAll(r.Body) + require.NoError(t, err) + require.Contains(t, string(body), "cpu value=42") + + w.WriteHeader(http.StatusNoContent) + return + default: + w.WriteHeader(http.StatusNotFound) + return + } + }), + ) + defer ts.Close() + + addr := &url.URL{ + Scheme: "http", + Host: ts.Listener.Addr().String(), + } + + config := &influxdb.HTTPConfig{ + URL: addr, + Bucket: "telegraf", + BucketTag: "bucket", + ExcludeBucketTag: true, + } + + client, err := influxdb.NewHTTPClient(config) + require.NoError(t, err) + + metrics := []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{ + "bucket": "foo", + }, + map[string]interface{}{ + "value": 42.0, + }, + time.Unix(0, 0), + ), + } + + ctx := context.Background() + err = client.Write(ctx, metrics) + require.NoError(t, err) + err = client.Write(ctx, metrics) + require.NoError(t, err) +} diff --git a/plugins/outputs/influxdb_v2/influxdb.go b/plugins/outputs/influxdb_v2/influxdb.go index 972773f797463..4e23146916cdb 100644 --- a/plugins/outputs/influxdb_v2/influxdb.go +++ b/plugins/outputs/influxdb_v2/influxdb.go @@ -96,8 +96,7 @@ type InfluxDB struct { UintSupport bool `toml:"influx_uint_support"` tls.ClientConfig - clients []Client - serializer *influx.Serializer + clients []Client } func (i *InfluxDB) Connect() error { @@ -107,11 +106,6 @@ func (i *InfluxDB) Connect() error { i.URLs = append(i.URLs, defaultURL) } - i.serializer = influx.NewSerializer() - if i.UintSupport { - i.serializer.SetFieldTypeSupport(influx.UintSupport) - } - for _, u := range i.URLs { parts, err := url.Parse(u) if err != nil { @@ -196,7 +190,7 @@ func (i *InfluxDB) getHTTPClient(ctx context.Context, url *url.URL, proxy *url.U UserAgent: i.UserAgent, ContentEncoding: i.ContentEncoding, TLSConfig: tlsConfig, - Serializer: i.serializer, + Serializer: i.newSerializer(), } c, err := NewHTTPClient(config) @@ -207,6 +201,15 @@ func (i *InfluxDB) getHTTPClient(ctx context.Context, url *url.URL, proxy *url.U return c, nil } +func (i *InfluxDB) newSerializer() *influx.Serializer { + serializer := influx.NewSerializer() + if i.UintSupport { + serializer.SetFieldTypeSupport(influx.UintSupport) + } + + return serializer +} + func init() { outputs.Add("influxdb_v2", func() telegraf.Output { return &InfluxDB{ diff --git a/plugins/outputs/kafka/README.md b/plugins/outputs/kafka/README.md index 25b173a0260f7..7b9fc0e303d7f 100644 --- a/plugins/outputs/kafka/README.md +++ b/plugins/outputs/kafka/README.md @@ -96,6 +96,9 @@ This plugin writes to a [Kafka Broker](http://kafka.apache.org/07/quickstart.htm # sasl_username = "kafka" # sasl_password = "secret" + ## SASL protocol version. When connecting to Azure EventHub set to 0. + # sasl_version = 1 + ## Data format to output. ## Each data format has its own unique set of configuration options, read ## more about them here: diff --git a/plugins/outputs/kafka/kafka.go b/plugins/outputs/kafka/kafka.go index 7ba457c5932c9..18a8925a54786 100644 --- a/plugins/outputs/kafka/kafka.go +++ b/plugins/outputs/kafka/kafka.go @@ -5,13 +5,15 @@ import ( "fmt" "log" "strings" + "time" "github.com/Shopify/sarama" + "github.com/gofrs/uuid" "github.com/influxdata/telegraf" tlsint "github.com/influxdata/telegraf/internal/tls" + "github.com/influxdata/telegraf/plugins/common/kafka" "github.com/influxdata/telegraf/plugins/outputs" "github.com/influxdata/telegraf/plugins/serializers" - uuid "github.com/satori/go.uuid" ) var ValidTopicSuffixMethods = []string{ @@ -20,6 +22,8 @@ var ValidTopicSuffixMethods = []string{ "tags", } +var zeroTime = time.Unix(0, 0) + type ( Kafka struct { Brokers []string @@ -43,12 +47,14 @@ type ( // TLS certificate authority CA string + EnableTLS *bool `toml:"enable_tls"` tlsint.ClientConfig - // SASL Username SASLUsername string `toml:"sasl_username"` - // SASL Password SASLPassword string `toml:"sasl_password"` + SASLVersion *int `toml:"sasl_version"` + + Log telegraf.Logger `toml:"-"` tlsConfig tls.Config producer sarama.SyncProducer @@ -168,6 +174,7 @@ var sampleConfig = ` # max_message_bytes = 1000000 ## Optional TLS Config + # enable_tls = true # tls_ca = "/etc/telegraf/ca.pem" # tls_cert = "/etc/telegraf/cert.pem" # tls_key = "/etc/telegraf/key.pem" @@ -178,6 +185,9 @@ var sampleConfig = ` # sasl_username = "kafka" # sasl_password = "secret" + ## SASL protocol version. When connecting to Azure EventHub set to 0. + # sasl_version = 1 + ## Data format to output. ## Each data format has its own unique set of configuration options, read ## more about them here: @@ -256,6 +266,10 @@ func (k *Kafka) Connect() error { k.TLSKey = k.Key } + if k.EnableTLS != nil && *k.EnableTLS { + config.Net.TLS.Enable = true + } + tlsConfig, err := k.ClientConfig.TLSConfig() if err != nil { return err @@ -263,13 +277,25 @@ func (k *Kafka) Connect() error { if tlsConfig != nil { config.Net.TLS.Config = tlsConfig - config.Net.TLS.Enable = true + + // To maintain backwards compatibility, if the enable_tls option is not + // set TLS is enabled if a non-default TLS config is used. + if k.EnableTLS == nil { + k.Log.Warnf("Use of deprecated configuration: enable_tls should be set when using TLS") + config.Net.TLS.Enable = true + } } if k.SASLUsername != "" && k.SASLPassword != "" { config.Net.SASL.User = k.SASLUsername config.Net.SASL.Password = k.SASLPassword config.Net.SASL.Enable = true + + version, err := kafka.SASLVersion(config.Version, k.SASLVersion) + if err != nil { + return err + } + config.Net.SASL.Version = version } producer, err := sarama.NewSyncProducer(k.Brokers, config) @@ -292,20 +318,23 @@ func (k *Kafka) Description() string { return "Configuration for the Kafka server to send metrics to" } -func (k *Kafka) routingKey(metric telegraf.Metric) string { +func (k *Kafka) routingKey(metric telegraf.Metric) (string, error) { if k.RoutingTag != "" { key, ok := metric.GetTag(k.RoutingTag) if ok { - return key + return key, nil } } if k.RoutingKey == "random" { - u := uuid.NewV4() - return u.String() + u, err := uuid.NewV4() + if err != nil { + return "", err + } + return u.String(), nil } - return k.RoutingKey + return k.RoutingKey, nil } func (k *Kafka) Write(metrics []telegraf.Metric) error { @@ -313,7 +342,7 @@ func (k *Kafka) Write(metrics []telegraf.Metric) error { for _, metric := range metrics { buf, err := k.serializer.Serialize(metric) if err != nil { - log.Printf("D! [outputs.kafka] Could not serialize metric: %v", err) + k.Log.Debugf("Could not serialize metric: %v", err) continue } @@ -321,7 +350,17 @@ func (k *Kafka) Write(metrics []telegraf.Metric) error { Topic: k.GetTopicName(metric), Value: sarama.ByteEncoder(buf), } - key := k.routingKey(metric) + + // Negative timestamps are not allowed by the Kafka protocol. + if !metric.Time().Before(zeroTime) { + m.Timestamp = metric.Time() + } + + key, err := k.routingKey(metric) + if err != nil { + return fmt.Errorf("could not generate routing key: %v", err) + } + if key != "" { m.Key = sarama.StringEncoder(key) } @@ -334,7 +373,11 @@ func (k *Kafka) Write(metrics []telegraf.Metric) error { if errs, ok := err.(sarama.ProducerErrors); ok { for _, prodErr := range errs { if prodErr.Err == sarama.ErrMessageSizeTooLarge { - log.Printf("E! Error writing to output [kafka]: Message too large, consider increasing `max_message_bytes`; dropping batch") + k.Log.Error("Message too large, consider increasing `max_message_bytes`; dropping batch") + return nil + } + if prodErr.Err == sarama.ErrInvalidTimestamp { + k.Log.Error("The timestamp of the message is out of acceptable range, consider increasing broker `message.timestamp.difference.max.ms`; dropping batch") return nil } return prodErr diff --git a/plugins/outputs/kafka/kafka_test.go b/plugins/outputs/kafka/kafka_test.go index ba900e32c6eaa..bac51c28d64d7 100644 --- a/plugins/outputs/kafka/kafka_test.go +++ b/plugins/outputs/kafka/kafka_test.go @@ -150,7 +150,8 @@ func TestRoutingKey(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - key := tt.kafka.routingKey(tt.metric) + key, err := tt.kafka.routingKey(tt.metric) + require.NoError(t, err) tt.check(t, key) }) } diff --git a/plugins/outputs/kinesis/kinesis.go b/plugins/outputs/kinesis/kinesis.go index 1b7b747e96b46..d2d482ff33f10 100644 --- a/plugins/outputs/kinesis/kinesis.go +++ b/plugins/outputs/kinesis/kinesis.go @@ -6,11 +6,11 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/kinesis" + "github.com/gofrs/uuid" "github.com/influxdata/telegraf" internalaws "github.com/influxdata/telegraf/internal/config/aws" "github.com/influxdata/telegraf/plugins/outputs" "github.com/influxdata/telegraf/plugins/serializers" - "github.com/satori/go.uuid" ) type ( @@ -183,7 +183,10 @@ func (k *KinesisOutput) getPartitionKey(metric telegraf.Metric) string { case "static": return k.Partition.Key case "random": - u := uuid.NewV4() + u, err := uuid.NewV4() + if err != nil { + return k.Partition.Default + } return u.String() case "measurement": return metric.Name() @@ -200,7 +203,10 @@ func (k *KinesisOutput) getPartitionKey(metric telegraf.Metric) string { } } if k.RandomPartitionKey { - u := uuid.NewV4() + u, err := uuid.NewV4() + if err != nil { + return k.Partition.Default + } return u.String() } return k.PartitionKey diff --git a/plugins/outputs/kinesis/kinesis_test.go b/plugins/outputs/kinesis/kinesis_test.go index 627a459dbd582..9d4f6729be53c 100644 --- a/plugins/outputs/kinesis/kinesis_test.go +++ b/plugins/outputs/kinesis/kinesis_test.go @@ -3,8 +3,8 @@ package kinesis import ( "testing" + "github.com/gofrs/uuid" "github.com/influxdata/telegraf/testutil" - uuid "github.com/satori/go.uuid" "github.com/stretchr/testify/assert" ) diff --git a/plugins/outputs/prometheus_client/README.md b/plugins/outputs/prometheus_client/README.md index 49030bb3c3336..7d4fe09b1f04f 100644 --- a/plugins/outputs/prometheus_client/README.md +++ b/plugins/outputs/prometheus_client/README.md @@ -1,6 +1,7 @@ -# Prometheus Client Service Output Plugin +# Prometheus Output Plugin -This plugin starts a [Prometheus](https://prometheus.io/) Client, it exposes all metrics on `/metrics` (default) to be polled by a Prometheus server. +This plugin starts a [Prometheus](https://prometheus.io/) Client, it exposes +all metrics on `/metrics` (default) to be polled by a Prometheus server. ## Configuration @@ -10,6 +11,14 @@ This plugin starts a [Prometheus](https://prometheus.io/) Client, it exposes all ## Address to listen on. listen = ":9273" + ## Metric version controls the mapping from Telegraf metrics into + ## Prometheus format. When using the prometheus input, use the same value in + ## both plugins to ensure metrics are round-tripped without modification. + ## + ## example: metric_version = 1; deprecated in 1.13 + ## metric_version = 2; recommended version + # metric_version = 1 + ## Use HTTP Basic Authentication. # basic_username = "Foo" # basic_password = "Bar" diff --git a/plugins/outputs/prometheus_client/prometheus_client.go b/plugins/outputs/prometheus_client/prometheus_client.go index 32dcdbb891f14..afdf7e1074545 100644 --- a/plugins/outputs/prometheus_client/prometheus_client.go +++ b/plugins/outputs/prometheus_client/prometheus_client.go @@ -1,18 +1,12 @@ -package prometheus_client +package prometheus import ( "context" - "crypto/subtle" "crypto/tls" "fmt" - "log" "net" "net/http" "net/url" - "regexp" - "sort" - "strconv" - "strings" "sync" "time" @@ -20,73 +14,30 @@ import ( "github.com/influxdata/telegraf/internal" tlsint "github.com/influxdata/telegraf/internal/tls" "github.com/influxdata/telegraf/plugins/outputs" + "github.com/influxdata/telegraf/plugins/outputs/prometheus_client/v1" + "github.com/influxdata/telegraf/plugins/outputs/prometheus_client/v2" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" ) var ( - invalidNameCharRE = regexp.MustCompile(`[^a-zA-Z0-9_:]`) - validNameCharRE = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*`) + defaultListen = ":9273" + defaultPath = "/metrics" + defaultExpirationInterval = internal.Duration{Duration: 60 * time.Second} ) -// SampleID uniquely identifies a Sample -type SampleID string - -// Sample represents the current value of a series. -type Sample struct { - // Labels are the Prometheus labels. - Labels map[string]string - // Value is the value in the Prometheus output. Only one of these will populated. - Value float64 - HistogramValue map[float64]uint64 - SummaryValue map[float64]float64 - // Histograms and Summaries need a count and a sum - Count uint64 - Sum float64 - // Metric timestamp - Timestamp time.Time - // Expiration is the deadline that this Sample is valid until. - Expiration time.Time -} - -// MetricFamily contains the data required to build valid prometheus Metrics. -type MetricFamily struct { - // Samples are the Sample belonging to this MetricFamily. - Samples map[SampleID]*Sample - // Need the telegraf ValueType because there isn't a Prometheus ValueType - // representing Histogram or Summary - TelegrafValueType telegraf.ValueType - // LabelSet is the label counts for all Samples. - LabelSet map[string]int -} - -type PrometheusClient struct { - Listen string - BasicUsername string `toml:"basic_username"` - BasicPassword string `toml:"basic_password"` - IPRange []string `toml:"ip_range"` - ExpirationInterval internal.Duration `toml:"expiration_interval"` - Path string `toml:"path"` - CollectorsExclude []string `toml:"collectors_exclude"` - StringAsLabel bool `toml:"string_as_label"` - ExportTimestamp bool `toml:"export_timestamp"` - - tlsint.ServerConfig - - server *http.Server - url string - - sync.Mutex - // fam is the non-expired MetricFamily by Prometheus metric name. - fam map[string]*MetricFamily - // now returns the current time. - now func() time.Time -} - var sampleConfig = ` ## Address to listen on listen = ":9273" + ## Metric version controls the mapping from Telegraf metrics into + ## Prometheus format. When using the prometheus input, use the same value in + ## both plugins to ensure metrics are round-tripped without modification. + ## + ## example: metric_version = 1; deprecated in 1.13 + ## metric_version = 2; recommended version + # metric_version = 1 + ## Use HTTP Basic Authentication. # basic_username = "Foo" # basic_password = "Bar" @@ -121,46 +72,42 @@ var sampleConfig = ` # export_timestamp = false ` -func (p *PrometheusClient) auth(h http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if p.BasicUsername != "" && p.BasicPassword != "" { - w.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`) - - username, password, ok := r.BasicAuth() - if !ok || - subtle.ConstantTimeCompare([]byte(username), []byte(p.BasicUsername)) != 1 || - subtle.ConstantTimeCompare([]byte(password), []byte(p.BasicPassword)) != 1 { - http.Error(w, "Not authorized", 401) - return - } - } +type Collector interface { + Describe(ch chan<- *prometheus.Desc) + Collect(ch chan<- prometheus.Metric) + Add(metrics []telegraf.Metric) error +} - if len(p.IPRange) > 0 { - matched := false - remoteIPs, _, _ := net.SplitHostPort(r.RemoteAddr) - remoteIP := net.ParseIP(remoteIPs) - for _, iprange := range p.IPRange { - _, ipNet, err := net.ParseCIDR(iprange) - if err != nil { - http.Error(w, "Config Error in ip_range setting", 500) - return - } - if ipNet.Contains(remoteIP) { - matched = true - break - } - } - if !matched { - http.Error(w, "Not authorized", 401) - return - } - } +type PrometheusClient struct { + Listen string `toml:"listen"` + MetricVersion int `toml:"metric_version"` + BasicUsername string `toml:"basic_username"` + BasicPassword string `toml:"basic_password"` + IPRange []string `toml:"ip_range"` + ExpirationInterval internal.Duration `toml:"expiration_interval"` + Path string `toml:"path"` + CollectorsExclude []string `toml:"collectors_exclude"` + StringAsLabel bool `toml:"string_as_label"` + ExportTimestamp bool `toml:"export_timestamp"` + tlsint.ServerConfig - h.ServeHTTP(w, r) - }) + Log telegraf.Logger `toml:"-"` + + server *http.Server + url *url.URL + collector Collector + wg sync.WaitGroup } -func (p *PrometheusClient) Connect() error { +func (p *PrometheusClient) Description() string { + return "Configuration for the Prometheus client to spawn" +} + +func (p *PrometheusClient) SampleConfig() string { + return sampleConfig +} + +func (p *PrometheusClient) Init() error { defaultCollectors := map[string]bool{ "gocollector": true, "process": true, @@ -181,421 +128,137 @@ func (p *PrometheusClient) Connect() error { } } - err := registry.Register(p) - if err != nil { - return err + switch p.MetricVersion { + default: + fallthrough + case 1: + p.Log.Warnf("Use of deprecated configuration: metric_version = 1; please update to metric_version = 2") + p.collector = v1.NewCollector(p.ExpirationInterval.Duration, p.StringAsLabel, p.Log) + err := registry.Register(p.collector) + if err != nil { + return err + } + case 2: + p.collector = v2.NewCollector(p.ExpirationInterval.Duration, p.StringAsLabel) + err := registry.Register(p.collector) + if err != nil { + return err + } } - if p.Listen == "" { - p.Listen = "localhost:9273" - } + ipRange := make([]*net.IPNet, 0, len(p.IPRange)) + for _, cidr := range p.IPRange { + _, ipNet, err := net.ParseCIDR(cidr) + if err != nil { + return fmt.Errorf("error parsing ip_range: %v", err) + } - if p.Path == "" { - p.Path = "/metrics" + ipRange = append(ipRange, ipNet) } + authHandler := internal.AuthHandler(p.BasicUsername, p.BasicPassword, onAuthError) + rangeHandler := internal.IPRangeHandler(ipRange, onError) + promHandler := promhttp.HandlerFor(registry, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}) + mux := http.NewServeMux() - mux.Handle(p.Path, p.auth(promhttp.HandlerFor( - registry, promhttp.HandlerOpts{ErrorHandling: promhttp.ContinueOnError}))) + if p.Path == "" { + p.Path = "/" + } + mux.Handle(p.Path, authHandler(rangeHandler(promHandler))) tlsConfig, err := p.TLSConfig() if err != nil { return err } + p.server = &http.Server{ Addr: p.Listen, Handler: mux, TLSConfig: tlsConfig, } - var listener net.Listener - if tlsConfig != nil { - listener, err = tls.Listen("tcp", p.Listen, tlsConfig) + return nil +} + +func (p *PrometheusClient) listen() (net.Listener, error) { + if p.server.TLSConfig != nil { + return tls.Listen("tcp", p.Listen, p.server.TLSConfig) } else { - listener, err = net.Listen("tcp", p.Listen) + return net.Listen("tcp", p.Listen) } +} + +func (p *PrometheusClient) Connect() error { + listener, err := p.listen() if err != nil { return err } - p.url = createURL(tlsConfig, listener, p.Path) + scheme := "http" + if p.server.TLSConfig != nil { + scheme = "https" + } + + p.url = &url.URL{ + Scheme: scheme, + Host: listener.Addr().String(), + Path: p.Path, + } + + p.Log.Infof("Listening on %s", p.URL()) + p.wg.Add(1) go func() { + defer p.wg.Done() err := p.server.Serve(listener) if err != nil && err != http.ErrServerClosed { - log.Printf("E! Error creating prometheus metric endpoint, err: %s\n", - err.Error()) + p.Log.Errorf("Server error: %v", err) } }() return nil } -// Address returns the address the plugin is listening on. If not listening -// an empty string is returned. -func (p *PrometheusClient) URL() string { - return p.url +func onAuthError(rw http.ResponseWriter, code int) { + rw.Header().Set("WWW-Authenticate", `Basic realm="Restricted"`) + http.Error(rw, http.StatusText(code), code) } -func createURL(tlsConfig *tls.Config, listener net.Listener, path string) string { - u := url.URL{ - Scheme: "http", - Host: listener.Addr().String(), - Path: path, - } +func onError(rw http.ResponseWriter, code int) { + http.Error(rw, http.StatusText(code), code) +} - if tlsConfig != nil { - u.Scheme = "https" +// Address returns the address the plugin is listening on. If not listening +// an empty string is returned. +func (p *PrometheusClient) URL() string { + if p.url != nil { + return p.url.String() } - return u.String() + return "" } func (p *PrometheusClient) Close() error { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() + err := p.server.Shutdown(ctx) - prometheus.Unregister(p) - p.url = "" + p.wg.Wait() + p.url = nil + prometheus.Unregister(p.collector) return err } -func (p *PrometheusClient) SampleConfig() string { - return sampleConfig -} - -func (p *PrometheusClient) Description() string { - return "Configuration for the Prometheus client to spawn" -} - -// Implements prometheus.Collector -func (p *PrometheusClient) Describe(ch chan<- *prometheus.Desc) { - prometheus.NewGauge(prometheus.GaugeOpts{Name: "Dummy", Help: "Dummy"}).Describe(ch) -} - -// Expire removes Samples that have expired. -func (p *PrometheusClient) Expire() { - now := p.now() - for name, family := range p.fam { - for key, sample := range family.Samples { - if p.ExpirationInterval.Duration != 0 && now.After(sample.Expiration) { - for k := range sample.Labels { - family.LabelSet[k]-- - } - delete(family.Samples, key) - - if len(family.Samples) == 0 { - delete(p.fam, name) - } - } - } - } -} - -// Collect implements prometheus.Collector -func (p *PrometheusClient) Collect(ch chan<- prometheus.Metric) { - p.Lock() - defer p.Unlock() - - p.Expire() - - for name, family := range p.fam { - // Get list of all labels on MetricFamily - var labelNames []string - for k, v := range family.LabelSet { - if v > 0 { - labelNames = append(labelNames, k) - } - } - desc := prometheus.NewDesc(name, "Telegraf collected metric", labelNames, nil) - - for _, sample := range family.Samples { - // Get labels for this sample; unset labels will be set to the - // empty string - var labels []string - for _, label := range labelNames { - v := sample.Labels[label] - labels = append(labels, v) - } - - var metric prometheus.Metric - var err error - switch family.TelegrafValueType { - case telegraf.Summary: - metric, err = prometheus.NewConstSummary(desc, sample.Count, sample.Sum, sample.SummaryValue, labels...) - case telegraf.Histogram: - metric, err = prometheus.NewConstHistogram(desc, sample.Count, sample.Sum, sample.HistogramValue, labels...) - default: - metric, err = prometheus.NewConstMetric(desc, getPromValueType(family.TelegrafValueType), sample.Value, labels...) - } - if err != nil { - log.Printf("E! Error creating prometheus metric, "+ - "key: %s, labels: %v,\nerr: %s\n", - name, labels, err.Error()) - continue - } - - if p.ExportTimestamp { - metric = prometheus.NewMetricWithTimestamp(sample.Timestamp, metric) - } - ch <- metric - } - } -} - -func sanitize(value string) string { - return invalidNameCharRE.ReplaceAllString(value, "_") -} - -func isValidTagName(tag string) bool { - return validNameCharRE.MatchString(tag) -} - -func getPromValueType(tt telegraf.ValueType) prometheus.ValueType { - switch tt { - case telegraf.Counter: - return prometheus.CounterValue - case telegraf.Gauge: - return prometheus.GaugeValue - default: - return prometheus.UntypedValue - } -} - -// CreateSampleID creates a SampleID based on the tags of a telegraf.Metric. -func CreateSampleID(tags map[string]string) SampleID { - pairs := make([]string, 0, len(tags)) - for k, v := range tags { - pairs = append(pairs, fmt.Sprintf("%s=%s", k, v)) - } - sort.Strings(pairs) - return SampleID(strings.Join(pairs, ",")) -} - -func addSample(fam *MetricFamily, sample *Sample, sampleID SampleID) { - - for k := range sample.Labels { - fam.LabelSet[k]++ - } - - fam.Samples[sampleID] = sample -} - -func (p *PrometheusClient) addMetricFamily(point telegraf.Metric, sample *Sample, mname string, sampleID SampleID) { - var fam *MetricFamily - var ok bool - if fam, ok = p.fam[mname]; !ok { - fam = &MetricFamily{ - Samples: make(map[SampleID]*Sample), - TelegrafValueType: point.Type(), - LabelSet: make(map[string]int), - } - p.fam[mname] = fam - } - - addSample(fam, sample, sampleID) -} - -// Sorted returns a copy of the metrics in time ascending order. A copy is -// made to avoid modifying the input metric slice since doing so is not -// allowed. -func sorted(metrics []telegraf.Metric) []telegraf.Metric { - batch := make([]telegraf.Metric, 0, len(metrics)) - for i := len(metrics) - 1; i >= 0; i-- { - batch = append(batch, metrics[i]) - } - sort.Slice(batch, func(i, j int) bool { - return batch[i].Time().Before(batch[j].Time()) - }) - return batch -} - func (p *PrometheusClient) Write(metrics []telegraf.Metric) error { - p.Lock() - defer p.Unlock() - - now := p.now() - - for _, point := range sorted(metrics) { - tags := point.Tags() - sampleID := CreateSampleID(tags) - - labels := make(map[string]string) - for k, v := range tags { - tName := sanitize(k) - if !isValidTagName(tName) { - continue - } - labels[tName] = v - } - - // Prometheus doesn't have a string value type, so convert string - // fields to labels if enabled. - if p.StringAsLabel { - for fn, fv := range point.Fields() { - switch fv := fv.(type) { - case string: - tName := sanitize(fn) - if !isValidTagName(tName) { - continue - } - labels[tName] = fv - } - } - } - - switch point.Type() { - case telegraf.Summary: - var mname string - var sum float64 - var count uint64 - summaryvalue := make(map[float64]float64) - for fn, fv := range point.Fields() { - var value float64 - switch fv := fv.(type) { - case int64: - value = float64(fv) - case uint64: - value = float64(fv) - case float64: - value = fv - default: - continue - } - - switch fn { - case "sum": - sum = value - case "count": - count = uint64(value) - default: - limit, err := strconv.ParseFloat(fn, 64) - if err == nil { - summaryvalue[limit] = value - } - } - } - sample := &Sample{ - Labels: labels, - SummaryValue: summaryvalue, - Count: count, - Sum: sum, - Timestamp: point.Time(), - Expiration: now.Add(p.ExpirationInterval.Duration), - } - mname = sanitize(point.Name()) - - if !isValidTagName(mname) { - continue - } - - p.addMetricFamily(point, sample, mname, sampleID) - - case telegraf.Histogram: - var mname string - var sum float64 - var count uint64 - histogramvalue := make(map[float64]uint64) - for fn, fv := range point.Fields() { - var value float64 - switch fv := fv.(type) { - case int64: - value = float64(fv) - case uint64: - value = float64(fv) - case float64: - value = fv - default: - continue - } - - switch fn { - case "sum": - sum = value - case "count": - count = uint64(value) - default: - limit, err := strconv.ParseFloat(fn, 64) - if err == nil { - histogramvalue[limit] = uint64(value) - } - } - } - sample := &Sample{ - Labels: labels, - HistogramValue: histogramvalue, - Count: count, - Sum: sum, - Timestamp: point.Time(), - Expiration: now.Add(p.ExpirationInterval.Duration), - } - mname = sanitize(point.Name()) - - if !isValidTagName(mname) { - continue - } - - p.addMetricFamily(point, sample, mname, sampleID) - - default: - for fn, fv := range point.Fields() { - // Ignore string and bool fields. - var value float64 - switch fv := fv.(type) { - case int64: - value = float64(fv) - case uint64: - value = float64(fv) - case float64: - value = fv - default: - continue - } - - sample := &Sample{ - Labels: labels, - Value: value, - Timestamp: point.Time(), - Expiration: now.Add(p.ExpirationInterval.Duration), - } - - // Special handling of value field; supports passthrough from - // the prometheus input. - var mname string - switch point.Type() { - case telegraf.Counter: - if fn == "counter" { - mname = sanitize(point.Name()) - } - case telegraf.Gauge: - if fn == "gauge" { - mname = sanitize(point.Name()) - } - } - if mname == "" { - if fn == "value" { - mname = sanitize(point.Name()) - } else { - mname = sanitize(fmt.Sprintf("%s_%s", point.Name(), fn)) - } - } - if !isValidTagName(mname) { - continue - } - p.addMetricFamily(point, sample, mname, sampleID) - - } - } - } - return nil + return p.collector.Add(metrics) } func init() { outputs.Add("prometheus_client", func() telegraf.Output { return &PrometheusClient{ - ExpirationInterval: internal.Duration{Duration: time.Second * 60}, + Listen: defaultListen, + Path: defaultPath, + ExpirationInterval: defaultExpirationInterval, StringAsLabel: true, - fam: make(map[string]*MetricFamily), - now: time.Now, } }) } diff --git a/plugins/outputs/prometheus_client/prometheus_client_test.go b/plugins/outputs/prometheus_client/prometheus_client_test.go deleted file mode 100644 index 211e24030dc56..0000000000000 --- a/plugins/outputs/prometheus_client/prometheus_client_test.go +++ /dev/null @@ -1,693 +0,0 @@ -package prometheus_client - -import ( - "testing" - "time" - - "github.com/influxdata/telegraf" - "github.com/influxdata/telegraf/internal" - "github.com/influxdata/telegraf/metric" - prometheus_input "github.com/influxdata/telegraf/plugins/inputs/prometheus" - "github.com/influxdata/telegraf/testutil" - "github.com/stretchr/testify/require" -) - -func setUnixTime(client *PrometheusClient, sec int64) { - client.now = func() time.Time { - return time.Unix(sec, 0) - } -} - -// NewClient initializes a PrometheusClient. -func NewClient() *PrometheusClient { - return &PrometheusClient{ - ExpirationInterval: internal.Duration{Duration: time.Second * 60}, - StringAsLabel: true, - fam: make(map[string]*MetricFamily), - now: time.Now, - } -} - -func TestWrite_Basic(t *testing.T) { - now := time.Now() - pt1, err := metric.New( - "foo", - make(map[string]string), - map[string]interface{}{"value": 0.0}, - now) - var metrics = []telegraf.Metric{ - pt1, - } - - client := NewClient() - err = client.Write(metrics) - require.NoError(t, err) - - fam, ok := client.fam["foo"] - require.True(t, ok) - require.Equal(t, telegraf.Untyped, fam.TelegrafValueType) - require.Equal(t, map[string]int{}, fam.LabelSet) - - sample, ok := fam.Samples[CreateSampleID(pt1.Tags())] - require.True(t, ok) - - require.Equal(t, 0.0, sample.Value) - require.True(t, now.Before(sample.Expiration)) -} - -func TestWrite_IntField(t *testing.T) { - client := NewClient() - - p1, err := metric.New( - "foo", - make(map[string]string), - map[string]interface{}{"value": 42}, - time.Now()) - err = client.Write([]telegraf.Metric{p1}) - require.NoError(t, err) - - fam, ok := client.fam["foo"] - require.True(t, ok) - for _, v := range fam.Samples { - require.Equal(t, 42.0, v.Value) - } - -} - -func TestWrite_FieldNotValue(t *testing.T) { - client := NewClient() - - p1, err := metric.New( - "foo", - make(map[string]string), - map[string]interface{}{"howdy": 0.0}, - time.Now()) - err = client.Write([]telegraf.Metric{p1}) - require.NoError(t, err) - - fam, ok := client.fam["foo_howdy"] - require.True(t, ok) - for _, v := range fam.Samples { - require.Equal(t, 0.0, v.Value) - } -} - -func TestWrite_SkipNonNumberField(t *testing.T) { - client := NewClient() - - p1, err := metric.New( - "foo", - make(map[string]string), - map[string]interface{}{"value": "howdy"}, - time.Now()) - err = client.Write([]telegraf.Metric{p1}) - require.NoError(t, err) - - _, ok := client.fam["foo"] - require.False(t, ok) -} - -func TestWrite_Counters(t *testing.T) { - type args struct { - measurement string - tags map[string]string - fields map[string]interface{} - valueType telegraf.ValueType - } - var tests = []struct { - name string - args args - err error - metricName string - valueType telegraf.ValueType - }{ - { - name: "field named value is not added to metric name", - args: args{ - measurement: "foo", - fields: map[string]interface{}{"value": 42}, - valueType: telegraf.Counter, - }, - metricName: "foo", - valueType: telegraf.Counter, - }, - { - name: "field named counter is not added to metric name", - args: args{ - measurement: "foo", - fields: map[string]interface{}{"counter": 42}, - valueType: telegraf.Counter, - }, - metricName: "foo", - valueType: telegraf.Counter, - }, - { - name: "field with any other name is added to metric name", - args: args{ - measurement: "foo", - fields: map[string]interface{}{"other": 42}, - valueType: telegraf.Counter, - }, - metricName: "foo_other", - valueType: telegraf.Counter, - }, - { - name: "uint64 fields are output", - args: args{ - measurement: "foo", - fields: map[string]interface{}{"value": uint64(42)}, - valueType: telegraf.Counter, - }, - metricName: "foo", - valueType: telegraf.Counter, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - m, err := metric.New( - tt.args.measurement, - tt.args.tags, - tt.args.fields, - time.Now(), - tt.args.valueType, - ) - client := NewClient() - err = client.Write([]telegraf.Metric{m}) - require.Equal(t, tt.err, err) - - fam, ok := client.fam[tt.metricName] - require.True(t, ok) - require.Equal(t, tt.valueType, fam.TelegrafValueType) - }) - } -} - -func TestWrite_Sanitize(t *testing.T) { - client := NewClient() - - p1, err := metric.New( - "foo.bar:colon", - map[string]string{"tag-with-dash": "localhost.local"}, - map[string]interface{}{"field-with-dash-and:colon": 42}, - time.Now(), - telegraf.Counter) - err = client.Write([]telegraf.Metric{p1}) - require.NoError(t, err) - - fam, ok := client.fam["foo_bar:colon_field_with_dash_and:colon"] - require.True(t, ok) - require.Equal(t, map[string]int{"tag_with_dash": 1}, fam.LabelSet) - - sample1, ok := fam.Samples[CreateSampleID(p1.Tags())] - require.True(t, ok) - - require.Equal(t, map[string]string{ - "tag_with_dash": "localhost.local"}, sample1.Labels) -} - -func TestWrite_Gauge(t *testing.T) { - type args struct { - measurement string - tags map[string]string - fields map[string]interface{} - valueType telegraf.ValueType - } - var tests = []struct { - name string - args args - err error - metricName string - valueType telegraf.ValueType - }{ - { - name: "field named value is not added to metric name", - args: args{ - measurement: "foo", - fields: map[string]interface{}{"value": 42}, - valueType: telegraf.Gauge, - }, - metricName: "foo", - valueType: telegraf.Gauge, - }, - { - name: "field named gauge is not added to metric name", - args: args{ - measurement: "foo", - fields: map[string]interface{}{"gauge": 42}, - valueType: telegraf.Gauge, - }, - metricName: "foo", - valueType: telegraf.Gauge, - }, - { - name: "field with any other name is added to metric name", - args: args{ - measurement: "foo", - fields: map[string]interface{}{"other": 42}, - valueType: telegraf.Gauge, - }, - metricName: "foo_other", - valueType: telegraf.Gauge, - }, - { - name: "uint64 fields are output", - args: args{ - measurement: "foo", - fields: map[string]interface{}{"value": uint64(42)}, - valueType: telegraf.Counter, - }, - metricName: "foo", - valueType: telegraf.Counter, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - m, err := metric.New( - tt.args.measurement, - tt.args.tags, - tt.args.fields, - time.Now(), - tt.args.valueType, - ) - client := NewClient() - err = client.Write([]telegraf.Metric{m}) - require.Equal(t, tt.err, err) - - fam, ok := client.fam[tt.metricName] - require.True(t, ok) - require.Equal(t, tt.valueType, fam.TelegrafValueType) - - }) - } -} - -func TestWrite_Summary(t *testing.T) { - client := NewClient() - - p1, err := metric.New( - "foo", - make(map[string]string), - map[string]interface{}{"sum": 84, "count": 42, "0": 2, "0.5": 3, "1": 4}, - time.Now(), - telegraf.Summary) - - err = client.Write([]telegraf.Metric{p1}) - require.NoError(t, err) - - fam, ok := client.fam["foo"] - require.True(t, ok) - require.Equal(t, 1, len(fam.Samples)) - - sample1, ok := fam.Samples[CreateSampleID(p1.Tags())] - require.True(t, ok) - - require.Equal(t, 84.0, sample1.Sum) - require.Equal(t, uint64(42), sample1.Count) - require.Equal(t, 3, len(sample1.SummaryValue)) -} - -func TestWrite_Histogram(t *testing.T) { - client := NewClient() - - p1, err := metric.New( - "foo", - make(map[string]string), - map[string]interface{}{"sum": 84, "count": 42, "0": 2, "0.5": 3, "1": 4}, - time.Now(), - telegraf.Histogram) - - err = client.Write([]telegraf.Metric{p1}) - require.NoError(t, err) - - fam, ok := client.fam["foo"] - require.True(t, ok) - require.Equal(t, 1, len(fam.Samples)) - - sample1, ok := fam.Samples[CreateSampleID(p1.Tags())] - require.True(t, ok) - - require.Equal(t, 84.0, sample1.Sum) - require.Equal(t, uint64(42), sample1.Count) - require.Equal(t, 3, len(sample1.HistogramValue)) -} - -func TestWrite_MixedValueType(t *testing.T) { - now := time.Now() - p1, err := metric.New( - "foo", - make(map[string]string), - map[string]interface{}{"value": 1.0}, - now, - telegraf.Counter) - p2, err := metric.New( - "foo", - make(map[string]string), - map[string]interface{}{"value": 2.0}, - now, - telegraf.Gauge) - var metrics = []telegraf.Metric{p1, p2} - - client := NewClient() - err = client.Write(metrics) - require.NoError(t, err) - - fam, ok := client.fam["foo"] - require.True(t, ok) - require.Equal(t, 1, len(fam.Samples)) -} - -func TestWrite_MixedValueTypeUpgrade(t *testing.T) { - now := time.Now() - p1, err := metric.New( - "foo", - map[string]string{"a": "x"}, - map[string]interface{}{"value": 1.0}, - now, - telegraf.Untyped) - p2, err := metric.New( - "foo", - map[string]string{"a": "y"}, - map[string]interface{}{"value": 2.0}, - now, - telegraf.Gauge) - var metrics = []telegraf.Metric{p1, p2} - - client := NewClient() - err = client.Write(metrics) - require.NoError(t, err) - - fam, ok := client.fam["foo"] - require.True(t, ok) - require.Equal(t, 2, len(fam.Samples)) -} - -func TestWrite_MixedValueTypeDowngrade(t *testing.T) { - now := time.Now() - p1, err := metric.New( - "foo", - map[string]string{"a": "x"}, - map[string]interface{}{"value": 1.0}, - now, - telegraf.Gauge) - p2, err := metric.New( - "foo", - map[string]string{"a": "y"}, - map[string]interface{}{"value": 2.0}, - now, - telegraf.Untyped) - var metrics = []telegraf.Metric{p1, p2} - - client := NewClient() - err = client.Write(metrics) - require.NoError(t, err) - - fam, ok := client.fam["foo"] - require.True(t, ok) - require.Equal(t, 2, len(fam.Samples)) -} - -func TestWrite_Tags(t *testing.T) { - now := time.Now() - p1, err := metric.New( - "foo", - make(map[string]string), - map[string]interface{}{"value": 1.0}, - now) - p2, err := metric.New( - "foo", - map[string]string{"host": "localhost"}, - map[string]interface{}{"value": 2.0}, - now) - var metrics = []telegraf.Metric{p1, p2} - - client := NewClient() - err = client.Write(metrics) - require.NoError(t, err) - - fam, ok := client.fam["foo"] - require.True(t, ok) - require.Equal(t, telegraf.Untyped, fam.TelegrafValueType) - - require.Equal(t, map[string]int{"host": 1}, fam.LabelSet) - - sample1, ok := fam.Samples[CreateSampleID(p1.Tags())] - require.True(t, ok) - - require.Equal(t, 1.0, sample1.Value) - require.True(t, now.Before(sample1.Expiration)) - - sample2, ok := fam.Samples[CreateSampleID(p2.Tags())] - require.True(t, ok) - - require.Equal(t, 2.0, sample2.Value) - require.True(t, now.Before(sample2.Expiration)) -} - -func TestWrite_StringFields(t *testing.T) { - now := time.Now() - p1, err := metric.New( - "foo", - make(map[string]string), - map[string]interface{}{"value": 1.0, "status": "good"}, - now, - telegraf.Counter) - p2, err := metric.New( - "bar", - make(map[string]string), - map[string]interface{}{"status": "needs numeric field"}, - now, - telegraf.Gauge) - var metrics = []telegraf.Metric{p1, p2} - - client := NewClient() - err = client.Write(metrics) - require.NoError(t, err) - - fam, ok := client.fam["foo"] - require.True(t, ok) - require.Equal(t, 1, fam.LabelSet["status"]) - - fam, ok = client.fam["bar"] - require.False(t, ok) -} - -func TestDoNotWrite_StringFields(t *testing.T) { - now := time.Now() - p1, err := metric.New( - "foo", - make(map[string]string), - map[string]interface{}{"value": 1.0, "status": "good"}, - now, - telegraf.Counter) - p2, err := metric.New( - "bar", - make(map[string]string), - map[string]interface{}{"status": "needs numeric field"}, - now, - telegraf.Gauge) - var metrics = []telegraf.Metric{p1, p2} - - client := &PrometheusClient{ - ExpirationInterval: internal.Duration{Duration: time.Second * 60}, - StringAsLabel: false, - fam: make(map[string]*MetricFamily), - now: time.Now, - } - - err = client.Write(metrics) - require.NoError(t, err) - - fam, ok := client.fam["foo"] - require.True(t, ok) - require.Equal(t, 0, fam.LabelSet["status"]) - - fam, ok = client.fam["bar"] - require.False(t, ok) -} - -func TestExpire(t *testing.T) { - client := NewClient() - - p1, err := metric.New( - "foo", - make(map[string]string), - map[string]interface{}{"value": 1.0}, - time.Now()) - setUnixTime(client, 0) - err = client.Write([]telegraf.Metric{p1}) - require.NoError(t, err) - - p2, err := metric.New( - "bar", - make(map[string]string), - map[string]interface{}{"value": 2.0}, - time.Now()) - setUnixTime(client, 1) - err = client.Write([]telegraf.Metric{p2}) - - setUnixTime(client, 61) - require.Equal(t, 2, len(client.fam)) - client.Expire() - require.Equal(t, 1, len(client.fam)) -} - -func TestExpire_TagsNoDecrement(t *testing.T) { - client := NewClient() - - p1, err := metric.New( - "foo", - make(map[string]string), - map[string]interface{}{"value": 1.0}, - time.Now()) - setUnixTime(client, 0) - err = client.Write([]telegraf.Metric{p1}) - require.NoError(t, err) - - p2, err := metric.New( - "foo", - map[string]string{"host": "localhost"}, - map[string]interface{}{"value": 2.0}, - time.Now()) - setUnixTime(client, 1) - err = client.Write([]telegraf.Metric{p2}) - - setUnixTime(client, 61) - fam, ok := client.fam["foo"] - require.True(t, ok) - require.Equal(t, 2, len(fam.Samples)) - client.Expire() - require.Equal(t, 1, len(fam.Samples)) - - require.Equal(t, map[string]int{"host": 1}, fam.LabelSet) -} - -func TestExpire_TagsWithDecrement(t *testing.T) { - client := NewClient() - - p1, err := metric.New( - "foo", - map[string]string{"host": "localhost"}, - map[string]interface{}{"value": 1.0}, - time.Now()) - setUnixTime(client, 0) - err = client.Write([]telegraf.Metric{p1}) - require.NoError(t, err) - - p2, err := metric.New( - "foo", - make(map[string]string), - map[string]interface{}{"value": 2.0}, - time.Now()) - setUnixTime(client, 1) - err = client.Write([]telegraf.Metric{p2}) - - setUnixTime(client, 61) - fam, ok := client.fam["foo"] - require.True(t, ok) - require.Equal(t, 2, len(fam.Samples)) - client.Expire() - require.Equal(t, 1, len(fam.Samples)) - - require.Equal(t, map[string]int{"host": 0}, fam.LabelSet) -} - -var pTesting *PrometheusClient - -func TestPrometheusWritePointEmptyTag(t *testing.T) { - if testing.Short() { - t.Skip("Skipping integration test in short mode") - } - - pClient, p, err := setupPrometheus() - require.NoError(t, err) - defer pClient.Close() - - now := time.Now() - tags := make(map[string]string) - pt1, _ := metric.New( - "test_point_1", - tags, - map[string]interface{}{"value": 0.0}, - now) - pt2, _ := metric.New( - "test_point_2", - tags, - map[string]interface{}{"value": 1.0}, - now) - var metrics = []telegraf.Metric{ - pt1, - pt2, - } - require.NoError(t, pClient.Write(metrics)) - - expected := []struct { - name string - value float64 - tags map[string]string - }{ - {"test_point_1", 0.0, tags}, - {"test_point_2", 1.0, tags}, - } - - var acc testutil.Accumulator - - require.NoError(t, p.Gather(&acc)) - for _, e := range expected { - acc.AssertContainsFields(t, e.name, - map[string]interface{}{"value": e.value}) - } - - tags = make(map[string]string) - tags["testtag"] = "testvalue" - pt3, _ := metric.New( - "test_point_3", - tags, - map[string]interface{}{"value": 0.0}, - now) - pt4, _ := metric.New( - "test_point_4", - tags, - map[string]interface{}{"value": 1.0}, - now) - metrics = []telegraf.Metric{ - pt3, - pt4, - } - require.NoError(t, pClient.Write(metrics)) - - expected2 := []struct { - name string - value float64 - }{ - {"test_point_3", 0.0}, - {"test_point_4", 1.0}, - } - - require.NoError(t, p.Gather(&acc)) - for _, e := range expected2 { - acc.AssertContainsFields(t, e.name, - map[string]interface{}{"value": e.value}) - } -} - -func setupPrometheus() (*PrometheusClient, *prometheus_input.Prometheus, error) { - if pTesting == nil { - pTesting = NewClient() - pTesting.Listen = "localhost:9127" - pTesting.Path = "/metrics" - err := pTesting.Connect() - if err != nil { - return nil, nil, err - } - } else { - pTesting.fam = make(map[string]*MetricFamily) - } - - time.Sleep(time.Millisecond * 200) - - p := &prometheus_input.Prometheus{ - URLs: []string{"http://localhost:9127/metrics"}, - } - - return pTesting, p, nil -} diff --git a/plugins/outputs/prometheus_client/prometheus_client_tls_test.go b/plugins/outputs/prometheus_client/prometheus_client_tls_test.go deleted file mode 100644 index bcbb4e70e4c9c..0000000000000 --- a/plugins/outputs/prometheus_client/prometheus_client_tls_test.go +++ /dev/null @@ -1,114 +0,0 @@ -package prometheus_client_test - -import ( - "crypto/tls" - "fmt" - "net/http" - "testing" - - inttls "github.com/influxdata/telegraf/internal/tls" - "github.com/influxdata/telegraf/plugins/outputs/prometheus_client" - "github.com/influxdata/telegraf/testutil" - "github.com/influxdata/toml" - "github.com/stretchr/testify/require" -) - -var pki = testutil.NewPKI("../../../testutil/pki") - -var configWithTLS = fmt.Sprintf(` - listen = "127.0.0.1:0" - tls_allowed_cacerts = ["%s"] - tls_cert = "%s" - tls_key = "%s" - tls_cipher_suites = ["%s"] - tls_min_version = "%s" -`, pki.TLSServerConfig().TLSAllowedCACerts[0], pki.TLSServerConfig().TLSCert, pki.TLSServerConfig().TLSKey, pki.CipherSuite(), pki.TLSMaxVersion()) - -var configWithoutTLS = ` - listen = "127.0.0.1:0" -` - -type PrometheusClientTestContext struct { - Output *prometheus_client.PrometheusClient - Accumulator *testutil.Accumulator - Client *http.Client -} - -func TestWorksWithoutTLS(t *testing.T) { - tc := buildTestContext(t, []byte(configWithoutTLS)) - err := tc.Output.Connect() - require.NoError(t, err) - defer tc.Output.Close() - - response, err := tc.Client.Get(tc.Output.URL()) - require.NoError(t, err) - - require.NoError(t, err) - require.Equal(t, response.StatusCode, http.StatusOK) -} - -func TestWorksWithTLS(t *testing.T) { - tc := buildTestContext(t, []byte(configWithTLS)) - err := tc.Output.Connect() - require.NoError(t, err) - defer tc.Output.Close() - - serverCiphers, err := inttls.ParseCiphers(tc.Output.ServerConfig.TLSCipherSuites) - require.NoError(t, err) - require.Equal(t, 1, len(serverCiphers)) - - tlsVersion, err := inttls.ParseTLSVersion(tc.Output.ServerConfig.TLSMinVersion) - require.NoError(t, err) - - response, err := tc.Client.Get(tc.Output.URL()) - require.NoError(t, err) - - require.NoError(t, err) - require.Equal(t, response.StatusCode, http.StatusOK) - - require.Equal(t, response.TLS.CipherSuite, serverCiphers[0]) - require.Equal(t, response.TLS.Version, tlsVersion) - - tr := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - } - - client := &http.Client{Transport: tr} - response, err = client.Get(tc.Output.URL()) - - require.Error(t, err) -} - -func buildTestContext(t *testing.T, config []byte) *PrometheusClientTestContext { - output := prometheus_client.NewClient() - err := toml.Unmarshal(config, output) - require.NoError(t, err) - - var ( - httpClient *http.Client - ) - - if len(output.TLSAllowedCACerts) != 0 { - httpClient = buildClientWithTLS(t, output) - } else { - httpClient = buildClientWithoutTLS() - } - - return &PrometheusClientTestContext{ - Output: output, - Accumulator: &testutil.Accumulator{}, - Client: httpClient, - } -} - -func buildClientWithoutTLS() *http.Client { - return &http.Client{} -} - -func buildClientWithTLS(t *testing.T, output *prometheus_client.PrometheusClient) *http.Client { - tlsConfig, err := pki.TLSClientConfig().TLSConfig() - require.NoError(t, err) - - transport := &http.Transport{TLSClientConfig: tlsConfig} - return &http.Client{Transport: transport} -} diff --git a/plugins/outputs/prometheus_client/prometheus_client_v1_test.go b/plugins/outputs/prometheus_client/prometheus_client_v1_test.go new file mode 100644 index 0000000000000..adf18c9f0f076 --- /dev/null +++ b/plugins/outputs/prometheus_client/prometheus_client_v1_test.go @@ -0,0 +1,402 @@ +package prometheus + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/influxdata/telegraf" + inputs "github.com/influxdata/telegraf/plugins/inputs/prometheus" + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/require" +) + +func TestMetricVersion1(t *testing.T) { + Logger := testutil.Logger{Name: "outputs.prometheus_client"} + tests := []struct { + name string + output *PrometheusClient + metrics []telegraf.Metric + expected []byte + }{ + { + name: "simple", + output: &PrometheusClient{ + Listen: ":0", + MetricVersion: 1, + CollectorsExclude: []string{"gocollector", "process"}, + Path: "/metrics", + Log: Logger, + }, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{ + "host": "example.org", + }, + map[string]interface{}{ + "time_idle": 42.0, + }, + time.Unix(0, 0), + ), + }, + expected: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle untyped +cpu_time_idle{host="example.org"} 42 +`), + }, + { + name: "prometheus untyped", + output: &PrometheusClient{ + Listen: ":0", + MetricVersion: 1, + CollectorsExclude: []string{"gocollector", "process"}, + Path: "/metrics", + Log: Logger, + }, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu_time_idle", + map[string]string{ + "host": "example.org", + }, + map[string]interface{}{ + "value": 42.0, + }, + time.Unix(0, 0), + ), + }, + expected: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle untyped +cpu_time_idle{host="example.org"} 42 +`), + }, + { + name: "prometheus counter", + output: &PrometheusClient{ + Listen: ":0", + MetricVersion: 1, + CollectorsExclude: []string{"gocollector", "process"}, + Path: "/metrics", + Log: Logger, + }, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu_time_idle", + map[string]string{ + "host": "example.org", + }, + map[string]interface{}{ + "counter": 42.0, + }, + time.Unix(0, 0), + telegraf.Counter, + ), + }, + expected: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle counter +cpu_time_idle{host="example.org"} 42 +`), + }, + { + name: "replace characters when using string as label", + output: &PrometheusClient{ + Listen: ":0", + MetricVersion: 1, + CollectorsExclude: []string{"gocollector", "process"}, + Path: "/metrics", + StringAsLabel: true, + Log: Logger, + }, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu_time_idle", + map[string]string{}, + map[string]interface{}{ + "host:name": "example.org", + "counter": 42.0, + }, + time.Unix(0, 0), + telegraf.Counter, + ), + }, + expected: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle counter +cpu_time_idle{host_name="example.org"} 42 +`), + }, + { + name: "prometheus gauge", + output: &PrometheusClient{ + Listen: ":0", + MetricVersion: 1, + CollectorsExclude: []string{"gocollector", "process"}, + Path: "/metrics", + Log: Logger, + }, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu_time_idle", + map[string]string{ + "host": "example.org", + }, + map[string]interface{}{ + "gauge": 42.0, + }, + time.Unix(0, 0), + telegraf.Gauge, + ), + }, + expected: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle gauge +cpu_time_idle{host="example.org"} 42 +`), + }, + { + name: "prometheus histogram", + output: &PrometheusClient{ + Listen: ":0", + MetricVersion: 1, + CollectorsExclude: []string{"gocollector", "process"}, + Path: "/metrics", + Log: Logger, + }, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "http_request_duration_seconds", + map[string]string{}, + map[string]interface{}{ + "sum": 53423, + "0.05": 24054, + "0.1": 33444, + "0.2": 100392, + "0.5": 129389, + "1": 133988, + "+Inf": 144320, + "count": 144320, + }, + time.Unix(0, 0), + telegraf.Histogram, + ), + }, + expected: []byte(` +# HELP http_request_duration_seconds Telegraf collected metric +# TYPE http_request_duration_seconds histogram +http_request_duration_seconds_bucket{le="0.05"} 24054 +http_request_duration_seconds_bucket{le="0.1"} 33444 +http_request_duration_seconds_bucket{le="0.2"} 100392 +http_request_duration_seconds_bucket{le="0.5"} 129389 +http_request_duration_seconds_bucket{le="1"} 133988 +http_request_duration_seconds_bucket{le="+Inf"} 144320 +http_request_duration_seconds_sum 53423 +http_request_duration_seconds_count 144320 +`), + }, + { + name: "prometheus summary", + output: &PrometheusClient{ + Listen: ":0", + MetricVersion: 1, + CollectorsExclude: []string{"gocollector", "process"}, + Path: "/metrics", + Log: Logger, + }, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "rpc_duration_seconds", + map[string]string{}, + map[string]interface{}{ + "0.01": 3102, + "0.05": 3272, + "0.5": 4773, + "0.9": 9001, + "0.99": 76656, + "count": 2693, + "sum": 17560473, + }, + time.Unix(0, 0), + telegraf.Summary, + ), + }, + expected: []byte(` +# HELP rpc_duration_seconds Telegraf collected metric +# TYPE rpc_duration_seconds summary +rpc_duration_seconds{quantile="0.01"} 3102 +rpc_duration_seconds{quantile="0.05"} 3272 +rpc_duration_seconds{quantile="0.5"} 4773 +rpc_duration_seconds{quantile="0.9"} 9001 +rpc_duration_seconds{quantile="0.99"} 76656 +rpc_duration_seconds_sum 1.7560473e+07 +rpc_duration_seconds_count 2693 +`), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.output.Init() + require.NoError(t, err) + + err = tt.output.Connect() + require.NoError(t, err) + + defer func() { + err := tt.output.Close() + require.NoError(t, err) + }() + + err = tt.output.Write(tt.metrics) + require.NoError(t, err) + + resp, err := http.Get(tt.output.URL()) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + + require.Equal(t, + strings.TrimSpace(string(tt.expected)), + strings.TrimSpace(string(body))) + }) + } +} + +func TestRoundTripMetricVersion1(t *testing.T) { + Logger := testutil.Logger{Name: "outputs.prometheus_client"} + tests := []struct { + name string + data []byte + }{ + { + name: "untyped", + data: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle untyped +cpu_time_idle{host="example.org"} 42 +`), + }, + { + name: "counter", + data: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle counter +cpu_time_idle{host="example.org"} 42 +`), + }, + { + name: "gauge", + data: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle gauge +cpu_time_idle{host="example.org"} 42 +`), + }, + { + name: "multi", + data: []byte(` +# HELP cpu_time_guest Telegraf collected metric +# TYPE cpu_time_guest gauge +cpu_time_guest{host="one.example.org"} 42 +cpu_time_guest{host="two.example.org"} 42 +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle gauge +cpu_time_idle{host="one.example.org"} 42 +cpu_time_idle{host="two.example.org"} 42 +`), + }, + { + name: "histogram", + data: []byte(` +# HELP http_request_duration_seconds Telegraf collected metric +# TYPE http_request_duration_seconds histogram +http_request_duration_seconds_bucket{le="0.05"} 24054 +http_request_duration_seconds_bucket{le="0.1"} 33444 +http_request_duration_seconds_bucket{le="0.2"} 100392 +http_request_duration_seconds_bucket{le="0.5"} 129389 +http_request_duration_seconds_bucket{le="1"} 133988 +http_request_duration_seconds_bucket{le="+Inf"} 144320 +http_request_duration_seconds_sum 53423 +http_request_duration_seconds_count 144320 +`), + }, + { + name: "summary", + data: []byte(` +# HELP rpc_duration_seconds Telegraf collected metric +# TYPE rpc_duration_seconds summary +rpc_duration_seconds{quantile="0.01"} 3102 +rpc_duration_seconds{quantile="0.05"} 3272 +rpc_duration_seconds{quantile="0.5"} 4773 +rpc_duration_seconds{quantile="0.9"} 9001 +rpc_duration_seconds{quantile="0.99"} 76656 +rpc_duration_seconds_sum 1.7560473e+07 +rpc_duration_seconds_count 2693 +`), + }, + } + + ts := httptest.NewServer(http.NotFoundHandler()) + defer ts.Close() + + url := fmt.Sprintf("http://%s", ts.Listener.Addr()) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ts.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(tt.data) + }) + + input := &inputs.Prometheus{ + URLs: []string{url}, + URLTag: "", + MetricVersion: 1, + } + var acc testutil.Accumulator + err := input.Start(&acc) + require.NoError(t, err) + err = input.Gather(&acc) + require.NoError(t, err) + input.Stop() + + metrics := acc.GetTelegrafMetrics() + + output := &PrometheusClient{ + Listen: "127.0.0.1:0", + Path: defaultPath, + MetricVersion: 1, + Log: Logger, + CollectorsExclude: []string{"gocollector", "process"}, + } + err = output.Init() + require.NoError(t, err) + err = output.Connect() + require.NoError(t, err) + defer func() { + err = output.Close() + require.NoError(t, err) + }() + err = output.Write(metrics) + require.NoError(t, err) + + resp, err := http.Get(output.URL()) + require.NoError(t, err) + + actual, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + + require.Equal(t, + strings.TrimSpace(string(tt.data)), + strings.TrimSpace(string(actual))) + }) + } +} diff --git a/plugins/outputs/prometheus_client/prometheus_client_v2_test.go b/plugins/outputs/prometheus_client/prometheus_client_v2_test.go new file mode 100644 index 0000000000000..755bd5dc40244 --- /dev/null +++ b/plugins/outputs/prometheus_client/prometheus_client_v2_test.go @@ -0,0 +1,376 @@ +package prometheus + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/influxdata/telegraf" + inputs "github.com/influxdata/telegraf/plugins/inputs/prometheus" + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/require" +) + +func TestMetricVersion2(t *testing.T) { + Logger := testutil.Logger{Name: "outputs.prometheus_client"} + tests := []struct { + name string + output *PrometheusClient + metrics []telegraf.Metric + expected []byte + }{ + { + name: "untyped telegraf metric", + output: &PrometheusClient{ + Listen: ":0", + MetricVersion: 2, + CollectorsExclude: []string{"gocollector", "process"}, + Path: "/metrics", + Log: Logger, + }, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{ + "host": "example.org", + }, + map[string]interface{}{ + "time_idle": 42.0, + }, + time.Unix(0, 0), + ), + }, + expected: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle untyped +cpu_time_idle{host="example.org"} 42 +`), + }, + { + name: "strings as labels", + output: &PrometheusClient{ + Listen: ":0", + MetricVersion: 2, + CollectorsExclude: []string{"gocollector", "process"}, + Path: "/metrics", + StringAsLabel: true, + Log: Logger, + }, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "time_idle": 42.0, + "host": "example.org", + }, + time.Unix(0, 0), + ), + }, + expected: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle untyped +cpu_time_idle{host="example.org"} 42 +`), + }, + { + name: "when strings as labels is false string fields are discarded", + output: &PrometheusClient{ + Listen: ":0", + MetricVersion: 2, + CollectorsExclude: []string{"gocollector", "process"}, + Path: "/metrics", + StringAsLabel: false, + Log: Logger, + }, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "time_idle": 42.0, + "host": "example.org", + }, + time.Unix(0, 0), + ), + }, + expected: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle untyped +cpu_time_idle 42 +`), + }, + { + name: "untype prometheus metric", + output: &PrometheusClient{ + Listen: ":0", + MetricVersion: 2, + CollectorsExclude: []string{"gocollector", "process"}, + Path: "/metrics", + Log: Logger, + }, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "prometheus", + map[string]string{ + "host": "example.org", + }, + map[string]interface{}{ + "cpu_time_idle": 42.0, + }, + time.Unix(0, 0), + ), + }, + expected: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle untyped +cpu_time_idle{host="example.org"} 42 +`), + }, + { + name: "telegraf histogram", + output: &PrometheusClient{ + Listen: ":0", + MetricVersion: 2, + CollectorsExclude: []string{"gocollector", "process"}, + Path: "/metrics", + Log: Logger, + }, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{ + "cpu": "cpu1", + }, + map[string]interface{}{ + "usage_idle_sum": 2000.0, + "usage_idle_count": 20.0, + }, + time.Unix(0, 0), + telegraf.Histogram, + ), + testutil.MustMetric( + "cpu", + map[string]string{ + "cpu": "cpu1", + "le": "0.0", + }, + map[string]interface{}{ + "usage_idle_bucket": 0.0, + }, + time.Unix(0, 0), + telegraf.Histogram, + ), + testutil.MustMetric( + "cpu", + map[string]string{ + "cpu": "cpu1", + "le": "50.0", + }, + map[string]interface{}{ + "usage_idle_bucket": 7.0, + }, + time.Unix(0, 0), + telegraf.Histogram, + ), + testutil.MustMetric( + "cpu", + map[string]string{ + "cpu": "cpu1", + "le": "100.0", + }, + map[string]interface{}{ + "usage_idle_bucket": 20.0, + }, + time.Unix(0, 0), + telegraf.Histogram, + ), + testutil.MustMetric( + "cpu", + map[string]string{ + "cpu": "cpu1", + "le": "+Inf", + }, + map[string]interface{}{ + "usage_idle_bucket": 20.0, + }, + time.Unix(0, 0), + telegraf.Histogram, + ), + }, + expected: []byte(` +# HELP cpu_usage_idle Telegraf collected metric +# TYPE cpu_usage_idle histogram +cpu_usage_idle_bucket{cpu="cpu1",le="0"} 0 +cpu_usage_idle_bucket{cpu="cpu1",le="50"} 7 +cpu_usage_idle_bucket{cpu="cpu1",le="100"} 20 +cpu_usage_idle_bucket{cpu="cpu1",le="+Inf"} 20 +cpu_usage_idle_sum{cpu="cpu1"} 2000 +cpu_usage_idle_count{cpu="cpu1"} 20 +`), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.output.Init() + require.NoError(t, err) + + err = tt.output.Connect() + require.NoError(t, err) + + defer func() { + err := tt.output.Close() + require.NoError(t, err) + }() + + err = tt.output.Write(tt.metrics) + require.NoError(t, err) + + resp, err := http.Get(tt.output.URL()) + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + + require.Equal(t, + strings.TrimSpace(string(tt.expected)), + strings.TrimSpace(string(body))) + }) + } +} + +func TestRoundTripMetricVersion2(t *testing.T) { + Logger := testutil.Logger{Name: "outputs.prometheus_client"} + tests := []struct { + name string + data []byte + }{ + { + name: "untyped", + data: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle untyped +cpu_time_idle{host="example.org"} 42 +`), + }, + { + name: "counter", + data: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle counter +cpu_time_idle{host="example.org"} 42 +`), + }, + { + name: "gauge", + data: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle gauge +cpu_time_idle{host="example.org"} 42 +`), + }, + { + name: "multi", + data: []byte(` +# HELP cpu_time_guest Telegraf collected metric +# TYPE cpu_time_guest gauge +cpu_time_guest{host="one.example.org"} 42 +cpu_time_guest{host="two.example.org"} 42 +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle gauge +cpu_time_idle{host="one.example.org"} 42 +cpu_time_idle{host="two.example.org"} 42 +`), + }, + { + name: "histogram", + data: []byte(` +# HELP http_request_duration_seconds Telegraf collected metric +# TYPE http_request_duration_seconds histogram +http_request_duration_seconds_bucket{le="0.05"} 24054 +http_request_duration_seconds_bucket{le="0.1"} 33444 +http_request_duration_seconds_bucket{le="0.2"} 100392 +http_request_duration_seconds_bucket{le="0.5"} 129389 +http_request_duration_seconds_bucket{le="1"} 133988 +http_request_duration_seconds_bucket{le="+Inf"} 144320 +http_request_duration_seconds_sum 53423 +http_request_duration_seconds_count 144320 +`), + }, + { + name: "summary", + data: []byte(` +# HELP rpc_duration_seconds Telegraf collected metric +# TYPE rpc_duration_seconds summary +rpc_duration_seconds{quantile="0.01"} 3102 +rpc_duration_seconds{quantile="0.05"} 3272 +rpc_duration_seconds{quantile="0.5"} 4773 +rpc_duration_seconds{quantile="0.9"} 9001 +rpc_duration_seconds{quantile="0.99"} 76656 +rpc_duration_seconds_sum 1.7560473e+07 +rpc_duration_seconds_count 2693 +`), + }, + } + + ts := httptest.NewServer(http.NotFoundHandler()) + defer ts.Close() + + url := fmt.Sprintf("http://%s", ts.Listener.Addr()) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ts.Config.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + w.Write(tt.data) + }) + + input := &inputs.Prometheus{ + URLs: []string{url}, + URLTag: "", + MetricVersion: 2, + } + var acc testutil.Accumulator + err := input.Start(&acc) + require.NoError(t, err) + err = input.Gather(&acc) + require.NoError(t, err) + input.Stop() + + metrics := acc.GetTelegrafMetrics() + + output := &PrometheusClient{ + Listen: "127.0.0.1:0", + Path: defaultPath, + MetricVersion: 2, + Log: Logger, + CollectorsExclude: []string{"gocollector", "process"}, + } + err = output.Init() + require.NoError(t, err) + err = output.Connect() + require.NoError(t, err) + defer func() { + err = output.Close() + require.NoError(t, err) + }() + err = output.Write(metrics) + require.NoError(t, err) + + resp, err := http.Get(output.URL()) + require.NoError(t, err) + + actual, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + + require.Equal(t, + strings.TrimSpace(string(tt.data)), + strings.TrimSpace(string(actual))) + }) + } +} diff --git a/plugins/outputs/prometheus_client/v1/collector.go b/plugins/outputs/prometheus_client/v1/collector.go new file mode 100644 index 0000000000000..7932bbc59f44d --- /dev/null +++ b/plugins/outputs/prometheus_client/v1/collector.go @@ -0,0 +1,392 @@ +package v1 + +import ( + "fmt" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/influxdata/telegraf" + serializer "github.com/influxdata/telegraf/plugins/serializers/prometheus" + "github.com/prometheus/client_golang/prometheus" +) + +var ( + invalidNameCharRE = regexp.MustCompile(`[^a-zA-Z0-9_:]`) + validNameCharRE = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_]*`) +) + +// SampleID uniquely identifies a Sample +type SampleID string + +// Sample represents the current value of a series. +type Sample struct { + // Labels are the Prometheus labels. + Labels map[string]string + // Value is the value in the Prometheus output. Only one of these will populated. + Value float64 + HistogramValue map[float64]uint64 + SummaryValue map[float64]float64 + // Histograms and Summaries need a count and a sum + Count uint64 + Sum float64 + // Metric timestamp + Timestamp time.Time + // Expiration is the deadline that this Sample is valid until. + Expiration time.Time +} + +// MetricFamily contains the data required to build valid prometheus Metrics. +type MetricFamily struct { + // Samples are the Sample belonging to this MetricFamily. + Samples map[SampleID]*Sample + // Need the telegraf ValueType because there isn't a Prometheus ValueType + // representing Histogram or Summary + TelegrafValueType telegraf.ValueType + // LabelSet is the label counts for all Samples. + LabelSet map[string]int +} + +type Collector struct { + ExpirationInterval time.Duration + StringAsLabel bool + ExportTimestamp bool + Log telegraf.Logger + + sync.Mutex + fam map[string]*MetricFamily +} + +func NewCollector(expire time.Duration, stringsAsLabel bool, logger telegraf.Logger) *Collector { + return &Collector{ + ExpirationInterval: expire, + StringAsLabel: stringsAsLabel, + Log: logger, + fam: make(map[string]*MetricFamily), + } +} + +func (c *Collector) Describe(ch chan<- *prometheus.Desc) { + prometheus.NewGauge(prometheus.GaugeOpts{Name: "Dummy", Help: "Dummy"}).Describe(ch) +} + +func (c *Collector) Collect(ch chan<- prometheus.Metric) { + c.Lock() + defer c.Unlock() + + c.Expire(time.Now(), c.ExpirationInterval) + + for name, family := range c.fam { + // Get list of all labels on MetricFamily + var labelNames []string + for k, v := range family.LabelSet { + if v > 0 { + labelNames = append(labelNames, k) + } + } + desc := prometheus.NewDesc(name, "Telegraf collected metric", labelNames, nil) + + for _, sample := range family.Samples { + // Get labels for this sample; unset labels will be set to the + // empty string + var labels []string + for _, label := range labelNames { + v := sample.Labels[label] + labels = append(labels, v) + } + + var metric prometheus.Metric + var err error + switch family.TelegrafValueType { + case telegraf.Summary: + metric, err = prometheus.NewConstSummary(desc, sample.Count, sample.Sum, sample.SummaryValue, labels...) + case telegraf.Histogram: + metric, err = prometheus.NewConstHistogram(desc, sample.Count, sample.Sum, sample.HistogramValue, labels...) + default: + metric, err = prometheus.NewConstMetric(desc, getPromValueType(family.TelegrafValueType), sample.Value, labels...) + } + if err != nil { + c.Log.Errorf("Error creating prometheus metric: "+ + "key: %s, labels: %v, err: %v", + name, labels, err) + continue + } + + if c.ExportTimestamp { + metric = prometheus.NewMetricWithTimestamp(sample.Timestamp, metric) + } + ch <- metric + } + } +} + +func sanitize(value string) string { + return invalidNameCharRE.ReplaceAllString(value, "_") +} + +func isValidTagName(tag string) bool { + return validNameCharRE.MatchString(tag) +} + +func getPromValueType(tt telegraf.ValueType) prometheus.ValueType { + switch tt { + case telegraf.Counter: + return prometheus.CounterValue + case telegraf.Gauge: + return prometheus.GaugeValue + default: + return prometheus.UntypedValue + } +} + +// CreateSampleID creates a SampleID based on the tags of a telegraf.Metric. +func CreateSampleID(tags map[string]string) SampleID { + pairs := make([]string, 0, len(tags)) + for k, v := range tags { + pairs = append(pairs, fmt.Sprintf("%s=%s", k, v)) + } + sort.Strings(pairs) + return SampleID(strings.Join(pairs, ",")) +} + +func addSample(fam *MetricFamily, sample *Sample, sampleID SampleID) { + + for k := range sample.Labels { + fam.LabelSet[k]++ + } + + fam.Samples[sampleID] = sample +} + +func (c *Collector) addMetricFamily(point telegraf.Metric, sample *Sample, mname string, sampleID SampleID) { + var fam *MetricFamily + var ok bool + if fam, ok = c.fam[mname]; !ok { + fam = &MetricFamily{ + Samples: make(map[SampleID]*Sample), + TelegrafValueType: point.Type(), + LabelSet: make(map[string]int), + } + c.fam[mname] = fam + } + + addSample(fam, sample, sampleID) +} + +// Sorted returns a copy of the metrics in time ascending order. A copy is +// made to avoid modifying the input metric slice since doing so is not +// allowed. +func sorted(metrics []telegraf.Metric) []telegraf.Metric { + batch := make([]telegraf.Metric, 0, len(metrics)) + for i := len(metrics) - 1; i >= 0; i-- { + batch = append(batch, metrics[i]) + } + sort.Slice(batch, func(i, j int) bool { + return batch[i].Time().Before(batch[j].Time()) + }) + return batch +} + +func (c *Collector) Add(metrics []telegraf.Metric) error { + c.Lock() + defer c.Unlock() + + now := time.Now() + + for _, point := range sorted(metrics) { + tags := point.Tags() + sampleID := CreateSampleID(tags) + + labels := make(map[string]string) + for k, v := range tags { + name, ok := serializer.SanitizeLabelName(k) + if !ok { + continue + } + labels[name] = v + } + + // Prometheus doesn't have a string value type, so convert string + // fields to labels if enabled. + if c.StringAsLabel { + for fn, fv := range point.Fields() { + switch fv := fv.(type) { + case string: + name, ok := serializer.SanitizeLabelName(fn) + if !ok { + continue + } + labels[name] = fv + } + } + } + + switch point.Type() { + case telegraf.Summary: + var mname string + var sum float64 + var count uint64 + summaryvalue := make(map[float64]float64) + for fn, fv := range point.Fields() { + var value float64 + switch fv := fv.(type) { + case int64: + value = float64(fv) + case uint64: + value = float64(fv) + case float64: + value = fv + default: + continue + } + + switch fn { + case "sum": + sum = value + case "count": + count = uint64(value) + default: + limit, err := strconv.ParseFloat(fn, 64) + if err == nil { + summaryvalue[limit] = value + } + } + } + sample := &Sample{ + Labels: labels, + SummaryValue: summaryvalue, + Count: count, + Sum: sum, + Timestamp: point.Time(), + Expiration: now.Add(c.ExpirationInterval), + } + mname = sanitize(point.Name()) + + if !isValidTagName(mname) { + continue + } + + c.addMetricFamily(point, sample, mname, sampleID) + + case telegraf.Histogram: + var mname string + var sum float64 + var count uint64 + histogramvalue := make(map[float64]uint64) + for fn, fv := range point.Fields() { + var value float64 + switch fv := fv.(type) { + case int64: + value = float64(fv) + case uint64: + value = float64(fv) + case float64: + value = fv + default: + continue + } + + switch fn { + case "sum": + sum = value + case "count": + count = uint64(value) + default: + limit, err := strconv.ParseFloat(fn, 64) + if err == nil { + histogramvalue[limit] = uint64(value) + } + } + } + sample := &Sample{ + Labels: labels, + HistogramValue: histogramvalue, + Count: count, + Sum: sum, + Timestamp: point.Time(), + Expiration: now.Add(c.ExpirationInterval), + } + mname = sanitize(point.Name()) + + if !isValidTagName(mname) { + continue + } + + c.addMetricFamily(point, sample, mname, sampleID) + + default: + for fn, fv := range point.Fields() { + // Ignore string and bool fields. + var value float64 + switch fv := fv.(type) { + case int64: + value = float64(fv) + case uint64: + value = float64(fv) + case float64: + value = fv + default: + continue + } + + sample := &Sample{ + Labels: labels, + Value: value, + Timestamp: point.Time(), + Expiration: now.Add(c.ExpirationInterval), + } + + // Special handling of value field; supports passthrough from + // the prometheus input. + var mname string + switch point.Type() { + case telegraf.Counter: + if fn == "counter" { + mname = sanitize(point.Name()) + } + case telegraf.Gauge: + if fn == "gauge" { + mname = sanitize(point.Name()) + } + } + if mname == "" { + if fn == "value" { + mname = sanitize(point.Name()) + } else { + mname = sanitize(fmt.Sprintf("%s_%s", point.Name(), fn)) + } + } + if !isValidTagName(mname) { + continue + } + c.addMetricFamily(point, sample, mname, sampleID) + + } + } + } + return nil +} + +func (c *Collector) Expire(now time.Time, age time.Duration) { + if age == 0 { + return + } + + for name, family := range c.fam { + for key, sample := range family.Samples { + if age != 0 && now.After(sample.Expiration) { + for k := range sample.Labels { + family.LabelSet[k]-- + } + delete(family.Samples, key) + + if len(family.Samples) == 0 { + delete(c.fam, name) + } + } + } + } +} diff --git a/plugins/outputs/prometheus_client/v2/collector.go b/plugins/outputs/prometheus_client/v2/collector.go new file mode 100644 index 0000000000000..45e1cb7a7a35e --- /dev/null +++ b/plugins/outputs/prometheus_client/v2/collector.go @@ -0,0 +1,87 @@ +package v2 + +import ( + "sync" + "time" + + "github.com/influxdata/telegraf" + serializer "github.com/influxdata/telegraf/plugins/serializers/prometheus" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" +) + +type Metric struct { + family *dto.MetricFamily + metric *dto.Metric +} + +func (m *Metric) Desc() *prometheus.Desc { + labelNames := make([]string, 0, len(m.metric.Label)) + for _, label := range m.metric.Label { + labelNames = append(labelNames, *label.Name) + } + + desc := prometheus.NewDesc(*m.family.Name, *m.family.Help, labelNames, nil) + + return desc +} + +func (m *Metric) Write(out *dto.Metric) error { + out.Label = m.metric.Label + out.Counter = m.metric.Counter + out.Untyped = m.metric.Untyped + out.Gauge = m.metric.Gauge + out.Histogram = m.metric.Histogram + out.Summary = m.metric.Summary + out.TimestampMs = m.metric.TimestampMs + return nil +} + +type Collector struct { + sync.Mutex + expireDuration time.Duration + coll *serializer.Collection +} + +func NewCollector(expire time.Duration, stringsAsLabel bool) *Collector { + config := serializer.FormatConfig{} + if stringsAsLabel { + config.StringHandling = serializer.StringAsLabel + } + return &Collector{ + expireDuration: expire, + coll: serializer.NewCollection(config), + } +} + +func (c *Collector) Describe(ch chan<- *prometheus.Desc) { + // Sending no descriptor at all marks the Collector as "unchecked", + // i.e. no checks will be performed at registration time, and the + // Collector may yield any Metric it sees fit in its Collect method. + return +} + +func (c *Collector) Collect(ch chan<- prometheus.Metric) { + c.Lock() + defer c.Unlock() + + for _, family := range c.coll.GetProto() { + for _, metric := range family.Metric { + ch <- &Metric{family: family, metric: metric} + } + } +} + +func (c *Collector) Add(metrics []telegraf.Metric) error { + c.Lock() + defer c.Unlock() + + for _, metric := range metrics { + c.coll.Add(metric) + } + + if c.expireDuration != 0 { + c.coll.Expire(time.Now(), c.expireDuration) + } + return nil +} diff --git a/plugins/outputs/syslog/syslog.go b/plugins/outputs/syslog/syslog.go index 013db94a1b211..582e8e920608f 100644 --- a/plugins/outputs/syslog/syslog.go +++ b/plugins/outputs/syslog/syslog.go @@ -8,8 +8,8 @@ import ( "strconv" "strings" - "github.com/influxdata/go-syslog/nontransparent" - "github.com/influxdata/go-syslog/rfc5424" + "github.com/influxdata/go-syslog/v2/nontransparent" + "github.com/influxdata/go-syslog/v2/rfc5424" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal" framing "github.com/influxdata/telegraf/internal/syslog" diff --git a/plugins/outputs/syslog/syslog_mapper.go b/plugins/outputs/syslog/syslog_mapper.go index ba6b0d6609790..4e4848205ca28 100644 --- a/plugins/outputs/syslog/syslog_mapper.go +++ b/plugins/outputs/syslog/syslog_mapper.go @@ -8,7 +8,7 @@ import ( "strings" "time" - "github.com/influxdata/go-syslog/rfc5424" + "github.com/influxdata/go-syslog/v2/rfc5424" "github.com/influxdata/telegraf" ) diff --git a/plugins/outputs/warp10/README.md b/plugins/outputs/warp10/README.md new file mode 100644 index 0000000000000..07e6cd25b92be --- /dev/null +++ b/plugins/outputs/warp10/README.md @@ -0,0 +1,50 @@ +# Warp10 Output Plugin + +The `warp10` output plugin writes metrics to [Warp 10][]. + +### Configuration + +```toml +[[outputs.warp10]] + # Prefix to add to the measurement. + prefix = "telegraf." + + # URL of the Warp 10 server + warp_url = "http://localhost:8080" + + # Write token to access your app on warp 10 + token = "Token" + + # Warp 10 query timeout + # timeout = "15s" + + ## Print Warp 10 error body + # print_error_body = false + + ## Max string error size + # max_string_error_size = 511 + + ## Optional TLS Config + # tls_ca = "/etc/telegraf/ca.pem" + # tls_cert = "/etc/telegraf/cert.pem" + # tls_key = "/etc/telegraf/key.pem" + ## Use TLS but skip chain & host verification + # insecure_skip_verify = false +``` + +### Output Format + +Metrics are converted and sent using the [Geo Time Series][] (GTS) input format. + +The class name of the reading is produced by combining the value of the +`prefix` option, the measurement name, and the field key. A dot (`.`) +character is used as the joining character. + +The GTS form provides support for the Telegraf integer, float, boolean, and +string types directly. Unsigned integer fields will be capped to the largest +64-bit integer (2^63-1) in case of overflow. + +Timestamps are sent in microsecond precision. + +[Warp 10]: https://www.warp10.io +[Geo Time Series]: https://www.warp10.io/content/03_Documentation/03_Interacting_with_Warp_10/03_Ingesting_data/02_GTS_input_format diff --git a/plugins/outputs/warp10/warp10.go b/plugins/outputs/warp10/warp10.go new file mode 100644 index 0000000000000..eead153e03e5a --- /dev/null +++ b/plugins/outputs/warp10/warp10.go @@ -0,0 +1,291 @@ +package warp10 + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "math" + "net/http" + "sort" + "strconv" + "strings" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" + "github.com/influxdata/telegraf/internal/tls" + "github.com/influxdata/telegraf/plugins/outputs" +) + +const ( + defaultClientTimeout = 15 * time.Second +) + +// Warp10 output plugin +type Warp10 struct { + Prefix string `toml:"prefix"` + WarpURL string `toml:"warp_url"` + Token string `toml:"token"` + Timeout internal.Duration `toml:"timeout"` + PrintErrorBody bool `toml:"print_error_body"` + MaxStringErrorSize int `toml:"max_string_error_size"` + client *http.Client + tls.ClientConfig +} + +var sampleConfig = ` + # Prefix to add to the measurement. + prefix = "telegraf." + + # URL of the Warp 10 server + warp_url = "http://localhost:8080" + + # Write token to access your app on warp 10 + token = "Token" + + # Warp 10 query timeout + # timeout = "15s" + + ## Print Warp 10 error body + # print_error_body = false + + ## Max string error size + # max_string_error_size = 511 + + ## Optional TLS Config + # tls_ca = "/etc/telegraf/ca.pem" + # tls_cert = "/etc/telegraf/cert.pem" + # tls_key = "/etc/telegraf/key.pem" + ## Use TLS but skip chain & host verification + # insecure_skip_verify = false +` + +// MetricLine Warp 10 metrics +type MetricLine struct { + Metric string + Timestamp int64 + Value string + Tags string +} + +func (w *Warp10) createClient() (*http.Client, error) { + tlsCfg, err := w.ClientConfig.TLSConfig() + if err != nil { + return nil, err + } + + if w.Timeout.Duration == 0 { + w.Timeout.Duration = defaultClientTimeout + } + + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsCfg, + Proxy: http.ProxyFromEnvironment, + }, + Timeout: w.Timeout.Duration, + } + + return client, nil +} + +// Connect to warp10 +func (w *Warp10) Connect() error { + client, err := w.createClient() + if err != nil { + return err + } + + w.client = client + return nil +} + +// GenWarp10Payload compute Warp 10 metrics payload +func (w *Warp10) GenWarp10Payload(metrics []telegraf.Metric) string { + collectString := make([]string, 0) + for _, mm := range metrics { + + for _, field := range mm.FieldList() { + + metric := &MetricLine{ + Metric: fmt.Sprintf("%s%s", w.Prefix, mm.Name()+"."+field.Key), + Timestamp: mm.Time().UnixNano() / 1000, + } + + metricValue, err := buildValue(field.Value) + if err != nil { + log.Printf("E! [outputs.warp10] Could not encode value: %v", err) + continue + } + metric.Value = metricValue + + tagsSlice := buildTags(mm.TagList()) + metric.Tags = strings.Join(tagsSlice, ",") + + messageLine := fmt.Sprintf("%d// %s{%s} %s\n", metric.Timestamp, metric.Metric, metric.Tags, metric.Value) + + collectString = append(collectString, messageLine) + } + } + return fmt.Sprint(strings.Join(collectString, "")) +} + +// Write metrics to Warp10 +func (w *Warp10) Write(metrics []telegraf.Metric) error { + payload := w.GenWarp10Payload(metrics) + if payload == "" { + return nil + } + + req, err := http.NewRequest("POST", w.WarpURL+"/api/v0/update", bytes.NewBufferString(payload)) + req.Header.Set("X-Warp10-Token", w.Token) + req.Header.Set("Content-Type", "text/plain") + + resp, err := w.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + if w.PrintErrorBody { + body, _ := ioutil.ReadAll(resp.Body) + return fmt.Errorf(w.WarpURL + ": " + w.HandleError(string(body), w.MaxStringErrorSize)) + } + + if len(resp.Status) < w.MaxStringErrorSize { + return fmt.Errorf(w.WarpURL + ": " + resp.Status) + } + + return fmt.Errorf(w.WarpURL + ": " + resp.Status[0:w.MaxStringErrorSize]) + } + + return nil +} + +func buildTags(tags []*telegraf.Tag) []string { + + tagsString := make([]string, len(tags)+1) + indexSource := 0 + for index, tag := range tags { + tagsString[index] = fmt.Sprintf("%s=%s", tag.Key, tag.Value) + indexSource = index + } + indexSource++ + tagsString[indexSource] = fmt.Sprintf("source=telegraf") + sort.Strings(tagsString) + return tagsString +} + +func buildValue(v interface{}) (string, error) { + var retv string + switch p := v.(type) { + case int64: + retv = intToString(p) + case string: + retv = fmt.Sprintf("'%s'", strings.Replace(p, "'", "\\'", -1)) + case bool: + retv = boolToString(p) + case uint64: + if p <= uint64(math.MaxInt64) { + retv = strconv.FormatInt(int64(p), 10) + } else { + retv = strconv.FormatInt(math.MaxInt64, 10) + } + case float64: + retv = floatToString(float64(p)) + default: + return "", fmt.Errorf("unsupported type: %T", v) + } + return retv, nil +} + +func intToString(inputNum int64) string { + return strconv.FormatInt(inputNum, 10) +} + +func boolToString(inputBool bool) string { + return strconv.FormatBool(inputBool) +} + +func uIntToString(inputNum uint64) string { + return strconv.FormatUint(inputNum, 10) +} + +func floatToString(inputNum float64) string { + return strconv.FormatFloat(inputNum, 'f', 6, 64) +} + +// SampleConfig get config +func (w *Warp10) SampleConfig() string { + return sampleConfig +} + +// Description get description +func (w *Warp10) Description() string { + return "Write metrics to Warp 10" +} + +// Close close +func (w *Warp10) Close() error { + return nil +} + +// Init Warp10 struct +func (w *Warp10) Init() error { + if w.MaxStringErrorSize <= 0 { + w.MaxStringErrorSize = 511 + } + return nil +} + +func init() { + outputs.Add("warp10", func() telegraf.Output { + return &Warp10{} + }) +} + +// HandleError read http error body and return a corresponding error +func (w *Warp10) HandleError(body string, maxStringSize int) string { + if body == "" { + return "Empty return" + } + + if strings.Contains(body, "Invalid token") { + return "Invalid token" + } + + if strings.Contains(body, "Write token missing") { + return "Write token missing" + } + + if strings.Contains(body, "Token Expired") { + return "Token Expired" + } + + if strings.Contains(body, "Token revoked") { + return "Token revoked" + } + + if strings.Contains(body, "exceed your Monthly Active Data Streams limit") || strings.Contains(body, "exceed the Monthly Active Data Streams limit") { + return "Exceeded Monthly Active Data Streams limit" + } + + if strings.Contains(body, "Daily Data Points limit being already exceeded") { + return "Exceeded Daily Data Points limit" + } + + if strings.Contains(body, "Application suspended or closed") { + return "Application suspended or closed" + } + + if strings.Contains(body, "broken pipe") { + return "broken pipe" + } + + if len(body) < maxStringSize { + return body + } + return body[0:maxStringSize] +} diff --git a/plugins/outputs/warp10/warp10_test.go b/plugins/outputs/warp10/warp10_test.go new file mode 100644 index 0000000000000..5b543b34c0d8b --- /dev/null +++ b/plugins/outputs/warp10/warp10_test.go @@ -0,0 +1,105 @@ +package warp10 + +import ( + "fmt" + "testing" + + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/require" +) + +type ErrorTest struct { + Message string + Expected string +} + +func TestWriteWarp10(t *testing.T) { + w := Warp10{ + Prefix: "unit.test", + WarpURL: "http://localhost:8090", + Token: "WRITE", + } + + payload := w.GenWarp10Payload(testutil.MockMetrics()) + require.Exactly(t, "1257894000000000// unit.testtest1.value{source=telegraf,tag1=value1} 1.000000\n", payload) +} + +func TestHandleWarp10Error(t *testing.T) { + w := Warp10{ + Prefix: "unit.test", + WarpURL: "http://localhost:8090", + Token: "WRITE", + } + tests := [...]*ErrorTest{ + { + Message: ` + + + + Error 500 io.warp10.script.WarpScriptException: Invalid token. + +

HTTP ERROR 500

+

Problem accessing /api/v0/update. Reason: + 

    io.warp10.script.WarpScriptException: Invalid token.

+ + + `, + Expected: fmt.Sprintf("Invalid token"), + }, + { + Message: ` + + + + Error 500 io.warp10.script.WarpScriptException: Token Expired. + +

HTTP ERROR 500

+

Problem accessing /api/v0/update. Reason: + 

    io.warp10.script.WarpScriptException: Token Expired.

+ + + `, + Expected: fmt.Sprintf("Token Expired"), + }, + { + Message: ` + + + + Error 500 io.warp10.script.WarpScriptException: Token revoked. + +

HTTP ERROR 500

+

Problem accessing /api/v0/update. Reason: + 

    io.warp10.script.WarpScriptException: Token revoked.

+ + + `, + Expected: fmt.Sprintf("Token revoked"), + }, + { + Message: ` + + + + Error 500 io.warp10.script.WarpScriptException: Write token missing. + +

HTTP ERROR 500

+

Problem accessing /api/v0/update. Reason: + 

    io.warp10.script.WarpScriptException: Write token missing.

+ + + `, + Expected: "Write token missing", + }, + { + Message: `Error 503: server unavailable`, + Expected: "Error 503: server unavailable", + }, + } + + for _, handledError := range tests { + payload := w.HandleError(handledError.Message, 511) + require.Exactly(t, handledError.Expected, payload) + } + +} diff --git a/plugins/parsers/csv/README.md b/plugins/parsers/csv/README.md index ec1ffa1cac4e9..1881248ee8233 100644 --- a/plugins/parsers/csv/README.md +++ b/plugins/parsers/csv/README.md @@ -29,6 +29,7 @@ values. ## For assigning explicit data types to columns. ## Supported types: "int", "float", "bool", "string". + ## Specify types in order by column (e.g. `["string", "int", "float"]`) ## If this is not specified, type conversion will be done on the types above. csv_column_types = [] diff --git a/plugins/parsers/csv/parser.go b/plugins/parsers/csv/parser.go index 861844488a68c..b59ea97999426 100644 --- a/plugins/parsers/csv/parser.go +++ b/plugins/parsers/csv/parser.go @@ -45,6 +45,7 @@ func (p *Parser) compile(r *bytes.Reader) (*csv.Reader, error) { if p.Comment != "" { csvReader.Comment = []rune(p.Comment)[0] } + csvReader.TrimLeadingSpace = p.TrimSpace return csvReader, nil } diff --git a/plugins/parsers/csv/parser_test.go b/plugins/parsers/csv/parser_test.go index 6a10c083439eb..1b6fb8f3bc11c 100644 --- a/plugins/parsers/csv/parser_test.go +++ b/plugins/parsers/csv/parser_test.go @@ -243,6 +243,30 @@ func TestTrimSpace(t *testing.T) { require.Equal(t, expectedFields, metrics[0].Fields()) } +func TestTrimSpaceDelimetedBySpace(t *testing.T) { + p := Parser{ + Delimiter: " ", + HeaderRowCount: 1, + TrimSpace: true, + TimeFunc: DefaultTime, + } + testCSV := ` first second third fourth +abcdefgh 0 2 false + abcdef 3.3 4 true + f 0 2 false` + + expectedFields := map[string]interface{}{ + "first": "abcdef", + "second": 3.3, + "third": int64(4), + "fourth": true, + } + + metrics, err := p.Parse([]byte(testCSV)) + require.NoError(t, err) + require.Equal(t, expectedFields, metrics[1].Fields()) +} + func TestSkipRows(t *testing.T) { p := Parser{ HeaderRowCount: 1, diff --git a/plugins/parsers/grok/README.md b/plugins/parsers/grok/README.md index 6263eecc91050..14c128f169d0a 100644 --- a/plugins/parsers/grok/README.md +++ b/plugins/parsers/grok/README.md @@ -50,6 +50,7 @@ You must capture at least one field per line. - ts-httpd ("02/Jan/2006:15:04:05 -0700") - ts-epoch (seconds since unix epoch, may contain decimal) - ts-epochnano (nanoseconds since unix epoch) + - ts-epochmilli (milliseconds since unix epoch) - ts-syslog ("Jan 02 15:04:05", parsed time is set to the current year) - ts-"CUSTOM" diff --git a/plugins/parsers/grok/parser.go b/plugins/parsers/grok/parser.go index ce9c0af5943a1..810190b9d2f12 100644 --- a/plugins/parsers/grok/parser.go +++ b/plugins/parsers/grok/parser.go @@ -28,12 +28,13 @@ var timeLayouts = map[string]string{ "ts-rfc3339": "2006-01-02T15:04:05Z07:00", "ts-rfc3339nano": "2006-01-02T15:04:05.999999999Z07:00", "ts-httpd": "02/Jan/2006:15:04:05 -0700", - // These three are not exactly "layouts", but they are special cases that + // These four are not exactly "layouts", but they are special cases that // will get handled in the ParseLine function. - "ts-epoch": "EPOCH", - "ts-epochnano": "EPOCH_NANO", - "ts-syslog": "SYSLOG_TIMESTAMP", - "ts": "GENERIC_TIMESTAMP", // try parsing all known timestamp layouts. + "ts-epoch": "EPOCH", + "ts-epochnano": "EPOCH_NANO", + "ts-epochmilli": "EPOCH_MILLI", + "ts-syslog": "SYSLOG_TIMESTAMP", + "ts": "GENERIC_TIMESTAMP", // try parsing all known timestamp layouts. } const ( @@ -45,6 +46,7 @@ const ( DURATION = "duration" DROP = "drop" EPOCH = "EPOCH" + EPOCH_MILLI = "EPOCH_MILLI" EPOCH_NANO = "EPOCH_NANO" SYSLOG_TIMESTAMP = "SYSLOG_TIMESTAMP" GENERIC_TIMESTAMP = "GENERIC_TIMESTAMP" @@ -297,6 +299,13 @@ func (p *Parser) ParseLine(line string) (telegraf.Metric, error) { ts = ts.Add(time.Duration(nanosec) * time.Nanosecond) } timestamp = ts + case EPOCH_MILLI: + ms, err := strconv.ParseInt(v, 10, 64) + if err != nil { + log.Printf("E! Error parsing %s to int: %s", v, err) + } else { + timestamp = time.Unix(0, ms*int64(time.Millisecond)) + } case EPOCH_NANO: iv, err := strconv.ParseInt(v, 10, 64) if err != nil { diff --git a/plugins/parsers/grok/parser_test.go b/plugins/parsers/grok/parser_test.go index e0b9575cb3245..ec5e47388f312 100644 --- a/plugins/parsers/grok/parser_test.go +++ b/plugins/parsers/grok/parser_test.go @@ -277,6 +277,28 @@ func TestParsePatternsWithoutCustom(t *testing.T) { assert.Equal(t, time.Unix(0, 1466004605359052000), metricA.Time()) } +func TestParseEpochMilli(t *testing.T) { + p := &Parser{ + Patterns: []string{"%{MYAPP}"}, + CustomPatterns: ` + MYAPP %{POSINT:ts:ts-epochmilli} response_time=%{POSINT:response_time:int} mymetric=%{NUMBER:metric:float} + `, + } + assert.NoError(t, p.Compile()) + + metricA, err := p.ParseLine(`1568540909963 response_time=20821 mymetric=10890.645`) + require.NotNil(t, metricA) + assert.NoError(t, err) + assert.Equal(t, + map[string]interface{}{ + "response_time": int64(20821), + "metric": float64(10890.645), + }, + metricA.Fields()) + assert.Equal(t, map[string]string{}, metricA.Tags()) + assert.Equal(t, time.Unix(0, 1568540909963000000), metricA.Time()) +} + func TestParseEpochNano(t *testing.T) { p := &Parser{ Patterns: []string{"%{MYAPP}"}, diff --git a/plugins/parsers/json/README.md b/plugins/parsers/json/README.md index 08aeef18edd69..b4975bcd334f4 100644 --- a/plugins/parsers/json/README.md +++ b/plugins/parsers/json/README.md @@ -18,6 +18,10 @@ ignored unless specified in the `tag_key` or `json_string_fields` options. ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md data_format = "json" + ## When strict is true and a JSON array is being parsed, all objects within the + ## array must be valid + strict = false + ## Query is a GJSON path that specifies a specific chunk of JSON to be ## parsed, if not specified the whole document will be parsed. ## @@ -25,7 +29,8 @@ ignored unless specified in the `tag_key` or `json_string_fields` options. ## https://github.com/tidwall/gjson/tree/v1.3.0#path-syntax json_query = "" - ## Tag keys is an array of keys that should be added as tags. + ## Tag keys is an array of keys that should be added as tags. Matching keys + ## are no longer saved as fields. tag_keys = [ "my_tag_1", "my_tag_2" diff --git a/plugins/parsers/json/parser.go b/plugins/parsers/json/parser.go index fb64997feeaa1..bba179e1b91cf 100644 --- a/plugins/parsers/json/parser.go +++ b/plugins/parsers/json/parser.go @@ -7,7 +7,6 @@ import ( "fmt" "log" "strconv" - "strings" "time" "github.com/influxdata/telegraf" @@ -32,6 +31,7 @@ type Config struct { TimeFormat string Timezone string DefaultTags map[string]string + Strict bool } type Parser struct { @@ -44,6 +44,7 @@ type Parser struct { timeFormat string timezone string defaultTags map[string]string + strict bool } func New(config *Config) (*Parser, error) { @@ -62,6 +63,7 @@ func New(config *Config) (*Parser, error) { timeFormat: config.TimeFormat, timezone: config.Timezone, defaultTags: config.DefaultTags, + strict: config.Strict, }, nil } @@ -73,7 +75,10 @@ func (p *Parser) parseArray(data []interface{}) ([]telegraf.Metric, error) { case map[string]interface{}: metrics, err := p.parseObject(v) if err != nil { - return nil, err + if p.strict { + return nil, err + } + continue } results = append(results, metrics...) default: @@ -254,19 +259,27 @@ func (f *JSONFlattener) FullFlattenJSON( if f.Fields == nil { f.Fields = make(map[string]interface{}) } - fieldname = strings.Trim(fieldname, "_") + switch t := v.(type) { case map[string]interface{}: for k, v := range t { - err := f.FullFlattenJSON(fieldname+"_"+k+"_", v, convertString, convertBool) + fieldkey := k + if fieldname != "" { + fieldkey = fieldname + "_" + fieldkey + } + + err := f.FullFlattenJSON(fieldkey, v, convertString, convertBool) if err != nil { return err } } case []interface{}: for i, v := range t { - k := strconv.Itoa(i) - err := f.FullFlattenJSON(fieldname+"_"+k+"_", v, convertString, convertBool) + fieldkey := strconv.Itoa(i) + if fieldname != "" { + fieldkey = fieldname + "_" + fieldkey + } + err := f.FullFlattenJSON(fieldkey, v, convertString, convertBool) if err != nil { return nil } diff --git a/plugins/parsers/json/parser_test.go b/plugins/parsers/json/parser_test.go index 44ae73af51801..4571de63a6201 100644 --- a/plugins/parsers/json/parser_test.go +++ b/plugins/parsers/json/parser_test.go @@ -17,6 +17,7 @@ const ( validJSONArrayMultiple = "[{\"a\": 5, \"b\": {\"c\": 6}}, {\"a\": 7, \"b\": {\"c\": 8}}]" invalidJSON = "I don't think this is JSON" invalidJSON2 = "{\"a\": 5, \"b\": \"c\": 6}}" + mixedValidityJSON = "[{\"a\": 5, \"time\": \"2006-01-02T15:04:05\"}, {\"a\": 2}]" ) const validJSONTags = ` @@ -152,6 +153,41 @@ func TestParseInvalidJSON(t *testing.T) { require.Error(t, err) } +func TestParseJSONImplicitStrictness(t *testing.T) { + parserImplicitNoStrict, err := New(&Config{ + MetricName: "json_test", + TimeKey: "time", + }) + require.NoError(t, err) + + _, err = parserImplicitNoStrict.Parse([]byte(mixedValidityJSON)) + require.NoError(t, err) +} + +func TestParseJSONExplicitStrictnessFalse(t *testing.T) { + parserNoStrict, err := New(&Config{ + MetricName: "json_test", + TimeKey: "time", + Strict: false, + }) + require.NoError(t, err) + + _, err = parserNoStrict.Parse([]byte(mixedValidityJSON)) + require.NoError(t, err) +} + +func TestParseJSONExplicitStrictnessTrue(t *testing.T) { + parserStrict, err := New(&Config{ + MetricName: "json_test", + TimeKey: "time", + Strict: true, + }) + require.NoError(t, err) + + _, err = parserStrict.Parse([]byte(mixedValidityJSON)) + require.Error(t, err) +} + func TestParseWithTagKeys(t *testing.T) { // Test that strings not matching tag keys are ignored parser, err := New(&Config{ @@ -779,110 +815,124 @@ func TestNameKey(t *testing.T) { require.Equal(t, "this is my name", metrics[0].Name()) } -func TestTimeKeyDelete(t *testing.T) { - data := `{ - "timestamp": 1541183052, - "value": 42 - }` - - parser, err := New(&Config{ - MetricName: "json", - TimeKey: "timestamp", - TimeFormat: "unix", - }) - require.NoError(t, err) +func TestParseArrayWithWrongType(t *testing.T) { + data := `[{"answer": 42}, 123]` - metrics, err := parser.Parse([]byte(data)) + parser, err := New(&Config{}) require.NoError(t, err) - expected := []telegraf.Metric{ - testutil.MustMetric("json", - map[string]string{}, - map[string]interface{}{"value": 42.0}, - time.Unix(1541183052, 0)), - } - testutil.RequireMetricsEqual(t, expected, metrics) + _, err = parser.Parse([]byte(data)) + require.Error(t, err) } -func TestStringFieldGlob(t *testing.T) { - data := ` +func TestParse(t *testing.T) { + tests := []struct { + name string + config *Config + input []byte + expected []telegraf.Metric + }{ + { + name: "tag keys with underscore issue 6705", + config: &Config{ + MetricName: "json", + TagKeys: []string{"metric___name__"}, + }, + input: []byte(`{"metric": {"__name__": "howdy", "time_idle": 42}}`), + expected: []telegraf.Metric{ + testutil.MustMetric( + "json", + map[string]string{ + "metric___name__": "howdy", + }, + map[string]interface{}{ + "metric_time_idle": 42.0, + }, + time.Unix(0, 0), + ), + }, + }, + { + name: "parse empty array", + config: &Config{}, + input: []byte(`[]`), + expected: []telegraf.Metric{}, + }, + { + name: "parse simple array", + config: &Config{ + MetricName: "json", + }, + input: []byte(`[{"answer": 42}]`), + expected: []telegraf.Metric{ + testutil.MustMetric( + "json", + map[string]string{}, + map[string]interface{}{ + "answer": 42.0, + }, + time.Unix(0, 0), + ), + }, + }, + { + name: "string field glob", + config: &Config{ + MetricName: "json", + StringFields: []string{"*"}, + }, + input: []byte(` { "color": "red", - "status": "error", - "time": "1541183052" + "status": "error" } -` - - parser, err := New(&Config{ - MetricName: "json", - StringFields: []string{"*"}, - TimeKey: "time", - TimeFormat: "unix", - }) - require.NoError(t, err) - - actual, err := parser.Parse([]byte(data)) - require.NoError(t, err) - - expected := []telegraf.Metric{ - testutil.MustMetric( - "json", - map[string]string{}, - map[string]interface{}{ - "color": "red", - "status": "error", +`), + expected: []telegraf.Metric{ + testutil.MustMetric( + "json", + map[string]string{}, + map[string]interface{}{ + "color": "red", + "status": "error", + }, + time.Unix(0, 0), + ), }, - time.Unix(1541183052, 0), - ), - } - - testutil.RequireMetricsEqual(t, expected, actual) -} - -func TestParseEmptyArray(t *testing.T) { - data := `[]` - - parser, err := New(&Config{}) - require.NoError(t, err) - - actual, err := parser.Parse([]byte(data)) - require.NoError(t, err) - - expected := []telegraf.Metric{} - testutil.RequireMetricsEqual(t, expected, actual) + }, + { + name: "time key is deleted from fields", + config: &Config{ + MetricName: "json", + TimeKey: "timestamp", + TimeFormat: "unix", + }, + input: []byte(` +{ + "value": 42, + "timestamp": 1541183052 } - -func TestParseSimpleArray(t *testing.T) { - data := `[{"answer": 42}]` - - parser, err := New(&Config{ - MetricName: "json", - }) - require.NoError(t, err) - - actual, err := parser.Parse([]byte(data)) - require.NoError(t, err) - - expected := []telegraf.Metric{ - testutil.MustMetric( - "json", - map[string]string{}, - map[string]interface{}{ - "answer": 42.0, +`), + expected: []telegraf.Metric{ + testutil.MustMetric( + "json", + map[string]string{}, + map[string]interface{}{ + "value": 42.0, + }, + time.Unix(1541183052, 0), + ), }, - time.Unix(0, 0), - ), + }, } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + parser, err := New(tt.config) + require.NoError(t, err) - testutil.RequireMetricsEqual(t, expected, actual, testutil.IgnoreTime()) -} - -func TestParseArrayWithWrongType(t *testing.T) { - data := `[{"answer": 42}, 123]` + actual, err := parser.Parse(tt.input) + require.NoError(t, err) - parser, err := New(&Config{}) - require.NoError(t, err) - - _, err = parser.Parse([]byte(data)) - require.Error(t, err) + testutil.RequireMetricsEqual(t, tt.expected, actual, testutil.IgnoreTime()) + }) + } } diff --git a/plugins/parsers/registry.go b/plugins/parsers/registry.go index 9e4ea2b1f8a43..1c3af27631efd 100644 --- a/plugins/parsers/registry.go +++ b/plugins/parsers/registry.go @@ -89,6 +89,9 @@ type Config struct { // default timezone JSONTimezone string `toml:"json_timezone"` + // Whether to continue if a JSON object can't be coerced + JSONStrict bool `toml:"json_strict"` + // Authentication file for collectd CollectdAuthFile string `toml:"collectd_auth_file"` // One of none (default), sign, or encrypt @@ -164,6 +167,7 @@ func NewParser(config *Config) (Parser, error) { TimeFormat: config.JSONTimeFormat, Timezone: config.JSONTimezone, DefaultTags: config.DefaultTags, + Strict: config.JSONStrict, }, ) case "value": diff --git a/plugins/processors/all/all.go b/plugins/processors/all/all.go index 47ff83f54840a..e0f69d7874121 100644 --- a/plugins/processors/all/all.go +++ b/plugins/processors/all/all.go @@ -1,6 +1,7 @@ package all import ( + _ "github.com/influxdata/telegraf/plugins/processors/clone" _ "github.com/influxdata/telegraf/plugins/processors/converter" _ "github.com/influxdata/telegraf/plugins/processors/date" _ "github.com/influxdata/telegraf/plugins/processors/enum" diff --git a/plugins/processors/clone/README.md b/plugins/processors/clone/README.md new file mode 100644 index 0000000000000..7ae33d36b235c --- /dev/null +++ b/plugins/processors/clone/README.md @@ -0,0 +1,38 @@ +# Clone Processor Plugin + +The clone processor plugin create a copy of each metric passing through it, +preserving untouched the original metric and allowing modifications in the +copied one. + +The modifications allowed are the ones supported by input plugins and aggregators: + +* name_override +* name_prefix +* name_suffix +* tags + +Select the metrics to modify using the standard +[measurement filtering](https://github.com/influxdata/telegraf/blob/master/docs/CONFIGURATION.md#measurement-filtering) +options. + +Values of *name_override*, *name_prefix*, *name_suffix* and already present +*tags* with conflicting keys will be overwritten. Absent *tags* will be +created. + +A typical use-case is gathering metrics once and cloning them to simulate +having several hosts (modifying ``host`` tag). + +### Configuration: + +```toml +# Apply metric modifications using override semantics. +[[processors.clone]] + ## All modifications on inputs and aggregators can be overridden: + # name_override = "new_name" + # name_prefix = "new_name_prefix" + # name_suffix = "new_name_suffix" + + ## Tags to be added (all values must be strings) + # [processors.clone.tags] + # additional_tag = "tag_value" +``` diff --git a/plugins/processors/clone/clone.go b/plugins/processors/clone/clone.go new file mode 100644 index 0000000000000..ad03fd3e4e2bc --- /dev/null +++ b/plugins/processors/clone/clone.go @@ -0,0 +1,60 @@ +package clone + +import ( + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/plugins/processors" +) + +var sampleConfig = ` + ## All modifications on inputs and aggregators can be overridden: + # name_override = "new_name" + # name_prefix = "new_name_prefix" + # name_suffix = "new_name_suffix" + + ## Tags to be added (all values must be strings) + # [processors.clone.tags] + # additional_tag = "tag_value" +` + +type Clone struct { + NameOverride string + NamePrefix string + NameSuffix string + Tags map[string]string +} + +func (c *Clone) SampleConfig() string { + return sampleConfig +} + +func (c *Clone) Description() string { + return "Clone metrics and apply modifications." +} + +func (c *Clone) Apply(in ...telegraf.Metric) []telegraf.Metric { + cloned := []telegraf.Metric{} + + for _, metric := range in { + cloned = append(cloned, metric.Copy()) + + if len(c.NameOverride) > 0 { + metric.SetName(c.NameOverride) + } + if len(c.NamePrefix) > 0 { + metric.AddPrefix(c.NamePrefix) + } + if len(c.NameSuffix) > 0 { + metric.AddSuffix(c.NameSuffix) + } + for key, value := range c.Tags { + metric.AddTag(key, value) + } + } + return append(in, cloned...) +} + +func init() { + processors.Add("clone", func() telegraf.Processor { + return &Clone{} + }) +} diff --git a/plugins/processors/clone/clone_test.go b/plugins/processors/clone/clone_test.go new file mode 100644 index 0000000000000..f1b8dc5b29c03 --- /dev/null +++ b/plugins/processors/clone/clone_test.go @@ -0,0 +1,83 @@ +package clone + +import ( + "testing" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/metric" + "github.com/stretchr/testify/assert" +) + +func createTestMetric() telegraf.Metric { + metric, _ := metric.New("m1", + map[string]string{"metric_tag": "from_metric"}, + map[string]interface{}{"value": int64(1)}, + time.Now(), + ) + return metric +} + +func calculateProcessedTags(processor Clone, metric telegraf.Metric) map[string]string { + processed := processor.Apply(metric) + return processed[0].Tags() +} + +func TestRetainsTags(t *testing.T) { + processor := Clone{} + + tags := calculateProcessedTags(processor, createTestMetric()) + + value, present := tags["metric_tag"] + assert.True(t, present, "Tag of metric was not present") + assert.Equal(t, "from_metric", value, "Value of Tag was changed") +} + +func TestAddTags(t *testing.T) { + processor := Clone{Tags: map[string]string{"added_tag": "from_config", "another_tag": ""}} + + tags := calculateProcessedTags(processor, createTestMetric()) + + value, present := tags["added_tag"] + assert.True(t, present, "Additional Tag of metric was not present") + assert.Equal(t, "from_config", value, "Value of Tag was changed") + assert.Equal(t, 3, len(tags), "Should have one previous and two added tags.") +} + +func TestOverwritesPresentTagValues(t *testing.T) { + processor := Clone{Tags: map[string]string{"metric_tag": "from_config"}} + + tags := calculateProcessedTags(processor, createTestMetric()) + + value, present := tags["metric_tag"] + assert.True(t, present, "Tag of metric was not present") + assert.Equal(t, 1, len(tags), "Should only have one tag.") + assert.Equal(t, "from_config", value, "Value of Tag was not changed") +} + +func TestOverridesName(t *testing.T) { + processor := Clone{NameOverride: "overridden"} + + processed := processor.Apply(createTestMetric()) + + assert.Equal(t, "overridden", processed[0].Name(), "Name was not overridden") + assert.Equal(t, "m1", processed[1].Name(), "Original metric was modified") +} + +func TestNamePrefix(t *testing.T) { + processor := Clone{NamePrefix: "Pre-"} + + processed := processor.Apply(createTestMetric()) + + assert.Equal(t, "Pre-m1", processed[0].Name(), "Prefix was not applied") + assert.Equal(t, "m1", processed[1].Name(), "Original metric was modified") +} + +func TestNameSuffix(t *testing.T) { + processor := Clone{NameSuffix: "-suff"} + + processed := processor.Apply(createTestMetric()) + + assert.Equal(t, "m1-suff", processed[0].Name(), "Suffix was not applied") + assert.Equal(t, "m1", processed[1].Name(), "Original metric was modified") +} diff --git a/plugins/processors/date/README.md b/plugins/processors/date/README.md index 1a68119e1174d..b04964b4a7165 100644 --- a/plugins/processors/date/README.md +++ b/plugins/processors/date/README.md @@ -19,6 +19,23 @@ A few example usecases include: ## Date format string, must be a representation of the Go "reference time" ## which is "Mon Jan 2 15:04:05 -0700 MST 2006". date_format = "Jan" + + ## Offset duration added to the date string when writing the new tag. + # date_offset = "0s" + + ## Timezone to use when generating the date. This can be set to one of + ## "Local", "UTC", or to a location name in the IANA Time Zone database. + ## example: timezone = "America/Los_Angeles" + # timezone = "UTC" +``` + +#### timezone + +On Windows, only the `Local` and `UTC` zones are available by default. To use +other timezones, set the `ZONEINFO` environment variable to the location of +[`zoneinfo.zip`][zoneinfo]: +``` +set ZONEINFO=C:\zoneinfo.zip ``` ### Example @@ -27,3 +44,5 @@ A few example usecases include: - throughput lower=10i,upper=1000i,mean=500i 1560540094000000000 + throughput,month=Jun lower=10i,upper=1000i,mean=500i 1560540094000000000 ``` + +[zoneinfo]: https://github.com/golang/go/raw/50bd1c4d4eb4fac8ddeb5f063c099daccfb71b26/lib/time/zoneinfo.zip diff --git a/plugins/processors/date/date.go b/plugins/processors/date/date.go index 479106ef2a1af..c8007323f511b 100644 --- a/plugins/processors/date/date.go +++ b/plugins/processors/date/date.go @@ -1,7 +1,10 @@ package date import ( + "time" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/plugins/processors" ) @@ -12,11 +15,25 @@ const sampleConfig = ` ## Date format string, must be a representation of the Go "reference time" ## which is "Mon Jan 2 15:04:05 -0700 MST 2006". date_format = "Jan" + + ## Offset duration added to the date string when writing the new tag. + # date_offset = "0s" + + ## Timezone to use when creating the tag. This can be set to one of + ## "UTC", "Local", or to a location name in the IANA Time Zone database. + ## example: timezone = "America/Los_Angeles" + # timezone = "UTC" ` +const defaultTimezone = "UTC" + type Date struct { - TagKey string `toml:"tag_key"` - DateFormat string `toml:"date_format"` + TagKey string `toml:"tag_key"` + DateFormat string `toml:"date_format"` + DateOffset internal.Duration `toml:"date_offset"` + Timezone string `toml:"timezone"` + + location *time.Location } func (d *Date) SampleConfig() string { @@ -27,9 +44,17 @@ func (d *Date) Description() string { return "Dates measurements, tags, and fields that pass through this filter." } +func (d *Date) Init() error { + var err error + // LoadLocation returns UTC if timezone is the empty string. + d.location, err = time.LoadLocation(d.Timezone) + return err +} + func (d *Date) Apply(in ...telegraf.Metric) []telegraf.Metric { for _, point := range in { - point.AddTag(d.TagKey, point.Time().Format(d.DateFormat)) + tm := point.Time().In(d.location).Add(d.DateOffset.Duration) + point.AddTag(d.TagKey, tm.Format(d.DateFormat)) } return in @@ -37,6 +62,8 @@ func (d *Date) Apply(in ...telegraf.Metric) []telegraf.Metric { func init() { processors.Add("date", func() telegraf.Processor { - return &Date{} + return &Date{ + Timezone: defaultTimezone, + } }) } diff --git a/plugins/processors/date/date_test.go b/plugins/processors/date/date_test.go index 98d88b351a396..d97cc2a9cccab 100644 --- a/plugins/processors/date/date_test.go +++ b/plugins/processors/date/date_test.go @@ -5,8 +5,11 @@ import ( "time" "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/internal" "github.com/influxdata/telegraf/metric" + "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func MustMetric(name string, tags map[string]string, fields map[string]interface{}, metricTime time.Time) telegraf.Metric { @@ -25,6 +28,8 @@ func TestMonthTag(t *testing.T) { TagKey: "month", DateFormat: "Jan", } + err := dateFormatMonth.Init() + require.NoError(t, err) currentTime := time.Now() month := currentTime.Format("Jan") @@ -43,6 +48,10 @@ func TestYearTag(t *testing.T) { TagKey: "year", DateFormat: "2006", } + + err := dateFormatYear.Init() + require.NoError(t, err) + currentTime := time.Now() year := currentTime.Format("2006") @@ -61,7 +70,46 @@ func TestOldDateTag(t *testing.T) { DateFormat: "2006", } + err := dateFormatYear.Init() + require.NoError(t, err) + m7 := MustMetric("foo", nil, nil, time.Date(1993, 05, 27, 0, 0, 0, 0, time.UTC)) customDateApply := dateFormatYear.Apply(m7) assert.Equal(t, map[string]string{"year": "1993"}, customDateApply[0].Tags(), "should add tag 'year'") } + +func TestDateOffset(t *testing.T) { + plugin := &Date{ + TagKey: "hour", + DateFormat: "15", + DateOffset: internal.Duration{Duration: 2 * time.Hour}, + } + + err := plugin.Init() + require.NoError(t, err) + + metric := testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "time_idle": 42.0, + }, + time.Unix(1578603600, 0), + ) + + expected := []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{ + "hour": "23", + }, + map[string]interface{}{ + "time_idle": 42.0, + }, + time.Unix(1578603600, 0), + ), + } + + actual := plugin.Apply(metric) + testutil.RequireMetricsEqual(t, expected, actual) +} diff --git a/plugins/processors/enum/README.md b/plugins/processors/enum/README.md index 29821e83dac9d..0f2a6135dea5f 100644 --- a/plugins/processors/enum/README.md +++ b/plugins/processors/enum/README.md @@ -25,8 +25,8 @@ source tag or field is overwritten. dest = "status_code" ## Default value to be used for all values not contained in the mapping - ## table. When unset, the unmodified value for the field will be used if no - ## match is found. + ## table. When unset and no match is found, the original field will remain + ## unmodified and the destination tag or field will not be created. # default = 0 ## Table of mappings @@ -42,3 +42,9 @@ source tag or field is overwritten. - xyzzy status="green" 1502489900000000000 + xyzzy status="green",status_code=1i 1502489900000000000 ``` + +With unknown value and no default set: +```diff +- xyzzy status="black" 1502489900000000000 ++ xyzzy status="black" 1502489900000000000 +``` diff --git a/plugins/processors/enum/enum_test.go b/plugins/processors/enum/enum_test.go index 06204523d559a..5f89510ca8c90 100644 --- a/plugins/processors/enum/enum_test.go +++ b/plugins/processors/enum/enum_test.go @@ -123,3 +123,14 @@ func TestWritesToDestination(t *testing.T) { assertFieldValue(t, "test", "string_value", fields) assertFieldValue(t, 1, "string_code", fields) } + +func TestDoNotWriteToDestinationWithoutDefaultOrDefinedMapping(t *testing.T) { + field := "string_code" + mapper := EnumMapper{Mappings: []Mapping{{Field: "string_value", Dest: field, ValueMappings: map[string]interface{}{"other": int64(1)}}}} + + fields := calculateProcessedValues(mapper, createTestMetric()) + + assertFieldValue(t, "test", "string_value", fields) + _, present := fields[field] + assert.False(t, present, "value of field '"+field+"' was present") +} diff --git a/plugins/processors/pivot/README.md b/plugins/processors/pivot/README.md index 7d2fa91b431b4..b3eb06fd3f7da 100644 --- a/plugins/processors/pivot/README.md +++ b/plugins/processors/pivot/README.md @@ -24,7 +24,7 @@ To perform the reverse operation use the [unpivot] processor. - cpu,cpu=cpu0,name=time_idle value=42i - cpu,cpu=cpu0,name=time_user value=43i + cpu,cpu=cpu0 time_idle=42i -+ cpu,cpu=cpu0 time_user=42i ++ cpu,cpu=cpu0 time_user=43i ``` [unpivot]: /plugins/processors/unpivot/README.md diff --git a/plugins/processors/strings/README.md b/plugins/processors/strings/README.md index 367732c6f9305..d00bf03db7ff0 100644 --- a/plugins/processors/strings/README.md +++ b/plugins/processors/strings/README.md @@ -12,6 +12,7 @@ Implemented functions are: - trim_suffix - replace - left +- base64decode Please note that in this implementation these are processed in the order that they appear above. @@ -68,6 +69,10 @@ If you'd like to apply multiple processings to the same `tag_key` or `field_key` # [[processors.strings.left]] # field = "message" # width = 10 + + ## Decode a base64 encoded utf-8 string + # [[processors.strings.base64decode]] + # field = "message" ``` #### Trim, TrimLeft, TrimRight diff --git a/plugins/processors/strings/strings.go b/plugins/processors/strings/strings.go index e185bdd3b4042..4a8a6e7ffda13 100644 --- a/plugins/processors/strings/strings.go +++ b/plugins/processors/strings/strings.go @@ -1,23 +1,26 @@ package strings import ( + "encoding/base64" "strings" "unicode" + "unicode/utf8" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/plugins/processors" ) type Strings struct { - Lowercase []converter `toml:"lowercase"` - Uppercase []converter `toml:"uppercase"` - Trim []converter `toml:"trim"` - TrimLeft []converter `toml:"trim_left"` - TrimRight []converter `toml:"trim_right"` - TrimPrefix []converter `toml:"trim_prefix"` - TrimSuffix []converter `toml:"trim_suffix"` - Replace []converter `toml:"replace"` - Left []converter `toml:"left"` + Lowercase []converter `toml:"lowercase"` + Uppercase []converter `toml:"uppercase"` + Trim []converter `toml:"trim"` + TrimLeft []converter `toml:"trim_left"` + TrimRight []converter `toml:"trim_right"` + TrimPrefix []converter `toml:"trim_prefix"` + TrimSuffix []converter `toml:"trim_suffix"` + Replace []converter `toml:"replace"` + Left []converter `toml:"left"` + Base64Decode []converter `toml:"base64decode"` converters []converter init bool @@ -86,6 +89,10 @@ const sampleConfig = ` # [[processors.strings.left]] # field = "message" # width = 10 + + ## Decode a base64 encoded utf-8 string + # [[processors.strings.base64decode]] + # field = "message" ` func (s *Strings) SampleConfig() string { @@ -288,6 +295,20 @@ func (s *Strings) initOnce() { } s.converters = append(s.converters, c) } + for _, c := range s.Base64Decode { + c := c + c.fn = func(s string) string { + data, err := base64.StdEncoding.DecodeString(s) + if err != nil { + return s + } + if utf8.Valid(data) { + return string(data) + } + return s + } + s.converters = append(s.converters, c) + } s.init = true } diff --git a/plugins/processors/strings/strings_test.go b/plugins/processors/strings/strings_test.go index 95d16c05eac6f..ae35acecf1a27 100644 --- a/plugins/processors/strings/strings_test.go +++ b/plugins/processors/strings/strings_test.go @@ -6,6 +6,7 @@ import ( "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/metric" + "github.com/influxdata/telegraf/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -892,3 +893,110 @@ func TestMeasurementCharDeletion(t *testing.T) { assert.Equal(t, "foofoofoo", results[1].Name(), "Should have refused to delete the whole string") assert.Equal(t, "barbarbar", results[2].Name(), "Should not have changed the input") } + +func TestBase64Decode(t *testing.T) { + tests := []struct { + name string + plugin *Strings + metric []telegraf.Metric + expected []telegraf.Metric + }{ + { + name: "base64decode success", + plugin: &Strings{ + Base64Decode: []converter{ + { + Field: "message", + }, + }, + }, + metric: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "message": "aG93ZHk=", + }, + time.Unix(0, 0), + ), + }, + expected: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "message": "howdy", + }, + time.Unix(0, 0), + ), + }, + }, + { + name: "base64decode not valid base64 returns original string", + plugin: &Strings{ + Base64Decode: []converter{ + { + Field: "message", + }, + }, + }, + metric: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "message": "_not_base64_", + }, + time.Unix(0, 0), + ), + }, + expected: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "message": "_not_base64_", + }, + time.Unix(0, 0), + ), + }, + }, + { + name: "base64decode not valid utf-8 returns original string", + plugin: &Strings{ + Base64Decode: []converter{ + { + Field: "message", + }, + }, + }, + metric: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "message": "//5oAG8AdwBkAHkA", + }, + time.Unix(0, 0), + ), + }, + expected: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "message": "//5oAG8AdwBkAHkA", + }, + time.Unix(0, 0), + ), + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + actual := tt.plugin.Apply(tt.metric...) + testutil.RequireMetricsEqual(t, tt.expected, actual) + }) + } +} diff --git a/plugins/serializers/prometheus/README.md b/plugins/serializers/prometheus/README.md new file mode 100644 index 0000000000000..9a0cdfea233e8 --- /dev/null +++ b/plugins/serializers/prometheus/README.md @@ -0,0 +1,68 @@ +# Prometheus + +The `prometheus` data format converts metrics into the Prometheus text +exposition format. When used with the `prometheus` input, the input should be +use the `metric_version = 2` option in order to properly round trip metrics. + +**Warning**: When generating histogram and summary types, output may +not be correct if the metric spans multiple batches. This issue can be +somewhat, but not fully, mitigated by using outputs that support writing in +"batch format". When using histogram and summary types, it is recommended to +use only the `prometheus_client` output. + +## Configuration + +```toml +[[outputs.file]] + files = ["stdout"] + use_batch_format = true + + ## Include the metric timestamp on each sample. + prometheus_export_timestamp = false + + ## Sort prometheus metric families and metric samples. Useful for + ## debugging. + prometheus_sort_metrics = false + + ## Output string fields as metric labels; when false string fields are + ## discarded. + prometheus_string_as_label = false + + ## Data format to output. + ## Each data format has its own unique set of configuration options, read + ## more about them here: + ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md + data_format = "prometheus" +``` + +### Example + +**Example Input** +``` +cpu,cpu=cpu0 time_guest=8022.6,time_system=26145.98,time_user=92512.89 1574317740000000000 +cpu,cpu=cpu1 time_guest=8097.88,time_system=25223.35,time_user=96519.58 1574317740000000000 +cpu,cpu=cpu2 time_guest=7386.28,time_system=24870.37,time_user=95631.59 1574317740000000000 +cpu,cpu=cpu3 time_guest=7434.19,time_system=24843.71,time_user=93753.88 1574317740000000000 +``` + +**Example Output** +``` +# HELP cpu_time_guest Telegraf collected metric +# TYPE cpu_time_guest counter +cpu_time_guest{cpu="cpu0"} 9582.54 +cpu_time_guest{cpu="cpu1"} 9660.88 +cpu_time_guest{cpu="cpu2"} 8946.45 +cpu_time_guest{cpu="cpu3"} 9002.31 +# HELP cpu_time_system Telegraf collected metric +# TYPE cpu_time_system counter +cpu_time_system{cpu="cpu0"} 28675.47 +cpu_time_system{cpu="cpu1"} 27779.34 +cpu_time_system{cpu="cpu2"} 27406.18 +cpu_time_system{cpu="cpu3"} 27404.97 +# HELP cpu_time_user Telegraf collected metric +# TYPE cpu_time_user counter +cpu_time_user{cpu="cpu0"} 99551.84 +cpu_time_user{cpu="cpu1"} 103468.52 +cpu_time_user{cpu="cpu2"} 102591.45 +cpu_time_user{cpu="cpu3"} 100717.05 +``` diff --git a/plugins/serializers/prometheus/collection.go b/plugins/serializers/prometheus/collection.go new file mode 100644 index 0000000000000..5c385caad0881 --- /dev/null +++ b/plugins/serializers/prometheus/collection.go @@ -0,0 +1,483 @@ +package prometheus + +import ( + "hash/fnv" + "sort" + "strconv" + "strings" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/influxdata/telegraf" + dto "github.com/prometheus/client_model/go" +) + +const helpString = "Telegraf collected metric" + +type MetricFamily struct { + Name string + Type telegraf.ValueType +} + +type Metric struct { + Labels []LabelPair + Time time.Time + Scaler *Scaler + Histogram *Histogram + Summary *Summary +} + +type LabelPair struct { + Name string + Value string +} + +type Scaler struct { + Value float64 +} + +type Bucket struct { + Bound float64 + Count uint64 +} + +type Quantile struct { + Quantile float64 + Value float64 +} + +type Histogram struct { + Buckets []Bucket + Count uint64 + Sum float64 +} + +func (h *Histogram) merge(b Bucket) { + for i := range h.Buckets { + if h.Buckets[i].Bound == b.Bound { + h.Buckets[i].Count = b.Count + return + } + } + h.Buckets = append(h.Buckets, b) +} + +type Summary struct { + Quantiles []Quantile + Count uint64 + Sum float64 +} + +func (s *Summary) merge(q Quantile) { + for i := range s.Quantiles { + if s.Quantiles[i].Quantile == q.Quantile { + s.Quantiles[i].Value = q.Value + return + } + } + s.Quantiles = append(s.Quantiles, q) +} + +type MetricKey uint64 + +func MakeMetricKey(labels []LabelPair) MetricKey { + h := fnv.New64a() + for _, label := range labels { + h.Write([]byte(label.Name)) + h.Write([]byte("\x00")) + h.Write([]byte(label.Value)) + h.Write([]byte("\x00")) + } + return MetricKey(h.Sum64()) +} + +type Entry struct { + Family MetricFamily + Metrics map[MetricKey]*Metric +} + +type Collection struct { + config FormatConfig + Entries map[MetricFamily]Entry +} + +func NewCollection(config FormatConfig) *Collection { + cache := &Collection{ + config: config, + Entries: make(map[MetricFamily]Entry), + } + return cache +} + +func hasLabel(name string, labels []LabelPair) bool { + for _, label := range labels { + if name == label.Name { + return true + } + } + return false +} + +func (c *Collection) createLabels(metric telegraf.Metric) []LabelPair { + labels := make([]LabelPair, 0, len(metric.TagList())) + for _, tag := range metric.TagList() { + // Ignore special tags for histogram and summary types. + switch metric.Type() { + case telegraf.Histogram: + if tag.Key == "le" { + continue + } + case telegraf.Summary: + if tag.Key == "quantile" { + continue + } + } + + name, ok := SanitizeLabelName(tag.Key) + if !ok { + continue + } + + labels = append(labels, LabelPair{Name: name, Value: tag.Value}) + } + + if c.config.StringHandling != StringAsLabel { + return labels + } + + addedFieldLabel := false + for _, field := range metric.FieldList() { + value, ok := field.Value.(string) + if !ok { + continue + } + + name, ok := SanitizeLabelName(field.Key) + if !ok { + continue + } + + // If there is a tag with the same name as the string field, discard + // the field and use the tag instead. + if hasLabel(name, labels) { + continue + } + + labels = append(labels, LabelPair{Name: name, Value: value}) + addedFieldLabel = true + + } + + if addedFieldLabel { + sort.Slice(labels, func(i, j int) bool { + return labels[i].Name < labels[j].Name + }) + } + + return labels +} + +func (c *Collection) Add(metric telegraf.Metric) { + labels := c.createLabels(metric) + for _, field := range metric.FieldList() { + metricName := MetricName(metric.Name(), field.Key, metric.Type()) + metricName, ok := SanitizeMetricName(metricName) + if !ok { + continue + } + + family := MetricFamily{ + Name: metricName, + Type: metric.Type(), + } + + entry, ok := c.Entries[family] + if !ok { + entry = Entry{ + Family: family, + Metrics: make(map[MetricKey]*Metric), + } + c.Entries[family] = entry + + } + + metricKey := MakeMetricKey(labels) + + m, ok := entry.Metrics[metricKey] + if ok { + // A batch of metrics can contain multiple values for a single + // Prometheus sample. If this metric is older than the existing + // sample then we can skip over it. + if metric.Time().Before(m.Time) { + continue + } + } + + switch metric.Type() { + case telegraf.Counter: + fallthrough + case telegraf.Gauge: + fallthrough + case telegraf.Untyped: + value, ok := SampleValue(field.Value) + if !ok { + continue + } + + m = &Metric{ + Labels: labels, + Time: metric.Time(), + Scaler: &Scaler{Value: value}, + } + + entry.Metrics[metricKey] = m + case telegraf.Histogram: + if m == nil { + m = &Metric{ + Labels: labels, + Time: metric.Time(), + Histogram: &Histogram{}, + } + } + switch { + case strings.HasSuffix(field.Key, "_bucket"): + le, ok := metric.GetTag("le") + if !ok { + continue + } + bound, err := strconv.ParseFloat(le, 64) + if err != nil { + continue + } + + count, ok := SampleCount(field.Value) + if !ok { + continue + } + + m.Histogram.merge(Bucket{ + Bound: bound, + Count: count, + }) + case strings.HasSuffix(field.Key, "_sum"): + sum, ok := SampleSum(field.Value) + if !ok { + continue + } + + m.Histogram.Sum = sum + case strings.HasSuffix(field.Key, "_count"): + count, ok := SampleCount(field.Value) + if !ok { + continue + } + + m.Histogram.Count = count + default: + continue + } + + entry.Metrics[metricKey] = m + case telegraf.Summary: + if m == nil { + m = &Metric{ + Labels: labels, + Time: metric.Time(), + Summary: &Summary{}, + } + } + switch { + case strings.HasSuffix(field.Key, "_sum"): + sum, ok := SampleSum(field.Value) + if !ok { + continue + } + + m.Summary.Sum = sum + case strings.HasSuffix(field.Key, "_count"): + count, ok := SampleCount(field.Value) + if !ok { + continue + } + + m.Summary.Count = count + default: + quantileTag, ok := metric.GetTag("quantile") + if !ok { + continue + } + quantile, err := strconv.ParseFloat(quantileTag, 64) + if err != nil { + continue + } + + value, ok := SampleValue(field.Value) + if !ok { + continue + } + + m.Summary.merge(Quantile{ + Quantile: quantile, + Value: value, + }) + } + + entry.Metrics[metricKey] = m + } + } +} + +func (c *Collection) Expire(now time.Time, age time.Duration) { + expireTime := now.Add(-age) + for _, entry := range c.Entries { + for key, metric := range entry.Metrics { + if metric.Time.Before(expireTime) { + delete(entry.Metrics, key) + if len(entry.Metrics) == 0 { + delete(c.Entries, entry.Family) + } + } + } + } +} + +func (c *Collection) GetEntries(order MetricSortOrder) []Entry { + entries := make([]Entry, 0, len(c.Entries)) + for _, entry := range c.Entries { + entries = append(entries, entry) + } + + switch order { + case SortMetrics: + sort.Slice(entries, func(i, j int) bool { + lhs := entries[i].Family + rhs := entries[j].Family + if lhs.Name != rhs.Name { + return lhs.Name < rhs.Name + } + + return lhs.Type < rhs.Type + }) + } + return entries +} + +func (c *Collection) GetMetrics(entry Entry, order MetricSortOrder) []*Metric { + metrics := make([]*Metric, 0, len(entry.Metrics)) + for _, metric := range entry.Metrics { + metrics = append(metrics, metric) + } + + switch order { + case SortMetrics: + sort.Slice(metrics, func(i, j int) bool { + lhs := metrics[i].Labels + rhs := metrics[j].Labels + if len(lhs) != len(rhs) { + return len(lhs) < len(rhs) + } + + for index := range lhs { + l := lhs[index] + r := rhs[index] + + if l.Name != r.Name { + return l.Name < r.Name + } + + if l.Value != r.Value { + return l.Value < r.Value + } + } + + return false + }) + } + + return metrics +} + +func (c *Collection) GetProto() []*dto.MetricFamily { + result := make([]*dto.MetricFamily, 0, len(c.Entries)) + + for _, entry := range c.GetEntries(c.config.MetricSortOrder) { + mf := &dto.MetricFamily{ + Name: proto.String(entry.Family.Name), + Help: proto.String(helpString), + Type: MetricType(entry.Family.Type), + } + + for _, metric := range c.GetMetrics(entry, c.config.MetricSortOrder) { + l := make([]*dto.LabelPair, 0, len(metric.Labels)) + for _, label := range metric.Labels { + l = append(l, &dto.LabelPair{ + Name: proto.String(label.Name), + Value: proto.String(label.Value), + }) + } + + m := &dto.Metric{ + Label: l, + } + + if c.config.TimestampExport == ExportTimestamp { + m.TimestampMs = proto.Int64(metric.Time.UnixNano() / int64(time.Millisecond)) + } + + switch entry.Family.Type { + case telegraf.Gauge: + m.Gauge = &dto.Gauge{Value: proto.Float64(metric.Scaler.Value)} + case telegraf.Counter: + m.Counter = &dto.Counter{Value: proto.Float64(metric.Scaler.Value)} + case telegraf.Untyped: + m.Untyped = &dto.Untyped{Value: proto.Float64(metric.Scaler.Value)} + case telegraf.Histogram: + buckets := make([]*dto.Bucket, 0, len(metric.Histogram.Buckets)) + for _, bucket := range metric.Histogram.Buckets { + buckets = append(buckets, &dto.Bucket{ + UpperBound: proto.Float64(bucket.Bound), + CumulativeCount: proto.Uint64(bucket.Count), + }) + } + + if len(buckets) == 0 { + continue + } + + m.Histogram = &dto.Histogram{ + Bucket: buckets, + SampleCount: proto.Uint64(metric.Histogram.Count), + SampleSum: proto.Float64(metric.Histogram.Sum), + } + case telegraf.Summary: + quantiles := make([]*dto.Quantile, 0, len(metric.Summary.Quantiles)) + for _, quantile := range metric.Summary.Quantiles { + quantiles = append(quantiles, &dto.Quantile{ + Quantile: proto.Float64(quantile.Quantile), + Value: proto.Float64(quantile.Value), + }) + } + + if len(quantiles) == 0 { + continue + } + + m.Summary = &dto.Summary{ + Quantile: quantiles, + SampleCount: proto.Uint64(metric.Summary.Count), + SampleSum: proto.Float64(metric.Summary.Sum), + } + default: + panic("unknown telegraf.ValueType") + } + + mf.Metric = append(mf.Metric, m) + } + + if len(mf.Metric) != 0 { + result = append(result, mf) + } + } + + return result +} diff --git a/plugins/serializers/prometheus/collection_test.go b/plugins/serializers/prometheus/collection_test.go new file mode 100644 index 0000000000000..70f26dac788d7 --- /dev/null +++ b/plugins/serializers/prometheus/collection_test.go @@ -0,0 +1,347 @@ +package prometheus + +import ( + "math" + "testing" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/testutil" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" +) + +func TestCollectionExpire(t *testing.T) { + tests := []struct { + name string + now time.Time + age time.Duration + metrics []telegraf.Metric + expected []*dto.MetricFamily + }{ + { + name: "not expired", + now: time.Unix(1, 0), + age: 10 * time.Second, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "time_idle": 42.0, + }, + time.Unix(0, 0), + ), + }, + expected: []*dto.MetricFamily{ + { + Name: proto.String("cpu_time_idle"), + Help: proto.String(helpString), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{}, + Untyped: &dto.Untyped{Value: proto.Float64(42.0)}, + }, + }, + }, + }, + }, + { + name: "update metric expiration", + now: time.Unix(20, 0), + age: 10 * time.Second, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "time_idle": 42.0, + }, + time.Unix(0, 0), + ), + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "time_idle": 43.0, + }, + time.Unix(12, 0), + ), + }, + expected: []*dto.MetricFamily{ + { + Name: proto.String("cpu_time_idle"), + Help: proto.String(helpString), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{}, + Untyped: &dto.Untyped{Value: proto.Float64(43.0)}, + }, + }, + }, + }, + }, + { + name: "update metric expiration descending order", + now: time.Unix(20, 0), + age: 10 * time.Second, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "time_idle": 42.0, + }, + time.Unix(12, 0), + ), + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "time_idle": 43.0, + }, + time.Unix(0, 0), + ), + }, + expected: []*dto.MetricFamily{ + { + Name: proto.String("cpu_time_idle"), + Help: proto.String(helpString), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{}, + Untyped: &dto.Untyped{Value: proto.Float64(42.0)}, + }, + }, + }, + }, + }, + { + name: "expired single metric in metric family", + now: time.Unix(20, 0), + age: 10 * time.Second, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "time_idle": 42.0, + }, + time.Unix(0, 0), + ), + }, + expected: []*dto.MetricFamily{}, + }, + { + name: "expired one metric in metric family", + now: time.Unix(20, 0), + age: 10 * time.Second, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "time_idle": 42.0, + }, + time.Unix(0, 0), + ), + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "time_guest": 42.0, + }, + time.Unix(15, 0), + ), + }, + expected: []*dto.MetricFamily{ + { + Name: proto.String("cpu_time_guest"), + Help: proto.String(helpString), + Type: dto.MetricType_UNTYPED.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{}, + Untyped: &dto.Untyped{Value: proto.Float64(42.0)}, + }, + }, + }, + }, + }, + { + name: "histogram bucket updates", + now: time.Unix(0, 0), + age: 10 * time.Second, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "prometheus", + map[string]string{}, + map[string]interface{}{ + "http_request_duration_seconds_sum": 10.0, + "http_request_duration_seconds_count": 2, + }, + time.Unix(0, 0), + telegraf.Histogram, + ), + testutil.MustMetric( + "prometheus", + map[string]string{"le": "0.05"}, + map[string]interface{}{ + "http_request_duration_seconds_bucket": 1.0, + }, + time.Unix(0, 0), + telegraf.Histogram, + ), + testutil.MustMetric( + "prometheus", + map[string]string{"le": "+Inf"}, + map[string]interface{}{ + "http_request_duration_seconds_bucket": 1.0, + }, + time.Unix(0, 0), + telegraf.Histogram, + ), + // Next interval + testutil.MustMetric( + "prometheus", + map[string]string{}, + map[string]interface{}{ + "http_request_duration_seconds_sum": 20.0, + "http_request_duration_seconds_count": 4, + }, + time.Unix(0, 0), + telegraf.Histogram, + ), + testutil.MustMetric( + "prometheus", + map[string]string{"le": "0.05"}, + map[string]interface{}{ + "http_request_duration_seconds_bucket": 2.0, + }, + time.Unix(0, 0), + telegraf.Histogram, + ), + testutil.MustMetric( + "prometheus", + map[string]string{"le": "+Inf"}, + map[string]interface{}{ + "http_request_duration_seconds_bucket": 2.0, + }, + time.Unix(0, 0), + telegraf.Histogram, + ), + }, + expected: []*dto.MetricFamily{ + { + Name: proto.String("http_request_duration_seconds"), + Help: proto.String(helpString), + Type: dto.MetricType_HISTOGRAM.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{}, + Histogram: &dto.Histogram{ + SampleCount: proto.Uint64(4), + SampleSum: proto.Float64(20.0), + Bucket: []*dto.Bucket{ + { + UpperBound: proto.Float64(0.05), + CumulativeCount: proto.Uint64(2), + }, + { + UpperBound: proto.Float64(math.Inf(1)), + CumulativeCount: proto.Uint64(2), + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "summary quantile updates", + now: time.Unix(0, 0), + age: 10 * time.Second, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "prometheus", + map[string]string{}, + map[string]interface{}{ + "rpc_duration_seconds_sum": 1.0, + "rpc_duration_seconds_count": 1, + }, + time.Unix(0, 0), + telegraf.Summary, + ), + testutil.MustMetric( + "prometheus", + map[string]string{"quantile": "0.01"}, + map[string]interface{}{ + "rpc_duration_seconds": 1.0, + }, + time.Unix(0, 0), + telegraf.Summary, + ), + // Updated Summary + testutil.MustMetric( + "prometheus", + map[string]string{}, + map[string]interface{}{ + "rpc_duration_seconds_sum": 2.0, + "rpc_duration_seconds_count": 2, + }, + time.Unix(0, 0), + telegraf.Summary, + ), + testutil.MustMetric( + "prometheus", + map[string]string{"quantile": "0.01"}, + map[string]interface{}{ + "rpc_duration_seconds": 2.0, + }, + time.Unix(0, 0), + telegraf.Summary, + ), + }, + expected: []*dto.MetricFamily{ + { + Name: proto.String("rpc_duration_seconds"), + Help: proto.String(helpString), + Type: dto.MetricType_SUMMARY.Enum(), + Metric: []*dto.Metric{ + { + Label: []*dto.LabelPair{}, + Summary: &dto.Summary{ + SampleCount: proto.Uint64(2), + SampleSum: proto.Float64(2.0), + Quantile: []*dto.Quantile{ + { + Quantile: proto.Float64(0.01), + Value: proto.Float64(2), + }, + }, + }, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := NewCollection(FormatConfig{}) + for _, metric := range tt.metrics { + c.Add(metric) + } + c.Expire(tt.now, tt.age) + + actual := c.GetProto() + + require.Equal(t, tt.expected, actual) + }) + } +} diff --git a/plugins/serializers/prometheus/convert.go b/plugins/serializers/prometheus/convert.go new file mode 100644 index 0000000000000..131ac31b8036c --- /dev/null +++ b/plugins/serializers/prometheus/convert.go @@ -0,0 +1,215 @@ +package prometheus + +import ( + "strings" + "unicode" + + "github.com/influxdata/telegraf" + dto "github.com/prometheus/client_model/go" +) + +type Table struct { + First *unicode.RangeTable + Rest *unicode.RangeTable +} + +var MetricNameTable = Table{ + First: &unicode.RangeTable{ + R16: []unicode.Range16{ + {0x003A, 0x003A, 1}, // : + {0x0041, 0x005A, 1}, // A-Z + {0x005F, 0x005F, 1}, // _ + {0x0061, 0x007A, 1}, // a-z + }, + LatinOffset: 4, + }, + Rest: &unicode.RangeTable{ + R16: []unicode.Range16{ + {0x0030, 0x003A, 1}, // 0-: + {0x0041, 0x005A, 1}, // A-Z + {0x005F, 0x005F, 1}, // _ + {0x0061, 0x007A, 1}, // a-z + }, + LatinOffset: 4, + }, +} + +var LabelNameTable = Table{ + First: &unicode.RangeTable{ + R16: []unicode.Range16{ + {0x0041, 0x005A, 1}, // A-Z + {0x005F, 0x005F, 1}, // _ + {0x0061, 0x007A, 1}, // a-z + }, + LatinOffset: 3, + }, + Rest: &unicode.RangeTable{ + R16: []unicode.Range16{ + {0x0030, 0x0039, 1}, // 0-9 + {0x0041, 0x005A, 1}, // A-Z + {0x005F, 0x005F, 1}, // _ + {0x0061, 0x007A, 1}, // a-z + }, + LatinOffset: 4, + }, +} + +func isValid(name string, table Table) bool { + if name == "" { + return false + } + + for i, r := range name { + switch { + case i == 0: + if !unicode.In(r, table.First) { + return false + } + default: + if !unicode.In(r, table.Rest) { + return false + } + } + } + + return true +} + +// Sanitize checks if the name is valid according to the table. If not, it +// attempts to replaces invalid runes with an underscore to create a valid +// name. +func sanitize(name string, table Table) (string, bool) { + if isValid(name, table) { + return name, true + } + + var b strings.Builder + + for i, r := range name { + switch { + case i == 0: + if unicode.In(r, table.First) { + b.WriteRune(r) + } + default: + if unicode.In(r, table.Rest) { + b.WriteRune(r) + } else { + b.WriteString("_") + } + } + } + + name = strings.Trim(b.String(), "_") + if name == "" { + return "", false + } + + return name, true +} + +// SanitizeMetricName checks if the name is a valid Prometheus metric name. If +// not, it attempts to replaces invalid runes with an underscore to create a +// valid name. +func SanitizeMetricName(name string) (string, bool) { + return sanitize(name, MetricNameTable) +} + +// SanitizeLabelName checks if the name is a valid Prometheus label name. If +// not, it attempts to replaces invalid runes with an underscore to create a +// valid name. +func SanitizeLabelName(name string) (string, bool) { + return sanitize(name, LabelNameTable) +} + +// MetricName returns the Prometheus metric name. +func MetricName(measurement, fieldKey string, valueType telegraf.ValueType) string { + switch valueType { + case telegraf.Histogram, telegraf.Summary: + switch { + case strings.HasSuffix(fieldKey, "_bucket"): + fieldKey = strings.TrimSuffix(fieldKey, "_bucket") + case strings.HasSuffix(fieldKey, "_sum"): + fieldKey = strings.TrimSuffix(fieldKey, "_sum") + case strings.HasSuffix(fieldKey, "_count"): + fieldKey = strings.TrimSuffix(fieldKey, "_count") + } + } + + if measurement == "prometheus" { + return fieldKey + } + return measurement + "_" + fieldKey +} + +func MetricType(valueType telegraf.ValueType) *dto.MetricType { + switch valueType { + case telegraf.Counter: + return dto.MetricType_COUNTER.Enum() + case telegraf.Gauge: + return dto.MetricType_GAUGE.Enum() + case telegraf.Summary: + return dto.MetricType_SUMMARY.Enum() + case telegraf.Untyped: + return dto.MetricType_UNTYPED.Enum() + case telegraf.Histogram: + return dto.MetricType_HISTOGRAM.Enum() + default: + panic("unknown telegraf.ValueType") + } +} + +// SampleValue converts a field value into a value suitable for a simple sample value. +func SampleValue(value interface{}) (float64, bool) { + switch v := value.(type) { + case float64: + return v, true + case int64: + return float64(v), true + case uint64: + return float64(v), true + case bool: + if v { + return 1.0, true + } + return 0.0, true + default: + return 0, false + } +} + +// SampleCount converts a field value into a count suitable for a metric family +// of the Histogram or Summary type. +func SampleCount(value interface{}) (uint64, bool) { + switch v := value.(type) { + case float64: + if v < 0 { + return 0, false + } + return uint64(v), true + case int64: + if v < 0 { + return 0, false + } + return uint64(v), true + case uint64: + return v, true + default: + return 0, false + } +} + +// SampleSum converts a field value into a sum suitable for a metric family +// of the Histogram or Summary type. +func SampleSum(value interface{}) (float64, bool) { + switch v := value.(type) { + case float64: + return v, true + case int64: + return float64(v), true + case uint64: + return float64(v), true + default: + return 0, false + } +} diff --git a/plugins/serializers/prometheus/prometheus.go b/plugins/serializers/prometheus/prometheus.go new file mode 100644 index 0000000000000..11c305aa40330 --- /dev/null +++ b/plugins/serializers/prometheus/prometheus.go @@ -0,0 +1,69 @@ +package prometheus + +import ( + "bytes" + + "github.com/influxdata/telegraf" + "github.com/prometheus/common/expfmt" +) + +// TimestampExport controls if the output contains timestamps. +type TimestampExport int + +const ( + NoExportTimestamp TimestampExport = iota + ExportTimestamp +) + +// MetricSortOrder controls if the output is sorted. +type MetricSortOrder int + +const ( + NoSortMetrics MetricSortOrder = iota + SortMetrics +) + +// StringHandling defines how to process string fields. +type StringHandling int + +const ( + DiscardStrings StringHandling = iota + StringAsLabel +) + +type FormatConfig struct { + TimestampExport TimestampExport + MetricSortOrder MetricSortOrder + StringHandling StringHandling +} + +type Serializer struct { + config FormatConfig +} + +func NewSerializer(config FormatConfig) (*Serializer, error) { + s := &Serializer{config: config} + return s, nil +} + +func (s *Serializer) Serialize(metric telegraf.Metric) ([]byte, error) { + return s.SerializeBatch([]telegraf.Metric{metric}) +} + +func (s *Serializer) SerializeBatch(metrics []telegraf.Metric) ([]byte, error) { + coll := NewCollection(s.config) + for _, metric := range metrics { + coll.Add(metric) + } + + var buf bytes.Buffer + for _, mf := range coll.GetProto() { + enc := expfmt.NewEncoder(&buf, expfmt.FmtText) + err := enc.Encode(mf) + if err != nil { + return nil, err + } + } + + return buf.Bytes(), nil +} diff --git a/plugins/serializers/prometheus/prometheus_test.go b/plugins/serializers/prometheus/prometheus_test.go new file mode 100644 index 0000000000000..ff082f7b26e3f --- /dev/null +++ b/plugins/serializers/prometheus/prometheus_test.go @@ -0,0 +1,667 @@ +package prometheus + +import ( + "strings" + "testing" + "time" + + "github.com/influxdata/telegraf" + "github.com/influxdata/telegraf/testutil" + "github.com/stretchr/testify/require" +) + +func TestSerialize(t *testing.T) { + tests := []struct { + name string + config FormatConfig + metric telegraf.Metric + expected []byte + }{ + { + name: "simple", + metric: testutil.MustMetric( + "cpu", + map[string]string{ + "host": "example.org", + }, + map[string]interface{}{ + "time_idle": 42.0, + }, + time.Unix(0, 0), + ), + expected: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle untyped +cpu_time_idle{host="example.org"} 42 +`), + }, + { + name: "prometheus input untyped", + metric: testutil.MustMetric( + "prometheus", + map[string]string{ + "code": "400", + "method": "post", + }, + map[string]interface{}{ + "http_requests_total": 3.0, + }, + time.Unix(0, 0), + telegraf.Untyped, + ), + expected: []byte(` +# HELP http_requests_total Telegraf collected metric +# TYPE http_requests_total untyped +http_requests_total{code="400",method="post"} 3 +`), + }, + { + name: "prometheus input counter", + metric: testutil.MustMetric( + "prometheus", + map[string]string{ + "code": "400", + "method": "post", + }, + map[string]interface{}{ + "http_requests_total": 3.0, + }, + time.Unix(0, 0), + telegraf.Counter, + ), + expected: []byte(` +# HELP http_requests_total Telegraf collected metric +# TYPE http_requests_total counter +http_requests_total{code="400",method="post"} 3 +`), + }, + { + name: "prometheus input gauge", + metric: testutil.MustMetric( + "prometheus", + map[string]string{ + "code": "400", + "method": "post", + }, + map[string]interface{}{ + "http_requests_total": 3.0, + }, + time.Unix(0, 0), + telegraf.Gauge, + ), + expected: []byte(` +# HELP http_requests_total Telegraf collected metric +# TYPE http_requests_total gauge +http_requests_total{code="400",method="post"} 3 +`), + }, + { + name: "prometheus input histogram no buckets", + metric: testutil.MustMetric( + "prometheus", + map[string]string{}, + map[string]interface{}{ + "http_request_duration_seconds_sum": 53423, + "http_request_duration_seconds_count": 144320, + }, + time.Unix(0, 0), + telegraf.Histogram, + ), + expected: []byte(` +`), + }, + { + name: "prometheus input histogram only bucket", + metric: testutil.MustMetric( + "prometheus", + map[string]string{ + "le": "0.5", + }, + map[string]interface{}{ + "http_request_duration_seconds_bucket": 129389.0, + }, + time.Unix(0, 0), + telegraf.Histogram, + ), + expected: []byte(` +# HELP http_request_duration_seconds Telegraf collected metric +# TYPE http_request_duration_seconds histogram +http_request_duration_seconds_bucket{le="0.5"} 129389 +http_request_duration_seconds_bucket{le="+Inf"} 0 +http_request_duration_seconds_sum 0 +http_request_duration_seconds_count 0 +`), + }, + { + name: "simple with timestamp", + config: FormatConfig{ + TimestampExport: ExportTimestamp, + }, + metric: testutil.MustMetric( + "cpu", + map[string]string{ + "host": "example.org", + }, + map[string]interface{}{ + "time_idle": 42.0, + }, + time.Unix(1574279268, 0), + ), + expected: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle untyped +cpu_time_idle{host="example.org"} 42 1574279268000 +`), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s, err := NewSerializer(FormatConfig{ + MetricSortOrder: SortMetrics, + TimestampExport: tt.config.TimestampExport, + StringHandling: tt.config.StringHandling, + }) + require.NoError(t, err) + actual, err := s.Serialize(tt.metric) + require.NoError(t, err) + + require.Equal(t, strings.TrimSpace(string(tt.expected)), + strings.TrimSpace(string(actual))) + }) + } +} + +func TestSerializeBatch(t *testing.T) { + tests := []struct { + name string + config FormatConfig + metrics []telegraf.Metric + expected []byte + }{ + { + name: "simple", + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{ + "host": "one.example.org", + }, + map[string]interface{}{ + "time_idle": 42.0, + }, + time.Unix(0, 0), + ), + testutil.MustMetric( + "cpu", + map[string]string{ + "host": "two.example.org", + }, + map[string]interface{}{ + "time_idle": 42.0, + }, + time.Unix(0, 0), + ), + }, + expected: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle untyped +cpu_time_idle{host="one.example.org"} 42 +cpu_time_idle{host="two.example.org"} 42 +`), + }, + { + name: "multiple metric families", + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{ + "host": "one.example.org", + }, + map[string]interface{}{ + "time_idle": 42.0, + "time_guest": 42.0, + }, + time.Unix(0, 0), + ), + }, + expected: []byte(` +# HELP cpu_time_guest Telegraf collected metric +# TYPE cpu_time_guest untyped +cpu_time_guest{host="one.example.org"} 42 +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle untyped +cpu_time_idle{host="one.example.org"} 42 +`), + }, + { + name: "histogram", + metrics: []telegraf.Metric{ + testutil.MustMetric( + "prometheus", + map[string]string{}, + map[string]interface{}{ + "http_request_duration_seconds_sum": 53423, + "http_request_duration_seconds_count": 144320, + }, + time.Unix(0, 0), + telegraf.Histogram, + ), + testutil.MustMetric( + "prometheus", + map[string]string{"le": "0.05"}, + map[string]interface{}{ + "http_request_duration_seconds_bucket": 24054.0, + }, + time.Unix(0, 0), + telegraf.Histogram, + ), + testutil.MustMetric( + "prometheus", + map[string]string{"le": "0.1"}, + map[string]interface{}{ + "http_request_duration_seconds_bucket": 33444.0, + }, + time.Unix(0, 0), + telegraf.Histogram, + ), + testutil.MustMetric( + "prometheus", + map[string]string{"le": "0.2"}, + map[string]interface{}{ + "http_request_duration_seconds_bucket": 100392.0, + }, + time.Unix(0, 0), + telegraf.Histogram, + ), + testutil.MustMetric( + "prometheus", + map[string]string{"le": "0.5"}, + map[string]interface{}{ + "http_request_duration_seconds_bucket": 129389.0, + }, + time.Unix(0, 0), + telegraf.Histogram, + ), + testutil.MustMetric( + "prometheus", + map[string]string{"le": "1.0"}, + map[string]interface{}{ + "http_request_duration_seconds_bucket": 133988.0, + }, + time.Unix(0, 0), + telegraf.Histogram, + ), + testutil.MustMetric( + "prometheus", + map[string]string{"le": "+Inf"}, + map[string]interface{}{ + "http_request_duration_seconds_bucket": 144320.0, + }, + time.Unix(0, 0), + telegraf.Histogram, + ), + }, + expected: []byte(` +# HELP http_request_duration_seconds Telegraf collected metric +# TYPE http_request_duration_seconds histogram +http_request_duration_seconds_bucket{le="0.05"} 24054 +http_request_duration_seconds_bucket{le="0.1"} 33444 +http_request_duration_seconds_bucket{le="0.2"} 100392 +http_request_duration_seconds_bucket{le="0.5"} 129389 +http_request_duration_seconds_bucket{le="1"} 133988 +http_request_duration_seconds_bucket{le="+Inf"} 144320 +http_request_duration_seconds_sum 53423 +http_request_duration_seconds_count 144320 +`), + }, + { + name: "", + metrics: []telegraf.Metric{ + testutil.MustMetric( + "prometheus", + map[string]string{}, + map[string]interface{}{ + "rpc_duration_seconds_sum": 1.7560473e+07, + "rpc_duration_seconds_count": 2693, + }, + time.Unix(0, 0), + telegraf.Summary, + ), + testutil.MustMetric( + "prometheus", + map[string]string{"quantile": "0.01"}, + map[string]interface{}{ + "rpc_duration_seconds": 3102.0, + }, + time.Unix(0, 0), + telegraf.Summary, + ), + testutil.MustMetric( + "prometheus", + map[string]string{"quantile": "0.05"}, + map[string]interface{}{ + "rpc_duration_seconds": 3272.0, + }, + time.Unix(0, 0), + telegraf.Summary, + ), + testutil.MustMetric( + "prometheus", + map[string]string{"quantile": "0.5"}, + map[string]interface{}{ + "rpc_duration_seconds": 4773.0, + }, + time.Unix(0, 0), + telegraf.Summary, + ), + testutil.MustMetric( + "prometheus", + map[string]string{"quantile": "0.9"}, + map[string]interface{}{ + "rpc_duration_seconds": 9001.0, + }, + time.Unix(0, 0), + telegraf.Summary, + ), + testutil.MustMetric( + "prometheus", + map[string]string{"quantile": "0.99"}, + map[string]interface{}{ + "rpc_duration_seconds": 76656.0, + }, + time.Unix(0, 0), + telegraf.Summary, + ), + }, + expected: []byte(` +# HELP rpc_duration_seconds Telegraf collected metric +# TYPE rpc_duration_seconds summary +rpc_duration_seconds{quantile="0.01"} 3102 +rpc_duration_seconds{quantile="0.05"} 3272 +rpc_duration_seconds{quantile="0.5"} 4773 +rpc_duration_seconds{quantile="0.9"} 9001 +rpc_duration_seconds{quantile="0.99"} 76656 +rpc_duration_seconds_sum 1.7560473e+07 +rpc_duration_seconds_count 2693 +`), + }, + { + name: "newer sample", + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "time_idle": 43.0, + }, + time.Unix(1, 0), + ), + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "time_idle": 42.0, + }, + time.Unix(0, 0), + ), + }, + expected: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle untyped +cpu_time_idle 43 +`), + }, + { + name: "colons are not replaced in metric name from measurement", + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu::xyzzy", + map[string]string{}, + map[string]interface{}{ + "time_idle": 42.0, + }, + time.Unix(0, 0), + ), + }, + expected: []byte(` +# HELP cpu::xyzzy_time_idle Telegraf collected metric +# TYPE cpu::xyzzy_time_idle untyped +cpu::xyzzy_time_idle 42 +`), + }, + { + name: "colons are not replaced in metric name from field", + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "time:idle": 42.0, + }, + time.Unix(0, 0), + ), + }, + expected: []byte(` +# HELP cpu_time:idle Telegraf collected metric +# TYPE cpu_time:idle untyped +cpu_time:idle 42 +`), + }, + { + name: "invalid label", + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{ + "host-name": "example.org", + }, + map[string]interface{}{ + "time_idle": 42.0, + }, + time.Unix(0, 0), + ), + }, + expected: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle untyped +cpu_time_idle{host_name="example.org"} 42 +`), + }, + { + name: "colons are replaced in label name", + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{ + "host:name": "example.org", + }, + map[string]interface{}{ + "time_idle": 42.0, + }, + time.Unix(0, 0), + ), + }, + expected: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle untyped +cpu_time_idle{host_name="example.org"} 42 +`), + }, + { + name: "discard strings", + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "time_idle": 42.0, + "cpu": "cpu0", + }, + time.Unix(0, 0), + ), + }, + expected: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle untyped +cpu_time_idle 42 +`), + }, + { + name: "string as label", + config: FormatConfig{ + StringHandling: StringAsLabel, + }, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "time_idle": 42.0, + "cpu": "cpu0", + }, + time.Unix(0, 0), + ), + }, + expected: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle untyped +cpu_time_idle{cpu="cpu0"} 42 +`), + }, + { + name: "string as label duplicate tag", + config: FormatConfig{ + StringHandling: StringAsLabel, + }, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{ + "cpu": "cpu0", + }, + map[string]interface{}{ + "time_idle": 42.0, + "cpu": "cpu1", + }, + time.Unix(0, 0), + ), + }, + expected: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle untyped +cpu_time_idle{cpu="cpu0"} 42 +`), + }, + { + name: "replace characters when using string as label", + config: FormatConfig{ + StringHandling: StringAsLabel, + }, + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{}, + map[string]interface{}{ + "host:name": "example.org", + "time_idle": 42.0, + }, + time.Unix(1574279268, 0), + ), + }, + expected: []byte(` +# HELP cpu_time_idle Telegraf collected metric +# TYPE cpu_time_idle untyped +cpu_time_idle{host_name="example.org"} 42 +`), + }, + { + name: "multiple fields grouping", + metrics: []telegraf.Metric{ + testutil.MustMetric( + "cpu", + map[string]string{ + "cpu": "cpu0", + }, + map[string]interface{}{ + "time_guest": 8106.04, + "time_system": 26271.4, + "time_user": 92904.33, + }, + time.Unix(0, 0), + ), + testutil.MustMetric( + "cpu", + map[string]string{ + "cpu": "cpu1", + }, + map[string]interface{}{ + "time_guest": 8181.63, + "time_system": 25351.49, + "time_user": 96912.57, + }, + time.Unix(0, 0), + ), + testutil.MustMetric( + "cpu", + map[string]string{ + "cpu": "cpu2", + }, + map[string]interface{}{ + "time_guest": 7470.04, + "time_system": 24998.43, + "time_user": 96034.08, + }, + time.Unix(0, 0), + ), + testutil.MustMetric( + "cpu", + map[string]string{ + "cpu": "cpu3", + }, + map[string]interface{}{ + "time_guest": 7517.95, + "time_system": 24970.82, + "time_user": 94148, + }, + time.Unix(0, 0), + ), + }, + expected: []byte(` +# HELP cpu_time_guest Telegraf collected metric +# TYPE cpu_time_guest untyped +cpu_time_guest{cpu="cpu0"} 8106.04 +cpu_time_guest{cpu="cpu1"} 8181.63 +cpu_time_guest{cpu="cpu2"} 7470.04 +cpu_time_guest{cpu="cpu3"} 7517.95 +# HELP cpu_time_system Telegraf collected metric +# TYPE cpu_time_system untyped +cpu_time_system{cpu="cpu0"} 26271.4 +cpu_time_system{cpu="cpu1"} 25351.49 +cpu_time_system{cpu="cpu2"} 24998.43 +cpu_time_system{cpu="cpu3"} 24970.82 +# HELP cpu_time_user Telegraf collected metric +# TYPE cpu_time_user untyped +cpu_time_user{cpu="cpu0"} 92904.33 +cpu_time_user{cpu="cpu1"} 96912.57 +cpu_time_user{cpu="cpu2"} 96034.08 +cpu_time_user{cpu="cpu3"} 94148 +`), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s, err := NewSerializer(FormatConfig{ + MetricSortOrder: SortMetrics, + TimestampExport: tt.config.TimestampExport, + StringHandling: tt.config.StringHandling, + }) + require.NoError(t, err) + actual, err := s.SerializeBatch(tt.metrics) + require.NoError(t, err) + + require.Equal(t, + strings.TrimSpace(string(tt.expected)), + strings.TrimSpace(string(actual))) + }) + } +} diff --git a/plugins/serializers/registry.go b/plugins/serializers/registry.go index cfdb784ccfe73..dc9859e3459a8 100644 --- a/plugins/serializers/registry.go +++ b/plugins/serializers/registry.go @@ -10,6 +10,7 @@ import ( "github.com/influxdata/telegraf/plugins/serializers/influx" "github.com/influxdata/telegraf/plugins/serializers/json" "github.com/influxdata/telegraf/plugins/serializers/nowmetric" + "github.com/influxdata/telegraf/plugins/serializers/prometheus" "github.com/influxdata/telegraf/plugins/serializers/splunkmetric" "github.com/influxdata/telegraf/plugins/serializers/wavefront" ) @@ -45,40 +46,54 @@ type Serializer interface { // and can be used to instantiate _any_ of the serializers. type Config struct { // Dataformat can be one of the serializer types listed in NewSerializer. - DataFormat string + DataFormat string `toml:"data_format"` // Support tags in graphite protocol - GraphiteTagSupport bool + GraphiteTagSupport bool `toml:"graphite_tag_support"` // Maximum line length in bytes; influx format only - InfluxMaxLineBytes int + InfluxMaxLineBytes int `toml:"influx_max_line_bytes"` // Sort field keys, set to true only when debugging as it less performant // than unsorted fields; influx format only - InfluxSortFields bool + InfluxSortFields bool `toml:"influx_sort_fields"` // Support unsigned integer output; influx format only - InfluxUintSupport bool + InfluxUintSupport bool `toml:"influx_uint_support"` // Prefix to add to all measurements, only supports Graphite - Prefix string + Prefix string `toml:"prefix"` // Template for converting telegraf metrics into Graphite // only supports Graphite - Template string + Template string `toml:"template"` // Timestamp units to use for JSON formatted output - TimestampUnits time.Duration + TimestampUnits time.Duration `toml:"timestamp_units"` // Include HEC routing fields for splunkmetric output - HecRouting bool + HecRouting bool `toml:"hec_routing"` + + // Enable Splunk MultiMetric output (Splunk 8.0+) + SplunkmetricMultiMetric bool `toml:"splunkmetric_multi_metric"` // Point tags to use as the source name for Wavefront (if none found, host will be used). - WavefrontSourceOverride []string + WavefrontSourceOverride []string `toml:"wavefront_source_override"` // Use Strict rules to sanitize metric and tag names from invalid characters for Wavefront // When enabled forward slash (/) and comma (,) will be accepted - WavefrontUseStrict bool + WavefrontUseStrict bool `toml:"wavefront_use_strict"` + + // Include the metric timestamp on each sample. + PrometheusExportTimestamp bool `toml:"prometheus_export_timestamp"` + + // Sort prometheus metric families and metric samples. Useful for + // debugging. + PrometheusSortMetrics bool `toml:"prometheus_sort_metrics"` + + // Output string fields as metric labels; when false string fields are + // discarded. + PrometheusStringAsLabel bool `toml:"prometheus_string_as_label"` } // NewSerializer a Serializer interface based on the given config. @@ -93,19 +108,44 @@ func NewSerializer(config *Config) (Serializer, error) { case "json": serializer, err = NewJsonSerializer(config.TimestampUnits) case "splunkmetric": - serializer, err = NewSplunkmetricSerializer(config.HecRouting) + serializer, err = NewSplunkmetricSerializer(config.HecRouting, config.SplunkmetricMultiMetric) case "nowmetric": serializer, err = NewNowSerializer() case "carbon2": serializer, err = NewCarbon2Serializer() case "wavefront": serializer, err = NewWavefrontSerializer(config.Prefix, config.WavefrontUseStrict, config.WavefrontSourceOverride) + case "prometheus": + serializer, err = NewPrometheusSerializer(config) default: err = fmt.Errorf("Invalid data format: %s", config.DataFormat) } return serializer, err } +func NewPrometheusSerializer(config *Config) (Serializer, error) { + exportTimestamp := prometheus.NoExportTimestamp + if config.PrometheusExportTimestamp { + exportTimestamp = prometheus.ExportTimestamp + } + + sortMetrics := prometheus.NoSortMetrics + if config.PrometheusExportTimestamp { + sortMetrics = prometheus.SortMetrics + } + + stringAsLabels := prometheus.DiscardStrings + if config.PrometheusStringAsLabel { + stringAsLabels = prometheus.StringAsLabel + } + + return prometheus.NewSerializer(prometheus.FormatConfig{ + TimestampExport: exportTimestamp, + MetricSortOrder: sortMetrics, + StringHandling: stringAsLabels, + }) +} + func NewWavefrontSerializer(prefix string, useStrict bool, sourceOverride []string) (Serializer, error) { return wavefront.NewSerializer(prefix, useStrict, sourceOverride) } @@ -118,8 +158,8 @@ func NewCarbon2Serializer() (Serializer, error) { return carbon2.NewSerializer() } -func NewSplunkmetricSerializer(splunkmetric_hec_routing bool) (Serializer, error) { - return splunkmetric.NewSerializer(splunkmetric_hec_routing) +func NewSplunkmetricSerializer(splunkmetric_hec_routing bool, splunkmetric_multimetric bool) (Serializer, error) { + return splunkmetric.NewSerializer(splunkmetric_hec_routing, splunkmetric_multimetric) } func NewNowSerializer() (Serializer, error) { diff --git a/plugins/serializers/splunkmetric/README.md b/plugins/serializers/splunkmetric/README.md index 552b90ea47b4b..47ad8e1bff6f0 100644 --- a/plugins/serializers/splunkmetric/README.md +++ b/plugins/serializers/splunkmetric/README.md @@ -27,6 +27,36 @@ In the above snippet, the following keys are dimensions: * dc * user +## Using Multimetric output + +Starting with Splunk Enterprise and Splunk Cloud 8.0, you can now send multiple metric values in one payload. This means, for example, that +you can send all of your CPU stats in one JSON struct, an example event looks like: + +```javascript +{ + "time": 1572469920, + "event": "metric", + "host": "mono.local", + "fields": { + "_config_hecRouting": false, + "_config_multiMetric": true, + "class": "osx", + "cpu": "cpu0", + "metric_name:telegraf.cpu.usage_guest": 0, + "metric_name:telegraf.cpu.usage_guest_nice": 0, + "metric_name:telegraf.cpu.usage_idle": 65.1, + "metric_name:telegraf.cpu.usage_iowait": 0, + "metric_name:telegraf.cpu.usage_irq": 0, + "metric_name:telegraf.cpu.usage_nice": 0, + "metric_name:telegraf.cpu.usage_softirq": 0, + "metric_name:telegraf.cpu.usage_steal": 0, + "metric_name:telegraf.cpu.usage_system": 10.2, + "metric_name:telegraf.cpu.usage_user": 24.7, + } +} +``` +In order to enable this mode, there's a new option `splunkmetric_multimetric` that you set in the appropriate output module you plan on using. + ## Using with the HTTP output To send this data to a Splunk HEC, you can use the HTTP output, there are some custom headers that you need to add @@ -61,6 +91,7 @@ to manage the HEC authorization, here's a sample config for an HTTP output: data_format = "splunkmetric" ## Provides time, index, source overrides for the HEC splunkmetric_hec_routing = true + # splunkmentric_multimetric = true ## Additional HTTP headers [outputs.http.headers] @@ -118,7 +149,6 @@ disabled = false INDEXED_EXTRACTIONS = json KV_MODE = none TIMESTAMP_FIELDS = time -TIME_FORMAT = %s.%3N ``` An example configuration of a file based output is: @@ -134,5 +164,6 @@ An example configuration of a file based output is: ## more about them here: ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md data_format = "splunkmetric" - hec_routing = false + splunkmetric_hec_routing = false + splunkmetric_multimetric = true ``` diff --git a/plugins/serializers/splunkmetric/splunkmetric.go b/plugins/serializers/splunkmetric/splunkmetric.go index cdcf6cc592567..77c724aa8570a 100644 --- a/plugins/serializers/splunkmetric/splunkmetric.go +++ b/plugins/serializers/splunkmetric/splunkmetric.go @@ -9,12 +9,33 @@ import ( ) type serializer struct { - HecRouting bool + HecRouting bool + SplunkmetricMultiMetric bool } -func NewSerializer(splunkmetric_hec_routing bool) (*serializer, error) { +type CommonTags struct { + Time float64 + Host string + Index string + Source string + Fields map[string]interface{} +} + +type HECTimeSeries struct { + Time float64 `json:"time"` + Event string `json:"event"` + Host string `json:"host,omitempty"` + Index string `json:"index,omitempty"` + Source string `json:"source,omitempty"` + Fields map[string]interface{} `json:"fields"` +} + +// NewSerializer Setup our new serializer +func NewSerializer(splunkmetric_hec_routing bool, splunkmetric_multimetric bool) (*serializer, error) { + /* Define output params */ s := &serializer{ - HecRouting: splunkmetric_hec_routing, + HecRouting: splunkmetric_hec_routing, + SplunkmetricMultiMetric: splunkmetric_multimetric, } return s, nil } @@ -45,26 +66,61 @@ func (s *serializer) SerializeBatch(metrics []telegraf.Metric) ([]byte, error) { return serialized, nil } -func (s *serializer) createObject(metric telegraf.Metric) (metricGroup []byte, err error) { +func (s *serializer) createMulti(metric telegraf.Metric, dataGroup HECTimeSeries, commonTags CommonTags) (metricGroup []byte, err error) { + /* When splunkmetric_multimetric is true, then we can write out multiple name=value pairs as part of the same + ** event payload. This only works when the time, host, and dimensions are the same for every name=value pair + ** in the timeseries data. + ** + ** The format for multimetric data is 'metric_name:nameOfMetric = valueOfMetric' + */ + var metricJSON []byte + + // Set the event data from the commonTags above. + dataGroup.Event = "metric" + dataGroup.Time = commonTags.Time + dataGroup.Host = commonTags.Host + dataGroup.Index = commonTags.Index + dataGroup.Source = commonTags.Source + dataGroup.Fields = commonTags.Fields + + // Stuff the metrid data into the structure. + for _, field := range metric.FieldList() { + value, valid := verifyValue(field.Value) - /* Splunk supports one metric json object, and does _not_ support an array of JSON objects. - ** Splunk has the following required names for the metric store: - ** metric_name: The name of the metric - ** _value: The value for the metric - ** time: The timestamp for the metric - ** All other index fields become dimensions. - */ - type HECTimeSeries struct { - Time float64 `json:"time"` - Event string `json:"event"` - Host string `json:"host,omitempty"` - Index string `json:"index,omitempty"` - Source string `json:"source,omitempty"` - Fields map[string]interface{} `json:"fields"` + if !valid { + log.Printf("D! Can not parse value: %v for key: %v", field.Value, field.Key) + continue + } + + dataGroup.Fields["metric_name:"+metric.Name()+"."+field.Key] = value } - dataGroup := HECTimeSeries{} - var metricJson []byte + // Manage the rest of the event details based upon HEC routing rules + switch s.HecRouting { + case true: + // Output the data as a fields array and host,index,time,source overrides for the HEC. + metricJSON, err = json.Marshal(dataGroup) + default: + // Just output the data and the time, useful for file based outuputs + dataGroup.Fields["time"] = dataGroup.Time + metricJSON, err = json.Marshal(dataGroup.Fields) + } + if err != nil { + return nil, err + } + // Let the JSON fall through to the return below + metricGroup = metricJSON + + return metricGroup, nil +} + +func (s *serializer) createSingle(metric telegraf.Metric, dataGroup HECTimeSeries, commonTags CommonTags) (metricGroup []byte, err error) { + /* The default mode is to generate one JSON entitiy per metric (required for pre-8.0 Splunks) + ** + ** The format for single metric is 'nameOfMetric = valueOfMetric' + */ + + var metricJSON []byte for _, field := range metric.FieldList() { @@ -75,39 +131,30 @@ func (s *serializer) createObject(metric telegraf.Metric) (metricGroup []byte, e continue } - obj := map[string]interface{}{} - obj["metric_name"] = metric.Name() + "." + field.Key - obj["_value"] = value - dataGroup.Event = "metric" - // Convert ns to float seconds since epoch. - dataGroup.Time = float64(metric.Time().UnixNano()) / float64(1000000000) - dataGroup.Fields = obj - - // Break tags out into key(n)=value(t) pairs - for n, t := range metric.Tags() { - if n == "host" { - dataGroup.Host = t - } else if n == "index" { - dataGroup.Index = t - } else if n == "source" { - dataGroup.Source = t - } else { - dataGroup.Fields[n] = t - } - } + + dataGroup.Time = commonTags.Time + + // Apply the common tags from above to every record. + dataGroup.Host = commonTags.Host + dataGroup.Index = commonTags.Index + dataGroup.Source = commonTags.Source + dataGroup.Fields = commonTags.Fields + + dataGroup.Fields["metric_name"] = metric.Name() + "." + field.Key + dataGroup.Fields["_value"] = value switch s.HecRouting { case true: // Output the data as a fields array and host,index,time,source overrides for the HEC. - metricJson, err = json.Marshal(dataGroup) + metricJSON, err = json.Marshal(dataGroup) default: // Just output the data and the time, useful for file based outuputs dataGroup.Fields["time"] = dataGroup.Time - metricJson, err = json.Marshal(dataGroup.Fields) + metricJSON, err = json.Marshal(dataGroup.Fields) } - metricGroup = append(metricGroup, metricJson...) + metricGroup = append(metricGroup, metricJSON...) if err != nil { return nil, err @@ -117,6 +164,52 @@ func (s *serializer) createObject(metric telegraf.Metric) (metricGroup []byte, e return metricGroup, nil } +func (s *serializer) createObject(metric telegraf.Metric) (metricGroup []byte, err error) { + + /* Splunk supports one metric json object, and does _not_ support an array of JSON objects. + ** Splunk has the following required names for the metric store: + ** metric_name: The name of the metric + ** _value: The value for the metric + ** time: The timestamp for the metric + ** All other index fields become dimensions. + */ + + dataGroup := HECTimeSeries{} + + // The tags are common to all events in this timeseries + commonTags := CommonTags{} + + commonObj := map[string]interface{}{} + + commonObj["config:hecRouting"] = s.HecRouting + commonObj["config:multiMetric"] = s.SplunkmetricMultiMetric + + commonTags.Fields = commonObj + + // Break tags out into key(n)=value(t) pairs + for n, t := range metric.Tags() { + if n == "host" { + commonTags.Host = t + } else if n == "index" { + commonTags.Index = t + } else if n == "source" { + commonTags.Source = t + } else { + commonTags.Fields[n] = t + } + } + commonTags.Time = float64(metric.Time().UnixNano()) / float64(1000000000) + switch s.SplunkmetricMultiMetric { + case true: + metricGroup, _ = s.createMulti(metric, dataGroup, commonTags) + default: + metricGroup, _ = s.createSingle(metric, dataGroup, commonTags) + } + + // Return the metric group regardless of if it's multimetric or single metric. + return metricGroup, nil +} + func verifyValue(v interface{}) (value interface{}, valid bool) { switch v.(type) { case string: diff --git a/plugins/serializers/splunkmetric/splunkmetric_test.go b/plugins/serializers/splunkmetric/splunkmetric_test.go index 04f6e6538294a..70037e28a43c8 100644 --- a/plugins/serializers/splunkmetric/splunkmetric_test.go +++ b/plugins/serializers/splunkmetric/splunkmetric_test.go @@ -29,11 +29,11 @@ func TestSerializeMetricFloat(t *testing.T) { m, err := metric.New("cpu", tags, fields, now) assert.NoError(t, err) - s, _ := NewSerializer(false) + s, _ := NewSerializer(false, false) var buf []byte buf, err = s.Serialize(m) assert.NoError(t, err) - expS := `{"_value":91.5,"cpu":"cpu0","metric_name":"cpu.usage_idle","time":1529875740.819}` + expS := `{"_value":91.5,"config:hecRouting":false,"config:multiMetric":false,"cpu":"cpu0","metric_name":"cpu.usage_idle","time":1529875740.819}` assert.Equal(t, string(expS), string(buf)) } @@ -49,11 +49,11 @@ func TestSerializeMetricFloatHec(t *testing.T) { m, err := metric.New("cpu", tags, fields, now) assert.NoError(t, err) - s, _ := NewSerializer(true) + s, _ := NewSerializer(true, false) var buf []byte buf, err = s.Serialize(m) assert.NoError(t, err) - expS := `{"time":1529875740.819,"event":"metric","fields":{"_value":91.5,"cpu":"cpu0","metric_name":"cpu.usage_idle"}}` + expS := `{"time":1529875740.819,"event":"metric","fields":{"_value":91.5,"config:hecRouting":true,"config:multiMetric":false,"cpu":"cpu0","metric_name":"cpu.usage_idle"}}` assert.Equal(t, string(expS), string(buf)) } @@ -68,12 +68,12 @@ func TestSerializeMetricInt(t *testing.T) { m, err := metric.New("cpu", tags, fields, now) assert.NoError(t, err) - s, _ := NewSerializer(false) + s, _ := NewSerializer(false, false) var buf []byte buf, err = s.Serialize(m) assert.NoError(t, err) - expS := `{"_value":90,"cpu":"cpu0","metric_name":"cpu.usage_idle","time":0}` + expS := `{"_value":90,"config:hecRouting":false,"config:multiMetric":false,"cpu":"cpu0","metric_name":"cpu.usage_idle","time":0}` assert.Equal(t, string(expS), string(buf)) } @@ -88,12 +88,12 @@ func TestSerializeMetricIntHec(t *testing.T) { m, err := metric.New("cpu", tags, fields, now) assert.NoError(t, err) - s, _ := NewSerializer(true) + s, _ := NewSerializer(true, false) var buf []byte buf, err = s.Serialize(m) assert.NoError(t, err) - expS := `{"time":0,"event":"metric","fields":{"_value":90,"cpu":"cpu0","metric_name":"cpu.usage_idle"}}` + expS := `{"time":0,"event":"metric","fields":{"_value":90,"config:hecRouting":true,"config:multiMetric":false,"cpu":"cpu0","metric_name":"cpu.usage_idle"}}` assert.Equal(t, string(expS), string(buf)) } @@ -108,12 +108,12 @@ func TestSerializeMetricBool(t *testing.T) { m, err := metric.New("docker", tags, fields, now) assert.NoError(t, err) - s, _ := NewSerializer(false) + s, _ := NewSerializer(false, false) var buf []byte buf, err = s.Serialize(m) assert.NoError(t, err) - expS := `{"_value":1,"container-name":"telegraf-test","metric_name":"docker.oomkiller","time":0}` + expS := `{"_value":1,"config:hecRouting":false,"config:multiMetric":false,"container-name":"telegraf-test","metric_name":"docker.oomkiller","time":0}` assert.Equal(t, string(expS), string(buf)) } @@ -128,12 +128,12 @@ func TestSerializeMetricBoolHec(t *testing.T) { m, err := metric.New("docker", tags, fields, now) assert.NoError(t, err) - s, _ := NewSerializer(true) + s, _ := NewSerializer(true, false) var buf []byte buf, err = s.Serialize(m) assert.NoError(t, err) - expS := `{"time":0,"event":"metric","fields":{"_value":0,"container-name":"telegraf-test","metric_name":"docker.oomkiller"}}` + expS := `{"time":0,"event":"metric","fields":{"_value":0,"config:hecRouting":true,"config:multiMetric":false,"container-name":"telegraf-test","metric_name":"docker.oomkiller"}}` assert.Equal(t, string(expS), string(buf)) } @@ -149,12 +149,12 @@ func TestSerializeMetricString(t *testing.T) { m, err := metric.New("cpu", tags, fields, now) assert.NoError(t, err) - s, _ := NewSerializer(false) + s, _ := NewSerializer(false, false) var buf []byte buf, err = s.Serialize(m) assert.NoError(t, err) - expS := `{"_value":5,"cpu":"cpu0","metric_name":"cpu.usage_idle","time":0}` + expS := `{"_value":5,"config:hecRouting":false,"config:multiMetric":false,"cpu":"cpu0","metric_name":"cpu.usage_idle","time":0}` assert.Equal(t, string(expS), string(buf)) assert.NoError(t, err) } @@ -182,11 +182,33 @@ func TestSerializeBatch(t *testing.T) { ) metrics := []telegraf.Metric{m, n} - s, _ := NewSerializer(false) + s, _ := NewSerializer(false, false) buf, err := s.SerializeBatch(metrics) assert.NoError(t, err) - expS := `{"_value":42,"metric_name":"cpu.value","time":0}` + `{"_value":92,"metric_name":"cpu.value","time":0}` + expS := `{"_value":42,"config:hecRouting":false,"config:multiMetric":false,"metric_name":"cpu.value","time":0}{"_value":92,"config:hecRouting":false,"config:multiMetric":false,"metric_name":"cpu.value","time":0}` + assert.Equal(t, string(expS), string(buf)) +} + +func TestSerializeMulti(t *testing.T) { + m := MustMetric( + metric.New( + "cpu", + map[string]string{}, + map[string]interface{}{ + "user": 42.0, + "system": 8.0, + }, + time.Unix(0, 0), + ), + ) + + metrics := []telegraf.Metric{m} + s, _ := NewSerializer(false, true) + buf, err := s.SerializeBatch(metrics) + assert.NoError(t, err) + + expS := `{"config:hecRouting":false,"config:multiMetric":true,"metric_name:cpu.system":8,"metric_name:cpu.user":42,"time":0}` assert.Equal(t, string(expS), string(buf)) } @@ -213,10 +235,32 @@ func TestSerializeBatchHec(t *testing.T) { ) metrics := []telegraf.Metric{m, n} - s, _ := NewSerializer(true) + s, _ := NewSerializer(true, false) + buf, err := s.SerializeBatch(metrics) + assert.NoError(t, err) + + expS := `{"time":0,"event":"metric","fields":{"_value":42,"config:hecRouting":true,"config:multiMetric":false,"metric_name":"cpu.value"}}{"time":0,"event":"metric","fields":{"_value":92,"config:hecRouting":true,"config:multiMetric":false,"metric_name":"cpu.value"}}` + assert.Equal(t, string(expS), string(buf)) +} + +func TestSerializeMultiHec(t *testing.T) { + m := MustMetric( + metric.New( + "cpu", + map[string]string{}, + map[string]interface{}{ + "usage": 42.0, + "system": 8.0, + }, + time.Unix(0, 0), + ), + ) + + metrics := []telegraf.Metric{m} + s, _ := NewSerializer(true, true) buf, err := s.SerializeBatch(metrics) assert.NoError(t, err) - expS := `{"time":0,"event":"metric","fields":{"_value":42,"metric_name":"cpu.value"}}` + `{"time":0,"event":"metric","fields":{"_value":92,"metric_name":"cpu.value"}}` + expS := `{"time":0,"event":"metric","fields":{"config:hecRouting":true,"config:multiMetric":true,"metric_name:cpu.system":8,"metric_name:cpu.usage":42}}` assert.Equal(t, string(expS), string(buf)) } diff --git a/scripts/build.py b/scripts/build.py index 4bde925148c16..2c2d0be763895 100755 --- a/scripts/build.py +++ b/scripts/build.py @@ -101,7 +101,7 @@ "freebsd": [ "tar" ] } -next_version = '1.13.0' +next_version = '1.14.0' ################ #### Telegraf Functions @@ -161,8 +161,8 @@ def go_get(branch, update=False, no_uncommitted=False): if local_changes() and no_uncommitted: logging.error("There are uncommitted changes in the current directory.") return False - logging.info("Retrieving dependencies with `dep`...") - run("dep ensure -v -vendor-only") + logging.info("Retrieving dependencies...") + run("go mod download") return True def run_tests(race, parallel, timeout, no_vet): diff --git a/scripts/ci-1.11.docker b/scripts/ci-1.11.docker deleted file mode 100644 index 93f2d64b6beae..0000000000000 --- a/scripts/ci-1.11.docker +++ /dev/null @@ -1,28 +0,0 @@ -FROM golang:1.11.13 - -RUN chmod -R 755 "$GOPATH" - -RUN DEBIAN_FRONTEND=noninteractive \ - apt update && apt install -y --no-install-recommends \ - autoconf \ - git \ - libtool \ - locales \ - make \ - python-boto \ - rpm \ - ruby \ - ruby-dev \ - zip && \ - rm -rf /var/lib/apt/lists/* - -RUN ln -sf /usr/share/zoneinfo/Etc/UTC /etc/localtime -RUN locale-gen C.UTF-8 || true -ENV LANG=C.UTF-8 - -RUN gem install fpm - -RUN go get -d github.com/golang/dep && \ - cd src/github.com/golang/dep && \ - git checkout -q v0.5.0 && \ - go install -ldflags="-X main.version=v0.5.0" ./cmd/dep diff --git a/scripts/ci-1.12.docker b/scripts/ci-1.12.docker index f5b093413d8dc..e68618dbcc11e 100644 --- a/scripts/ci-1.12.docker +++ b/scripts/ci-1.12.docker @@ -1,4 +1,4 @@ -FROM golang:1.12.9 +FROM golang:1.12.14 RUN chmod -R 755 "$GOPATH" diff --git a/scripts/ci-1.10.docker b/scripts/ci-1.13.docker similarity index 96% rename from scripts/ci-1.10.docker rename to scripts/ci-1.13.docker index 54c30f382048b..ad71addb9d6ba 100644 --- a/scripts/ci-1.10.docker +++ b/scripts/ci-1.13.docker @@ -1,4 +1,4 @@ -FROM golang:1.10.8 +FROM golang:1.13.5 RUN chmod -R 755 "$GOPATH" diff --git a/scripts/init.sh b/scripts/init.sh index fc71536f93985..d01e16a7ca6f2 100755 --- a/scripts/init.sh +++ b/scripts/init.sh @@ -122,11 +122,11 @@ case $1 in # Checked the PID file exists and check the actual status of process if [ -e "$pidfile" ]; then if pidofproc -p $pidfile $daemon > /dev/null; then - log_failure_msg "$name process is running" - else - log_failure_msg "$name pidfile has no corresponding process; ensure $name is stopped and remove $pidfile" - fi - exit 0 + log_failure_msg "$name process is running" + else + log_failure_msg "$name pidfile has no corresponding process; ensure $name is stopped and remove $pidfile" + fi + exit 0 fi # Bump the file limits, before launching the daemon. These will carry over to diff --git a/selfstat/selfstat.go b/selfstat/selfstat.go index 98ecbb4d42b90..821db1c94a9d5 100644 --- a/selfstat/selfstat.go +++ b/selfstat/selfstat.go @@ -32,9 +32,6 @@ type Stat interface { // Tags is a tag map. Each time this is called a new map is allocated. Tags() map[string]string - // Key is the unique measurement+tags key of the stat. - Key() uint64 - // Incr increments a regular stat by 'v'. // in the case of a timing stat, increment adds the timing to the cache. Incr(v int64) @@ -56,11 +53,7 @@ type Stat interface { // The returned Stat can be incremented by the consumer of Register(), and it's // value will be returned as a telegraf metric when Metrics() is called. func Register(measurement, field string, tags map[string]string) Stat { - return registry.register(&stat{ - measurement: "internal_" + measurement, - field: field, - tags: tags, - }) + return registry.register("internal_"+measurement, field, tags) } // RegisterTiming registers the given measurement, field, and tags in the selfstat @@ -80,11 +73,7 @@ func Register(measurement, field string, tags map[string]string) Stat { // The returned Stat can be incremented by the consumer of Register(), and it's // value will be returned as a telegraf metric when Metrics() is called. func RegisterTiming(measurement, field string, tags map[string]string) Stat { - return registry.register(&timingStat{ - measurement: "internal_" + measurement, - field: field, - tags: tags, - }) + return registry.registerTiming("internal_"+measurement, field, tags) } // Metrics returns all registered stats as telegraf metrics. @@ -125,22 +114,71 @@ type rgstry struct { mu sync.Mutex } -func (r *rgstry) register(s Stat) Stat { +func (r *rgstry) register(measurement, field string, tags map[string]string) Stat { r.mu.Lock() defer r.mu.Unlock() - if stats, ok := r.stats[s.Key()]; ok { - // measurement exists - if stat, ok := stats[s.FieldName()]; ok { - // field already exists, so don't create a new one - return stat - } - r.stats[s.Key()][s.FieldName()] = s - return s - } else { - // creating a new unique metric - r.stats[s.Key()] = map[string]Stat{s.FieldName(): s} - return s + + key := key(measurement, tags) + if stat, ok := registry.get(key, field); ok { + return stat + } + + t := make(map[string]string, len(tags)) + for k, v := range tags { + t[k] = v + } + + s := &stat{ + measurement: measurement, + field: field, + tags: t, + } + registry.set(key, s) + return s +} + +func (r *rgstry) registerTiming(measurement, field string, tags map[string]string) Stat { + r.mu.Lock() + defer r.mu.Unlock() + + key := key(measurement, tags) + if stat, ok := registry.get(key, field); ok { + return stat } + + t := make(map[string]string, len(tags)) + for k, v := range tags { + t[k] = v + } + + s := &timingStat{ + measurement: measurement, + field: field, + tags: t, + } + registry.set(key, s) + return s +} + +func (r *rgstry) get(key uint64, field string) (Stat, bool) { + if _, ok := r.stats[key]; !ok { + return nil, false + } + + if stat, ok := r.stats[key][field]; ok { + return stat, true + } + + return nil, false +} + +func (r *rgstry) set(key uint64, s Stat) { + if _, ok := r.stats[key]; !ok { + r.stats[key] = make(map[string]Stat) + } + + r.stats[key][s.FieldName()] = s + return } func key(measurement string, tags map[string]string) uint64 { diff --git a/selfstat/selfstat_test.go b/selfstat/selfstat_test.go index 2de2bd3811680..10ce327286e37 100644 --- a/selfstat/selfstat_test.go +++ b/selfstat/selfstat_test.go @@ -5,8 +5,8 @@ import ( "testing" "github.com/influxdata/telegraf/testutil" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) var ( @@ -109,32 +109,17 @@ func TestRegisterTimingAndIncrAndSet(t *testing.T) { } func TestStatKeyConsistency(t *testing.T) { - s := &stat{ - measurement: "internal_stat", - field: "myfield", - tags: map[string]string{ - "foo": "bar", - "bar": "baz", - "whose": "first", - }, - } - k := s.Key() - for i := 0; i < 5000; i++ { - // assert that the Key() func doesn't change anything. - assert.Equal(t, k, s.Key()) - - // assert that two identical measurements always produce the same key. - tmp := &stat{ - measurement: "internal_stat", - field: "myfield", - tags: map[string]string{ - "foo": "bar", - "bar": "baz", - "whose": "first", - }, - } - assert.Equal(t, k, tmp.Key()) - } + lhs := key("internal_stats", map[string]string{ + "foo": "bar", + "bar": "baz", + "whose": "first", + }) + rhs := key("internal_stats", map[string]string{ + "foo": "bar", + "bar": "baz", + "whose": "first", + }) + require.Equal(t, lhs, rhs) } func TestRegisterMetricsAndVerify(t *testing.T) { @@ -219,3 +204,10 @@ func TestRegisterMetricsAndVerify(t *testing.T) { }, ) } + +func TestRegisterCopy(t *testing.T) { + tags := map[string]string{"input": "mem", "alias": "mem1"} + stat := Register("gather", "metrics_gathered", tags) + tags["new"] = "value" + require.NotEqual(t, tags, stat.Tags()) +} diff --git a/selfstat/stat.go b/selfstat/stat.go index d7ec60a2bb53b..e1905baf57878 100644 --- a/selfstat/stat.go +++ b/selfstat/stat.go @@ -41,10 +41,3 @@ func (s *stat) Tags() map[string]string { } return m } - -func (s *stat) Key() uint64 { - if s.key == 0 { - s.key = key(s.measurement, s.tags) - } - return s.key -} diff --git a/selfstat/timingStat.go b/selfstat/timingStat.go index ef0ee05aa6106..13f8400bc7a48 100644 --- a/selfstat/timingStat.go +++ b/selfstat/timingStat.go @@ -57,10 +57,3 @@ func (s *timingStat) Tags() map[string]string { } return m } - -func (s *timingStat) Key() uint64 { - if s.key == 0 { - s.key = key(s.measurement, s.tags) - } - return s.key -} diff --git a/testutil/accumulator.go b/testutil/accumulator.go index e33959a83ec9b..65592b5a0531d 100644 --- a/testutil/accumulator.go +++ b/testutil/accumulator.go @@ -28,6 +28,7 @@ type Metric struct { Tags map[string]string Fields map[string]interface{} Time time.Time + Type telegraf.ValueType } func (p *Metric) String() string { @@ -75,11 +76,11 @@ func (a *Accumulator) ClearMetrics() { a.Metrics = make([]*Metric, 0) } -// AddFields adds a measurement point with a specified timestamp. -func (a *Accumulator) AddFields( +func (a *Accumulator) addFields( measurement string, - fields map[string]interface{}, tags map[string]string, + fields map[string]interface{}, + tp telegraf.ValueType, timestamp ...time.Time, ) { a.Lock() @@ -132,18 +133,29 @@ func (a *Accumulator) AddFields( Fields: fieldsCopy, Tags: tagsCopy, Time: t, + Type: tp, } a.Metrics = append(a.Metrics, p) } +// AddFields adds a measurement point with a specified timestamp. +func (a *Accumulator) AddFields( + measurement string, + fields map[string]interface{}, + tags map[string]string, + timestamp ...time.Time, +) { + a.addFields(measurement, tags, fields, telegraf.Untyped, timestamp...) +} + func (a *Accumulator) AddCounter( measurement string, fields map[string]interface{}, tags map[string]string, timestamp ...time.Time, ) { - a.AddFields(measurement, fields, tags, timestamp...) + a.addFields(measurement, tags, fields, telegraf.Counter, timestamp...) } func (a *Accumulator) AddGauge( @@ -152,12 +164,12 @@ func (a *Accumulator) AddGauge( tags map[string]string, timestamp ...time.Time, ) { - a.AddFields(measurement, fields, tags, timestamp...) + a.addFields(measurement, tags, fields, telegraf.Gauge, timestamp...) } func (a *Accumulator) AddMetrics(metrics []telegraf.Metric) { for _, m := range metrics { - a.AddFields(m.Name(), m.Fields(), m.Tags(), m.Time()) + a.addFields(m.Name(), m.Tags(), m.Fields(), m.Type(), m.Time()) } } @@ -167,7 +179,7 @@ func (a *Accumulator) AddSummary( tags map[string]string, timestamp ...time.Time, ) { - a.AddFields(measurement, fields, tags, timestamp...) + a.addFields(measurement, tags, fields, telegraf.Summary, timestamp...) } func (a *Accumulator) AddHistogram( @@ -176,11 +188,11 @@ func (a *Accumulator) AddHistogram( tags map[string]string, timestamp ...time.Time, ) { - a.AddFields(measurement, fields, tags, timestamp...) + a.addFields(measurement, tags, fields, telegraf.Histogram, timestamp...) } func (a *Accumulator) AddMetric(m telegraf.Metric) { - a.AddFields(m.Name(), m.Fields(), m.Tags(), m.Time()) + a.addFields(m.Name(), m.Tags(), m.Fields(), m.Type(), m.Time()) } func (a *Accumulator) WithTracking(maxTracked int) telegraf.TrackingAccumulator { @@ -258,6 +270,18 @@ func (a *Accumulator) HasTag(measurement string, key string) bool { return false } +func (a *Accumulator) TagSetValue(measurement string, key string) string { + for _, p := range a.Metrics { + if p.Measurement == measurement { + v, ok := p.Tags[key] + if ok { + return v + } + } + } + return "" +} + func (a *Accumulator) TagValue(measurement string, key string) string { for _, p := range a.Metrics { if p.Measurement == measurement { @@ -691,3 +715,22 @@ func (a *Accumulator) BoolField(measurement string, field string) (bool, bool) { return false, false } + +// NopAccumulator is used for benchmarking to isolate the plugin from the internal +// telegraf accumulator machinary. +type NopAccumulator struct{} + +func (n *NopAccumulator) AddFields(measurement string, fields map[string]interface{}, tags map[string]string, t ...time.Time) { +} +func (n *NopAccumulator) AddGauge(measurement string, fields map[string]interface{}, tags map[string]string, t ...time.Time) { +} +func (n *NopAccumulator) AddCounter(measurement string, fields map[string]interface{}, tags map[string]string, t ...time.Time) { +} +func (n *NopAccumulator) AddSummary(measurement string, fields map[string]interface{}, tags map[string]string, t ...time.Time) { +} +func (n *NopAccumulator) AddHistogram(measurement string, fields map[string]interface{}, tags map[string]string, t ...time.Time) { +} +func (n *NopAccumulator) AddMetric(telegraf.Metric) {} +func (n *NopAccumulator) SetPrecision(precision time.Duration) {} +func (n *NopAccumulator) AddError(err error) {} +func (n *NopAccumulator) WithTracking(maxTracked int) telegraf.TrackingAccumulator { return nil } diff --git a/testutil/metric.go b/testutil/metric.go index da3ace0f22fba..36ba63af9338f 100644 --- a/testutil/metric.go +++ b/testutil/metric.go @@ -197,7 +197,7 @@ func MustMetric( } func FromTestMetric(met *Metric) telegraf.Metric { - m, err := metric.New(met.Measurement, met.Tags, met.Fields, met.Time) + m, err := metric.New(met.Measurement, met.Tags, met.Fields, met.Time, met.Type) if err != nil { panic("MustMetric") }