diff --git a/.travis.yml b/.travis.yml index 081ff2830b3..f218e38df15 100644 --- a/.travis.yml +++ b/.travis.yml @@ -44,6 +44,7 @@ before_script: - tools/travis-setup-db.sh - if [ $PRESET = 'ldap_mnesia' ]; then sudo tools/travis-setup-ldap.sh; fi - if [ $PRESET = 'mysql_redis' ]; then sudo tools/travis-setup-rmq.sh; fi + - if [ ! -z "$HUMIO_PASSWORD" ]; then tools/travis-setup-humio.sh; fi script: - SKIP_AUTO_COMPILE=true KEEP_COVER_RUNNING=1 tools/travis-test.sh -p $PRESET -s $RUN_SMALL_TESTS -e $RUN_SMALL_TESTS @@ -54,6 +55,7 @@ after_failure: - tail -100 _build/mim2/rel/mongooseim/log/mongooseim.log.1 after_script: + - if [ ! -z "$HUMIO_PASSWORD" ]; then docker logs mongooseim-filebeat || echo "ok"; fi - cat travis-pull-db.log # Upload logs to s3 for debugging - test 1 = "$SKIP_REPORT_UPLOAD" || diff --git a/doc/operation-and-maintenance/Humio.md b/doc/operation-and-maintenance/Humio.md new file mode 100644 index 00000000000..849c11e1f91 --- /dev/null +++ b/doc/operation-and-maintenance/Humio.md @@ -0,0 +1,50 @@ +# Humio and MongooseIM + +## Getting Humio's ingest token + +Visit this [url](https://cloud.humio.com/sandbox/settings/ingest-tokens) to create a new sandbox's ingest token. + +The URL is: + +``` +https://cloud.humio.com/YOUR_REPOSITORY_NAME_HERE/settings/ingest-tokens +``` + +## Configure Filebeat + +Configure Filebeat, using this config file [priv/filebeat.mongooseim.humio.yml](https://github.com/esl/MongooseIM/blob/master/priv/filebeat.humio.yml). + +We recommend to use the Filebeat docker container. You have to use an open-source version +of Filebeat, which has the `oss` suffix. + +This example mounts a log directory `$(pwd)/_build/mim1/rel/mongooseim/log` as a volume for Filebeat. +It also mounts a configuration file `$(pwd)/priv/filebeat.mongooseim.humio.yml`. +Most likely these paths would be different on your machine. + +Pass your Humio ingest token as a password argument. +Or uncomment and change it inside the `filebeat.mongooseim.humio.yml` file. + +```bash +docker run -d \ + --name mongooseim-filebeat \ + -v "$(pwd)/_build/mim1/rel/mongooseim/log:/usr/lib/mongooseim/log" \ + -v "$(pwd)/priv/filebeat.mongooseim.humio.yml:/usr/share/filebeat/filebeat.yml:ro" \ + docker.elastic.co/beats/filebeat-oss:7.9.2 \ + filebeat -e -E output.elasticsearch.password="abc12345-xxxx-yyyy-zzzz-123456789abc" +``` + +Argument `-e` enables debugging information for Filebeat that can be visible using the `docker logs mongooseim-filebeat` command. + +## Viewing logs + +Navigate to [https://cloud.humio.com/sandbox/search](https://cloud.humio.com/sandbox/search) to see the Sandbox's dashboard. + + +A list of log messages: + +![A list of log messages](humio_list.png) + + +Structured log message: + +![Structured log message](humio_message.png) diff --git a/doc/operation-and-maintenance/Logging.md b/doc/operation-and-maintenance/Logging.md index 321a5a70a0c..f869ada2525 100644 --- a/doc/operation-and-maintenance/Logging.md +++ b/doc/operation-and-maintenance/Logging.md @@ -122,7 +122,11 @@ Options: # JSON file log handler -JSON formatted file. It could be used to store messages in ELK or in Splunk. +JSON formatted file. It could be used to store messages in ELK, in Humio or in Splunk. + +Check this [tutorial](Humio.md) to configure MongooseIM with Humio. +Check below information to configure MongooseIM with ELK. + You can use [Filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-log.html#filebeat-input-log-config-json) to send messages from the file into ELK. diff --git a/doc/operation-and-maintenance/humio_list.png b/doc/operation-and-maintenance/humio_list.png new file mode 100644 index 00000000000..84de37f9147 Binary files /dev/null and b/doc/operation-and-maintenance/humio_list.png differ diff --git a/doc/operation-and-maintenance/humio_message.png b/doc/operation-and-maintenance/humio_message.png new file mode 100644 index 00000000000..236918bc76e Binary files /dev/null and b/doc/operation-and-maintenance/humio_message.png differ diff --git a/mkdocs.yml b/mkdocs.yml index 0ca0a741e09..733efa74c9c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -75,6 +75,7 @@ pages: - 'Cluster configuration and node management': 'operation-and-maintenance/Cluster-configuration-and-node-management.md' - 'Logging & monitoring': 'operation-and-maintenance/Logging-&-monitoring.md' - 'Logging configuration': 'operation-and-maintenance/Logging.md' + - 'Logging with Humio': 'operation-and-maintenance/Humio.md' - 'Logging fields': 'operation-and-maintenance/Logging-fields.md' - 'Reloading configuration on a running system': 'operation-and-maintenance/Reloading-configuration-on-a-running-system.md' - 'Metrics': 'operation-and-maintenance/Mongoose-metrics.md' diff --git a/priv/filebeat.mongooseim.humio.yml b/priv/filebeat.mongooseim.humio.yml new file mode 100644 index 00000000000..f5fd0c4ec1c --- /dev/null +++ b/priv/filebeat.mongooseim.humio.yml @@ -0,0 +1,19 @@ +# https://docs.humio.com/integrations/ingest-logs-with-a-data-shipper/beats/filebeat/ +filebeat.inputs: +- paths: + - /usr/lib/mongooseim/log/mongooseim.json.1 + encoding: utf-8 + +queue.mem: + events: 8000 + flush.min_events: 1000 + flush.timeout: 1s + +output: + elasticsearch: + hosts: ["https://cloud.humio.com:443/api/v1/ingest/elastic-bulk"] + username: anything + compression_level: 5 + bulk_max_size: 200 + worker: 5 +# password: "abc12345-xxxx-yyyy-zzzz-123456789abc" diff --git a/tools/test_runner/list_presets.erl b/tools/test_runner/list_presets.erl index a902d787018..f815beddd8a 100644 --- a/tools/test_runner/list_presets.erl +++ b/tools/test_runner/list_presets.erl @@ -12,8 +12,9 @@ main(_) -> -spec read_presets() -> [atom()]. read_presets() -> {ok, Config} = file:consult("big_tests/test.config"), - Presets = proplists:get_value(ejabberd_presets, Config, []), - lists:map(fun({Name,_}) -> Name end, Presets). + Presets = proplists:get_value(presets, Config, []), + Toml = proplists:get_value(toml, Presets, []), + lists:map(fun({Name,_}) -> Name end, Toml). print_presets(PresetNames) -> [io:format("~p~n", [P]) || P <- PresetNames]. diff --git a/tools/travis-setup-humio.sh b/tools/travis-setup-humio.sh new file mode 100755 index 00000000000..02695d899b7 --- /dev/null +++ b/tools/travis-setup-humio.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +mkdir -p _build/mim1/rel/mongooseim/log + +docker run -d \ + --name mongooseim-filebeat \ + -v "$(pwd)/_build/mim1/rel/mongooseim/log:/usr/lib/mongooseim/log" \ + -v="$(pwd)/priv/filebeat.mongooseim.humio.yml:/usr/share/filebeat/filebeat.yml:ro" \ + docker.elastic.co/beats/filebeat-oss:7.9.2 \ + filebeat -e -strict.perms=false -E output.elasticsearch.password="$HUMIO_PASSWORD"