Merge pull request #2952 from esl/mu-only-humio

Add humio - reworked PR

.travis.yml

@@ -44,6 +44,7 @@ before_script:
         - tools/travis-setup-db.sh
         - if [ $PRESET = 'ldap_mnesia' ]; then sudo tools/travis-setup-ldap.sh; fi
         - if [ $PRESET = 'mysql_redis' ]; then sudo tools/travis-setup-rmq.sh; fi
+        - if [ ! -z "$HUMIO_PASSWORD" ]; then tools/travis-setup-humio.sh; fi
 script:
     - SKIP_AUTO_COMPILE=true KEEP_COVER_RUNNING=1 tools/travis-test.sh -p $PRESET -s $RUN_SMALL_TESTS -e $RUN_SMALL_TESTS
 
@@ -54,6 +55,7 @@ after_failure:
         - tail -100 _build/mim2/rel/mongooseim/log/mongooseim.log.1
 
 after_script:
+        - if [ ! -z "$HUMIO_PASSWORD" ]; then docker logs mongooseim-filebeat || echo "ok"; fi
         - cat travis-pull-db.log
         # Upload logs to s3 for debugging
         - test 1 = "$SKIP_REPORT_UPLOAD" ||

doc/operation-and-maintenance/Humio.md

@@ -0,0 +1,50 @@
+# Humio and MongooseIM
+
+## Getting Humio's ingest token
+
+Visit this [url](https://cloud.humio.com/sandbox/settings/ingest-tokens) to create a new sandbox's ingest token.
+
+The URL is:
+
+```
+https://cloud.humio.com/YOUR_REPOSITORY_NAME_HERE/settings/ingest-tokens
+```
+
+## Configure Filebeat
+
+Configure Filebeat, using this config file [priv/filebeat.mongooseim.humio.yml](https://github.com/esl/MongooseIM/blob/master/priv/filebeat.humio.yml).
+
+We recommend to use the Filebeat docker container. You have to use an open-source version
+of Filebeat, which has the `oss` suffix.
+
+This example mounts a log directory `$(pwd)/_build/mim1/rel/mongooseim/log` as a volume for Filebeat.
+It also mounts a configuration file `$(pwd)/priv/filebeat.mongooseim.humio.yml`.
+Most likely these paths would be different on your machine.
+
+Pass your Humio ingest token as a password argument.
+Or uncomment and change it inside the `filebeat.mongooseim.humio.yml` file.
+
+```bash
+docker run -d \
+    --name mongooseim-filebeat \
+    -v "$(pwd)/_build/mim1/rel/mongooseim/log:/usr/lib/mongooseim/log" \
+    -v "$(pwd)/priv/filebeat.mongooseim.humio.yml:/usr/share/filebeat/filebeat.yml:ro" \
+    docker.elastic.co/beats/filebeat-oss:7.9.2 \
+    filebeat -e -E output.elasticsearch.password="abc12345-xxxx-yyyy-zzzz-123456789abc"
+```
+
+Argument `-e` enables debugging information for Filebeat that can be visible using the `docker logs mongooseim-filebeat` command.
+
+## Viewing logs
+
+Navigate to [https://cloud.humio.com/sandbox/search](https://cloud.humio.com/sandbox/search) to see the Sandbox's dashboard.
+
+
+A list of log messages:
+
+![A list of log messages](humio_list.png)
+
+
+Structured log message:
+
+![Structured log message](humio_message.png)

doc/operation-and-maintenance/Logging.md

@@ -122,7 +122,11 @@ Options:
 
 # JSON file log handler
 
-JSON formatted file. It could be used to store messages in ELK or in Splunk.
+JSON formatted file. It could be used to store messages in ELK, in Humio or in Splunk.
+
+Check this [tutorial](Humio.md) to configure MongooseIM with Humio.
+Check below information to configure MongooseIM with ELK.
+
 You can use [Filebeat](https://www.elastic.co/guide/en/beats/filebeat/current/filebeat-input-log.html#filebeat-input-log-config-json)
 to send messages from the file into ELK.
 

mkdocs.yml

@@ -76,6 +76,7 @@ pages:
       - 'Cluster configuration and node management': 'operation-and-maintenance/Cluster-configuration-and-node-management.md'
       - 'Logging & monitoring': 'operation-and-maintenance/Logging-&-monitoring.md'
       - 'Logging configuration': 'operation-and-maintenance/Logging.md'
+      - 'Logging with Humio': 'operation-and-maintenance/Humio.md'
       - 'Logging fields': 'operation-and-maintenance/Logging-fields.md'
       - 'Reloading configuration on a running system': 'operation-and-maintenance/Reloading-configuration-on-a-running-system.md'
       - 'Metrics': 'operation-and-maintenance/Mongoose-metrics.md'

priv/filebeat.mongooseim.humio.yml

@@ -0,0 +1,19 @@
+# https://docs.humio.com/integrations/ingest-logs-with-a-data-shipper/beats/filebeat/
+filebeat.inputs:
+- paths:
+   - /usr/lib/mongooseim/log/mongooseim.json.1
+  encoding: utf-8
+
+queue.mem:
+  events: 8000
+  flush.min_events: 1000
+  flush.timeout: 1s
+
+output:
+  elasticsearch:
+    hosts: ["https://cloud.humio.com:443/api/v1/ingest/elastic-bulk"]
+    username: anything
+    compression_level: 5
+    bulk_max_size: 200
+    worker: 5
+#   password: "abc12345-xxxx-yyyy-zzzz-123456789abc"

tools/test_runner/list_presets.erl

@@ -12,8 +12,9 @@ main(_) ->
 -spec read_presets() -> [atom()].
 read_presets() ->
     {ok, Config} = file:consult("big_tests/test.config"),
-    Presets = proplists:get_value(ejabberd_presets, Config, []),
-    lists:map(fun({Name,_}) -> Name end, Presets).
+    Presets = proplists:get_value(presets, Config, []),
+    Toml = proplists:get_value(toml, Presets, []),
+    lists:map(fun({Name,_}) -> Name end, Toml).
 
 print_presets(PresetNames) ->
     [io:format("~p~n", [P]) || P <- PresetNames].

tools/travis-setup-humio.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+mkdir -p _build/mim1/rel/mongooseim/log
+
+docker run -d \
+    --name mongooseim-filebeat \
+    -v "$(pwd)/_build/mim1/rel/mongooseim/log:/usr/lib/mongooseim/log" \
+    -v="$(pwd)/priv/filebeat.mongooseim.humio.yml:/usr/share/filebeat/filebeat.yml:ro" \
+    docker.elastic.co/beats/filebeat-oss:7.9.2 \
+    filebeat -e -strict.perms=false -E output.elasticsearch.password="$HUMIO_PASSWORD"