diff --git a/Dockerfile b/Dockerfile index a6aaa1a..2f84b23 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,18 @@ -FROM fluent/fluentd:v0.12.34 +FROM fluent/fluentd:v0.14.17-debian WORKDIR /home/fluent ENV PATH /home/fluent/.gem/ruby/2.3.0/bin:$PATH USER root -RUN apk --no-cache --update add sudo build-base ruby-dev libffi-dev && \ - sudo -u fluent gem install fluent-plugin-record-reformer fluent-plugin-kubernetes_metadata_filter fluent-plugin-sumologic_output && \ - rm -rf /home/fluent/.gem/ruby/2.3.0/cache/*.gem && sudo -u fluent gem sources -c && \ - apk del sudo build-base ruby-dev && rm -rf /var/cache/apk/* +# New fluent image dynamically creates user in entrypoint +RUN [ -f /bin/entrypoint.sh ] && /bin/entrypoint.sh echo || : && \ + apt-get update && \ + apt-get install -y build-essential ruby-dev libffi-dev libsystemd-dev && \ + gem install fluent-plugin-systemd fluent-plugin-record-reformer fluent-plugin-kubernetes_metadata_filter fluent-plugin-sumologic_output && \ + rm -rf /home/fluent/.gem/ruby/2.3.0/cache/*.gem && \ + gem sources -c && \ + apt-get remove --purge -y build-essential ruby-dev libffi-dev libsystemd-dev && \ + rm -rf /var/lib/apt/lists/* RUN mkdir -p /mnt/pos EXPOSE 24284 @@ -26,9 +31,12 @@ ENV SOURCE_CATEGORY_REPLACE_DASH "/" ENV SOURCE_NAME "%{namespace}.%{pod}.%{container}" ENV KUBERNETES_META "true" ENV READ_FROM_HEAD "true" +ENV FLUENTD_SOURCE "file" +ENV FLUENTD_USER_CONFIG_DIR "/fluentd/conf.d/user" -COPY ./conf.d/* /fluentd/conf.d/ +COPY ./conf.d/ /fluentd/conf.d/ COPY ./etc/* /fluentd/etc/ COPY ./plugins/* /fluentd/plugins/ +COPY ./entrypoint.sh /fluentd/ -CMD exec fluentd -c /fluentd/etc/$FLUENTD_CONF -p /fluentd/plugins $FLUENTD_OPT \ No newline at end of file +ENTRYPOINT ["/fluentd/entrypoint.sh"] diff --git a/README.md b/README.md index df9736d..f0e86a0 100644 --- a/README.md +++ b/README.md @@ -25,10 +25,20 @@ And finally, you need to deploy the container. I will presume you have your own kubectl create -f fluentd.daemonset.yaml ``` +#### Helm + +A helm chart can also install the daemonset, secret, etc. + +``` +helm install --name sumo --set sumologic.collectorUrl=YOUR-URL-HERE stable/sumologic-fluentd +``` + ## Options The following options can be configured as environment variables on the DaemonSet +* `FLUENTD_SOURCE` - Fluentd can tail files or query systemd (default `file`) +* `FLUENTD_USER_CONFIG_DIR` - A directory of user defined fluentd configuration files, which must in in `*.conf` * `FLUSH_INTERVAL` - How frequently to push logs to SumoLogic (default `5s`) * `NUM_THREADS` - Increase number of http threads to Sumo. May be required in heavy logging clusters (default `1`) * `SOURCE_NAME` - Set the `_sourceName` metadata field in SumoLogic. (Default `"%{namespace}.%{pod}.%{container}"`) @@ -53,16 +63,22 @@ The following options can be configured as environment variables on the DaemonSe * `EXCLUDE_POD_REGEX` - A Regex pattern for pods. All matching pods will be excluded from Sumo Logic. The logs will still be sent to FluentD. * `EXCLUDE_CONTAINER_REGEX` - A Regex pattern for containers. All matching containers will be excluded from Sumo Logic. The logs will still be sent to FluentD. * `EXCLUDE_HOST_REGEX` - A Regex pattern for hosts. All matching hosts will be excluded from Sumo Logic. The logs will still be sent to FluentD. + * `EXCLUDE_FACILITY_REGEX` - A Regex pattern for syslog [faclilities](https://en.wikipedia.org/wiki/Syslog#Facility). All matching facilities will be excluded from Sumo Logic. The logs will still be sent to FluentD. + * `EXCLUDE_PRIORITY_REGEX` - A Regex pattern for syslog [priorities](https://en.wikipedia.org/wiki/Syslog#Severity_level). All matching priorities will be excluded from Sumo Logic. The logs will still be sent to FluentD. + * `EXCLUDE_UNIT_REGEX` - A Regex pattern for systemd [units](https://www.freedesktop.org/software/systemd/man/systemd.unit.html). All matching units will be excluded from Sumo Logic. The logs will still be sent to FluentD. The following table show which environment variables affect fluent sources -| Environment Variable | Containers | Docker | Kubernetes | -|----------------------|------------|--------|------------| -| `EXCLUDE_CONTAINER_REGEX` | ✔ | ✘ | ✘ | -| `EXCLUDE_HOST_REGEX `| ✔ | ✘ | ✘ | -| `EXCLUDE_NAMESPACE_REGEX` | ✔ | ✘ | ✔ | -| `EXCLUDE_PATH` | ✔ | ✔ | ✔ | -| `EXCLUDE_POD_REGEX` | ✔ | ✘ | ✘ | +| Environment Variable | Containers | Docker | Kubernetes | Systemd | +|----------------------|------------|--------|------------|---------| +| `EXCLUDE_CONTAINER_REGEX` | ✔ | ✘ | ✘ | ✘ | +| `EXCLUDE_FACILITY_REGEX` | ✘ | ✘ | ✘ | ✔ | +| `EXCLUDE_HOST_REGEX `| ✔ | ✘ | ✘ | ✔ | +| `EXCLUDE_NAMESPACE_REGEX` | ✔ | ✘ | ✔ | ✘ | +| `EXCLUDE_PATH` | ✔ | ✔ | ✔ | ✘ | +| `EXCLUDE_PRIORITY_REGEX` | ✘ | ✘ | ✘ | ✔ | +| `EXCLUDE_POD_REGEX` | ✔ | ✘ | ✘ | ✘ | +| `EXCLUDE_UNIT_REGEX` | ✘ | ✘ | ✘ | ✔ | The `LOG_FORMAT`, `SOURCE_CATEGORY` and `SOURCE_NAME` can be overridden per pod using [annotations](http://kubernetes.io/v1.0/docs/user-guide/annotations.html). For example diff --git a/conf.d/source.containers.conf b/conf.d/file/source.containers.conf similarity index 93% rename from conf.d/source.containers.conf rename to conf.d/file/source.containers.conf index de94671..e4e33f3 100644 --- a/conf.d/source.containers.conf +++ b/conf.d/file/source.containers.conf @@ -1,5 +1,5 @@ - type tail + @type tail format json time_key time path /mnt/log/containers/*.log @@ -11,7 +11,7 @@ - type kubernetes_metadata + @type kubernetes_metadata annotation_match ["sumologic\.com.*"] de_dot false tag_to_kubernetes_name_regexp '.+?\.containers\.(?[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?[^_]+)_(?.+)-(?[a-z0-9]{64})\.log$' @@ -19,7 +19,7 @@ - type kubernetes_sumologic + @type kubernetes_sumologic source_name "#{ENV['SOURCE_NAME']}" log_format "#{ENV['LOG_FORMAT']}" kubernetes_meta "#{ENV['KUBERNETES_META']}" @@ -30,4 +30,4 @@ exclude_pod_regex "#{ENV['EXCLUDE_POD_REGEX']}" exclude_container_regex "#{ENV['EXCLUDE_CONTAINER_REGEX']}" exclude_host_regex "#{ENV['EXCLUDE_HOST_REGEX']}" - \ No newline at end of file + diff --git a/conf.d/source.docker.conf b/conf.d/file/source.docker.conf similarity index 93% rename from conf.d/source.docker.conf rename to conf.d/file/source.docker.conf index 5bb3fd9..07e159e 100644 --- a/conf.d/source.docker.conf +++ b/conf.d/file/source.docker.conf @@ -2,7 +2,7 @@ # time="2016-02-04T06:51:03.053580605Z" level=info msg="GET /containers/json" # time="2016-02-04T07:53:57.505612354Z" level=error msg="HTTP Error" err="No such image: -f" statusCode=404 - type tail + @type tail format /^time="(?