systemd poc

Dockerfile

@@ -1,13 +1,18 @@
-FROM fluent/fluentd:v0.12.34
+FROM fluent/fluentd:v0.14.17-debian
 WORKDIR /home/fluent
 ENV PATH /home/fluent/.gem/ruby/2.3.0/bin:$PATH
 
 USER root
 
-RUN apk --no-cache --update add sudo build-base ruby-dev libffi-dev && \
-    sudo -u fluent gem install fluent-plugin-record-reformer fluent-plugin-kubernetes_metadata_filter fluent-plugin-sumologic_output && \
-    rm -rf /home/fluent/.gem/ruby/2.3.0/cache/*.gem && sudo -u fluent gem sources -c && \
-    apk del sudo build-base ruby-dev && rm -rf /var/cache/apk/*
+# New fluent image dynamically creates user in entrypoint
+RUN [ -f /bin/entrypoint.sh ] && /bin/entrypoint.sh echo || : && \
+    apt-get update && \
+    apt-get install -y build-essential ruby-dev libffi-dev libsystemd-dev && \
+    gem install fluent-plugin-systemd fluent-plugin-record-reformer fluent-plugin-kubernetes_metadata_filter fluent-plugin-sumologic_output && \
+    rm -rf /home/fluent/.gem/ruby/2.3.0/cache/*.gem && \
+    gem sources -c && \
+    apt-get remove --purge -y build-essential ruby-dev libffi-dev libsystemd-dev && \
+    rm -rf /var/lib/apt/lists/*
 
 RUN mkdir -p /mnt/pos
 EXPOSE 24284
@@ -26,9 +31,12 @@ ENV SOURCE_CATEGORY_REPLACE_DASH "/"
 ENV SOURCE_NAME "%{namespace}.%{pod}.%{container}"
 ENV KUBERNETES_META "true"
 ENV READ_FROM_HEAD "true"
+ENV FLUENTD_SOURCE "file"
+ENV FLUENTD_USER_CONFIG_DIR "/fluentd/conf.d/user"
 
-COPY ./conf.d/* /fluentd/conf.d/
+COPY ./conf.d/ /fluentd/conf.d/
 COPY ./etc/* /fluentd/etc/
 COPY ./plugins/* /fluentd/plugins/
+COPY ./entrypoint.sh /fluentd/
 
-CMD exec fluentd -c /fluentd/etc/$FLUENTD_CONF -p /fluentd/plugins $FLUENTD_OPT
+ENTRYPOINT ["/fluentd/entrypoint.sh"]

README.md

@@ -25,10 +25,20 @@ And finally, you need to deploy the container. I will presume you have your own
 kubectl create -f fluentd.daemonset.yaml
 ```
 
+#### Helm
+
+A helm chart can also install the daemonset, secret, etc. 
+
+```
+helm install --name sumo --set sumologic.collectorUrl=YOUR-URL-HERE stable/sumologic-fluentd
+```
+
 ## Options
 
 The following options can be configured as environment variables on the DaemonSet
 
+* `FLUENTD_SOURCE` - Fluentd can tail files or query systemd (default `file`)
+* `FLUENTD_USER_CONFIG_DIR` - A directory of user defined fluentd configuration files, which must in in `*.conf`
 * `FLUSH_INTERVAL` - How frequently to push logs to SumoLogic (default `5s`)
 * `NUM_THREADS` - Increase number of http threads to Sumo. May be required in heavy logging clusters (default `1`)
 * `SOURCE_NAME` - Set the `_sourceName` metadata field in SumoLogic. (Default `"%{namespace}.%{pod}.%{container}"`)
@@ -53,16 +63,22 @@ The following options can be configured as environment variables on the DaemonSe
  * `EXCLUDE_POD_REGEX` - A Regex pattern for pods.  All matching pods will be excluded from Sumo Logic.  The logs will still be sent to FluentD.
  * `EXCLUDE_CONTAINER_REGEX` - A Regex pattern for containers.  All matching containers will be excluded from Sumo Logic.  The logs will still be sent to FluentD.
  * `EXCLUDE_HOST_REGEX` - A Regex pattern for hosts.  All matching hosts will be excluded from Sumo Logic.  The logs will still be sent to FluentD.
+ * `EXCLUDE_FACILITY_REGEX` - A Regex pattern for syslog [faclilities](https://en.wikipedia.org/wiki/Syslog#Facility).  All matching facilities will be excluded from Sumo Logic.  The logs will still be sent to FluentD.
+ * `EXCLUDE_PRIORITY_REGEX` - A Regex pattern for syslog [priorities](https://en.wikipedia.org/wiki/Syslog#Severity_level).  All matching priorities will be excluded from Sumo Logic.  The logs will still be sent to FluentD.
+ * `EXCLUDE_UNIT_REGEX` - A Regex pattern for systemd [units](https://www.freedesktop.org/software/systemd/man/systemd.unit.html).  All matching units will be excluded from Sumo Logic.  The logs will still be sent to FluentD.
 
 The following table show which  environment variables affect fluent sources
 
-| Environment Variable | Containers | Docker | Kubernetes |
-|----------------------|------------|--------|------------|
-| `EXCLUDE_CONTAINER_REGEX` | ✔ | ✘ | ✘ |
-| `EXCLUDE_HOST_REGEX `| ✔ | ✘ | ✘ |
-| `EXCLUDE_NAMESPACE_REGEX` | ✔ | ✘ | ✔ |
-| `EXCLUDE_PATH` | ✔ | ✔ | ✔ |
-| `EXCLUDE_POD_REGEX` | ✔ | ✘ | ✘ |
+| Environment Variable | Containers | Docker | Kubernetes | Systemd |
+|----------------------|------------|--------|------------|---------|
+| `EXCLUDE_CONTAINER_REGEX` | ✔ | ✘ | ✘ | ✘ |
+| `EXCLUDE_FACILITY_REGEX` | ✘ | ✘ | ✘ | ✔ |
+| `EXCLUDE_HOST_REGEX `| ✔ | ✘ | ✘ | ✔ |
+| `EXCLUDE_NAMESPACE_REGEX` | ✔ | ✘ | ✔ | ✘ |
+| `EXCLUDE_PATH` | ✔ | ✔ | ✔ | ✘ |
+| `EXCLUDE_PRIORITY_REGEX` | ✘ | ✘ | ✘ | ✔ |
+| `EXCLUDE_POD_REGEX` | ✔ | ✘ | ✘ | ✘ |
+| `EXCLUDE_UNIT_REGEX` | ✘ | ✘ | ✘ | ✔ |
 
 The `LOG_FORMAT`, `SOURCE_CATEGORY` and `SOURCE_NAME` can be overridden per pod using [annotations](http://kubernetes.io/v1.0/docs/user-guide/annotations.html). For example
 

conf.d/source.containers.conf → conf.d/file/source.containers.conf

@@ -1,5 +1,5 @@
 <source>
-  type tail
+  @type tail
   format json
   time_key time
   path /mnt/log/containers/*.log
@@ -11,15 +11,15 @@
 </source>
 
 <filter containers.**>
-  type kubernetes_metadata
+  @type kubernetes_metadata
   annotation_match ["sumologic\.com.*"]
   de_dot false
   tag_to_kubernetes_name_regexp '.+?\.containers\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
   merge_json_log false
 </filter>
 
 <filter containers.**>
-  type kubernetes_sumologic
+  @type kubernetes_sumologic
   source_name "#{ENV['SOURCE_NAME']}"
   log_format "#{ENV['LOG_FORMAT']}"
   kubernetes_meta "#{ENV['KUBERNETES_META']}"
@@ -30,4 +30,4 @@
   exclude_pod_regex "#{ENV['EXCLUDE_POD_REGEX']}"
   exclude_container_regex "#{ENV['EXCLUDE_CONTAINER_REGEX']}"
   exclude_host_regex "#{ENV['EXCLUDE_HOST_REGEX']}"
-</filter>
+</filter>

conf.d/source.docker.conf → conf.d/file/source.docker.conf

@@ -2,7 +2,7 @@
 # time="2016-02-04T06:51:03.053580605Z" level=info msg="GET /containers/json"
 # time="2016-02-04T07:53:57.505612354Z" level=error msg="HTTP Error" err="No such image: -f" statusCode=404
 <source>
-  type tail
+  @type tail
   format /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/
   time_format %Y-%m-%dT%H:%M:%S.%NZ
   path /var/lib/docker.log
@@ -12,7 +12,7 @@
 </source>
 
 <filter docker.**>
-  type kubernetes_sumologic
+  @type kubernetes_sumologic
   source_category docker
   source_name k8s_docker
   source_category_prefix "#{ENV['SOURCE_CATEGORY_PREFIX']}"

conf.d/source.kubernetes.conf → conf.d/file/source.kubernetes.conf

@@ -1,7 +1,7 @@
 # Example:
 # 2015-12-21 23:17:22,066 [salt.state       ][INFO    ] Completed state [net.ipv4.ip_forward] at time 23:17:22.066081
 <source>
-  type tail
+  @type tail
   format /^(?<time>[^ ]* [^ ,]*)[^\[]*\[[^\]]*\]\[(?<severity>[^ \]]*) *\] (?<message>.*)$/
   time_format %Y-%m-%d %H:%M:%S
   path /mnt/log/salt/minion
@@ -11,7 +11,7 @@
 </source>
 
 <filter salt.**>
-  type kubernetes_sumologic
+  @type kubernetes_sumologic
   source_category salt
   source_name k8s_salt
   source_category_prefix "#{ENV['SOURCE_CATEGORY_PREFIX']}"
@@ -22,7 +22,7 @@
 # Example:
 # Dec 21 23:17:22 gke-foo-1-1-4b5cbd14-node-4eoj startupscript: Finished running startup script /var/run/google.startup.script
 <source>
-  type tail
+  @type tail
   format syslog
   path /mnt/log/startupscript.log
   exclude_path "#{ENV['EXCLUDE_PATH']}"
@@ -31,7 +31,7 @@
 </source>
 
 <filter startupscript.**>
-  type kubernetes_sumologic
+  @type kubernetes_sumologic
   source_category startupscript
   source_name k8s_startupscript
   source_category_prefix "#{ENV['SOURCE_CATEGORY_PREFIX']}"
@@ -46,7 +46,7 @@
 # Example:
 # I0204 07:32:30.020537    3368 server.go:1048] POST /stats/container/: (13.972191ms) 200 [[Go-http-client/1.1] 10.244.1.3:40537]
 <source>
-  type tail
+  @type tail
   format multiline
   multiline_flush_interval 5s
   format_firstline /^\w\d{4}/
@@ -59,7 +59,7 @@
 </source>
 
 <filter kubelet.**>
-  type kubernetes_sumologic
+  @type kubernetes_sumologic
   source_category kubelet
   source_name k8s_kubelet
   source_category_prefix "#{ENV['SOURCE_CATEGORY_PREFIX']}"
@@ -69,7 +69,7 @@
 # Example:
 # I0204 07:00:19.604280       5 handlers.go:131] GET /api/v1/nodes: (1.624207ms) 200 [[kube-controller-manager/v1.1.3 (linux/amd64) kubernetes/6a81b50] 127.0.0.1:38266]
 <source>
-  type tail
+  @type tail
   format multiline
   multiline_flush_interval 5s
   format_firstline /^\w\d{4}/
@@ -82,7 +82,7 @@
 </source>
 
 <filter kube-apiserver.**>
-  type kubernetes_sumologic
+  @type kubernetes_sumologic
   source_category kube-apiserver
   source_name k8s_kube-apiserver
   source_category_prefix "#{ENV['SOURCE_CATEGORY_PREFIX']}"
@@ -93,7 +93,7 @@
 # Example:
 # I0204 06:55:31.872680       5 servicecontroller.go:277] LB already exists and doesn't need update for service kube-system/kube-ui
 <source>
-  type tail
+  @type tail
   format multiline
   multiline_flush_interval 5s
   format_firstline /^\w\d{4}/
@@ -106,7 +106,7 @@
 </source>
 
 <filter kube-controller-manager.**>
-  type kubernetes_sumologic
+  @type kubernetes_sumologic
   source_category kube-controller-manager
   source_name k8s_kube-controller-manager
   source_category_prefix "#{ENV['SOURCE_CATEGORY_PREFIX']}"
@@ -117,7 +117,7 @@
 # Example:
 # W0204 06:49:18.239674       7 reflector.go:245] pkg/scheduler/factory/factory.go:193: watch of *api.Service ended with: 401: The event in requested index is outdated and cleared (the requested history has been cleared [2578313/2577886]) [2579312]
 <source>
-  type tail
+  @type tail
   format multiline
   multiline_flush_interval 5s
   format_firstline /^\w\d{4}/
@@ -130,7 +130,7 @@
 </source>
 
 <filter kube-scheduler.**>
-  type kubernetes_sumologic
+  @type kubernetes_sumologic
   source_category kube-scheduler
   source_name k8s_kube-scheduler
   source_category_prefix "#{ENV['SOURCE_CATEGORY_PREFIX']}"
@@ -142,7 +142,7 @@
 # Example:
 # I0603 15:31:05.793605       6 cluster_manager.go:230] Reading config from path /etc/gce.conf
 <source>
-  type tail
+  @type tail
   format multiline
   multiline_flush_interval 5s
   format_firstline /^\w\d{4}/
@@ -155,7 +155,7 @@
 </source>
 
 <filter glbc.**>
-  type kubernetes_sumologic
+  @type kubernetes_sumologic
   source_category glbc
   source_name k8s_glbc
   source_category_prefix "#{ENV['SOURCE_CATEGORY_PREFIX']}"
@@ -166,7 +166,7 @@
 # Example:
 # I0603 15:31:05.793605       6 cluster_manager.go:230] Reading config from path /etc/gce.conf
 <source>
-  type tail
+  @type tail
   format multiline
   multiline_flush_interval 5s
   format_firstline /^\w\d{4}/
@@ -180,7 +180,7 @@
 
 
 <filter cluster-autoscaler.**>
-  type kubernetes_sumologic
+  @type kubernetes_sumologic
   source_category cluster-autoscaler
   source_name k8s_cluster-autoscaler
   source_category_prefix "#{ENV['SOURCE_CATEGORY_PREFIX']}"

conf.d/out.sumo.conf

@@ -1,9 +1,9 @@
 <match **>
- type sumologic
+ @type sumologic
  log_key log
  endpoint "#{ENV['COLLECTOR_URL']}"
  verify_ssl "#{ENV['VERIFY_SSL']}"
  log_format "#{ENV['LOG_FORMAT']}"
  flush_interval "#{ENV['FLUSH_INTERVAL']}"
  num_threads "#{ENV['NUM_THREADS']}"
-</match>
+</match>

conf.d/systemd/source.containers.conf

@@ -0,0 +1,33 @@
+<source>
+  @type tail
+  format json
+  time_key time
+  path /mnt/log/containers/*.log
+  exclude_path "#{ENV['EXCLUDE_PATH']}"
+  pos_file /mnt/pos/ggcp-containers.log.pos
+  time_format %Y-%m-%dT%H:%M:%S.%NZ
+  tag containers.*
+  read_from_head true
+</source>
+
+<filter containers.**>
+  @type kubernetes_metadata
+  annotation_match ["sumologic\.com.*"]
+  de_dot false
+  tag_to_kubernetes_name_regexp '.+?\.containers\.(?<pod_name>[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*)_(?<namespace>[^_]+)_(?<container_name>.+)-(?<docker_id>[a-z0-9]{64})\.log$'
+  merge_json_log false
+</filter>
+
+<filter containers.**>
+  @type kubernetes_sumologic
+  source_name "#{ENV['SOURCE_NAME']}"
+  log_format "#{ENV['LOG_FORMAT']}"
+  kubernetes_meta "#{ENV['KUBERNETES_META']}"
+  source_category "#{ENV['SOURCE_CATEGORY']}"
+  source_category_prefix "#{ENV['SOURCE_CATEGORY_PREFIX']}"
+  source_category_replace_dash "#{ENV['SOURCE_CATEGORY_REPLACE_DASH']}"
+  exclude_namespace_regex "#{ENV['EXCLUDE_NAMESPACE_REGEX']}"
+  exclude_pod_regex "#{ENV['EXCLUDE_POD_REGEX']}"
+  exclude_container_regex "#{ENV['EXCLUDE_CONTAINER_REGEX']}"
+  exclude_host_regex "#{ENV['EXCLUDE_HOST_REGEX']}"
+</filter>