logs-dispatcher.fluent-conf.configmap.yaml

apiVersion: v1
kind: ConfigMap
metadata:
  name: {{ include "lagoon-logging.logsDispatcher.fullname" . }}-fluent-conf
  labels:
    {{- include "lagoon-logging.logsDispatcher.labels" . | nindent 4 }}
data:
  fluent.conf: |
    # vi: ft=fluentd
    <system>
      workers 2
      # comment out this line to see warnings
      # it is set to error because fluentd is quite chatty
      log_level error
    </system>

    # prometheus metrics
    <source>
      @type prometheus
    </source>
    <source>
      @type prometheus_monitor
    </source>
    <source>
      @type prometheus_output_monitor
    </source>

    # container logs collected by the logging-operator
    <source>
      @type  forward
      @id    in_container
      tag    process.container
    </source>
    # application logs emitted by the lagoon_logs drupal module
    <source>
      @type  udp
      @id    in_application
      tag    "lagoon.#{ENV['CLUSTER_NAME']}.application"
      port   5140
      # max IPv4 UDP payload size
      message_length_limit 65507
      <parse>
        @type json
      </parse>
    </source>
{{- if .Values.openshiftHaproxyLogsCollector.enabled }}
    # router logs emitted by the openshift routers
    <source>
      @type forward
      @id   in_router_openshift
      port  24225
      tag   "lagoon.#{ENV['CLUSTER_NAME']}.router.openshift"
    </source>
{{- end }}
{{- if .Values.cdnLogsCollector.enabled }}
    # cdn logs collected via cdn-logs-collector
    <source>
      @type  forward
      @id    in_cdn
      port   24226
      tag   "lagoon.#{ENV['CLUSTER_NAME']}.cdn"
    </source>
{{- end }}

    #
    # optional sources which can be enabled in the chart
    #
    @include source.d/*.conf

    #
    # pre-processing for nginx_router logs
    #
    # the reason for having the two match blocks is because we have two checks
    # to distinguish nginx_router logs:
    # * app label is "nginx-ingress"
    # * namespace is "syn-nginx-ingress"
    # if either of those checks fails the message is tagged as a regular
    # container log.
    #
    # check app name first. if app name didn't match, set tag to container log.
    <match process.container>
      @type rewrite_tag_filter
      <rule>
        key     $.kubernetes.labels.app
        pattern ^nginx-ingress$
        tag     "process.app_nginx_ingress"
      </rule>
      <rule>
        key     $['kubernetes']['labels']['app.kubernetes.io/name']
        pattern ^ingress-nginx$
        tag     "process.app_nginx_ingress"
      </rule>
      # Last rule: catchall
      <rule>
        invert  true
        key     $.kubernetes.labels.app
        pattern ^nginx-ingress$
        tag     "lagoon.#{ENV['CLUSTER_NAME']}.container"
      </rule>
    </match>
    # check namespace_name. if it is okay too, tag as router log.
    # if namespace didn't match, set tag to container log.
    <match process.app_nginx_ingress>
      @type rewrite_tag_filter
      <rule>
        key     $.kubernetes.namespace_name
        pattern ^syn-nginx-ingress$
        tag     "lagoon.#{ENV['CLUSTER_NAME']}.router.nginx"
      </rule>
      <rule>
        key     $.kubernetes.namespace_name
        pattern ^ingress-nginx$
        tag     "lagoon.#{ENV['CLUSTER_NAME']}.router.nginx"
      </rule>
      <rule>
        key     $.kubernetes.namespace_name
        pattern ^sigsci-ingress-nginx$
        tag     "lagoon.#{ENV['CLUSTER_NAME']}.router.nginx"
      </rule>
      # Last rule: catchall
      <rule>
        invert  true
        key     $.kubernetes.namespace_name
        pattern ^syn-nginx-ingress$
        tag     "lagoon.#{ENV['CLUSTER_NAME']}.container"
      </rule>
    </match>

    # prometheus monitoring
    <filter lagoon.**>
      @type prometheus
      <metric>
        name fluentd_input_status_num_records_total
        type counter
        desc The total number of incoming records
        <labels>
          tag ${tag}
          hostname ${hostname}
        </labels>
      </metric>
    </filter>

    #
    # process container logs
    #
    # restructure so the kubernetes_metadata plugin can find the keys it needs
    <filter lagoon.*.container>
      @type record_modifier
      remove_keys _dummy_
      <record>
        _dummy_ ${record['docker'] = {'container_id' => "#{record.dig('kubernetes','docker_id')}"}; nil}
      </record>
    </filter>
    # enrich with k8s metadata (will get the namespace labels)
    <filter lagoon.*.container>
      @type kubernetes_metadata
      @log_level warn
      skip_container_metadata true
      skip_master_url         true
    </filter>
    # strip the duplicate information so that it doesn't appear in logs
    <filter lagoon.*.container>
      @type record_modifier
      remove_keys docker
    </filter>
    # post-process to try to eke some more structure out of the logs.
    # the last "format none" block is a catch-all for unmatched messages.
    <filter lagoon.*.container>
      @type parser
{{- if eq .Values.containerRuntime "docker" }}
      key_name log
{{- else }}
      key_name message
{{- end }}
      reserve_data true
      <parse>
        @type multi_format
        <pattern>
          format regexp
          expression /^(?<remote>[^ ]*) (?<host>[^ ]*) (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^\"]*?)(?: +\S*)?)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)"(?:\s+"(?<forwarded_for>[^"]+)")?)?$/
          time_format %d/%b/%Y:%H:%M:%S %z
          types  size:integer
        </pattern>
        <pattern>
          format none
        </pattern>
      </parse>
    </filter>

    #
    # process application logs
    #
    # Restructure so the kubernetes_metadata plugin can find the keys it needs.
    # Also add some dummy data required by the kubernetes_metadata plugin.
    <filter lagoon.*.application>
      @type record_modifier
      remove_keys _dummy_,type
      <record>
        # modify the 'type' record before removal to convert underscores to
        # hyphens as the former are illegal in k8s names. Lagoon does this same
        # modification when creating the namespace.
        _dummy_ ${record['openshift_project'] = record['type']&.gsub!('_', '-'); record['kubernetes'] = {'namespace_name' => record['type'], 'pod_name' => record['host'], 'container_name' => 'unknown'}; record['docker'] = {'container_id' => "#{record['type']}_#{record['host']}"}; nil}
      </record>
    </filter>
    # enrich with k8s metadata (will get the namespace labels)
    <filter lagoon.*.application>
      @type kubernetes_metadata
      @log_level warn
      skip_container_metadata true
      skip_master_url         true
    </filter>
    # strip the dummy information so that it doesn't appear in logs
    <filter lagoon.*.application>
      @type record_modifier
      remove_keys _dummy_,docker
      <record>
        _dummy_ ${record['kubernetes'].delete('pod_name'); record['kubernetes'].delete('container_name'); record['kubernetes'].delete('pod_id'); nil}
      </record>
    </filter>

    #
    # process nginx_router logs
    #
    # The message field may be json-encoded router logs, so parse that and put the
    # keys in the top-level log object.
    <filter lagoon.*.router.nginx>
      @type parser
{{- if eq .Values.containerRuntime "docker" }}
      key_name log
{{- else }}
      key_name message
{{- end }}
      reserve_time true
      reserve_data true
      remove_key_name_field true
      <parse>
        @type multi_format
        <pattern>
          format json
        </pattern>
        <pattern>
          format none
        </pattern>
      </parse>
    </filter>
    # match the nginx router logs here and relabel them based on whether they
    # were successfully parsed as json above or not
    <match lagoon.*.router.nginx>
      @type rewrite_tag_filter
      <rule>
        # if the host key doesn't exist then this was not parsed as JSON, so we
        # just send to @OUTPUT directly because it is an actual nginx
        # controller log. These logs will appear in index_name
        # router-logs-ingress-nginx_.* if the keepIngressNginxController value
        # is set to true.
        invert true
        key host
        pattern /.+/
        tag ${tag}
{{- if .Values.keepIngressNginxController }}
        label @OUTPUT
{{- else }}
        label @DISCARD
{{- end }}
      </rule>
      <rule>
        # host key exists, so this is a HTTP request log
        key host
        pattern /.+/
        tag ${tag}
        label @NGINX_ROUTER_OUTPUT
      </rule>
    </match>
    <label @NGINX_ROUTER_OUTPUT>
      # Strip the nginx-ingress namespace info and add enough dummy information
      # so that kubernetes_metadata plugin can get the namespace labels.
      <filter lagoon.*.router.nginx>
        @type record_modifier
        remove_keys _dummy_
        <record>
          _dummy_ ${record['kubernetes'] = {'namespace_name' => record['namespace'], 'pod_name' => 'nopod', 'container_name' => 'nocontainer'}; record['docker'] = {'container_id' => "#{record['namespace']}_#{record['ingress_name']}"}; nil}
        </record>
      </filter>
      # enrich with k8s metadata (will get the namespace labels)
      <filter lagoon.*.router.nginx>
        @type kubernetes_metadata
        @log_level warn
        skip_container_metadata true
        skip_master_url         true
      </filter>
      # strip the dummy information so that it doesn't appear in logs
      <filter lagoon.*.router.nginx>
        @type record_modifier
        remove_keys _dummy_,docker
        <record>
          _dummy_ ${record['kubernetes'].delete('pod_name'); record['kubernetes'].delete('container_name'); record['kubernetes'].delete('pod_id'); nil}
        </record>
      </filter>
      {{- with .Values.routerLogsPostProcess }}
        {{- . | nindent 6 }}
      {{- end }}
      <match lagoon.*.router.nginx>
        @type relabel
        @label @OUTPUT
      </match>
    </label>

{{- if .Values.openshiftHaproxyLogsCollector.enabled }}
    #
    # process openshift router logs
    #
    # retructure the record enough for the kubernetes_metadata plugin to get
    # namespace labels
    <filter lagoon.*.router.openshift.**>
      @type record_modifier
      remove_keys _dummy_,kubernetes_namespace_name,kubernetes_pod_name,kubernetes_container_name,docker_container_id
      <record>
        _dummy_ ${record['kubernetes'] = {'namespace_name' => record['kubernetes_namespace_name'], 'pod_name' => record['kubernetes_pod_name'], 'container_name' => record['kubernetes_container_name']}; record['docker'] = {'container_id' => record['docker_container_id']}; nil}
      </record>
    </filter>
    # enrich with k8s metadata
    <filter lagoon.*.router.openshift.**>
      @type kubernetes_metadata
      @log_level warn
      skip_container_metadata true
      skip_master_url         true
    </filter>

{{- end }}

    <match lagoon.**>
      @type relabel
      @label @OUTPUT
    </match>

    <label @OUTPUT>
      #
      # add cluster identifier
      #
      <filter lagoon.**>
        @type record_modifier
        <record>
          cluster "#{ENV['CLUSTER_NAME']}"
        </record>
      </filter>
      #
      # add the container logs index_name
      #
      <filter lagoon.*.container>
        @type record_modifier
        <record>
          index_name container-logs-${record.dig('kubernetes','namespace_labels','lagoon.sh/project')&.gsub("_", "-") || "#{record.dig('kubernetes','namespace_name') || 'unknown_project'}_#{ENV['CLUSTER_NAME']}"}-_-${record.dig('kubernetes','namespace_labels','lagoon.sh/environmentType') || "unknown_environmenttype"}-_-${Time.at(time).strftime("%Y.%m")}
          log ${record['log'] || ""}
        </record>
      </filter>
      #
      # add the application logs index_name
      #
      <filter lagoon.*.application>
        @type record_modifier
        <record>
          index_name application-logs-${record.dig('kubernetes','namespace_labels','lagoon.sh/project')&.gsub("_", "-") || "#{record.dig('kubernetes','namespace_name') || 'unknown_project'}_#{ENV['CLUSTER_NAME']}"}-_-${record.dig('kubernetes','namespace_labels','lagoon.sh/environmentType') || "unknown_environmenttype"}-_-${Time.at(time).strftime("%Y.%m")}
        </record>
      </filter>
      #
      # add the router logs index_name
      #
      <filter lagoon.*.router.**>
        @type record_modifier
        <record>
          index_name router-logs-${record.dig('kubernetes','namespace_labels','lagoon.sh/project')&.gsub("_", "-") || "#{record.dig('kubernetes','namespace_name') || 'unknown_project'}_#{ENV['CLUSTER_NAME']}"}-_-${record.dig('kubernetes','namespace_labels','lagoon.sh/environmentType') || "unknown_environmenttype"}-_-${Time.at(time).strftime("%Y.%m")}
        </record>
      </filter>
      {{- if .Values.cdnLogsCollector.enabled }}
      #
      # add the cdn logs index_name
      #
      <filter lagoon.*.cdn>
        @type record_modifier
        <record>
          index_name cdn-logs-_-${ENV['CLUSTER_NAME']}-_-${Time.at(time).strftime("%Y.%m")}
        </record>
      </filter>
      {{- end }}
      {{- if .Values.consolidateServiceIndices }}
      # some cluster services generate namespaces with a random suffix, so
      # consolidate those in a single index for each service
      <filter lagoon.*.router.openshift.**>
        @type record_modifier
        <replace>
          key index_name
          expression /^(?<prefix>router-logs-e2e)-[^-]{10}-[^-]{26}_(?<suffix>.+)$/
          replace \k<prefix>_\k<suffix>
        </replace>
      </filter>
      <filter lagoon.*.container>
        @type record_modifier
        <replace>
          key index_name
          expression /^(?<prefix>container-logs-e2e)-[^-]{10}-[^-]{26}_(?<suffix>.+)$/
          replace \k<prefix>_\k<suffix>
        </replace>
        <replace>
          key index_name
          expression /^(?<prefix>container-logs-.+-apb)-(?:prov|depr)-[^-]{5}_(?<suffix>.+)$/
          replace \k<prefix>_\k<suffix>
        </replace>
        <replace>
          key index_name
          expression /^(?<prefix>container-logs-openshift-build-test)-[0-9]+-[^-]{5}$/
          replace \k<prefix>
        </replace>
      </filter>
      {{- end }}
      #
      # add the lagoon logs index_name
      # the source for this tag is included when lagoonLogs.enabled is true
      #
      <filter lagoon.*.lagoon>
        @type record_modifier
        <record>
          index_name lagoon-logs-${record['project']}-_-all_environments-_-${Time.at(time).strftime("%Y.%m")}
        </record>
      </filter>
      #
      # exclude the lagoon logs for running builds, and legacy task: updates
      #
      <filter lagoon.*.lagoon>
        @type grep
        <or>
          <exclude>
            key $.meta.buildPhase
            pattern ^running$
          </exclude>
          <exclude>
            key $.meta.buildStatus
            pattern ^running$
          </exclude>
          <exclude>
            key $.event
            pattern ^task:builddeploy-kubernetes:*
          </exclude>
        </or>
      </filter>
      #
      # forward all to logs-concentrator
      #
      <match lagoon.**>
        @type copy
        {{- if .Values.enableDefaultForwarding }}
        <store>
          @type forward
          @id out_forward
          # error out early
          verify_connection_at_startup {{ .Values.forward.verifyConnectionAtStartup }}
          # tls
          transport tls
          tls_cert_path /fluentd/tls/ca.crt
          tls_client_cert_path /fluentd/tls/client.crt
          tls_client_private_key_path /fluentd/tls/client.key
          tls_verify_hostname {{ .Values.forward.tlsVerifyHostname }}
          # endpoint
          keepalive true # makes sure the connection is not recreated every second
          keepalive_timeout 10m # reconnect after 10mins in order to handle DNS changes, etc.
          # avoid persistent DNS cache in case the server IP changes
          expire_dns_cache 3600 # refresh cached DNS every hour
          <server>
            port "#{ENV['LOGS_FORWARD_HOST_PORT']}"
            host "#{ENV['LOGS_FORWARD_HOST']}"
            name "#{ENV['LOGS_FORWARD_HOSTNAME']}"
            username "#{ENV['LOGS_FORWARD_USERNAME']}"
            password "#{ENV['LOGS_FORWARD_PASSWORD']}"
          </server>
          # authentication
          <security>
            self_hostname "#{ENV['LOGS_FORWARD_SELF_HOSTNAME']}"
            shared_key "#{ENV['LOGS_FORWARD_SHARED_KEY']}"
          </security>
          # buffer chunks by tag
          <buffer tag>
            @type file
            path /fluentd/buffer/forward
            # buffer params (per worker)
            total_limit_size 7500MB
            # flush params
            flush_thread_count 8 # 8 threads that flush, makes sure we have enough flushers for all the different buffers
            flush_interval 2s # flush every 2 seconds
            flush_thread_burst_interval 0 # don't sleep if there is more data to flush
            retry_max_interval 30s # limit exponential backoff period
            overflow_action drop_oldest_chunk
          </buffer>
        </store>
        {{- end }}
        {{- if .Values.extraConf }}
        <store>
          @type relabel
          @label @EXTRACONF
        </store>
        {{- end }}
        @include store.d/*.conf
      </match>
    </label>

    <label @DISCARD>
      <match lagoon.**>
        @type null
      </match>
    </label>
    {{- with .Values.extraConf }}

    <label @EXTRACONF>
      {{- . | nindent 6 }}
    </label>
    {{- end }}