[Agent] Add agent standalone manifests for system module & Pod's log …

…collection (#23938)
elastic · Feb 22, 2021 · 7f92834 · 7f92834
1 parent e315d66
commit 7f92834
Show file tree

Hide file tree

Showing 8 changed files with 739 additions and 80 deletions.
diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml
diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml
@@ -0,0 +1,231 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: agent-node-datastreams
+  namespace: kube-system
+  labels:
+    k8s-app: elastic-agent
+data:
+  agent.yml: |-
+    outputs:
+      default:
+        type: elasticsearch
+        hosts:
+          - >-
+            ${ES_HOST}
+        username: ${ES_USERNAME}
+        password: ${ES_PASSWORD}
+    agent:
+      monitoring:
+        enabled: true
+        use_output: default
+        logs: true
+        metrics: true
+    providers.kubernetes:
+      node: ${NODE_NAME}
+      scope: node
+    inputs:
+      - name: log-1
+        type: logfile
+        use_output: default
+        meta:
+          package:
+            name: log
+            version: 0.4.6
+        data_stream:
+          namespace: default
+        streams:
+          - data_stream:
+              dataset: generic
+            symlinks: true
+            paths:
+              - /var/log/containers/*${kubernetes.container.id}.log
+      - name: system-3
+        type: system/metrics
+        use_output: default
+        meta:
+          package:
+            name: system
+            version: 0.10.9
+        data_stream:
+          namespace: default
+        streams:
+          - data_stream:
+              dataset: system.core
+              type: metrics
+            metricsets:
+              - core
+            core.metrics:
+              - percentages
+          - data_stream:
+              dataset: system.cpu
+              type: metrics
+            period: 10s
+            cpu.metrics:
+              - percentages
+              - normalized_percentages
+            metricsets:
+              - cpu
+          - data_stream:
+              dataset: system.diskio
+              type: metrics
+            period: 10s
+            diskio.include_devices: null
+            metricsets:
+              - diskio
+          - data_stream:
+              dataset: system.filesystem
+              type: metrics
+            period: 1m
+            metricsets:
+              - filesystem
+            processors:
+              - drop_event.when.regexp:
+                  system.filesystem.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)
+          - data_stream:
+              dataset: system.fsstat
+              type: metrics
+            period: 1m
+            metricsets:
+              - fsstat
+            processors:
+              - drop_event.when.regexp:
+                  system.fsstat.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)
+          - data_stream:
+              dataset: system.load
+              type: metrics
+            period: 10s
+            metricsets:
+              - load
+          - data_stream:
+              dataset: system.memory
+              type: metrics
+            period: 10s
+            metricsets:
+              - memory
+          - data_stream:
+              dataset: system.network
+              type: metrics
+            period: 10s
+            network.interfaces: null
+            metricsets:
+              - network
+          - data_stream:
+              dataset: system.process
+              type: metrics
+            process.include_top_n.by_memory: 5
+            period: 10s
+            processes:
+              - .*
+            process.include_top_n.by_cpu: 5
+            process.cgroups.enabled: false
+            process.cmdline.cache.enabled: true
+            metricsets:
+              - process
+            process.include_cpu_ticks: false
+            system.hostfs: /hostfs
+          - data_stream:
+              dataset: system.process_summary
+              type: metrics
+            period: 10s
+            metricsets:
+              - process_summary
+            system.hostfs: /hostfs
+          - data_stream:
+              dataset: system.socket_summary
+              type: metrics
+            period: 10s
+            metricsets:
+              - socket_summary
+            system.hostfs: /hostfs
+      - name: kubernetes-node-metrics
+        type: kubernetes/metrics
+        use_output: default
+        meta:
+          package:
+            name: kubernetes
+            version: 0.2.8
+        data_stream:
+          namespace: default
+        streams:
+          - data_stream:
+              dataset: kubernetes.controllermanager
+              type: metrics
+            metricsets:
+              - controllermanager
+            hosts:
+              - '${kubernetes.pod.ip}:10252'
+            period: 10s
+            condition: ${kubernetes.pod.labels.component} == 'kube-controller-manager'
+          - data_stream:
+              dataset: kubernetes.scheduler
+              type: metrics
+            metricsets:
+              - scheduler
+            hosts:
+              - '${kubernetes.pod.ip}:10251'
+            period: 10s
+            condition: ${kubernetes.pod.labels.component} == 'kube-scheduler'
+          - data_stream:
+              dataset: kubernetes.proxy
+              type: metrics
+            metricsets:
+              - proxy
+            hosts:
+              - 'localhost:10249'
+            period: 10s
+          - data_stream:
+              dataset: kubernetes.container
+              type: metrics
+            metricsets:
+              - container
+            add_metadata: true
+            bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+            hosts:
+              - 'https://${env.NODE_NAME}:10250'
+            period: 10s
+            ssl.verification_mode: none
+          - data_stream:
+              dataset: kubernetes.node
+              type: metrics
+            metricsets:
+              - node
+            add_metadata: true
+            bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+            hosts:
+              - 'https://${env.NODE_NAME}:10250'
+            period: 10s
+            ssl.verification_mode: none
+          - data_stream:
+              dataset: kubernetes.pod
+              type: metrics
+            metricsets:
+              - pod
+            add_metadata: true
+            bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+            hosts:
+              - 'https://${env.NODE_NAME}:10250'
+            period: 10s
+            ssl.verification_mode: none
+          - data_stream:
+              dataset: kubernetes.system
+              type: metrics
+            metricsets:
+              - system
+            add_metadata: true
+            bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+            hosts:
+              - 'https://${env.NODE_NAME}:10250'
+            period: 10s
+            ssl.verification_mode: none
+          - data_stream:
+              dataset: kubernetes.volume
+              type: metrics
+            metricsets:
+              - volume
+            add_metadata: true
+            bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
+            hosts:
+              - 'https://${env.NODE_NAME}:10250'
+            period: 10s
+            ssl.verification_mode: none
diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml
@@ -0,0 +1,70 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: elastic-agent
+  namespace: kube-system
+  labels:
+    app: elastic-agent
+spec:
+  selector:
+    matchLabels:
+      app: elastic-agent
+  template:
+    metadata:
+      labels:
+        app: elastic-agent
+    spec:
+      tolerations:
+        - key: node-role.kubernetes.io/master
+          effect: NoSchedule
+      serviceAccountName: elastic-agent
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      containers:
+        - name: elastic-agent
+          image: docker.elastic.co/beats/elastic-agent:%VERSION%
+          args: [
+            "-c", "/etc/agent.yml",
+            "-e",
+          ]
+          env:
+            - name: ES_USERNAME
+              value: "elastic"
+            - name: ES_PASSWORD
+              value: ""
+            - name: ES_HOST
+              value: ""
+            - name: NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          securityContext:
+            runAsUser: 0
+          resources:
+            limits:
+              memory: 200Mi
+            requests:
+              cpu: 100m
+              memory: 100Mi
+          volumeMounts:
+            - name: datastreams
+              mountPath: /etc/agent.yml
+              readOnly: true
+              subPath: agent.yml
+            - name: proc
+              mountPath: /hostfs/proc
+              readOnly: true
+            - name: cgroup
+              mountPath: /hostfs/sys/fs/cgroup
+              readOnly: true
+      volumes:
+        - name: datastreams
+          configMap:
+            defaultMode: 0640
+            name: agent-node-datastreams
+        - name: proc
+          hostPath:
+            path: /proc
+        - name: cgroup
+          hostPath:
+            path: /sys/fs/cgroup