From fc9e7fe61fea44d323e1635f840d6e19f442f8b2 Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 9 Feb 2021 18:01:30 +0200 Subject: [PATCH 01/11] Add agent standalone manifests for system module Signed-off-by: chrismark --- .../elastic-agent-standalone-kubernetes.yml | 270 ++++++++++++++++++ 1 file changed, 270 insertions(+) create mode 100644 deploy/kubernetes/elastic-agent-standalone-kubernetes.yml diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml new file mode 100644 index 00000000000..865646692a1 --- /dev/null +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml @@ -0,0 +1,270 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: elastic-agent + namespace: kube-system + labels: + app: elastic-agent +spec: + selector: + matchLabels: + app: elastic-agent + template: + metadata: + labels: + app: elastic-agent + spec: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + serviceAccountName: elastic-agent + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: elastic-agent + image: docker.elastic.co/beats/elastic-agent:7.12.0-SNAPSHOT + args: [ + "-c", "/etc/agent.yml", + "-e", "-d", "*", + ] + env: + - name: ES_USERNAME + value: "elastic" + - name: ES_PASSWORD + value: "changeme" + - name: ES_HOST + value: "http://xxxxxx:9200/" + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + runAsUser: 0 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: datastreams + mountPath: /etc/agent.yml + readOnly: true + subPath: agent.yml + - name: proc + mountPath: /hostfs/proc + readOnly: true + - name: cgroup + mountPath: /hostfs/sys/fs/cgroup + readOnly: true + volumes: + - name: datastreams + configMap: + defaultMode: 0640 + name: agent-node-datastreams + - name: proc + hostPath: + path: /proc + - name: cgroup + hostPath: + path: /sys/fs/cgroup +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: agent-node-datastreams + namespace: kube-system + labels: + k8s-app: elastic-agent +data: + agent.yml: |- + id: ef9cc740-5bf0-11eb-8b51-39775155c3f5 + revision: 2 + outputs: + default: + type: elasticsearch + hosts: + - >- + ${ES_HOST} + username: ${ES_USERNAME} + password: ${ES_PASSWORD} + agent: + monitoring: + enabled: true + use_output: default + logs: true + metrics: true + inputs: + - id: 1f6112ef-9847-4294-aba4-0dc9e7240551 + name: system-3 + revision: 2 + type: system/metrics + use_output: default + meta: + package: + name: system + version: 0.10.9 + data_stream: + namespace: default + streams: + - id: system/metrics-system.core-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.core + type: metrics + metricsets: + - core + core.metrics: + - percentages + - id: system/metrics-system.cpu-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.cpu + type: metrics + period: 10s + cpu.metrics: + - percentages + - normalized_percentages + metricsets: + - cpu + - id: system/metrics-system.diskio-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.diskio + type: metrics + period: 10s + diskio.include_devices: null + metricsets: + - diskio + - id: system/metrics-system.filesystem-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.filesystem + type: metrics + period: 1m + metricsets: + - filesystem + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/) + - id: system/metrics-system.fsstat-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.fsstat + type: metrics + period: 1m + metricsets: + - fsstat + processors: + - drop_event.when.regexp: + system.fsstat.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/) + - id: system/metrics-system.load-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.load + type: metrics + period: 10s + metricsets: + - load + - id: system/metrics-system.memory-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.memory + type: metrics + period: 10s + metricsets: + - memory + - id: system/metrics-system.network-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.network + type: metrics + period: 10s + network.interfaces: null + metricsets: + - network + - id: system/metrics-system.process-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.process + type: metrics + process.include_top_n.by_memory: 5 + period: 10s + processes: + - .* + process.include_top_n.by_cpu: 5 + process.cgroups.enabled: false + process.cmdline.cache.enabled: true + metricsets: + - process + process.include_cpu_ticks: false + system.hostfs: /hostfs + - id: >- + system/metrics-system.process_summary-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.process_summary + type: metrics + period: 10s + metricsets: + - process_summary + system.hostfs: /hostfs + - id: >- + system/metrics-system.socket_summary-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.socket_summary + type: metrics + period: 10s + metricsets: + - socket_summary + system.hostfs: /hostfs +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: elastic-agent +subjects: + - kind: ServiceAccount + name: elastic-agent + namespace: kube-system +roleRef: + kind: ClusterRole + name: elastic-agent + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: elastic-agent + labels: + k8s-app: elastic-agent +rules: + - apiGroups: [""] + resources: + - nodes + - namespaces + - events + - pods + - secrets + verbs: ["get", "list", "watch"] + - apiGroups: ["extensions"] + resources: + - replicasets + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: + - statefulsets + - deployments + - replicasets + verbs: ["get", "list", "watch"] + - apiGroups: + - "" + resources: + - nodes/stats + verbs: + - get + # required for apiserver + - nonResourceURLs: + - "/metrics" + verbs: + - get +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: elastic-agent + namespace: kube-system + labels: + k8s-app: elastic-agent +--- From 3aec0b320fc4e0ac8858ac9d9ddd2bac532d2eb9 Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 16 Feb 2021 14:56:20 +0200 Subject: [PATCH 02/11] Rename file Signed-off-by: chrismark --- ...ernetes.yml => elastic-agent-standalone-kubernetes-system.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename deploy/kubernetes/{elastic-agent-standalone-kubernetes.yml => elastic-agent-standalone-kubernetes-system.yml} (100%) diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml b/deploy/kubernetes/elastic-agent-standalone-kubernetes-system.yml similarity index 100% rename from deploy/kubernetes/elastic-agent-standalone-kubernetes.yml rename to deploy/kubernetes/elastic-agent-standalone-kubernetes-system.yml From 5fdd0994a3aa4557b1edd3b9d2c730aa171e5799 Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 16 Feb 2021 16:08:49 +0200 Subject: [PATCH 03/11] rebase on top of k8s module Signed-off-by: chrismark --- ...tic-agent-standalone-kubernetes-system.yml | 270 ------------------ .../elastic-agent-standalone-kubernetes.yml | 125 ++++++++ 2 files changed, 125 insertions(+), 270 deletions(-) delete mode 100644 deploy/kubernetes/elastic-agent-standalone-kubernetes-system.yml diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes-system.yml b/deploy/kubernetes/elastic-agent-standalone-kubernetes-system.yml deleted file mode 100644 index 865646692a1..00000000000 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes-system.yml +++ /dev/null @@ -1,270 +0,0 @@ ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: elastic-agent - namespace: kube-system - labels: - app: elastic-agent -spec: - selector: - matchLabels: - app: elastic-agent - template: - metadata: - labels: - app: elastic-agent - spec: - tolerations: - - key: node-role.kubernetes.io/master - effect: NoSchedule - serviceAccountName: elastic-agent - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: elastic-agent - image: docker.elastic.co/beats/elastic-agent:7.12.0-SNAPSHOT - args: [ - "-c", "/etc/agent.yml", - "-e", "-d", "*", - ] - env: - - name: ES_USERNAME - value: "elastic" - - name: ES_PASSWORD - value: "changeme" - - name: ES_HOST - value: "http://xxxxxx:9200/" - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - securityContext: - runAsUser: 0 - resources: - limits: - memory: 200Mi - requests: - cpu: 100m - memory: 100Mi - volumeMounts: - - name: datastreams - mountPath: /etc/agent.yml - readOnly: true - subPath: agent.yml - - name: proc - mountPath: /hostfs/proc - readOnly: true - - name: cgroup - mountPath: /hostfs/sys/fs/cgroup - readOnly: true - volumes: - - name: datastreams - configMap: - defaultMode: 0640 - name: agent-node-datastreams - - name: proc - hostPath: - path: /proc - - name: cgroup - hostPath: - path: /sys/fs/cgroup ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: agent-node-datastreams - namespace: kube-system - labels: - k8s-app: elastic-agent -data: - agent.yml: |- - id: ef9cc740-5bf0-11eb-8b51-39775155c3f5 - revision: 2 - outputs: - default: - type: elasticsearch - hosts: - - >- - ${ES_HOST} - username: ${ES_USERNAME} - password: ${ES_PASSWORD} - agent: - monitoring: - enabled: true - use_output: default - logs: true - metrics: true - inputs: - - id: 1f6112ef-9847-4294-aba4-0dc9e7240551 - name: system-3 - revision: 2 - type: system/metrics - use_output: default - meta: - package: - name: system - version: 0.10.9 - data_stream: - namespace: default - streams: - - id: system/metrics-system.core-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: - dataset: system.core - type: metrics - metricsets: - - core - core.metrics: - - percentages - - id: system/metrics-system.cpu-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: - dataset: system.cpu - type: metrics - period: 10s - cpu.metrics: - - percentages - - normalized_percentages - metricsets: - - cpu - - id: system/metrics-system.diskio-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: - dataset: system.diskio - type: metrics - period: 10s - diskio.include_devices: null - metricsets: - - diskio - - id: system/metrics-system.filesystem-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: - dataset: system.filesystem - type: metrics - period: 1m - metricsets: - - filesystem - processors: - - drop_event.when.regexp: - system.filesystem.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/) - - id: system/metrics-system.fsstat-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: - dataset: system.fsstat - type: metrics - period: 1m - metricsets: - - fsstat - processors: - - drop_event.when.regexp: - system.fsstat.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/) - - id: system/metrics-system.load-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: - dataset: system.load - type: metrics - period: 10s - metricsets: - - load - - id: system/metrics-system.memory-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: - dataset: system.memory - type: metrics - period: 10s - metricsets: - - memory - - id: system/metrics-system.network-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: - dataset: system.network - type: metrics - period: 10s - network.interfaces: null - metricsets: - - network - - id: system/metrics-system.process-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: - dataset: system.process - type: metrics - process.include_top_n.by_memory: 5 - period: 10s - processes: - - .* - process.include_top_n.by_cpu: 5 - process.cgroups.enabled: false - process.cmdline.cache.enabled: true - metricsets: - - process - process.include_cpu_ticks: false - system.hostfs: /hostfs - - id: >- - system/metrics-system.process_summary-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: - dataset: system.process_summary - type: metrics - period: 10s - metricsets: - - process_summary - system.hostfs: /hostfs - - id: >- - system/metrics-system.socket_summary-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: - dataset: system.socket_summary - type: metrics - period: 10s - metricsets: - - socket_summary - system.hostfs: /hostfs ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: elastic-agent -subjects: - - kind: ServiceAccount - name: elastic-agent - namespace: kube-system -roleRef: - kind: ClusterRole - name: elastic-agent - apiGroup: rbac.authorization.k8s.io ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: elastic-agent - labels: - k8s-app: elastic-agent -rules: - - apiGroups: [""] - resources: - - nodes - - namespaces - - events - - pods - - secrets - verbs: ["get", "list", "watch"] - - apiGroups: ["extensions"] - resources: - - replicasets - verbs: ["get", "list", "watch"] - - apiGroups: ["apps"] - resources: - - statefulsets - - deployments - - replicasets - verbs: ["get", "list", "watch"] - - apiGroups: - - "" - resources: - - nodes/stats - verbs: - - get - # required for apiserver - - nonResourceURLs: - - "/metrics" - verbs: - - get ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: elastic-agent - namespace: kube-system - labels: - k8s-app: elastic-agent ---- diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml index f99281b6889..0d08fe88604 100644 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml @@ -52,11 +52,23 @@ spec: mountPath: /etc/agent.yml readOnly: true subPath: agent.yml + - name: proc + mountPath: /hostfs/proc + readOnly: true + - name: cgroup + mountPath: /hostfs/sys/fs/cgroup + readOnly: true volumes: - name: datastreams configMap: defaultMode: 0640 name: agent-node-datastreams + - name: proc + hostPath: + path: /proc + - name: cgroup + hostPath: + path: /sys/fs/cgroup --- apiVersion: v1 kind: ConfigMap @@ -87,6 +99,119 @@ data: node: ${NODE_NAME} scope: node inputs: + - id: 1f6112ef-9847-4294-aba4-0dc9e7240551 + name: system-3 + revision: 2 + type: system/metrics + use_output: default + meta: + package: + name: system + version: 0.10.9 + data_stream: + namespace: default + streams: + - id: system/metrics-system.core-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.core + type: metrics + metricsets: + - core + core.metrics: + - percentages + - id: system/metrics-system.cpu-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.cpu + type: metrics + period: 10s + cpu.metrics: + - percentages + - normalized_percentages + metricsets: + - cpu + - id: system/metrics-system.diskio-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.diskio + type: metrics + period: 10s + diskio.include_devices: null + metricsets: + - diskio + - id: system/metrics-system.filesystem-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.filesystem + type: metrics + period: 1m + metricsets: + - filesystem + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/) + - id: system/metrics-system.fsstat-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.fsstat + type: metrics + period: 1m + metricsets: + - fsstat + processors: + - drop_event.when.regexp: + system.fsstat.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/) + - id: system/metrics-system.load-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.load + type: metrics + period: 10s + metricsets: + - load + - id: system/metrics-system.memory-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.memory + type: metrics + period: 10s + metricsets: + - memory + - id: system/metrics-system.network-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.network + type: metrics + period: 10s + network.interfaces: null + metricsets: + - network + - id: system/metrics-system.process-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.process + type: metrics + process.include_top_n.by_memory: 5 + period: 10s + processes: + - .* + process.include_top_n.by_cpu: 5 + process.cgroups.enabled: false + process.cmdline.cache.enabled: true + metricsets: + - process + process.include_cpu_ticks: false + system.hostfs: /hostfs + - id: >- + system/metrics-system.process_summary-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.process_summary + type: metrics + period: 10s + metricsets: + - process_summary + system.hostfs: /hostfs + - id: >- + system/metrics-system.socket_summary-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.socket_summary + type: metrics + period: 10s + metricsets: + - socket_summary + system.hostfs: /hostfs - id: 934ef8aa-ed19-405b-8160-ebf62e3d32f8 name: kubernetes-node-metrics revision: 1 From 34da3b28669637889f06f2c7946bb4c30f800264 Mon Sep 17 00:00:00 2001 From: chrismark Date: Tue, 16 Feb 2021 16:50:24 +0200 Subject: [PATCH 04/11] Split manifests as per pattern Signed-off-by: chrismark --- .../elastic-agent-standalone-kubernetes.yml | 4 +- ...-agent-standalone-daemonset-configmap.yaml | 193 ++++++++++++++++++ .../elastic-agent-standalone-daemonset.yaml | 70 +++++++ ...agent-standalone-deployment-configmap.yaml | 193 ++++++++++++++++++ .../elastic-agent-standalone-deployment.yaml | 58 ++++++ ...elastic-agent-standalone-role-binding.yaml | 12 ++ .../elastic-agent-standalone-role.yaml | 36 ++++ ...stic-agent-standalone-service-account.yaml | 7 + 8 files changed, 571 insertions(+), 2 deletions(-) create mode 100644 deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml create mode 100644 deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml create mode 100644 deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment-configmap.yaml create mode 100644 deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment.yaml create mode 100644 deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role-binding.yaml create mode 100644 deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml create mode 100644 deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-service-account.yaml diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml index 0d08fe88604..a49e92407c3 100644 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml @@ -23,7 +23,7 @@ spec: dnsPolicy: ClusterFirstWithHostNet containers: - name: elastic-agent - image: docker.elastic.co/beats/elastic-agent:7.12.0-SNAPSHOT + image: docker.elastic.co/beats/elastic-agent:8.0.0-SNAPSHOT args: [ "-c", "/etc/agent.yml", "-e", "-d", "composable.providers.kubernetes", @@ -340,7 +340,7 @@ spec: serviceAccountName: elastic-agent containers: - name: elastic-agent - image: docker.elastic.co/beats/elastic-agent:7.12.0-SNAPSHOT + image: docker.elastic.co/beats/elastic-agent:8.0.0-SNAPSHOT args: [ "-c", "/etc/agent.yml", "-e", "-d", "composable.providers.kubernetes", diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml new file mode 100644 index 00000000000..47ab58841b2 --- /dev/null +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml @@ -0,0 +1,193 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: agent-deployment-datastreams + namespace: kube-system + labels: + k8s-app: elastic-agent +data: + # This part requires `kube-state-metrics` up and running under `kube-system` namespace + agent.yml: |- + id: ef9cc740-5bf0-11eb-8b51-39775155c3f5 + revision: 2 + outputs: + default: + type: elasticsearch + hosts: + - >- + ${ES_HOST} + username: ${ES_USERNAME} + password: ${ES_PASSWORD} + agent: + monitoring: + enabled: true + use_output: default + logs: true + metrics: true + inputs: + - id: 934ef8aa-ed19-405b-8160-ebf62e3d32f9 + name: kubernetes-cluster-metrics + revision: 1 + type: kubernetes/metrics + use_output: default + meta: + package: + name: kubernetes + version: 0.2.8 + data_stream: + namespace: default + streams: + - id: >- + kubernetes/metrics-kubernetes.apiserver-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.apiserver + type: metrics + metricsets: + - apiserver + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + hosts: + - 'https://${env.KUBERNETES_SERVICE_HOST}:${env.KUBERNETES_SERVICE_PORT}' + period: 30s + ssl.certificate_authorities: + - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - id: >- + kubernetes/metrics-kubernetes.event-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.event + type: metrics + metricsets: + - event + period: 10s + add_metadata: true + - id: >- + kubernetes/metrics-kubernetes.state_container-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_container + type: metrics + metricsets: + - state_container + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_cronjob-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_cronjob + type: metrics + metricsets: + - state_cronjob + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_deployment-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_deployment + type: metrics + metricsets: + - state_deployment + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_node-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_node + type: metrics + metricsets: + - state_node + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_persistentvolume-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_persistentvolume + type: metrics + metricsets: + - state_persistentvolume + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_persistentvolumeclaim-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_persistentvolumeclaim + type: metrics + metricsets: + - state_persistentvolumeclaim + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_pod-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_pod + type: metrics + metricsets: + - state_pod + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_replicaset-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_replicaset + type: metrics + metricsets: + - state_replicaset + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_resourcequota-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_resourcequota + type: metrics + metricsets: + - state_resourcequota + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_service-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_service + type: metrics + metricsets: + - state_service + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_statefulset-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_statefulset + type: metrics + metricsets: + - state_statefulset + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_storageclass-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_storageclass + type: metrics + metricsets: + - state_storageclass + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml new file mode 100644 index 00000000000..6e2d33799ec --- /dev/null +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml @@ -0,0 +1,70 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: elastic-agent + namespace: kube-system + labels: + app: elastic-agent +spec: + selector: + matchLabels: + app: elastic-agent + template: + metadata: + labels: + app: elastic-agent + spec: + tolerations: + - key: node-role.kubernetes.io/master + effect: NoSchedule + serviceAccountName: elastic-agent + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + containers: + - name: elastic-agent + image: docker.elastic.co/beats/elastic-agent:8.0.0-SNAPSHOT + args: [ + "-c", "/etc/agent.yml", + "-e", "-d", "composable.providers.kubernetes", + ] + env: + - name: ES_USERNAME + value: "elastic" + - name: ES_PASSWORD + value: "" + - name: ES_HOST + value: "" + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + runAsUser: 0 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: datastreams + mountPath: /etc/agent.yml + readOnly: true + subPath: agent.yml + - name: proc + mountPath: /hostfs/proc + readOnly: true + - name: cgroup + mountPath: /hostfs/sys/fs/cgroup + readOnly: true + volumes: + - name: datastreams + configMap: + defaultMode: 0640 + name: agent-node-datastreams + - name: proc + hostPath: + path: /proc + - name: cgroup + hostPath: + path: /sys/fs/cgroup diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment-configmap.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment-configmap.yaml new file mode 100644 index 00000000000..47ab58841b2 --- /dev/null +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment-configmap.yaml @@ -0,0 +1,193 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: agent-deployment-datastreams + namespace: kube-system + labels: + k8s-app: elastic-agent +data: + # This part requires `kube-state-metrics` up and running under `kube-system` namespace + agent.yml: |- + id: ef9cc740-5bf0-11eb-8b51-39775155c3f5 + revision: 2 + outputs: + default: + type: elasticsearch + hosts: + - >- + ${ES_HOST} + username: ${ES_USERNAME} + password: ${ES_PASSWORD} + agent: + monitoring: + enabled: true + use_output: default + logs: true + metrics: true + inputs: + - id: 934ef8aa-ed19-405b-8160-ebf62e3d32f9 + name: kubernetes-cluster-metrics + revision: 1 + type: kubernetes/metrics + use_output: default + meta: + package: + name: kubernetes + version: 0.2.8 + data_stream: + namespace: default + streams: + - id: >- + kubernetes/metrics-kubernetes.apiserver-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.apiserver + type: metrics + metricsets: + - apiserver + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + hosts: + - 'https://${env.KUBERNETES_SERVICE_HOST}:${env.KUBERNETES_SERVICE_PORT}' + period: 30s + ssl.certificate_authorities: + - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + - id: >- + kubernetes/metrics-kubernetes.event-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.event + type: metrics + metricsets: + - event + period: 10s + add_metadata: true + - id: >- + kubernetes/metrics-kubernetes.state_container-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_container + type: metrics + metricsets: + - state_container + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_cronjob-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_cronjob + type: metrics + metricsets: + - state_cronjob + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_deployment-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_deployment + type: metrics + metricsets: + - state_deployment + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_node-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_node + type: metrics + metricsets: + - state_node + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_persistentvolume-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_persistentvolume + type: metrics + metricsets: + - state_persistentvolume + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_persistentvolumeclaim-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_persistentvolumeclaim + type: metrics + metricsets: + - state_persistentvolumeclaim + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_pod-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_pod + type: metrics + metricsets: + - state_pod + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_replicaset-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_replicaset + type: metrics + metricsets: + - state_replicaset + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_resourcequota-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_resourcequota + type: metrics + metricsets: + - state_resourcequota + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_service-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_service + type: metrics + metricsets: + - state_service + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_statefulset-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_statefulset + type: metrics + metricsets: + - state_statefulset + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s + - id: >- + kubernetes/metrics-kubernetes.state_storageclass-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + data_stream: + dataset: kubernetes.state_storageclass + type: metrics + metricsets: + - state_storageclass + add_metadata: true + hosts: + - 'kube-state-metrics:8080' + period: 10s diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment.yaml new file mode 100644 index 00000000000..dfa8474a09f --- /dev/null +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment.yaml @@ -0,0 +1,58 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elastic-agent + namespace: kube-system + labels: + app: elastic-agent +spec: + selector: + matchLabels: + app: elastic-agent + template: + metadata: + labels: + app: elastic-agent + spec: + serviceAccountName: elastic-agent + containers: + - name: elastic-agent + image: docker.elastic.co/beats/elastic-agent:8.0.0-SNAPSHOT + args: [ + "-c", "/etc/agent.yml", + "-e", "-d", "composable.providers.kubernetes", + ] + env: + - name: ES_USERNAME + value: "elastic" + - name: ES_PASSWORD + value: "" + - name: ES_HOST + value: "" + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # this is needed because we cannot use hostNetwork + - name: HOSTNAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + securityContext: + runAsUser: 0 + resources: + limits: + memory: 200Mi + requests: + cpu: 100m + memory: 100Mi + volumeMounts: + - name: datastreams + mountPath: /etc/agent.yml + readOnly: true + subPath: agent.yml + volumes: + - name: datastreams + configMap: + defaultMode: 0640 + name: agent-deployment-datastreams diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role-binding.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role-binding.yaml new file mode 100644 index 00000000000..b352b2901d0 --- /dev/null +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role-binding.yaml @@ -0,0 +1,12 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: elastic-agent +subjects: + - kind: ServiceAccount + name: elastic-agent + namespace: kube-system +roleRef: + kind: ClusterRole + name: elastic-agent + apiGroup: rbac.authorization.k8s.io diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml new file mode 100644 index 00000000000..510ea6eeb10 --- /dev/null +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml @@ -0,0 +1,36 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: elastic-agent + labels: + k8s-app: elastic-agent +rules: + - apiGroups: [""] + resources: + - nodes + - namespaces + - events + - pods + - secrets + verbs: ["get", "list", "watch"] + - apiGroups: ["extensions"] + resources: + - replicasets + verbs: ["get", "list", "watch"] + - apiGroups: ["apps"] + resources: + - statefulsets + - deployments + - replicasets + verbs: ["get", "list", "watch"] + - apiGroups: + - "" + resources: + - nodes/stats + verbs: + - get + # required for apiserver + - nonResourceURLs: + - "/metrics" + verbs: + - get diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-service-account.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-service-account.yaml new file mode 100644 index 00000000000..43372b547d0 --- /dev/null +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-service-account.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: elastic-agent + namespace: kube-system + labels: + k8s-app: elastic-agent From e414c5972489eaba90279ca854de84ad24f1a706 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 17 Feb 2021 10:17:54 +0200 Subject: [PATCH 05/11] Add Pod's logs collection Signed-off-by: chrismark --- .../elastic-agent-standalone-kubernetes.yml | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml index a49e92407c3..b2e64451057 100644 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml @@ -58,6 +58,12 @@ spec: - name: cgroup mountPath: /hostfs/sys/fs/cgroup readOnly: true + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: varlog + mountPath: /var/log + readOnly: true volumes: - name: datastreams configMap: @@ -69,6 +75,12 @@ spec: - name: cgroup hostPath: path: /sys/fs/cgroup + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: varlog + hostPath: + path: /var/log --- apiVersion: v1 kind: ConfigMap @@ -99,6 +111,24 @@ data: node: ${NODE_NAME} scope: node inputs: + - id: 4ae27079-6cd4-4ab7-a459-abbae74ffc44 + name: log-1 + revision: 1 + type: logfile + use_output: default + meta: + package: + name: log + version: 0.4.6 + data_stream: + namespace: default + streams: + - id: logfile-log.log-4ae27079-6cd4-4ab7-a459-abbae74ffc44 + data_stream: + dataset: generic + symlinks: true + paths: + - /var/log/containers/*${kubernetes.container.id}.log - id: 1f6112ef-9847-4294-aba4-0dc9e7240551 name: system-3 revision: 2 From 757398781eabf9593b291891bfc675a665d10157 Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 17 Feb 2021 17:14:28 +0200 Subject: [PATCH 06/11] Update configmap subfile Signed-off-by: chrismark --- ...-agent-standalone-daemonset-configmap.yaml | 247 ++++++++++++------ 1 file changed, 161 insertions(+), 86 deletions(-) diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml index 47ab58841b2..60783ce3e32 100644 --- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml @@ -1,12 +1,11 @@ apiVersion: v1 kind: ConfigMap metadata: - name: agent-deployment-datastreams + name: agent-node-datastreams namespace: kube-system labels: k8s-app: elastic-agent data: - # This part requires `kube-state-metrics` up and running under `kube-system` namespace agent.yml: |- id: ef9cc740-5bf0-11eb-8b51-39775155c3f5 revision: 2 @@ -24,170 +23,246 @@ data: use_output: default logs: true metrics: true + providers.kubernetes: + node: ${NODE_NAME} + scope: node inputs: - - id: 934ef8aa-ed19-405b-8160-ebf62e3d32f9 - name: kubernetes-cluster-metrics + - id: 4ae27079-6cd4-4ab7-a459-abbae74ffc44 + name: log-1 revision: 1 - type: kubernetes/metrics + type: logfile use_output: default meta: package: - name: kubernetes - version: 0.2.8 + name: log + version: 0.4.6 data_stream: namespace: default streams: - - id: >- - kubernetes/metrics-kubernetes.apiserver-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + - id: logfile-log.log-4ae27079-6cd4-4ab7-a459-abbae74ffc44 data_stream: - dataset: kubernetes.apiserver + dataset: generic + symlinks: true + paths: + - /var/log/containers/*${kubernetes.container.id}.log + - id: 1f6112ef-9847-4294-aba4-0dc9e7240551 + name: system-3 + revision: 2 + type: system/metrics + use_output: default + meta: + package: + name: system + version: 0.10.9 + data_stream: + namespace: default + streams: + - id: system/metrics-system.core-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.core type: metrics metricsets: - - apiserver - bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token - hosts: - - 'https://${env.KUBERNETES_SERVICE_HOST}:${env.KUBERNETES_SERVICE_PORT}' - period: 30s - ssl.certificate_authorities: - - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - - id: >- - kubernetes/metrics-kubernetes.event-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + - core + core.metrics: + - percentages + - id: system/metrics-system.cpu-1f6112ef-9847-4294-aba4-0dc9e7240551 data_stream: - dataset: kubernetes.event + dataset: system.cpu type: metrics + period: 10s + cpu.metrics: + - percentages + - normalized_percentages metricsets: - - event + - cpu + - id: system/metrics-system.diskio-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.diskio + type: metrics period: 10s - add_metadata: true - - id: >- - kubernetes/metrics-kubernetes.state_container-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + diskio.include_devices: null + metricsets: + - diskio + - id: system/metrics-system.filesystem-1f6112ef-9847-4294-aba4-0dc9e7240551 data_stream: - dataset: kubernetes.state_container + dataset: system.filesystem type: metrics + period: 1m metricsets: - - state_container - add_metadata: true - hosts: - - 'kube-state-metrics:8080' - period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_cronjob-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + - filesystem + processors: + - drop_event.when.regexp: + system.filesystem.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/) + - id: system/metrics-system.fsstat-1f6112ef-9847-4294-aba4-0dc9e7240551 data_stream: - dataset: kubernetes.state_cronjob + dataset: system.fsstat type: metrics + period: 1m metricsets: - - state_cronjob - add_metadata: true - hosts: - - 'kube-state-metrics:8080' + - fsstat + processors: + - drop_event.when.regexp: + system.fsstat.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/) + - id: system/metrics-system.load-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.load + type: metrics period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_deployment-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + metricsets: + - load + - id: system/metrics-system.memory-1f6112ef-9847-4294-aba4-0dc9e7240551 data_stream: - dataset: kubernetes.state_deployment + dataset: system.memory type: metrics + period: 10s metricsets: - - state_deployment - add_metadata: true - hosts: - - 'kube-state-metrics:8080' + - memory + - id: system/metrics-system.network-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.network + type: metrics period: 10s + network.interfaces: null + metricsets: + - network + - id: system/metrics-system.process-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.process + type: metrics + process.include_top_n.by_memory: 5 + period: 10s + processes: + - .* + process.include_top_n.by_cpu: 5 + process.cgroups.enabled: false + process.cmdline.cache.enabled: true + metricsets: + - process + process.include_cpu_ticks: false + system.hostfs: /hostfs - id: >- - kubernetes/metrics-kubernetes.state_node-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + system/metrics-system.process_summary-1f6112ef-9847-4294-aba4-0dc9e7240551 data_stream: - dataset: kubernetes.state_node + dataset: system.process_summary type: metrics + period: 10s metricsets: - - state_node - add_metadata: true - hosts: - - 'kube-state-metrics:8080' + - process_summary + system.hostfs: /hostfs + - id: >- + system/metrics-system.socket_summary-1f6112ef-9847-4294-aba4-0dc9e7240551 + data_stream: + dataset: system.socket_summary + type: metrics period: 10s + metricsets: + - socket_summary + system.hostfs: /hostfs + - id: 934ef8aa-ed19-405b-8160-ebf62e3d32f8 + name: kubernetes-node-metrics + revision: 1 + type: kubernetes/metrics + use_output: default + meta: + package: + name: kubernetes + version: 0.2.8 + data_stream: + namespace: default + streams: - id: >- - kubernetes/metrics-kubernetes.state_persistentvolume-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + kubernetes/metrics-kubernetes.controllermanager-3d50c483-2327-40e7-b3e5-d877d4763fe1 data_stream: - dataset: kubernetes.state_persistentvolume + dataset: kubernetes.controllermanager type: metrics metricsets: - - state_persistentvolume - add_metadata: true + - controllermanager hosts: - - 'kube-state-metrics:8080' + - '${kubernetes.pod.ip}:10252' period: 10s + condition: ${kubernetes.pod.labels.component} == 'kube-controller-manager' - id: >- - kubernetes/metrics-kubernetes.state_persistentvolumeclaim-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + kubernetes/metrics-kubernetes.scheduler-3d50c483-2327-40e7-b3e5-d877d4763fe1 data_stream: - dataset: kubernetes.state_persistentvolumeclaim + dataset: kubernetes.scheduler type: metrics metricsets: - - state_persistentvolumeclaim - add_metadata: true + - scheduler hosts: - - 'kube-state-metrics:8080' + - '${kubernetes.pod.ip}:10251' period: 10s + condition: ${kubernetes.pod.labels.component} == 'kube-scheduler' - id: >- - kubernetes/metrics-kubernetes.state_pod-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + kubernetes/metrics-kubernetes.proxy-3d50c483-2327-40e7-b3e5-d877d4763fe1 data_stream: - dataset: kubernetes.state_pod + dataset: kubernetes.proxy type: metrics metricsets: - - state_pod - add_metadata: true + - proxy hosts: - - 'kube-state-metrics:8080' + - 'localhost:10249' period: 10s - id: >- - kubernetes/metrics-kubernetes.state_replicaset-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + kubernetes/metrics-kubernetes.container-934ef8aa-ed19-405b-8160-ebf62e3d32f8 data_stream: - dataset: kubernetes.state_replicaset + dataset: kubernetes.container type: metrics metricsets: - - state_replicaset + - container add_metadata: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token hosts: - - 'kube-state-metrics:8080' + - 'https://${env.NODE_NAME}:10250' period: 10s + ssl.verification_mode: none - id: >- - kubernetes/metrics-kubernetes.state_resourcequota-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + kubernetes/metrics-kubernetes.node-934ef8aa-ed19-405b-8160-ebf62e3d32f8 data_stream: - dataset: kubernetes.state_resourcequota + dataset: kubernetes.node type: metrics metricsets: - - state_resourcequota + - node add_metadata: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token hosts: - - 'kube-state-metrics:8080' + - 'https://${env.NODE_NAME}:10250' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_service-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + ssl.verification_mode: none + - id: kubernetes/metrics-kubernetes.pod-934ef8aa-ed19-405b-8160-ebf62e3d32f8 data_stream: - dataset: kubernetes.state_service + dataset: kubernetes.pod type: metrics metricsets: - - state_service + - pod add_metadata: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token hosts: - - 'kube-state-metrics:8080' + - 'https://${env.NODE_NAME}:10250' period: 10s + ssl.verification_mode: none - id: >- - kubernetes/metrics-kubernetes.state_statefulset-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + kubernetes/metrics-kubernetes.system-934ef8aa-ed19-405b-8160-ebf62e3d32f8 data_stream: - dataset: kubernetes.state_statefulset + dataset: kubernetes.system type: metrics metricsets: - - state_statefulset + - system add_metadata: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token hosts: - - 'kube-state-metrics:8080' + - 'https://${env.NODE_NAME}:10250' period: 10s + ssl.verification_mode: none - id: >- - kubernetes/metrics-kubernetes.state_storageclass-934ef8aa-ed19-405b-8160-ebf62e3d32f8 + kubernetes/metrics-kubernetes.volume-934ef8aa-ed19-405b-8160-ebf62e3d32f8 data_stream: - dataset: kubernetes.state_storageclass + dataset: kubernetes.volume type: metrics metricsets: - - state_storageclass + - volume add_metadata: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token hosts: - - 'kube-state-metrics:8080' + - 'https://${env.NODE_NAME}:10250' period: 10s + ssl.verification_mode: none From 97018db23fed0b76867eedfc5469a6fbe33e7c3e Mon Sep 17 00:00:00 2001 From: chrismark Date: Wed, 17 Feb 2021 18:15:38 +0200 Subject: [PATCH 07/11] Add more verobse logging Signed-off-by: chrismark --- deploy/kubernetes/elastic-agent-standalone-kubernetes.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml index b2e64451057..36bb5475b98 100644 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml @@ -26,7 +26,7 @@ spec: image: docker.elastic.co/beats/elastic-agent:8.0.0-SNAPSHOT args: [ "-c", "/etc/agent.yml", - "-e", "-d", "composable.providers.kubernetes", + "-e", "-d", "*", ] env: - name: ES_USERNAME @@ -373,7 +373,7 @@ spec: image: docker.elastic.co/beats/elastic-agent:8.0.0-SNAPSHOT args: [ "-c", "/etc/agent.yml", - "-e", "-d", "composable.providers.kubernetes", + "-e", "-d", "*", ] env: - name: ES_USERNAME From c5340e364c8b83045034b8565c26f6b4c855934a Mon Sep 17 00:00:00 2001 From: chrismark Date: Thu, 18 Feb 2021 10:42:10 +0200 Subject: [PATCH 08/11] Fix version placeholders Signed-off-by: chrismark --- deploy/kubernetes/elastic-agent-standalone-kubernetes.yml | 4 ++-- .../elastic-agent-standalone-daemonset.yaml | 2 +- .../elastic-agent-standalone-deployment.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml index 36bb5475b98..c2bf33851f5 100644 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml @@ -23,7 +23,7 @@ spec: dnsPolicy: ClusterFirstWithHostNet containers: - name: elastic-agent - image: docker.elastic.co/beats/elastic-agent:8.0.0-SNAPSHOT + image: docker.elastic.co/beats/elastic-agent:8.0.0 args: [ "-c", "/etc/agent.yml", "-e", "-d", "*", @@ -370,7 +370,7 @@ spec: serviceAccountName: elastic-agent containers: - name: elastic-agent - image: docker.elastic.co/beats/elastic-agent:8.0.0-SNAPSHOT + image: docker.elastic.co/beats/elastic-agent:8.0.0 args: [ "-c", "/etc/agent.yml", "-e", "-d", "*", diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml index 6e2d33799ec..93ae5cfe46f 100644 --- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml @@ -22,7 +22,7 @@ spec: dnsPolicy: ClusterFirstWithHostNet containers: - name: elastic-agent - image: docker.elastic.co/beats/elastic-agent:8.0.0-SNAPSHOT + image: docker.elastic.co/beats/elastic-agent:%VERSION% args: [ "-c", "/etc/agent.yml", "-e", "-d", "composable.providers.kubernetes", diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment.yaml index dfa8474a09f..03bb5c2535c 100644 --- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment.yaml +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment.yaml @@ -17,7 +17,7 @@ spec: serviceAccountName: elastic-agent containers: - name: elastic-agent - image: docker.elastic.co/beats/elastic-agent:8.0.0-SNAPSHOT + image: docker.elastic.co/beats/elastic-agent:%VERSION% args: [ "-c", "/etc/agent.yml", "-e", "-d", "composable.providers.kubernetes", From 75c6f1622b5973df8ce59be4cfe881c9f90c97b8 Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 19 Feb 2021 10:30:16 +0200 Subject: [PATCH 09/11] Remove ids & set log selector Signed-off-by: chrismark --- .../elastic-agent-standalone-kubernetes.yml | 149 +++++------------- ...-agent-standalone-daemonset-configmap.yaml | 83 +++------- ...agent-standalone-deployment-configmap.yaml | 62 ++------ 3 files changed, 78 insertions(+), 216 deletions(-) diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml index c2bf33851f5..4e243eb0ad8 100644 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml @@ -26,7 +26,7 @@ spec: image: docker.elastic.co/beats/elastic-agent:8.0.0 args: [ "-c", "/etc/agent.yml", - "-e", "-d", "*", + "-e", "-d", "composable.providers.kubernetes", ] env: - name: ES_USERNAME @@ -91,8 +91,6 @@ metadata: k8s-app: elastic-agent data: agent.yml: |- - id: ef9cc740-5bf0-11eb-8b51-39775155c3f5 - revision: 2 outputs: default: type: elasticsearch @@ -111,9 +109,7 @@ data: node: ${NODE_NAME} scope: node inputs: - - id: 4ae27079-6cd4-4ab7-a459-abbae74ffc44 - name: log-1 - revision: 1 + - name: log-1 type: logfile use_output: default meta: @@ -123,15 +119,12 @@ data: data_stream: namespace: default streams: - - id: logfile-log.log-4ae27079-6cd4-4ab7-a459-abbae74ffc44 - data_stream: + - data_stream: dataset: generic symlinks: true paths: - /var/log/containers/*${kubernetes.container.id}.log - - id: 1f6112ef-9847-4294-aba4-0dc9e7240551 - name: system-3 - revision: 2 + - name: system-3 type: system/metrics use_output: default meta: @@ -141,16 +134,14 @@ data: data_stream: namespace: default streams: - - id: system/metrics-system.core-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.core type: metrics metricsets: - core core.metrics: - percentages - - id: system/metrics-system.cpu-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.cpu type: metrics period: 10s @@ -159,16 +150,14 @@ data: - normalized_percentages metricsets: - cpu - - id: system/metrics-system.diskio-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.diskio type: metrics period: 10s diskio.include_devices: null metricsets: - diskio - - id: system/metrics-system.filesystem-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.filesystem type: metrics period: 1m @@ -177,8 +166,7 @@ data: processors: - drop_event.when.regexp: system.filesystem.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/) - - id: system/metrics-system.fsstat-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.fsstat type: metrics period: 1m @@ -187,30 +175,26 @@ data: processors: - drop_event.when.regexp: system.fsstat.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/) - - id: system/metrics-system.load-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.load type: metrics period: 10s metricsets: - load - - id: system/metrics-system.memory-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.memory type: metrics period: 10s metricsets: - memory - - id: system/metrics-system.network-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.network type: metrics period: 10s network.interfaces: null metricsets: - network - - id: system/metrics-system.process-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.process type: metrics process.include_top_n.by_memory: 5 @@ -224,27 +208,21 @@ data: - process process.include_cpu_ticks: false system.hostfs: /hostfs - - id: >- - system/metrics-system.process_summary-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.process_summary type: metrics period: 10s metricsets: - process_summary system.hostfs: /hostfs - - id: >- - system/metrics-system.socket_summary-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.socket_summary type: metrics period: 10s metricsets: - socket_summary system.hostfs: /hostfs - - id: 934ef8aa-ed19-405b-8160-ebf62e3d32f8 - name: kubernetes-node-metrics - revision: 1 + - name: kubernetes-node-metrics type: kubernetes/metrics use_output: default meta: @@ -254,9 +232,7 @@ data: data_stream: namespace: default streams: - - id: >- - kubernetes/metrics-kubernetes.controllermanager-3d50c483-2327-40e7-b3e5-d877d4763fe1 - data_stream: + - data_stream: dataset: kubernetes.controllermanager type: metrics metricsets: @@ -265,9 +241,7 @@ data: - '${kubernetes.pod.ip}:10252' period: 10s condition: ${kubernetes.pod.labels.component} == 'kube-controller-manager' - - id: >- - kubernetes/metrics-kubernetes.scheduler-3d50c483-2327-40e7-b3e5-d877d4763fe1 - data_stream: + - data_stream: dataset: kubernetes.scheduler type: metrics metricsets: @@ -276,9 +250,7 @@ data: - '${kubernetes.pod.ip}:10251' period: 10s condition: ${kubernetes.pod.labels.component} == 'kube-scheduler' - - id: >- - kubernetes/metrics-kubernetes.proxy-3d50c483-2327-40e7-b3e5-d877d4763fe1 - data_stream: + - data_stream: dataset: kubernetes.proxy type: metrics metricsets: @@ -286,9 +258,7 @@ data: hosts: - 'localhost:10249' period: 10s - - id: >- - kubernetes/metrics-kubernetes.container-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.container type: metrics metricsets: @@ -299,9 +269,7 @@ data: - 'https://${env.NODE_NAME}:10250' period: 10s ssl.verification_mode: none - - id: >- - kubernetes/metrics-kubernetes.node-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.node type: metrics metricsets: @@ -312,8 +280,7 @@ data: - 'https://${env.NODE_NAME}:10250' period: 10s ssl.verification_mode: none - - id: kubernetes/metrics-kubernetes.pod-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.pod type: metrics metricsets: @@ -324,9 +291,7 @@ data: - 'https://${env.NODE_NAME}:10250' period: 10s ssl.verification_mode: none - - id: >- - kubernetes/metrics-kubernetes.system-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.system type: metrics metricsets: @@ -337,9 +302,7 @@ data: - 'https://${env.NODE_NAME}:10250' period: 10s ssl.verification_mode: none - - id: >- - kubernetes/metrics-kubernetes.volume-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.volume type: metrics metricsets: @@ -373,7 +336,7 @@ spec: image: docker.elastic.co/beats/elastic-agent:8.0.0 args: [ "-c", "/etc/agent.yml", - "-e", "-d", "*", + "-e", "-d", "composable.providers.kubernetes", ] env: - name: ES_USERNAME @@ -420,8 +383,6 @@ metadata: data: # This part requires `kube-state-metrics` up and running under `kube-system` namespace agent.yml: |- - id: ef9cc740-5bf0-11eb-8b51-39775155c3f5 - revision: 2 outputs: default: type: elasticsearch @@ -437,9 +398,7 @@ data: logs: true metrics: true inputs: - - id: 934ef8aa-ed19-405b-8160-ebf62e3d32f9 - name: kubernetes-cluster-metrics - revision: 1 + - name: kubernetes-cluster-metrics type: kubernetes/metrics use_output: default meta: @@ -449,9 +408,7 @@ data: data_stream: namespace: default streams: - - id: >- - kubernetes/metrics-kubernetes.apiserver-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.apiserver type: metrics metricsets: @@ -462,18 +419,14 @@ data: period: 30s ssl.certificate_authorities: - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - - id: >- - kubernetes/metrics-kubernetes.event-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.event type: metrics metricsets: - event period: 10s add_metadata: true - - id: >- - kubernetes/metrics-kubernetes.state_container-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_container type: metrics metricsets: @@ -482,9 +435,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_cronjob-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_cronjob type: metrics metricsets: @@ -493,9 +444,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_deployment-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_deployment type: metrics metricsets: @@ -504,9 +453,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_node-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_node type: metrics metricsets: @@ -515,9 +462,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_persistentvolume-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_persistentvolume type: metrics metricsets: @@ -526,9 +471,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_persistentvolumeclaim-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_persistentvolumeclaim type: metrics metricsets: @@ -537,9 +480,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_pod-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_pod type: metrics metricsets: @@ -548,9 +489,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_replicaset-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_replicaset type: metrics metricsets: @@ -559,9 +498,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_resourcequota-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_resourcequota type: metrics metricsets: @@ -570,9 +507,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_service-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_service type: metrics metricsets: @@ -581,9 +516,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_statefulset-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_statefulset type: metrics metricsets: @@ -592,9 +525,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_storageclass-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_storageclass type: metrics metricsets: diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml index 60783ce3e32..11c2992c93c 100644 --- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml @@ -7,8 +7,6 @@ metadata: k8s-app: elastic-agent data: agent.yml: |- - id: ef9cc740-5bf0-11eb-8b51-39775155c3f5 - revision: 2 outputs: default: type: elasticsearch @@ -27,9 +25,7 @@ data: node: ${NODE_NAME} scope: node inputs: - - id: 4ae27079-6cd4-4ab7-a459-abbae74ffc44 - name: log-1 - revision: 1 + - name: log-1 type: logfile use_output: default meta: @@ -39,15 +35,12 @@ data: data_stream: namespace: default streams: - - id: logfile-log.log-4ae27079-6cd4-4ab7-a459-abbae74ffc44 - data_stream: + - data_stream: dataset: generic symlinks: true paths: - /var/log/containers/*${kubernetes.container.id}.log - - id: 1f6112ef-9847-4294-aba4-0dc9e7240551 - name: system-3 - revision: 2 + - name: system-3 type: system/metrics use_output: default meta: @@ -57,16 +50,14 @@ data: data_stream: namespace: default streams: - - id: system/metrics-system.core-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.core type: metrics metricsets: - core core.metrics: - percentages - - id: system/metrics-system.cpu-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.cpu type: metrics period: 10s @@ -75,16 +66,14 @@ data: - normalized_percentages metricsets: - cpu - - id: system/metrics-system.diskio-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.diskio type: metrics period: 10s diskio.include_devices: null metricsets: - diskio - - id: system/metrics-system.filesystem-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.filesystem type: metrics period: 1m @@ -93,8 +82,7 @@ data: processors: - drop_event.when.regexp: system.filesystem.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/) - - id: system/metrics-system.fsstat-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.fsstat type: metrics period: 1m @@ -103,30 +91,26 @@ data: processors: - drop_event.when.regexp: system.fsstat.mount_point: ^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/) - - id: system/metrics-system.load-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.load type: metrics period: 10s metricsets: - load - - id: system/metrics-system.memory-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.memory type: metrics period: 10s metricsets: - memory - - id: system/metrics-system.network-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.network type: metrics period: 10s network.interfaces: null metricsets: - network - - id: system/metrics-system.process-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.process type: metrics process.include_top_n.by_memory: 5 @@ -140,27 +124,21 @@ data: - process process.include_cpu_ticks: false system.hostfs: /hostfs - - id: >- - system/metrics-system.process_summary-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.process_summary type: metrics period: 10s metricsets: - process_summary system.hostfs: /hostfs - - id: >- - system/metrics-system.socket_summary-1f6112ef-9847-4294-aba4-0dc9e7240551 - data_stream: + - data_stream: dataset: system.socket_summary type: metrics period: 10s metricsets: - socket_summary system.hostfs: /hostfs - - id: 934ef8aa-ed19-405b-8160-ebf62e3d32f8 - name: kubernetes-node-metrics - revision: 1 + - name: kubernetes-node-metrics type: kubernetes/metrics use_output: default meta: @@ -170,9 +148,7 @@ data: data_stream: namespace: default streams: - - id: >- - kubernetes/metrics-kubernetes.controllermanager-3d50c483-2327-40e7-b3e5-d877d4763fe1 - data_stream: + - data_stream: dataset: kubernetes.controllermanager type: metrics metricsets: @@ -181,9 +157,7 @@ data: - '${kubernetes.pod.ip}:10252' period: 10s condition: ${kubernetes.pod.labels.component} == 'kube-controller-manager' - - id: >- - kubernetes/metrics-kubernetes.scheduler-3d50c483-2327-40e7-b3e5-d877d4763fe1 - data_stream: + - data_stream: dataset: kubernetes.scheduler type: metrics metricsets: @@ -192,9 +166,7 @@ data: - '${kubernetes.pod.ip}:10251' period: 10s condition: ${kubernetes.pod.labels.component} == 'kube-scheduler' - - id: >- - kubernetes/metrics-kubernetes.proxy-3d50c483-2327-40e7-b3e5-d877d4763fe1 - data_stream: + - data_stream: dataset: kubernetes.proxy type: metrics metricsets: @@ -202,9 +174,7 @@ data: hosts: - 'localhost:10249' period: 10s - - id: >- - kubernetes/metrics-kubernetes.container-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.container type: metrics metricsets: @@ -215,9 +185,7 @@ data: - 'https://${env.NODE_NAME}:10250' period: 10s ssl.verification_mode: none - - id: >- - kubernetes/metrics-kubernetes.node-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.node type: metrics metricsets: @@ -228,8 +196,7 @@ data: - 'https://${env.NODE_NAME}:10250' period: 10s ssl.verification_mode: none - - id: kubernetes/metrics-kubernetes.pod-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.pod type: metrics metricsets: @@ -240,9 +207,7 @@ data: - 'https://${env.NODE_NAME}:10250' period: 10s ssl.verification_mode: none - - id: >- - kubernetes/metrics-kubernetes.system-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.system type: metrics metricsets: @@ -253,9 +218,7 @@ data: - 'https://${env.NODE_NAME}:10250' period: 10s ssl.verification_mode: none - - id: >- - kubernetes/metrics-kubernetes.volume-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.volume type: metrics metricsets: diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment-configmap.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment-configmap.yaml index 47ab58841b2..58a92665e4e 100644 --- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment-configmap.yaml +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment-configmap.yaml @@ -8,8 +8,6 @@ metadata: data: # This part requires `kube-state-metrics` up and running under `kube-system` namespace agent.yml: |- - id: ef9cc740-5bf0-11eb-8b51-39775155c3f5 - revision: 2 outputs: default: type: elasticsearch @@ -25,9 +23,7 @@ data: logs: true metrics: true inputs: - - id: 934ef8aa-ed19-405b-8160-ebf62e3d32f9 - name: kubernetes-cluster-metrics - revision: 1 + - name: kubernetes-cluster-metrics type: kubernetes/metrics use_output: default meta: @@ -37,9 +33,7 @@ data: data_stream: namespace: default streams: - - id: >- - kubernetes/metrics-kubernetes.apiserver-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.apiserver type: metrics metricsets: @@ -50,18 +44,14 @@ data: period: 30s ssl.certificate_authorities: - /var/run/secrets/kubernetes.io/serviceaccount/ca.crt - - id: >- - kubernetes/metrics-kubernetes.event-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.event type: metrics metricsets: - event period: 10s add_metadata: true - - id: >- - kubernetes/metrics-kubernetes.state_container-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_container type: metrics metricsets: @@ -70,9 +60,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_cronjob-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_cronjob type: metrics metricsets: @@ -81,9 +69,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_deployment-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_deployment type: metrics metricsets: @@ -92,9 +78,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_node-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_node type: metrics metricsets: @@ -103,9 +87,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_persistentvolume-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_persistentvolume type: metrics metricsets: @@ -114,9 +96,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_persistentvolumeclaim-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_persistentvolumeclaim type: metrics metricsets: @@ -125,9 +105,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_pod-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_pod type: metrics metricsets: @@ -136,9 +114,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_replicaset-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_replicaset type: metrics metricsets: @@ -147,9 +123,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_resourcequota-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_resourcequota type: metrics metricsets: @@ -158,9 +132,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_service-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_service type: metrics metricsets: @@ -169,9 +141,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_statefulset-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_statefulset type: metrics metricsets: @@ -180,9 +150,7 @@ data: hosts: - 'kube-state-metrics:8080' period: 10s - - id: >- - kubernetes/metrics-kubernetes.state_storageclass-934ef8aa-ed19-405b-8160-ebf62e3d32f8 - data_stream: + - data_stream: dataset: kubernetes.state_storageclass type: metrics metricsets: From e2b008f1b12cc6c7ab54603086c37e813d5e77e5 Mon Sep 17 00:00:00 2001 From: chrismark Date: Fri, 19 Feb 2021 16:16:02 +0200 Subject: [PATCH 10/11] Remove verbose logging Signed-off-by: chrismark --- deploy/kubernetes/elastic-agent-standalone-kubernetes.yml | 4 ++-- .../elastic-agent-standalone-daemonset.yaml | 2 +- .../elastic-agent-standalone-deployment.yaml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml index 4e243eb0ad8..d956db3c38b 100644 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml @@ -26,7 +26,7 @@ spec: image: docker.elastic.co/beats/elastic-agent:8.0.0 args: [ "-c", "/etc/agent.yml", - "-e", "-d", "composable.providers.kubernetes", + "-e", ] env: - name: ES_USERNAME @@ -336,7 +336,7 @@ spec: image: docker.elastic.co/beats/elastic-agent:8.0.0 args: [ "-c", "/etc/agent.yml", - "-e", "-d", "composable.providers.kubernetes", + "-e", ] env: - name: ES_USERNAME diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml index 93ae5cfe46f..e97e0743926 100644 --- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset.yaml @@ -25,7 +25,7 @@ spec: image: docker.elastic.co/beats/elastic-agent:%VERSION% args: [ "-c", "/etc/agent.yml", - "-e", "-d", "composable.providers.kubernetes", + "-e", ] env: - name: ES_USERNAME diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment.yaml index 03bb5c2535c..0def8b88571 100644 --- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment.yaml +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-deployment.yaml @@ -20,7 +20,7 @@ spec: image: docker.elastic.co/beats/elastic-agent:%VERSION% args: [ "-c", "/etc/agent.yml", - "-e", "-d", "composable.providers.kubernetes", + "-e", ] env: - name: ES_USERNAME From 292c36082ca9a8befbe009cdac2336bdf30a2c3c Mon Sep 17 00:00:00 2001 From: chrismark Date: Mon, 22 Feb 2021 12:22:52 +0200 Subject: [PATCH 11/11] Remove secrets from clusterrole Signed-off-by: chrismark --- deploy/kubernetes/elastic-agent-standalone-kubernetes.yml | 1 - .../elastic-agent-standalone/elastic-agent-standalone-role.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml index d956db3c38b..46b625fa0ce 100644 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml @@ -561,7 +561,6 @@ rules: - namespaces - events - pods - - secrets verbs: ["get", "list", "watch"] - apiGroups: ["extensions"] resources: diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml index 510ea6eeb10..dcf2b4a5ff2 100644 --- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-role.yaml @@ -11,7 +11,6 @@ rules: - namespaces - events - pods - - secrets verbs: ["get", "list", "watch"] - apiGroups: ["extensions"] resources: