fix: fix reloader endpoint from post to get

- fix reloader endpoint from post to get - fix linting and test errors - reduce time for sleep to check more often Signed-off-by: Brian Davis <[email protected]>
vmware · Jun 5, 2023 · 3f2b3af · 3f2b3af
1 parent 727b621
commit 3f2b3af
Show file tree

Hide file tree

Showing 7 changed files with 34 additions and 14 deletions.
diff --git a/base-image/entrypoint.sh b/base-image/entrypoint.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
-# Copyright © 2022 VMware, Inc. All Rights Reserved.
+# Copyright © 2023 VMware, Inc. All Rights Reserved.
 # SPDX-License-Identifier: BSD-2-Clause
 
 # roughly based on https://github.com/fluent/fluentd-kubernetes-daemonset/blob/master/docker-image/v0.12/debian-elasticsearch/entrypoint.sh
@@ -21,7 +21,7 @@ main() {
       break
     fi
     echo "Waiting for config file to become available: $attempt of $retries"
-    sleep 10
+    sleep 5
   done
   if [[ "$ready" != "true" ]]; then
     return 1

diff --git a/config-reloader/datasource/kubedatasource/fluentdconfig/crd/crd.go b/config-reloader/datasource/kubedatasource/fluentdconfig/crd/crd.go
@@ -49,7 +49,7 @@ func CheckAndInstallCRD(ctx context.Context, config *rest.Config) error {
 	}
 
 	logrus.Infof("%s CRD is installed. Checking availability...", crdManager.GetCRDName(ctx))
-	if err := monitorCRDAvailability(crdManager); err != nil {
+	if err := monitorCRDAvailability(ctx, crdManager); err != nil {
 		return err
 	}
 	logrus.Infof("%s CRD is available", crdManager.GetCRDName(ctx))
@@ -90,7 +90,7 @@ func isV1CRDAvailable(config *rest.Config) (bool, error) {
 	return v1Available, nil
 }
 
-func monitorCRDAvailability(crdManager manager) error {
+func monitorCRDAvailability(ctx context.Context, crdManager manager) error {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second*30)
 	defer cancel()
 

diff --git a/config-reloader/fluentd/reloader.go b/config-reloader/fluentd/reloader.go
@@ -33,9 +33,9 @@ func (r *Reloader) ReloadConfiguration() {
 
 	logrus.Debugf("Reloading fluentd configuration gracefully via POST to /api/config.gracefulReload")
 
-	resp, err := http.Post(fmt.Sprintf("http://127.0.0.1:%d/api/config.gracefulReload", r.port), "application/json", nil)
+	resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/api/config.gracefulReload", r.port))
 	if err != nil {
-		logrus.Errorf("fluentd config.gracefulReload post request failed: %+v", err)
+		logrus.Errorf("fluentd config.gracefulReload request failed: %+v", err)
 	} else if resp.StatusCode != 200 {
 		defer resp.Body.Close()
 		body, _ := ioutil.ReadAll(resp.Body)

diff --git a/config-reloader/fluentd/reloader_test.go b/config-reloader/fluentd/reloader_test.go
@@ -19,13 +19,13 @@ func TestNullReloader(t *testing.T) {
 }
 func TestReloaderCalls(t *testing.T) {
 	ctx := context.Background()
-	port := 11543
+	port := 24444
 
 	counter := 0
 
 	handler := func(w http.ResponseWriter, r *http.Request) {
 		fmt.Printf("req %+v", r)
-		if r.Method == "POST" && r.RequestURI == "/api/config.gracefulReload" {
+		if r.Method == "GET" && r.RequestURI == "/api/config.gracefulReload" {
 			counter++
 		}
 	}

diff --git a/config-reloader/templates/kubernetes-postprocess.conf b/config-reloader/templates/kubernetes-postprocess.conf
@@ -7,8 +7,7 @@
 # kubernetes.container_name, so this exists to flatten the fields
 # so we can use them in our rewrite_tag_filter
 <filter kubernetes.**>
-  @type record_transformer
-  enable_ruby true
+  @type record_modifier
 
   # use the format namespace.pod_name.container_name
   <record>
@@ -41,7 +40,7 @@
 
 # Remove the unnecessary field as the information is already available on other fields.
 <filter kube.*.*.*>
-  @type record_transformer
+  @type record_modifier
   remove_keys kubernetes_namespace_container_name
 </filter>
 

diff --git a/config-reloader/templates/kubernetes.conf b/config-reloader/templates/kubernetes.conf
@@ -6,7 +6,8 @@
   @id in_tail_container_logs
   path /var/log/containers/*.log
   pos_file /var/log/{{.ID}}-fluentd-containers.log.pos
-  pos_file_compaction_interval 1h
+  pos_file_compaction_interval 3m
+  skip_refresh_on_startup true
   tag kubernetes.*
   read_from_head true
   read_bytes_limit_per_second 8192
@@ -51,6 +52,8 @@
   @id in_tail_minion
   path /var/log/salt/minion
   pos_file /var/log/{{.ID}}-fluentd-salt.pos
+  pos_file_compaction_interval 3m
+  skip_refresh_on_startup true
   tag salt
   <parse>
     @type regexp
@@ -65,6 +68,8 @@
   path /var/log/startupscript.log
   pos_file /var/log/{{.ID}}-fluentd-startupscript.log.pos
   tag startupscript
+  pos_file_compaction_interval 3m
+  skip_refresh_on_startup true
   <parse>
     @type syslog
   </parse>
@@ -75,6 +80,8 @@
   @id in_tail_docker
   path /var/log/docker.log
   pos_file /var/log/{{.ID}}-fluentd-docker.log.pos
+  pos_file_compaction_interval 3m
+  skip_refresh_on_startup true
   tag docker
   <parse>
     @type regexp
@@ -87,6 +94,8 @@
   @id in_tail_etcd
   path /var/log/etcd.log
   pos_file /var/log/{{.ID}}-fluentd-etcd.log.pos
+  pos_file_compaction_interval 3m
+  skip_refresh_on_startup true
   tag k8s.etcd
   <parse>
     @type none
@@ -111,6 +120,8 @@
   multiline_flush_interval 5s
   path /var/log/kube-proxy.log
   pos_file /var/log/{{.ID}}-fluentd-kube-proxy.log.pos
+  pos_file_compaction_interval 3m
+  skip_refresh_on_startup true
   tag k8s.kube-proxy
   <parse>
     @type kubernetes
@@ -123,6 +134,8 @@
   multiline_flush_interval 5s
   path /var/log/kube-apiserver.log
   pos_file /var/log/{{.ID}}-fluentd-kube-apiserver.log.pos
+  pos_file_compaction_interval 3m
+  skip_refresh_on_startup true
   tag k8s.kube-apiserver
   <parse>
     @type kubernetes
@@ -147,6 +160,8 @@
   multiline_flush_interval 5s
   path /var/log/kube-scheduler.log
   pos_file /var/log/{{.ID}}-fluentd-kube-scheduler.log.pos
+  pos_file_compaction_interval 3m
+  skip_refresh_on_startup true
   tag k8s.kube-scheduler
   <parse>
     @type kubernetes
@@ -159,6 +174,8 @@
   multiline_flush_interval 5s
   path /var/log/rescheduler.log
   pos_file /var/log/{{.ID}}-fluentd-rescheduler.log.pos
+  pos_file_compaction_interval 3m
+  skip_refresh_on_startup true
   tag k8s.rescheduler
   <parse>
     @type kubernetes
@@ -171,6 +188,8 @@
   multiline_flush_interval 5s
   path /var/log/glbc.log
   pos_file /var/log/{{.ID}}-fluentd-glbc.log.pos
+  pos_file_compaction_interval 3m
+  skip_refresh_on_startup true
   tag k8s.glbc
   <parse>
     @type kubernetes
@@ -183,6 +202,8 @@
   multiline_flush_interval 5s
   path /var/log/cluster-autoscaler.log
   pos_file /var/log/{{.ID}}-fluentd-cluster-autoscaler.log.pos
+  pos_file_compaction_interval 3m
+  skip_refresh_on_startup true
   tag k8s.cluster-autoscaler
   <parse>
     @type kubernetes
@@ -198,6 +219,8 @@
   multiline_flush_interval 5s
   path /var/log/kubernetes/kube-apiserver-audit.log
   pos_file /var/log/{{.ID}}-kube-apiserver-audit.log.pos
+  pos_file_compaction_interval 3m
+  skip_refresh_on_startup true
   tag k8s.kube-apiserver-audit
   <parse>
     @type multiline

diff --git a/config-reloader/util/util_test.go b/config-reloader/util/util_test.go
@@ -109,7 +109,6 @@ func TestMatch(t *testing.T) {
 }
 
 func TestEnsureDirExits(t *testing.T) {
-
 	type testDirConfig struct {
 		expectErr  bool
 		folderName string
@@ -129,7 +128,6 @@ func TestEnsureDirExits(t *testing.T) {
 		if config.expectErr == true {
 			assert.NoDirExists(t, config.folderName, EnsureDirExists(config.folderName))
 			assert.Error(t, EnsureDirExists(config.folderName))
-
 		} else {
 			assert.EqualValues(t, nil, EnsureDirExists(config.folderName))
 			assert.DirExists(t, config.folderName, EnsureDirExists(config.folderName))