Revise TidbMonitor controller (#1500)

* Revise TidbMonitor controller Co-Authored-By: Aylei <[email protected]> Co-authored-by: pingcap-github-bot <[email protected]>
pingcap · Jan 15, 2020 · d168502 · d168502
1 parent bfa1983
commit d168502
Show file tree

Hide file tree

Showing 3 changed files with 34 additions and 5 deletions.
diff --git a/pkg/controller/tidbmonitor/tidb_monitor_controller.go b/pkg/controller/tidbmonitor/tidb_monitor_controller.go
@@ -63,7 +63,7 @@ func NewController(
 	tidbMonitorInformer := informerFactory.Pingcap().V1alpha1().TidbMonitors()
 	deploymentInformer := kubeInformerFactory.Apps().V1().Deployments()
 	typedControl := controller.NewTypedControl(controller.NewRealGenericControl(genericCli, recorder))
-	monitorManager := monitor.NewMonitorManager(informerFactory, kubeInformerFactory, typedControl)
+	monitorManager := monitor.NewMonitorManager(informerFactory, kubeInformerFactory, typedControl, recorder)
 
 	tmc := &Controller{
 		cli:      genericCli,
@@ -115,7 +115,7 @@ func (tmc *Controller) processNextWorkItem() bool {
 		if perrors.Find(err, controller.IsRequeueError) != nil {
 			klog.Infof("TidbMonitor: %v, still need sync: %v, requeuing", key.(string), err)
 		} else {
-			utilruntime.HandleError(fmt.Errorf("TidbMonitor: %v, sync failed, err: %v, requeuing", key.(string), err))
+			utilruntime.HandleError(fmt.Errorf("TidbMonitor: %v, sync failed, err: %v", key.(string), err))
 		}
 		tmc.queue.AddRateLimited(key)
 	} else {

diff --git a/pkg/monitor/monitor/monitor_manager.go b/pkg/monitor/monitor/monitor_manager.go
@@ -22,23 +22,32 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	kubeinformers "k8s.io/client-go/informers"
 	appslisters "k8s.io/client-go/listers/apps/v1"
+	"k8s.io/client-go/tools/record"
 	"k8s.io/klog"
 )
 
 type MonitorManager struct {
 	typedControl     controller.TypedControlInterface
 	deploymentLister appslisters.DeploymentLister
 	tcLister         v1alpha1listers.TidbClusterLister
+	recorder         record.EventRecorder
 }
 
+const (
+	FailedSync  = "FailedSync"
+	SuccessSync = "SuccessSync"
+)
+
 func NewMonitorManager(
 	informerFactory informers.SharedInformerFactory,
 	kubeInformerFactory kubeinformers.SharedInformerFactory,
-	typedControl controller.TypedControlInterface) *MonitorManager {
+	typedControl controller.TypedControlInterface,
+	recorder record.EventRecorder) *MonitorManager {
 	return &MonitorManager{
 		typedControl:     typedControl,
 		deploymentLister: kubeInformerFactory.Apps().V1().Deployments().Lister(),
 		tcLister:         informerFactory.Pingcap().V1alpha1().TidbClusters().Lister(),
+		recorder:         recorder,
 	}
 }
 
@@ -50,18 +59,29 @@ func (mm *MonitorManager) Sync(monitor *v1alpha1.TidbMonitor) error {
 
 	// Sync Service
 	if err := mm.syncTidbMonitorService(monitor); err != nil {
+		message := fmt.Sprintf("Sync TidbMonitor[%s/%s] Service failed, err: %v", monitor.Namespace, monitor.Name, err)
+		mm.recorder.Event(monitor, corev1.EventTypeWarning, FailedSync, message)
 		return err
 	}
 	klog.V(4).Infof("tm[%s/%s]'s service synced", monitor.Namespace, monitor.Name)
 	// Sync PVC
 	if monitor.Spec.Persistent {
 		if err := mm.syncTidbMonitorPVC(monitor); err != nil {
+			message := fmt.Sprintf("Sync TidbMonitor[%s/%s] PVC failed,err:%v", monitor.Namespace, monitor.Name, err)
+			mm.recorder.Event(monitor, corev1.EventTypeWarning, FailedSync, message)
 			return err
 		}
 	}
 	klog.V(4).Infof("tm[%s/%s]'s pvc synced", monitor.Namespace, monitor.Name)
+
 	// Sync Deployment
-	return mm.syncTidbMonitorDeployment(monitor)
+	if err := mm.syncTidbMonitorDeployment(monitor); err != nil {
+		message := fmt.Sprintf("Sync TidbMonitor[%s/%s] Deployment failed,err:%v", monitor.Namespace, monitor.Name, err)
+		mm.recorder.Event(monitor, corev1.EventTypeWarning, FailedSync, message)
+		return err
+	}
+	klog.V(4).Infof("tm[%s/%s]'s deployment synced", monitor.Namespace, monitor.Name)
+	return nil
 }
 
 func (mm *MonitorManager) syncTidbMonitorService(monitor *v1alpha1.TidbMonitor) error {

diff --git a/pkg/monitor/monitor/util.go b/pkg/monitor/monitor/util.go
@@ -33,6 +33,8 @@ func GetMonitorObjectName(monitor *v1alpha1.TidbMonitor) string {
 	return fmt.Sprintf("%s-monitor", monitor.Name)
 }
 
+// getMonitorConfigMap generate the Prometheus config and Grafana config for TidbMonitor,
+// If the namespace in ClusterRef is empty, we would set the TidbMonitor's namespace in the default
 func getMonitorConfigMap(tc *v1alpha1.TidbCluster, monitor *v1alpha1.TidbMonitor) (*core.ConfigMap, error) {
 
 	var releaseNamespaces []string
@@ -51,6 +53,10 @@ func getMonitorConfigMap(tc *v1alpha1.TidbCluster, monitor *v1alpha1.TidbMonitor
 		model.AlertmanagerURL = *monitor.Spec.AlertmanagerURL
 	}
 
+	if len(model.ReleaseNamespaces) < 1 {
+		model.ReleaseNamespaces = append(model.ReleaseNamespaces, monitor.Namespace)
+	}
+
 	content, err := RenderPrometheusConfig(model)
 	if err != nil {
 		return nil, err
@@ -366,7 +372,6 @@ func getMonitorPrometheusContainer(monitor *v1alpha1.TidbMonitor, tc *v1alpha1.T
 			"/bin/prometheus",
 			"--web.enable-admin-api",
 			"--web.enable-lifecycle",
-			fmt.Sprintf("--log.level=%s", monitor.Spec.Prometheus.LogLevel),
 			"--config.file=/etc/prometheus/prometheus.yml",
 			"--storage.tsdb.path=/data/prometheus",
 			fmt.Sprintf("--storage.tsdb.retention=%dd", monitor.Spec.Prometheus.ReserveDays),
@@ -401,6 +406,10 @@ func getMonitorPrometheusContainer(monitor *v1alpha1.TidbMonitor, tc *v1alpha1.T
 			},
 		},
 	}
+	if len(monitor.Spec.Prometheus.LogLevel) > 0 {
+		c.Command = append(c.Command, fmt.Sprintf("--log.level=%s", monitor.Spec.Prometheus.LogLevel))
+	}
+
 	if tc.IsTLSClusterEnabled() {
 		c.VolumeMounts = append(c.VolumeMounts, core.VolumeMount{
 			Name:      "tls-pd-client",