Metric collector must fail on error (kubeflow#405)

* Fail when unable to collect logs * Set backlimit to 0 for jobs
arrikto · Feb 26, 2019 · 26da3ea · 26da3ea
1 parent 6b75138
commit 26da3ea
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 4 deletions.
diff --git a/cmd/metricscollector/main.go b/cmd/metricscollector/main.go
@@ -78,17 +78,15 @@ func main() {
 	}
 	mls, err := mc.CollectWorkerLog(*workerID, *workerKind, screp.StudyConfig.ObjectiveValueName, screp.StudyConfig.Metrics, *namespace)
 	if err != nil {
-		log.Printf("Failed to collect logs: %v", err)
-		return
+		log.Fatalf("Failed to collect logs: %v", err)
 	}
 	rmreq := &api.ReportMetricsLogsRequest{
 		StudyId:        *studyID,
 		MetricsLogSets: []*api.MetricsLogSet{mls},
 	}
 	_, err = c.ReportMetricsLogs(ctx, rmreq)
 	if err != nil {
-		log.Printf("Failed to Report logs: %v", err)
-		return
+		log.Fatalf("Failed to Report logs: %v", err)
 	}
 	log.Printf("Metrics reported. :\n%v", mls)
 	return

diff --git a/manifests/studyjobcontroller/metricsControllerConfigMap.yaml b/manifests/studyjobcontroller/metricsControllerConfigMap.yaml
@@ -16,6 +16,7 @@ data:
       failedJobsHistoryLimit: 1
       jobTemplate:
         spec:
+          backoffLimit: 0
           template:
             spec:
               serviceAccountName: metrics-collector