Merge pull request #48 from livepeer/SRE-221-reduce-logs

[SRE-221] Reduce logs printend on the default verbosity level

.github/workflows/test.yaml

@@ -47,7 +47,7 @@ jobs:
 
       - name: Run tests with coverage
         run: |
-          go test ./... --race --covermode=atomic --coverprofile=coverage.out
+          go test ./... -v --race --covermode=atomic --coverprofile=coverage.out
 
       - name: Upload coverage reports
         uses: codecov/codecov-action@v3

catalyst-uploader.go

@@ -3,10 +3,10 @@ package main
 import (
 	"encoding/json"
 	"flag"
-	"log"
 	"os"
 	"time"
 
+	"github.com/golang/glog"
 	"github.com/livepeer/catalyst-uploader/core"
 	"github.com/livepeer/go-tools/drivers"
 )
@@ -32,7 +32,7 @@ func run() int {
 	}
 
 	if flag.NArg() == 0 {
-		log.Fatal("Destination URI is not specified. See -h for usage.")
+		glog.Fatal("Destination URI is not specified. See -h for usage.")
 		return 1
 	}
 
@@ -42,20 +42,20 @@ func run() int {
 
 	uri := flag.Arg(0)
 	if uri == "" {
-		log.Fatalf("Could not parse object store URI: %s", uri)
+		glog.Fatalf("Could not parse object store URI: %s", uri)
 		return 1
 	}
 
 	err := core.Upload(os.Stdin, uri, WaitBetweenWrites, *timeout)
 	if err != nil {
-		log.Fatalf("Uploader failed for %s: %s", uri, err)
+		glog.Fatalf("Uploader failed for %s: %s", uri, err)
 		return 1
 	}
 
 	// success, write uploaded file details to stdout
 	err = json.NewEncoder(stdout).Encode(map[string]string{"uri": uri})
 	if err != nil {
-		log.Println(err)
+		glog.Fatal(err)
 		return 1
 	}
 

core/uploader.go

@@ -6,14 +6,14 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/cenkalti/backoff/v4"
+	"github.com/golang/glog"
 	"github.com/livepeer/go-tools/drivers"
 )
 
@@ -62,7 +62,7 @@ func Upload(input io.Reader, outputURI string, waitBetweenWrites, writeTimeout t
 		err = backoff.Retry(func() error {
 			_, err := session.SaveData(context.Background(), "", bytes.NewReader(fileContents), fields, writeTimeout)
 			if err != nil {
-				log.Printf("failed upload attempt: %s", err)
+				glog.Errorf("failed upload attempt: %v", err)
 			}
 			return err
 		}, UploadRetryBackoff())
@@ -71,7 +71,7 @@ func Upload(input io.Reader, outputURI string, waitBetweenWrites, writeTimeout t
 		}
 
 		if err = extractThumb(session, outputURI, fileContents); err != nil {
-			log.Printf("extracting thumbnail failed: %s", err)
+			glog.Errorf("extracting thumbnail failed: %v", err)
 		}
 		return nil
 	}
@@ -99,16 +99,16 @@ func Upload(input io.Reader, outputURI string, waitBetweenWrites, writeTimeout t
 		b := scanner.Bytes()
 		fileContents = append(fileContents, b...)
 		if strings.Contains(outputURI, "m3u8") {
-			log.Printf("Received new bytes for %s: %s", outputURI, string(b))
+			glog.V(5).Infof("Received new bytes for %s: %s", outputURI, string(b))
 		}
 
 		// Only write the latest version of the data that's been piped in if enough time has elapsed since the last write
 		if lastWrite.Add(waitBetweenWrites).Before(time.Now()) {
 			if _, err := session.SaveData(context.Background(), "", bytes.NewReader(fileContents), fields, writeTimeout); err != nil {
 				// Just log this error, since it'll effectively be retried after the next interval
-				log.Printf("Failed to write: %s", err)
+				glog.Errorf("Failed to write: %v", err)
 			} else {
-				log.Printf("Wrote %s to storage: %d bytes", outputURI, len(b))
+				glog.V(5).Infof("Wrote %s to storage: %d bytes", outputURI, len(b))
 			}
 			lastWrite = time.Now()
 		}
@@ -122,7 +122,7 @@ func Upload(input io.Reader, outputURI string, waitBetweenWrites, writeTimeout t
 		// Don't ignore this error, since there won't be any further attempts to write
 		return fmt.Errorf("failed to write final save: %w", err)
 	}
-
+	glog.Infof("Completed writing %s to storage", outputURI)
 	return nil
 }
 

go.mod

@@ -3,6 +3,8 @@ module github.com/livepeer/catalyst-uploader
 go 1.17
 
 require (
+	github.com/cenkalti/backoff/v4 v4.2.1
+	github.com/golang/glog v1.1.0
 	github.com/google/uuid v1.3.0
 	github.com/livepeer/go-tools v0.3.3
 	github.com/stretchr/testify v1.8.4
@@ -15,7 +17,6 @@ require (
 	cloud.google.com/go/iam v1.1.0 // indirect
 	cloud.google.com/go/storage v1.30.1 // indirect
 	github.com/aws/aws-sdk-go v1.44.273 // indirect
-	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect

go.sum

@@ -852,6 +852,7 @@ github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
+github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE=
 github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=