Use constant backoff for DownloadOSURL

clients/object_store_client.go

@@ -7,11 +7,13 @@ import (
 	"time"
 
 	"github.com/cenkalti/backoff/v4"
+	"github.com/livepeer/catalyst-api/config"
 	"github.com/livepeer/catalyst-api/metrics"
 	"github.com/livepeer/go-tools/drivers"
 )
 
-var backOff = newBackOffExecutor()
+var exponentialBackOff = newExponentialBackOffExecutor()
+var constantBackOff = newConstantBackOffExecutor()
 
 func DownloadOSURL(osURL string) (io.ReadCloser, error) {
 	storageDriver, err := drivers.ParseOSURL(osURL, true)
@@ -37,7 +39,7 @@ func DownloadOSURL(osURL string) (io.ReadCloser, error) {
 	})
 
 	start := time.Now()
-	err = backoff.Retry(readOperation, backoff.WithMaxRetries(backOff, 2))
+	err = backoff.Retry(readOperation, backoff.WithMaxRetries(constantBackOff, config.DownloadOSURLRetries))
 	if err != nil {
 		metrics.Metrics.ObjectStoreClient.FailureCount.WithLabelValues(url, "read").Inc()
 		return nil, fmt.Errorf("failed to read from OS URL %q: %s", osURL, err)
@@ -73,7 +75,7 @@ func UploadToOSURL(osURL, filename string, data io.Reader) error {
 	})
 
 	start := time.Now()
-	err = backoff.Retry(writeOperation, backoff.WithMaxRetries(backOff, 2))
+	err = backoff.Retry(writeOperation, backoff.WithMaxRetries(exponentialBackOff, 2))
 	if err != nil {
 		metrics.Metrics.ObjectStoreClient.FailureCount.WithLabelValues(url, "write").Inc()
 		return fmt.Errorf("failed to write file %q to OS URL %q: %s", filename, osURL, err)
@@ -87,14 +89,18 @@ func UploadToOSURL(osURL, filename string, data io.Reader) error {
 	return nil
 }
 
-func newBackOffExecutor() *backoff.ExponentialBackOff {
+func newExponentialBackOffExecutor() *backoff.ExponentialBackOff {
 	backOff := backoff.NewExponentialBackOff()
 	backOff.InitialInterval = 200 * time.Millisecond
 	backOff.MaxInterval = 1 * time.Second
 
 	return backOff
 }
 
+func newConstantBackOffExecutor() *backoff.ConstantBackOff {
+	return backoff.NewConstantBackOff(1 * time.Second)
+}
+
 var makeOperation = func(fn func() error) func() error {
 	return fn
 }

clients/object_store_client_test.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/livepeer/catalyst-api/config"
 	"github.com/stretchr/testify/require"
 )
 
@@ -79,8 +80,15 @@ func TestItRetriesReadingData(t *testing.T) {
 }
 
 func TestItFailsAfterMaxReadsReached(t *testing.T) {
-	var retries = 0
+	var retries uint64 = 0
 	var original = makeOperation
+
+	var originalRetries = config.DownloadOSURLRetries
+	config.DownloadOSURLRetries = 3
+	defer func() {
+		config.DownloadOSURLRetries = originalRetries
+	}()
+
 	makeOperation = func(fn func() error) func() error {
 		return func() error {
 			retries++
@@ -101,7 +109,7 @@ func TestItFailsAfterMaxReadsReached(t *testing.T) {
 	_, err = DownloadOSURL(f.Name())
 
 	require.Error(t, err)
-	require.Equal(t, 3, retries)
+	require.Equal(t, config.DownloadOSURLRetries+1, retries)
 }
 
 func TestItRetriesSavingData(t *testing.T) {

config/config.go

@@ -25,3 +25,5 @@ var RecordingCallback string = "http://127.0.0.1:8008/recording/status"
 var TranscodingParallelJobs int = 2
 
 var TranscodingParallelSleep time.Duration = 713 * time.Millisecond
+
+var DownloadOSURLRetries uint64 = 15