chore: let e2e test retry sending POST request if receiving an error …

…from http server (#513)

We start seeing flaky e2e tests after enabling source data transformer. The error message is 

```
Post "https://10.42.0.33:8443/vertices/in": dial tcp 10.42.0.33:8443: connect: connection refused
    e2eapi.go:68: Deleting Pipeline event-time-filter
    suite.go:73: test panicked: 500 Internal Server Error
```

My understanding is with transformer sidecar getting added to source vertex, it might take a bit longer for the source vertex to become fully functional. This change add retry so that test client retry sending POST to wait for source vertex to be ready.

Signed-off-by: Keran Yang <[email protected]>

test/fixtures/e2eapi.go

@@ -23,6 +23,9 @@ import (
 	"log"
 	"net/http"
 	"strings"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/wait"
 )
 
 func InvokeE2EAPI(format string, args ...interface{}) string {
@@ -51,7 +54,27 @@ func InvokeE2EAPI(format string, args ...interface{}) string {
 
 func InvokeE2EAPIPOST(format string, body string, args ...interface{}) string {
 	url := "http://127.0.0.1:8378" + fmt.Sprintf(format, args...)
-	resp, err := http.Post(url, "application/json", strings.NewReader(body))
+
+	var err error
+	var resp *http.Response
+	// Invoking POST can fail due to "500 Internal Server Error". It's because the server is still booting up and not ready to serve requests.
+	// To prevent such issue, we apply retry strategy.
+	// 3 attempts with 5 second fixed wait time are tested sufficient for it.
+	var retryBackOff = wait.Backoff{
+		Factor:   1,
+		Jitter:   0,
+		Steps:    3,
+		Duration: time.Second * 5,
+	}
+	_ = wait.ExponentialBackoff(retryBackOff, func() (done bool, err error) {
+		resp, err = http.Post(url, "application/json", strings.NewReader(body))
+		if err == nil && resp.StatusCode < 300 {
+			return true, nil
+		}
+		fmt.Printf("Got error %v, response %v, retrying.\n", err, *resp)
+		return false, nil
+	})
+
 	if err != nil {
 		panic(err)
 	}