Treat ErrPolicyForbidden and ErrPodNotFound as permanent errors durin…

…g retry backoff (#250)

* Treat forbidden policies as permanent errors

* Translate from gRPC to internal errors in gateway

fetchCredentials shouldn't need to know that interaction with the server
is happening over gRPC

* Add test for assuming a forbidden role

pkg/aws/metadata/handler_credentials.go

@@ -75,6 +75,9 @@ func (c *credentialsHandler) fetchCredentials(ctx context.Context, ip, requested
 	op := func() error {
 		creds, err := c.client.GetCredentials(ctx, ip, requestedRole)
 		if err != nil {
+			if err == server.ErrPolicyForbidden {
+				return backoff.Permanent(err)
+			}
 			return err
 		}
 		credsCh <- creds

pkg/aws/metadata/handler_credentials_test.go

@@ -103,3 +103,29 @@ func TestReturnsCredentialsWithRetryAfterError(t *testing.T) {
 		t.Error("unexpected status", rr.Code)
 	}
 }
+
+func TestForbiddenRole(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	defer cancel()
+	defer leaktest.Check(t)()
+
+	r, _ := http.NewRequest("GET", "/latest/meta-data/iam/security-credentials/role", nil)
+	rr := httptest.NewRecorder()
+
+	valid := st.GetCredentialsResult{&sts.Credentials{}, nil}
+	e := st.GetCredentialsResult{nil, server.ErrPolicyForbidden}
+	client := st.NewStubClient().WithRoles(st.GetRoleResult{"role", nil}).WithCredentials(e, valid)
+	handler := newCredentialsHandler(client, getBlankClientIP)
+	router := mux.NewRouter()
+	handler.Install(router)
+
+	router.ServeHTTP(rr, r.WithContext(ctx))
+
+	if rr.Code != http.StatusInternalServerError {
+		t.Error("unexpected status", rr.Code)
+	}
+
+	if !strings.Contains(rr.Body.String(), "forbidden by policy") {
+		t.Error("unexpected error", rr.Body.String())
+	}
+}

pkg/server/gateway.go

@@ -31,6 +31,8 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/balancer/roundrobin"
 	"google.golang.org/grpc/credentials"
+
+	status "google.golang.org/grpc/status"
 )
 
 // Client is the Server's client interface
@@ -123,6 +125,15 @@ func (g *KiamGateway) GetCredentials(ctx context.Context, ip, role string) (*sts
 	}
 	credentials, err := g.client.GetPodCredentials(ctx, &pb.GetPodCredentialsRequest{Ip: ip, Role: role})
 	if err != nil {
+		if grpcStatus, ok := status.FromError(err); ok {
+			switch grpcStatus.Message() {
+			case ErrPolicyForbidden.Error():
+				return nil, ErrPolicyForbidden
+			case ErrPodNotFound.Error():
+				return nil, ErrPodNotFound
+			}
+		}
+
 		return nil, err
 	}
 	return &sts.Credentials{