diff --git a/pkg/forwardingrules/forwarding_rules.go b/pkg/forwardingrules/forwarding_rules.go new file mode 100644 index 0000000000..2e1070dacf --- /dev/null +++ b/pkg/forwardingrules/forwarding_rules.go @@ -0,0 +1,62 @@ +package forwardingrules + +import ( + "fmt" + + "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" + "k8s.io/ingress-gce/pkg/composite" + "k8s.io/ingress-gce/pkg/utils" + "k8s.io/klog" + "k8s.io/legacy-cloud-providers/gce" +) + +type ForwardingRulesService struct { + cloud *gce.Cloud + version meta.Version + scope meta.KeyType +} + +func NewForwardingRulesService(cloud *gce.Cloud, version meta.Version, scope meta.KeyType) *ForwardingRulesService { + return &ForwardingRulesService{ + cloud: cloud, + version: version, + scope: scope, + } +} + +func (frc *ForwardingRulesService) createKey(name string) (*meta.Key, error) { + return composite.CreateKey(frc.cloud, name, frc.scope) +} + +func (frc *ForwardingRulesService) CreateForwardingRule(forwardingRule *composite.ForwardingRule) error { + key, err := frc.createKey(forwardingRule.Name) + if err != nil { + klog.Errorf("Failed to create key for creating forwarding rule %s, err: %v", forwardingRule.Name, err) + return nil + } + return composite.CreateForwardingRule(frc.cloud, key, forwardingRule) +} + +func (frc *ForwardingRulesService) GetForwardingRule(name string) (*composite.ForwardingRule, error) { + key, err := frc.createKey(name) + if err != nil { + return nil, fmt.Errorf("Failed to create key for fetching forwarding rule %s, err: %w", name, err) + } + fr, err := composite.GetForwardingRule(frc.cloud, key, frc.version) + if utils.IgnoreHTTPNotFound(err) != nil { + return nil, fmt.Errorf("Failed to get existing forwarding rule %s, err: %w", name, err) + } + return fr, nil +} + +func (frc *ForwardingRulesService) DeleteForwardingRule(name string) error { + key, err := frc.createKey(name) + if err != nil { + return fmt.Errorf("Failed to create key for deleting forwarding rule %s, err: %w", name, err) + } + err = composite.DeleteForwardingRule(frc.cloud, key, frc.version) + if utils.IgnoreHTTPNotFound(err) != nil { + return fmt.Errorf("Failed to delete forwarding rule %s, err: %w", name, err) + } + return nil +} diff --git a/pkg/forwardingrules/forwarding_rules_test.go b/pkg/forwardingrules/forwarding_rules_test.go new file mode 100644 index 0000000000..8cfea50348 --- /dev/null +++ b/pkg/forwardingrules/forwarding_rules_test.go @@ -0,0 +1,217 @@ +package forwardingrules + +import ( + "testing" + + "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud" + "github.com/GoogleCloudPlatform/k8s-cloud-provider/pkg/cloud/meta" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "k8s.io/ingress-gce/pkg/composite" + "k8s.io/ingress-gce/pkg/utils" + "k8s.io/legacy-cloud-providers/gce" +) + +func TestCreateForwardingRule(t *testing.T) { + testCases := []struct { + frRule *composite.ForwardingRule + desc string + }{ + { + frRule: &composite.ForwardingRule{ + Name: "elb", + Description: "elb description", + LoadBalancingScheme: string(cloud.SchemeExternal), + }, + desc: "Test creating external forwarding rule", + }, + { + frRule: &composite.ForwardingRule{ + Name: "ilb", + Description: "ilb description", + LoadBalancingScheme: string(cloud.SchemeInternal), + }, + desc: "Test creating internal forwarding rule", + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + fakeGCE := gce.NewFakeGCECloud(gce.DefaultTestClusterValues()) + frc := NewForwardingRulesService(fakeGCE, meta.VersionGA, meta.Regional) + + err := frc.CreateForwardingRule(tc.frRule) + if err != nil { + t.Fatalf("frc.CreateForwardingRule(%v), returned error %v, want nil", tc.frRule, err) + } + + verifyForwardingRuleExists(t, fakeGCE, tc.frRule.Name) + }) + } +} + +func TestGetForwardingRule(t *testing.T) { + elbForwardingRule := &composite.ForwardingRule{ + Name: "elb", + Version: meta.VersionGA, + Scope: meta.Regional, + LoadBalancingScheme: string(cloud.SchemeExternal), + } + ilbForwardingRule := &composite.ForwardingRule{ + Name: "ilb", + Version: meta.VersionGA, + Scope: meta.Regional, + LoadBalancingScheme: string(cloud.SchemeInternal), + } + + testCases := []struct { + existingFwdRules []*composite.ForwardingRule + getFwdRuleName string + expectedFwdRule *composite.ForwardingRule + desc string + }{ + { + existingFwdRules: []*composite.ForwardingRule{elbForwardingRule, ilbForwardingRule}, + getFwdRuleName: elbForwardingRule.Name, + expectedFwdRule: elbForwardingRule, + desc: "Test getting external forwarding rule", + }, + { + existingFwdRules: []*composite.ForwardingRule{elbForwardingRule, ilbForwardingRule}, + getFwdRuleName: ilbForwardingRule.Name, + expectedFwdRule: ilbForwardingRule, + desc: "Test getting internal forwarding rule", + }, + { + existingFwdRules: []*composite.ForwardingRule{elbForwardingRule, ilbForwardingRule}, + getFwdRuleName: "non-existent-rule", + expectedFwdRule: nil, + desc: "Test getting non existent forwarding rule", + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + fakeGCE := gce.NewFakeGCECloud(gce.DefaultTestClusterValues()) + for _, fr := range tc.existingFwdRules { + mustCreateForwardingRule(t, fakeGCE, fr) + } + frc := NewForwardingRulesService(fakeGCE, meta.VersionGA, meta.Regional) + + fr, err := frc.GetForwardingRule(tc.getFwdRuleName) + if err != nil { + t.Fatalf("frc.GetForwardingRule(%v), returned error %v, want nil", tc.getFwdRuleName, err) + } + + if !cmp.Equal(fr, tc.expectedFwdRule, cmpopts.IgnoreFields(composite.ForwardingRule{}, "SelfLink", "Region")) { + diff := cmp.Diff(fr, tc.expectedFwdRule, cmpopts.IgnoreFields(composite.ForwardingRule{}, "SelfLink", "Region")) + t.Errorf("frc.GetForwardingRule(s) returned %v, not equal to expectedFwdRule %v, diff: %v", fr, tc.expectedFwdRule, diff) + } + }) + } +} + +func TestDeleteForwardingRule(t *testing.T) { + elbForwardingRule := &composite.ForwardingRule{ + Name: "elb", + LoadBalancingScheme: string(cloud.SchemeExternal), + } + ilbForwardingRule := &composite.ForwardingRule{ + Name: "ilb", + LoadBalancingScheme: string(cloud.SchemeInternal), + } + + testCases := []struct { + existingFwdRules []*composite.ForwardingRule + deleteFwdRuleName string + shouldExistFwdRules []*composite.ForwardingRule + desc string + }{ + { + existingFwdRules: []*composite.ForwardingRule{elbForwardingRule, ilbForwardingRule}, + deleteFwdRuleName: elbForwardingRule.Name, + shouldExistFwdRules: []*composite.ForwardingRule{}, + desc: "Delete elb forwarding rule", + }, + { + existingFwdRules: []*composite.ForwardingRule{elbForwardingRule, ilbForwardingRule}, + deleteFwdRuleName: ilbForwardingRule.Name, + shouldExistFwdRules: []*composite.ForwardingRule{elbForwardingRule}, + desc: "Delete ilb forwarding rule", + }, + { + existingFwdRules: []*composite.ForwardingRule{elbForwardingRule}, + deleteFwdRuleName: elbForwardingRule.Name, + shouldExistFwdRules: []*composite.ForwardingRule{}, + desc: "Delete single elb forwarding rule", + }, + { + existingFwdRules: []*composite.ForwardingRule{elbForwardingRule, ilbForwardingRule}, + deleteFwdRuleName: "non-existent", + shouldExistFwdRules: []*composite.ForwardingRule{elbForwardingRule, ilbForwardingRule}, + desc: "Delete non existent forwarding rule", + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + fakeGCE := gce.NewFakeGCECloud(gce.DefaultTestClusterValues()) + for _, fr := range tc.existingFwdRules { + mustCreateForwardingRule(t, fakeGCE, fr) + } + frc := NewForwardingRulesService(fakeGCE, meta.VersionGA, meta.Regional) + + err := frc.DeleteForwardingRule(tc.deleteFwdRuleName) + if err != nil { + t.Fatalf("frc.DeleteForwardingRule(%v), returned error %v, want nil", tc.deleteFwdRuleName, err) + } + + verifyForwardingRuleNotExists(t, fakeGCE, tc.deleteFwdRuleName) + for _, fw := range tc.shouldExistFwdRules { + verifyForwardingRuleExists(t, fakeGCE, fw.Name) + } + }) + } +} + +func verifyForwardingRuleExists(t *testing.T, cloud *gce.Cloud, name string) { + t.Helper() + verifyForwardingRuleShouldExists(t, cloud, name, true) +} + +func verifyForwardingRuleNotExists(t *testing.T, cloud *gce.Cloud, name string) { + t.Helper() + verifyForwardingRuleShouldExists(t, cloud, name, false) +} + +func verifyForwardingRuleShouldExists(t *testing.T, cloud *gce.Cloud, name string, shouldExist bool) { + t.Helper() + + key, err := composite.CreateKey(cloud, name, meta.Regional) + if err != nil { + t.Fatalf("Failed to create key for fetching forwarding rule %s, err: %v", name, err) + } + _, err = composite.GetForwardingRule(cloud, key, meta.VersionGA) + if err != nil { + if utils.IsNotFoundError(err) { + if shouldExist { + t.Errorf("Forwarding rule %s was not found", name) + } + return + } + t.Fatalf("composite.GetForwardingRule(_, %v, %v) returned error %v, want nil", key, meta.VersionGA, err) + } + if !shouldExist { + t.Errorf("Forwarding rule %s exists, expected to be not found", name) + } +} + +func mustCreateForwardingRule(t *testing.T, cloud *gce.Cloud, fr *composite.ForwardingRule) { + t.Helper() + + key := meta.RegionalKey(fr.Name, cloud.Region()) + err := composite.CreateForwardingRule(cloud, key, fr) + if err != nil { + t.Fatalf("composite.CreateForwardingRule(_, %s, %v) returned error %v, want nil", key, fr, err) + } +} diff --git a/pkg/forwardingrules/interfaces.go b/pkg/forwardingrules/interfaces.go new file mode 100644 index 0000000000..a85582b3f9 --- /dev/null +++ b/pkg/forwardingrules/interfaces.go @@ -0,0 +1,11 @@ +package forwardingrules + +import ( + "k8s.io/ingress-gce/pkg/composite" +) + +type ForwardingRules interface { + GetForwardingRule(name string) (*composite.ForwardingRule, error) + CreateForwardingRule(forwardingRule *composite.ForwardingRule) error + DeleteForwardingRule(name string) error +} diff --git a/pkg/l4lb/l4controller.go b/pkg/l4lb/l4controller.go index 4d0007900d..9b838a975d 100644 --- a/pkg/l4lb/l4controller.go +++ b/pkg/l4lb/l4controller.go @@ -178,7 +178,7 @@ func (l4c *L4Controller) shouldProcessService(service *v1.Service, l4 *loadbalan frName := utils.LegacyForwardingRuleName(service) // Processing should continue if an external forwarding rule exists. This can happen if the service is transitioning from External to Internal. // The external forwarding rule might not be deleted by the time this controller starts processing the service. - if fr := l4.GetForwardingRule(frName, meta.VersionGA); fr != nil && fr.LoadBalancingScheme == string(cloud.SchemeInternal) { + if fr := l4.GetForwardingRule(frName); fr != nil && fr.LoadBalancingScheme == string(cloud.SchemeInternal) { klog.Warningf("Ignoring update for service %s:%s as it contains legacy forwarding rule %q", service.Namespace, service.Name, frName) return false } diff --git a/pkg/l4lb/l4netlbcontroller.go b/pkg/l4lb/l4netlbcontroller.go index f9cbb1a51c..40beb446df 100644 --- a/pkg/l4lb/l4netlbcontroller.go +++ b/pkg/l4lb/l4netlbcontroller.go @@ -276,7 +276,7 @@ func (lc *L4NetLBController) hasTargetPoolForwardingRule(service *v1.Service) bo return false } - existingFR := l4netlb.GetForwardingRule(frName, meta.VersionGA) + existingFR := l4netlb.GetForwardingRule(frName) if existingFR != nil && existingFR.Target != "" { return true } @@ -323,7 +323,7 @@ func (lc *L4NetLBController) hasRBSForwardingRule(svc *v1.Service) bool { if lc.hasForwardingRuleAnnotation(svc, frName) { return true } - existingFR := l4netlb.GetForwardingRule(frName, meta.VersionGA) + existingFR := l4netlb.GetForwardingRule(frName) return existingFR != nil && existingFR.LoadBalancingScheme == string(cloud.SchemeExternal) && existingFR.BackendService != "" } diff --git a/pkg/loadbalancers/forwarding_rules.go b/pkg/loadbalancers/forwarding_rules.go index 93e342e1f9..61776e283c 100644 --- a/pkg/loadbalancers/forwarding_rules.go +++ b/pkg/loadbalancers/forwarding_rules.go @@ -196,10 +196,6 @@ func (l *L7) getEffectiveIP() (string, bool, error) { // ensureForwardingRule creates a forwarding rule with the given name, if it does not exist. It updates the existing // forwarding rule if needed. func (l *L4) ensureForwardingRule(loadBalancerName, bsLink string, options gce.ILBOptions, existingFwdRule *composite.ForwardingRule) (*composite.ForwardingRule, error) { - key, err := l.CreateKey(loadBalancerName) - if err != nil { - return nil, err - } // version used for creating the existing forwarding rule. version := meta.VersionGA @@ -213,6 +209,10 @@ func (l *L4) ensureForwardingRule(loadBalancerName, bsLink string, options gce.I // Changes to subnet annotation will be picked up and reflected in the forwarding rule. // Removing the annotation will set the forwarding rule to use the default subnet. if options.SubnetName != "" { + key, err := l.CreateKey(loadBalancerName) + if err != nil { + return nil, err + } subnetKey := *key subnetKey.Name = options.SubnetName subnetworkURL = cloud.SelfLink(meta.VersionGA, l.cloud.NetworkProjectID(), "subnetworks", &subnetKey) @@ -228,6 +228,7 @@ func (l *L4) ensureForwardingRule(loadBalancerName, bsLink string, options gce.I nm := types.NamespacedName{Namespace: l.Service.Namespace, Name: l.Service.Name}.String() // ILB can be created only in Premium Tier addrMgr = newAddressManager(l.cloud, nm, l.cloud.Region(), subnetworkURL, loadBalancerName, ipToUse, cloud.SchemeInternal, cloud.NetworkTierPremium) + var err error ipToUse, _, err = addrMgr.HoldAddress() if err != nil { return nil, err @@ -237,7 +238,7 @@ func (l *L4) ensureForwardingRule(loadBalancerName, bsLink string, options gce.I // Release the address that was reserved, in all cases. If the forwarding rule was successfully created, // the ephemeral IP is not needed anymore. If it was not created, the address should be released to prevent leaks. if err := addrMgr.ReleaseAddress(); err != nil { - klog.Errorf("ensureInternalLoadBalancer: failed to release address reservation, possibly causing an orphan: %v", err) + klog.Errorf("ensureForwardingRule: failed to release address reservation, possibly causing an orphan: %v", err) } }() } @@ -284,54 +285,38 @@ func (l *L4) ensureForwardingRule(loadBalancerName, bsLink string, options gce.I // If the forwarding rule pointed to a backend service which does not match the controller naming scheme, // that resouce could be leaked. It is not being deleted here because that is a user-managed resource. klog.V(2).Infof("ensureForwardingRule: forwarding rule changed - Existing - %+v\n, New - %+v\n, Diff(-existing, +new) - %s\n. Deleting existing forwarding rule.", existingFwdRule, fr, frDiff) - if err = utils.IgnoreHTTPNotFound(composite.DeleteForwardingRule(l.cloud, key, version)); err != nil { + if err = l.forwardingRules.DeleteForwardingRule(existingFwdRule.Name); err != nil { return nil, err } - l.recorder.Eventf(l.Service, corev1.EventTypeNormal, events.SyncIngress, "ForwardingRule %q deleted", key.Name) + l.recorder.Eventf(l.Service, corev1.EventTypeNormal, events.SyncIngress, "ForwardingRule %q deleted", existingFwdRule.Name) } klog.V(2).Infof("ensureForwardingRule: Creating/Recreating forwarding rule - %s", fr.Name) - if err = composite.CreateForwardingRule(l.cloud, key, fr); err != nil { + if err = l.forwardingRules.CreateForwardingRule(fr); err != nil { return nil, err } - return composite.GetForwardingRule(l.cloud, key, fr.Version) + return l.forwardingRules.GetForwardingRule(fr.Name) } -func (l *L4) GetForwardingRule(name string, version meta.Version) *composite.ForwardingRule { - key, err := l.CreateKey(name) +func (l *L4) GetForwardingRule(name string) *composite.ForwardingRule { + fr, err := l.forwardingRules.GetForwardingRule(name) if err != nil { - klog.Errorf("Failed to create key for fetching existing forwarding rule %s, err: %v", name, err) - return nil - } - fr, err := composite.GetForwardingRule(l.cloud, key, version) - if utils.IgnoreHTTPNotFound(err) != nil { klog.Errorf("Failed to lookup existing forwarding rule %s, err: %v", name, err) return nil } return fr } -func (l *L4) deleteForwardingRule(name string, version meta.Version) { - key, err := l.CreateKey(name) - if err != nil { - klog.Errorf("Failed to create key for deleting forwarding rule %s, err: %v", name, err) - return - } - if err := utils.IgnoreHTTPNotFound(composite.DeleteForwardingRule(l.cloud, key, version)); err != nil { - klog.Errorf("Failed to delete forwarding rule %s, err: %v", name, err) - } -} - // ensureExternalForwardingRule creates a forwarding rule with the given name for L4NetLB, // if it does not exist. It updates the existing forwarding rule if needed. func (l4netlb *L4NetLB) ensureExternalForwardingRule(bsLink string) (*composite.ForwardingRule, IPAddressType, error) { frName := l4netlb.GetFRName() - key, err := l4netlb.createKey(frName) + // version used for creating the existing forwarding rule. + version := meta.VersionGA + existingFwdRule, err := l4netlb.forwardingRules.GetForwardingRule(frName) if err != nil { + klog.Errorf("l4netlb.forwardingRules.GetForwardingRule(%s) returned error %v", frName, err) return nil, IPAddrUndefined, err } - // version used for creating the existing forwarding rule. - version := meta.VersionGA - existingFwdRule := l4netlb.GetForwardingRule(frName, version) // Determine IP which will be used for this LB. If no forwarding rule has been established // or specified in the Service spec, then requestedIP = "". @@ -358,7 +343,7 @@ func (l4netlb *L4NetLB) ensureExternalForwardingRule(bsLink string) (*composite. if err != nil { return nil, IPAddrUndefined, err } - klog.V(2).Infof("ensureForwardingRule(%v): reserved IP %q for the forwarding rule", frName, ipToUse) + klog.V(2).Infof("ensureExternalForwardingRule(%v): reserved IP %q for the forwarding rule", frName, ipToUse) defer func() { // Release the address that was reserved, in all cases. If the forwarding rule was successfully created, // the ephemeral IP is not needed anymore. If it was not created, the address should be released to prevent leaks. @@ -406,52 +391,39 @@ func (l4netlb *L4NetLB) ensureExternalForwardingRule(bsLink string) (*composite. // If the forwarding rule pointed to a backend service which does not match the controller naming scheme, // that resource could be leaked. It is not being deleted here because that is a user-managed resource. klog.V(2).Infof("ensureExternalForwardingRule: forwarding rule changed - Existing - %+v\n, New - %+v\n, Diff(-existing, +new) - %s\n. Deleting existing forwarding rule.", existingFwdRule, fr, frDiff) - if err = utils.IgnoreHTTPNotFound(composite.DeleteForwardingRule(l4netlb.cloud, key, version)); err != nil { + if err = l4netlb.forwardingRules.DeleteForwardingRule(existingFwdRule.Name); err != nil { return nil, IPAddrUndefined, err } - l4netlb.recorder.Eventf(l4netlb.Service, corev1.EventTypeNormal, events.SyncIngress, "ForwardingRule %q deleted", key.Name) + l4netlb.recorder.Eventf(l4netlb.Service, corev1.EventTypeNormal, events.SyncIngress, "ForwardingRule %q deleted", existingFwdRule.Name) } klog.V(2).Infof("ensureExternalForwardingRule: Creating/Recreating forwarding rule - %s", fr.Name) - if err = composite.CreateForwardingRule(l4netlb.cloud, key, fr); err != nil { + if err = l4netlb.forwardingRules.CreateForwardingRule(fr); err != nil { return nil, IPAddrUndefined, err } - createdFr, err := composite.GetForwardingRule(l4netlb.cloud, key, fr.Version) + createdFr, err := l4netlb.forwardingRules.GetForwardingRule(fr.Name) return createdFr, isIPManaged, err } // tearDownResourcesWithWrongNetworkTier removes forwarding rule or IP address if its Network Tier differs from desired. func (l4netlb *L4NetLB) tearDownResourcesWithWrongNetworkTier(existingFwdRule *composite.ForwardingRule, svcNetTier cloud.NetworkTier, am *addressManager) error { if existingFwdRule != nil && existingFwdRule.NetworkTier != svcNetTier.ToGCEValue() { - l4netlb.deleteForwardingRule(existingFwdRule.Name, meta.VersionGA) + err := l4netlb.forwardingRules.DeleteForwardingRule(existingFwdRule.Name) + if err != nil { + klog.Errorf("l4netlb.forwardingRules.DeleteForwardingRule(%s) returned error %v, want nil", existingFwdRule.Name, err) + } } return am.TearDownAddressIPIfNetworkTierMismatch() } -func (l4netlb *L4NetLB) GetForwardingRule(name string, version meta.Version) *composite.ForwardingRule { - key, err := l4netlb.createKey(name) +func (l4netlb *L4NetLB) GetForwardingRule(name string) *composite.ForwardingRule { + fr, err := l4netlb.forwardingRules.GetForwardingRule(name) if err != nil { - klog.Errorf("Failed to create key for fetching existing forwarding rule %s, err: %v", name, err) - return nil - } - fr, err := composite.GetForwardingRule(l4netlb.cloud, key, version) - if utils.IgnoreHTTPNotFound(err) != nil { klog.Errorf("Failed to lookup existing forwarding rule %s, err: %v", name, err) return nil } return fr } -func (l4netlb *L4NetLB) deleteForwardingRule(name string, version meta.Version) { - key, err := l4netlb.createKey(name) - if err != nil { - klog.Errorf("Failed to create key for deleting forwarding rule %s, err: %v", name, err) - return - } - if err := utils.IgnoreHTTPNotFound(composite.DeleteForwardingRule(l4netlb.cloud, key, version)); err != nil { - klog.Errorf("Failed to delete forwarding rule %s, err: %v", name, err) - } -} - func Equal(fr1, fr2 *composite.ForwardingRule) (bool, error) { id1, err := cloud.ParseResourceURL(fr1.BackendService) if err != nil { diff --git a/pkg/loadbalancers/l4.go b/pkg/loadbalancers/l4.go index bcce040449..6c8f407c4d 100644 --- a/pkg/loadbalancers/l4.go +++ b/pkg/loadbalancers/l4.go @@ -31,6 +31,7 @@ import ( "k8s.io/ingress-gce/pkg/backends" "k8s.io/ingress-gce/pkg/composite" "k8s.io/ingress-gce/pkg/firewalls" + "k8s.io/ingress-gce/pkg/forwardingrules" "k8s.io/ingress-gce/pkg/healthchecks" "k8s.io/ingress-gce/pkg/metrics" "k8s.io/ingress-gce/pkg/utils" @@ -47,11 +48,12 @@ type L4 struct { scope meta.KeyType namer namer.L4ResourcesNamer // recorder is used to generate k8s Events. - recorder record.EventRecorder - Service *corev1.Service - ServicePort utils.ServicePort - NamespacedName types.NamespacedName - l4HealthChecks healthchecks.L4HealthChecks + recorder record.EventRecorder + Service *corev1.Service + ServicePort utils.ServicePort + NamespacedName types.NamespacedName + l4HealthChecks healthchecks.L4HealthChecks + forwardingRules forwardingrules.ForwardingRules } // L4ILBSyncResult contains information about the outcome of an L4 ILB sync. It stores the list of resource name annotations, @@ -69,12 +71,13 @@ type L4ILBSyncResult struct { // NewL4Handler creates a new L4Handler for the given L4 service. func NewL4Handler(service *corev1.Service, cloud *gce.Cloud, scope meta.KeyType, namer namer.L4ResourcesNamer, recorder record.EventRecorder) *L4 { l := &L4{ - cloud: cloud, - scope: scope, - namer: namer, - recorder: recorder, - Service: service, - l4HealthChecks: healthchecks.L4(), + cloud: cloud, + scope: scope, + namer: namer, + recorder: recorder, + Service: service, + l4HealthChecks: healthchecks.L4(), + forwardingRules: forwardingrules.NewForwardingRulesService(cloud, meta.VersionGA, scope), } l.NamespacedName = types.NamespacedName{Name: service.Name, Namespace: service.Namespace} l.backendPool = backends.NewPool(l.cloud, l.namer) @@ -222,12 +225,16 @@ func (l *L4) EnsureInternalLoadBalancer(nodeNames []string, svc *corev1.Service) if err != nil { klog.Errorf("Failed to lookup existing backend service, ignoring err: %v", err) } - existingFR := l.GetForwardingRule(l.GetFRName(), meta.VersionGA) + existingFR, err := l.forwardingRules.GetForwardingRule(l.GetFRName()) if existingBS != nil && existingBS.Protocol != string(protocol) { klog.Infof("Protocol changed from %q to %q for service %s", existingBS.Protocol, string(protocol), l.NamespacedName) // Delete forwarding rule if it exists - existingFR = l.GetForwardingRule(l.getFRNameWithProtocol(existingBS.Protocol), meta.VersionGA) - l.deleteForwardingRule(l.getFRNameWithProtocol(existingBS.Protocol), meta.VersionGA) + frName := l.getFRNameWithProtocol(existingBS.Protocol) + existingFR, err = l.forwardingRules.GetForwardingRule(frName) + err := l.forwardingRules.DeleteForwardingRule(frName) + if err != nil { + klog.Errorf("Failed to delete forwarding rule %s, err %v", frName, err) + } } // ensure backend service diff --git a/pkg/loadbalancers/l4netlb.go b/pkg/loadbalancers/l4netlb.go index 05822645da..4d76e66987 100644 --- a/pkg/loadbalancers/l4netlb.go +++ b/pkg/loadbalancers/l4netlb.go @@ -30,6 +30,7 @@ import ( "k8s.io/ingress-gce/pkg/backends" "k8s.io/ingress-gce/pkg/composite" "k8s.io/ingress-gce/pkg/firewalls" + "k8s.io/ingress-gce/pkg/forwardingrules" "k8s.io/ingress-gce/pkg/healthchecks" "k8s.io/ingress-gce/pkg/metrics" "k8s.io/ingress-gce/pkg/utils" @@ -45,11 +46,12 @@ type L4NetLB struct { scope meta.KeyType namer namer.L4ResourcesNamer // recorder is used to generate k8s Events. - recorder record.EventRecorder - Service *corev1.Service - ServicePort utils.ServicePort - NamespacedName types.NamespacedName - l4HealthChecks healthchecks.L4HealthChecks + recorder record.EventRecorder + Service *corev1.Service + ServicePort utils.ServicePort + NamespacedName types.NamespacedName + l4HealthChecks healthchecks.L4HealthChecks + forwardingRules forwardingrules.ForwardingRules } // L4NetLBSyncResult contains information about the outcome of an L4 NetLB sync. It stores the list of resource name annotations, @@ -83,13 +85,14 @@ func (r *L4NetLBSyncResult) SetMetricsForSuccessfulServiceSync() { // NewL4NetLB creates a new Handler for the given L4NetLB service. func NewL4NetLB(service *corev1.Service, cloud *gce.Cloud, scope meta.KeyType, namer namer.L4ResourcesNamer, recorder record.EventRecorder) *L4NetLB { l4netlb := &L4NetLB{cloud: cloud, - scope: scope, - namer: namer, - recorder: recorder, - Service: service, - NamespacedName: types.NamespacedName{Name: service.Name, Namespace: service.Namespace}, - backendPool: backends.NewPool(cloud, namer), - l4HealthChecks: healthchecks.L4(), + scope: scope, + namer: namer, + recorder: recorder, + Service: service, + NamespacedName: types.NamespacedName{Name: service.Name, Namespace: service.Namespace}, + backendPool: backends.NewPool(cloud, namer), + l4HealthChecks: healthchecks.L4(), + forwardingRules: forwardingrules.NewForwardingRulesService(cloud, meta.VersionGA, scope), } portId := utils.ServicePortID{Service: l4netlb.NamespacedName} l4netlb.ServicePort = utils.ServicePort{ diff --git a/vendor/github.com/google/go-cmp/cmp/cmpopts/equate.go b/vendor/github.com/google/go-cmp/cmp/cmpopts/equate.go new file mode 100644 index 0000000000..e4ffca838a --- /dev/null +++ b/vendor/github.com/google/go-cmp/cmp/cmpopts/equate.go @@ -0,0 +1,148 @@ +// Copyright 2017, The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cmpopts provides common options for the cmp package. +package cmpopts + +import ( + "math" + "reflect" + "time" + + "github.com/google/go-cmp/cmp" +) + +func equateAlways(_, _ interface{}) bool { return true } + +// EquateEmpty returns a Comparer option that determines all maps and slices +// with a length of zero to be equal, regardless of whether they are nil. +// +// EquateEmpty can be used in conjunction with SortSlices and SortMaps. +func EquateEmpty() cmp.Option { + return cmp.FilterValues(isEmpty, cmp.Comparer(equateAlways)) +} + +func isEmpty(x, y interface{}) bool { + vx, vy := reflect.ValueOf(x), reflect.ValueOf(y) + return (x != nil && y != nil && vx.Type() == vy.Type()) && + (vx.Kind() == reflect.Slice || vx.Kind() == reflect.Map) && + (vx.Len() == 0 && vy.Len() == 0) +} + +// EquateApprox returns a Comparer option that determines float32 or float64 +// values to be equal if they are within a relative fraction or absolute margin. +// This option is not used when either x or y is NaN or infinite. +// +// The fraction determines that the difference of two values must be within the +// smaller fraction of the two values, while the margin determines that the two +// values must be within some absolute margin. +// To express only a fraction or only a margin, use 0 for the other parameter. +// The fraction and margin must be non-negative. +// +// The mathematical expression used is equivalent to: +// |x-y| ≤ max(fraction*min(|x|, |y|), margin) +// +// EquateApprox can be used in conjunction with EquateNaNs. +func EquateApprox(fraction, margin float64) cmp.Option { + if margin < 0 || fraction < 0 || math.IsNaN(margin) || math.IsNaN(fraction) { + panic("margin or fraction must be a non-negative number") + } + a := approximator{fraction, margin} + return cmp.Options{ + cmp.FilterValues(areRealF64s, cmp.Comparer(a.compareF64)), + cmp.FilterValues(areRealF32s, cmp.Comparer(a.compareF32)), + } +} + +type approximator struct{ frac, marg float64 } + +func areRealF64s(x, y float64) bool { + return !math.IsNaN(x) && !math.IsNaN(y) && !math.IsInf(x, 0) && !math.IsInf(y, 0) +} +func areRealF32s(x, y float32) bool { + return areRealF64s(float64(x), float64(y)) +} +func (a approximator) compareF64(x, y float64) bool { + relMarg := a.frac * math.Min(math.Abs(x), math.Abs(y)) + return math.Abs(x-y) <= math.Max(a.marg, relMarg) +} +func (a approximator) compareF32(x, y float32) bool { + return a.compareF64(float64(x), float64(y)) +} + +// EquateNaNs returns a Comparer option that determines float32 and float64 +// NaN values to be equal. +// +// EquateNaNs can be used in conjunction with EquateApprox. +func EquateNaNs() cmp.Option { + return cmp.Options{ + cmp.FilterValues(areNaNsF64s, cmp.Comparer(equateAlways)), + cmp.FilterValues(areNaNsF32s, cmp.Comparer(equateAlways)), + } +} + +func areNaNsF64s(x, y float64) bool { + return math.IsNaN(x) && math.IsNaN(y) +} +func areNaNsF32s(x, y float32) bool { + return areNaNsF64s(float64(x), float64(y)) +} + +// EquateApproxTime returns a Comparer option that determines two non-zero +// time.Time values to be equal if they are within some margin of one another. +// If both times have a monotonic clock reading, then the monotonic time +// difference will be used. The margin must be non-negative. +func EquateApproxTime(margin time.Duration) cmp.Option { + if margin < 0 { + panic("margin must be a non-negative number") + } + a := timeApproximator{margin} + return cmp.FilterValues(areNonZeroTimes, cmp.Comparer(a.compare)) +} + +func areNonZeroTimes(x, y time.Time) bool { + return !x.IsZero() && !y.IsZero() +} + +type timeApproximator struct { + margin time.Duration +} + +func (a timeApproximator) compare(x, y time.Time) bool { + // Avoid subtracting times to avoid overflow when the + // difference is larger than the largest representible duration. + if x.After(y) { + // Ensure x is always before y + x, y = y, x + } + // We're within the margin if x+margin >= y. + // Note: time.Time doesn't have AfterOrEqual method hence the negation. + return !x.Add(a.margin).Before(y) +} + +// AnyError is an error that matches any non-nil error. +var AnyError anyError + +type anyError struct{} + +func (anyError) Error() string { return "any error" } +func (anyError) Is(err error) bool { return err != nil } + +// EquateErrors returns a Comparer option that determines errors to be equal +// if errors.Is reports them to match. The AnyError error can be used to +// match any non-nil error. +func EquateErrors() cmp.Option { + return cmp.FilterValues(areConcreteErrors, cmp.Comparer(compareErrors)) +} + +// areConcreteErrors reports whether x and y are types that implement error. +// The input types are deliberately of the interface{} type rather than the +// error type so that we can handle situations where the current type is an +// interface{}, but the underlying concrete types both happen to implement +// the error interface. +func areConcreteErrors(x, y interface{}) bool { + _, ok1 := x.(error) + _, ok2 := y.(error) + return ok1 && ok2 +} diff --git a/vendor/github.com/google/go-cmp/cmp/cmpopts/errors_go113.go b/vendor/github.com/google/go-cmp/cmp/cmpopts/errors_go113.go new file mode 100644 index 0000000000..26fe25d6af --- /dev/null +++ b/vendor/github.com/google/go-cmp/cmp/cmpopts/errors_go113.go @@ -0,0 +1,15 @@ +// Copyright 2021, The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.13 + +package cmpopts + +import "errors" + +func compareErrors(x, y interface{}) bool { + xe := x.(error) + ye := y.(error) + return errors.Is(xe, ye) || errors.Is(ye, xe) +} diff --git a/vendor/github.com/google/go-cmp/cmp/cmpopts/errors_xerrors.go b/vendor/github.com/google/go-cmp/cmp/cmpopts/errors_xerrors.go new file mode 100644 index 0000000000..6eeb8d6e65 --- /dev/null +++ b/vendor/github.com/google/go-cmp/cmp/cmpopts/errors_xerrors.go @@ -0,0 +1,18 @@ +// Copyright 2021, The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !go1.13 + +// TODO(≥go1.13): For support on = 0 && (ss.less(v, start, i-1) || ss.less(v, i-1, start)) { + panic(fmt.Sprintf("incomparable values detected: want equal elements: %v", v.Slice(start, i))) + } + start = -1 + } else if start == -1 { + start = i + } + } +} +func (ss sliceSorter) less(v reflect.Value, i, j int) bool { + vx, vy := v.Index(i), v.Index(j) + return ss.fnc.Call([]reflect.Value{vx, vy})[0].Bool() +} + +// SortMaps returns a Transformer option that flattens map[K]V types to be a +// sorted []struct{K, V}. The less function must be of the form +// "func(T, T) bool" which is used to sort any map with key K that is +// assignable to T. +// +// Flattening the map into a slice has the property that cmp.Equal is able to +// use Comparers on K or the K.Equal method if it exists. +// +// The less function must be: +// • Deterministic: less(x, y) == less(x, y) +// • Irreflexive: !less(x, x) +// • Transitive: if !less(x, y) and !less(y, z), then !less(x, z) +// • Total: if x != y, then either less(x, y) or less(y, x) +// +// SortMaps can be used in conjunction with EquateEmpty. +func SortMaps(lessFunc interface{}) cmp.Option { + vf := reflect.ValueOf(lessFunc) + if !function.IsType(vf.Type(), function.Less) || vf.IsNil() { + panic(fmt.Sprintf("invalid less function: %T", lessFunc)) + } + ms := mapSorter{vf.Type().In(0), vf} + return cmp.FilterValues(ms.filter, cmp.Transformer("cmpopts.SortMaps", ms.sort)) +} + +type mapSorter struct { + in reflect.Type // T + fnc reflect.Value // func(T, T) bool +} + +func (ms mapSorter) filter(x, y interface{}) bool { + vx, vy := reflect.ValueOf(x), reflect.ValueOf(y) + return (x != nil && y != nil && vx.Type() == vy.Type()) && + (vx.Kind() == reflect.Map && vx.Type().Key().AssignableTo(ms.in)) && + (vx.Len() != 0 || vy.Len() != 0) +} +func (ms mapSorter) sort(x interface{}) interface{} { + src := reflect.ValueOf(x) + outType := reflect.StructOf([]reflect.StructField{ + {Name: "K", Type: src.Type().Key()}, + {Name: "V", Type: src.Type().Elem()}, + }) + dst := reflect.MakeSlice(reflect.SliceOf(outType), src.Len(), src.Len()) + for i, k := range src.MapKeys() { + v := reflect.New(outType).Elem() + v.Field(0).Set(k) + v.Field(1).Set(src.MapIndex(k)) + dst.Index(i).Set(v) + } + sort.Slice(dst.Interface(), func(i, j int) bool { return ms.less(dst, i, j) }) + ms.checkSort(dst) + return dst.Interface() +} +func (ms mapSorter) checkSort(v reflect.Value) { + for i := 1; i < v.Len(); i++ { + if !ms.less(v, i-1, i) { + panic(fmt.Sprintf("partial order detected: want %v < %v", v.Index(i-1), v.Index(i))) + } + } +} +func (ms mapSorter) less(v reflect.Value, i, j int) bool { + vx, vy := v.Index(i).Field(0), v.Index(j).Field(0) + return ms.fnc.Call([]reflect.Value{vx, vy})[0].Bool() +} diff --git a/vendor/github.com/google/go-cmp/cmp/cmpopts/struct_filter.go b/vendor/github.com/google/go-cmp/cmp/cmpopts/struct_filter.go new file mode 100644 index 0000000000..a09829c3af --- /dev/null +++ b/vendor/github.com/google/go-cmp/cmp/cmpopts/struct_filter.go @@ -0,0 +1,187 @@ +// Copyright 2017, The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmpopts + +import ( + "fmt" + "reflect" + "strings" + + "github.com/google/go-cmp/cmp" +) + +// filterField returns a new Option where opt is only evaluated on paths that +// include a specific exported field on a single struct type. +// The struct type is specified by passing in a value of that type. +// +// The name may be a dot-delimited string (e.g., "Foo.Bar") to select a +// specific sub-field that is embedded or nested within the parent struct. +func filterField(typ interface{}, name string, opt cmp.Option) cmp.Option { + // TODO: This is currently unexported over concerns of how helper filters + // can be composed together easily. + // TODO: Add tests for FilterField. + + sf := newStructFilter(typ, name) + return cmp.FilterPath(sf.filter, opt) +} + +type structFilter struct { + t reflect.Type // The root struct type to match on + ft fieldTree // Tree of fields to match on +} + +func newStructFilter(typ interface{}, names ...string) structFilter { + // TODO: Perhaps allow * as a special identifier to allow ignoring any + // number of path steps until the next field match? + // This could be useful when a concrete struct gets transformed into + // an anonymous struct where it is not possible to specify that by type, + // but the transformer happens to provide guarantees about the names of + // the transformed fields. + + t := reflect.TypeOf(typ) + if t == nil || t.Kind() != reflect.Struct { + panic(fmt.Sprintf("%T must be a non-pointer struct", typ)) + } + var ft fieldTree + for _, name := range names { + cname, err := canonicalName(t, name) + if err != nil { + panic(fmt.Sprintf("%s: %v", strings.Join(cname, "."), err)) + } + ft.insert(cname) + } + return structFilter{t, ft} +} + +func (sf structFilter) filter(p cmp.Path) bool { + for i, ps := range p { + if ps.Type().AssignableTo(sf.t) && sf.ft.matchPrefix(p[i+1:]) { + return true + } + } + return false +} + +// fieldTree represents a set of dot-separated identifiers. +// +// For example, inserting the following selectors: +// Foo +// Foo.Bar.Baz +// Foo.Buzz +// Nuka.Cola.Quantum +// +// Results in a tree of the form: +// {sub: { +// "Foo": {ok: true, sub: { +// "Bar": {sub: { +// "Baz": {ok: true}, +// }}, +// "Buzz": {ok: true}, +// }}, +// "Nuka": {sub: { +// "Cola": {sub: { +// "Quantum": {ok: true}, +// }}, +// }}, +// }} +type fieldTree struct { + ok bool // Whether this is a specified node + sub map[string]fieldTree // The sub-tree of fields under this node +} + +// insert inserts a sequence of field accesses into the tree. +func (ft *fieldTree) insert(cname []string) { + if ft.sub == nil { + ft.sub = make(map[string]fieldTree) + } + if len(cname) == 0 { + ft.ok = true + return + } + sub := ft.sub[cname[0]] + sub.insert(cname[1:]) + ft.sub[cname[0]] = sub +} + +// matchPrefix reports whether any selector in the fieldTree matches +// the start of path p. +func (ft fieldTree) matchPrefix(p cmp.Path) bool { + for _, ps := range p { + switch ps := ps.(type) { + case cmp.StructField: + ft = ft.sub[ps.Name()] + if ft.ok { + return true + } + if len(ft.sub) == 0 { + return false + } + case cmp.Indirect: + default: + return false + } + } + return false +} + +// canonicalName returns a list of identifiers where any struct field access +// through an embedded field is expanded to include the names of the embedded +// types themselves. +// +// For example, suppose field "Foo" is not directly in the parent struct, +// but actually from an embedded struct of type "Bar". Then, the canonical name +// of "Foo" is actually "Bar.Foo". +// +// Suppose field "Foo" is not directly in the parent struct, but actually +// a field in two different embedded structs of types "Bar" and "Baz". +// Then the selector "Foo" causes a panic since it is ambiguous which one it +// refers to. The user must specify either "Bar.Foo" or "Baz.Foo". +func canonicalName(t reflect.Type, sel string) ([]string, error) { + var name string + sel = strings.TrimPrefix(sel, ".") + if sel == "" { + return nil, fmt.Errorf("name must not be empty") + } + if i := strings.IndexByte(sel, '.'); i < 0 { + name, sel = sel, "" + } else { + name, sel = sel[:i], sel[i:] + } + + // Type must be a struct or pointer to struct. + if t.Kind() == reflect.Ptr { + t = t.Elem() + } + if t.Kind() != reflect.Struct { + return nil, fmt.Errorf("%v must be a struct", t) + } + + // Find the canonical name for this current field name. + // If the field exists in an embedded struct, then it will be expanded. + sf, _ := t.FieldByName(name) + if !isExported(name) { + // Avoid using reflect.Type.FieldByName for unexported fields due to + // buggy behavior with regard to embeddeding and unexported fields. + // See https://golang.org/issue/4876 for details. + sf = reflect.StructField{} + for i := 0; i < t.NumField() && sf.Name == ""; i++ { + if t.Field(i).Name == name { + sf = t.Field(i) + } + } + } + if sf.Name == "" { + return []string{name}, fmt.Errorf("does not exist") + } + var ss []string + for i := range sf.Index { + ss = append(ss, t.FieldByIndex(sf.Index[:i+1]).Name) + } + if sel == "" { + return ss, nil + } + ssPost, err := canonicalName(sf.Type, sel) + return append(ss, ssPost...), err +} diff --git a/vendor/github.com/google/go-cmp/cmp/cmpopts/xform.go b/vendor/github.com/google/go-cmp/cmp/cmpopts/xform.go new file mode 100644 index 0000000000..4eb49d63db --- /dev/null +++ b/vendor/github.com/google/go-cmp/cmp/cmpopts/xform.go @@ -0,0 +1,35 @@ +// Copyright 2018, The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cmpopts + +import ( + "github.com/google/go-cmp/cmp" +) + +type xformFilter struct{ xform cmp.Option } + +func (xf xformFilter) filter(p cmp.Path) bool { + for _, ps := range p { + if t, ok := ps.(cmp.Transform); ok && t.Option() == xf.xform { + return false + } + } + return true +} + +// AcyclicTransformer returns a Transformer with a filter applied that ensures +// that the transformer cannot be recursively applied upon its own output. +// +// An example use case is a transformer that splits a string by lines: +// AcyclicTransformer("SplitLines", func(s string) []string{ +// return strings.Split(s, "\n") +// }) +// +// Had this been an unfiltered Transformer instead, this would result in an +// infinite cycle converting a string to []string to [][]string and so on. +func AcyclicTransformer(name string, xformFunc interface{}) cmp.Option { + xf := xformFilter{cmp.Transformer(name, xformFunc)} + return cmp.FilterPath(xf.filter, xf.xform) +} diff --git a/vendor/golang.org/x/xerrors/LICENSE b/vendor/golang.org/x/xerrors/LICENSE new file mode 100644 index 0000000000..e4a47e17f1 --- /dev/null +++ b/vendor/golang.org/x/xerrors/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2019 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/golang.org/x/xerrors/PATENTS b/vendor/golang.org/x/xerrors/PATENTS new file mode 100644 index 0000000000..733099041f --- /dev/null +++ b/vendor/golang.org/x/xerrors/PATENTS @@ -0,0 +1,22 @@ +Additional IP Rights Grant (Patents) + +"This implementation" means the copyrightable works distributed by +Google as part of the Go project. + +Google hereby grants to You a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable (except as stated in this section) +patent license to make, have made, use, offer to sell, sell, import, +transfer and otherwise run, modify and propagate the contents of this +implementation of Go, where such license applies only to those patent +claims, both currently owned or controlled by Google and acquired in +the future, licensable by Google that are necessarily infringed by this +implementation of Go. This grant does not include claims that would be +infringed only as a consequence of further modification of this +implementation. If you or your agent or exclusive licensee institute or +order or agree to the institution of patent litigation against any +entity (including a cross-claim or counterclaim in a lawsuit) alleging +that this implementation of Go or any code incorporated within this +implementation of Go constitutes direct or contributory patent +infringement, or inducement of patent infringement, then any patent +rights granted to you under this License for this implementation of Go +shall terminate as of the date such litigation is filed. diff --git a/vendor/golang.org/x/xerrors/README b/vendor/golang.org/x/xerrors/README new file mode 100644 index 0000000000..aac7867a56 --- /dev/null +++ b/vendor/golang.org/x/xerrors/README @@ -0,0 +1,2 @@ +This repository holds the transition packages for the new Go 1.13 error values. +See golang.org/design/29934-error-values. diff --git a/vendor/golang.org/x/xerrors/adaptor.go b/vendor/golang.org/x/xerrors/adaptor.go new file mode 100644 index 0000000000..4317f24833 --- /dev/null +++ b/vendor/golang.org/x/xerrors/adaptor.go @@ -0,0 +1,193 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xerrors + +import ( + "bytes" + "fmt" + "io" + "reflect" + "strconv" +) + +// FormatError calls the FormatError method of f with an errors.Printer +// configured according to s and verb, and writes the result to s. +func FormatError(f Formatter, s fmt.State, verb rune) { + // Assuming this function is only called from the Format method, and given + // that FormatError takes precedence over Format, it cannot be called from + // any package that supports errors.Formatter. It is therefore safe to + // disregard that State may be a specific printer implementation and use one + // of our choice instead. + + // limitations: does not support printing error as Go struct. + + var ( + sep = " " // separator before next error + p = &state{State: s} + direct = true + ) + + var err error = f + + switch verb { + // Note that this switch must match the preference order + // for ordinary string printing (%#v before %+v, and so on). + + case 'v': + if s.Flag('#') { + if stringer, ok := err.(fmt.GoStringer); ok { + io.WriteString(&p.buf, stringer.GoString()) + goto exit + } + // proceed as if it were %v + } else if s.Flag('+') { + p.printDetail = true + sep = "\n - " + } + case 's': + case 'q', 'x', 'X': + // Use an intermediate buffer in the rare cases that precision, + // truncation, or one of the alternative verbs (q, x, and X) are + // specified. + direct = false + + default: + p.buf.WriteString("%!") + p.buf.WriteRune(verb) + p.buf.WriteByte('(') + switch { + case err != nil: + p.buf.WriteString(reflect.TypeOf(f).String()) + default: + p.buf.WriteString("") + } + p.buf.WriteByte(')') + io.Copy(s, &p.buf) + return + } + +loop: + for { + switch v := err.(type) { + case Formatter: + err = v.FormatError((*printer)(p)) + case fmt.Formatter: + v.Format(p, 'v') + break loop + default: + io.WriteString(&p.buf, v.Error()) + break loop + } + if err == nil { + break + } + if p.needColon || !p.printDetail { + p.buf.WriteByte(':') + p.needColon = false + } + p.buf.WriteString(sep) + p.inDetail = false + p.needNewline = false + } + +exit: + width, okW := s.Width() + prec, okP := s.Precision() + + if !direct || (okW && width > 0) || okP { + // Construct format string from State s. + format := []byte{'%'} + if s.Flag('-') { + format = append(format, '-') + } + if s.Flag('+') { + format = append(format, '+') + } + if s.Flag(' ') { + format = append(format, ' ') + } + if okW { + format = strconv.AppendInt(format, int64(width), 10) + } + if okP { + format = append(format, '.') + format = strconv.AppendInt(format, int64(prec), 10) + } + format = append(format, string(verb)...) + fmt.Fprintf(s, string(format), p.buf.String()) + } else { + io.Copy(s, &p.buf) + } +} + +var detailSep = []byte("\n ") + +// state tracks error printing state. It implements fmt.State. +type state struct { + fmt.State + buf bytes.Buffer + + printDetail bool + inDetail bool + needColon bool + needNewline bool +} + +func (s *state) Write(b []byte) (n int, err error) { + if s.printDetail { + if len(b) == 0 { + return 0, nil + } + if s.inDetail && s.needColon { + s.needNewline = true + if b[0] == '\n' { + b = b[1:] + } + } + k := 0 + for i, c := range b { + if s.needNewline { + if s.inDetail && s.needColon { + s.buf.WriteByte(':') + s.needColon = false + } + s.buf.Write(detailSep) + s.needNewline = false + } + if c == '\n' { + s.buf.Write(b[k:i]) + k = i + 1 + s.needNewline = true + } + } + s.buf.Write(b[k:]) + if !s.inDetail { + s.needColon = true + } + } else if !s.inDetail { + s.buf.Write(b) + } + return len(b), nil +} + +// printer wraps a state to implement an xerrors.Printer. +type printer state + +func (s *printer) Print(args ...interface{}) { + if !s.inDetail || s.printDetail { + fmt.Fprint((*state)(s), args...) + } +} + +func (s *printer) Printf(format string, args ...interface{}) { + if !s.inDetail || s.printDetail { + fmt.Fprintf((*state)(s), format, args...) + } +} + +func (s *printer) Detail() bool { + s.inDetail = true + return s.printDetail +} diff --git a/vendor/golang.org/x/xerrors/codereview.cfg b/vendor/golang.org/x/xerrors/codereview.cfg new file mode 100644 index 0000000000..3f8b14b64e --- /dev/null +++ b/vendor/golang.org/x/xerrors/codereview.cfg @@ -0,0 +1 @@ +issuerepo: golang/go diff --git a/vendor/golang.org/x/xerrors/doc.go b/vendor/golang.org/x/xerrors/doc.go new file mode 100644 index 0000000000..eef99d9d54 --- /dev/null +++ b/vendor/golang.org/x/xerrors/doc.go @@ -0,0 +1,22 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package xerrors implements functions to manipulate errors. +// +// This package is based on the Go 2 proposal for error values: +// https://golang.org/design/29934-error-values +// +// These functions were incorporated into the standard library's errors package +// in Go 1.13: +// - Is +// - As +// - Unwrap +// +// Also, Errorf's %w verb was incorporated into fmt.Errorf. +// +// Use this package to get equivalent behavior in all supported Go versions. +// +// No other features of this package were included in Go 1.13, and at present +// there are no plans to include any of them. +package xerrors // import "golang.org/x/xerrors" diff --git a/vendor/golang.org/x/xerrors/errors.go b/vendor/golang.org/x/xerrors/errors.go new file mode 100644 index 0000000000..e88d3772d8 --- /dev/null +++ b/vendor/golang.org/x/xerrors/errors.go @@ -0,0 +1,33 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xerrors + +import "fmt" + +// errorString is a trivial implementation of error. +type errorString struct { + s string + frame Frame +} + +// New returns an error that formats as the given text. +// +// The returned error contains a Frame set to the caller's location and +// implements Formatter to show this information when printed with details. +func New(text string) error { + return &errorString{text, Caller(1)} +} + +func (e *errorString) Error() string { + return e.s +} + +func (e *errorString) Format(s fmt.State, v rune) { FormatError(e, s, v) } + +func (e *errorString) FormatError(p Printer) (next error) { + p.Print(e.s) + e.frame.Format(p) + return nil +} diff --git a/vendor/golang.org/x/xerrors/fmt.go b/vendor/golang.org/x/xerrors/fmt.go new file mode 100644 index 0000000000..829862ddf6 --- /dev/null +++ b/vendor/golang.org/x/xerrors/fmt.go @@ -0,0 +1,187 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xerrors + +import ( + "fmt" + "strings" + "unicode" + "unicode/utf8" + + "golang.org/x/xerrors/internal" +) + +const percentBangString = "%!" + +// Errorf formats according to a format specifier and returns the string as a +// value that satisfies error. +// +// The returned error includes the file and line number of the caller when +// formatted with additional detail enabled. If the last argument is an error +// the returned error's Format method will return it if the format string ends +// with ": %s", ": %v", or ": %w". If the last argument is an error and the +// format string ends with ": %w", the returned error implements an Unwrap +// method returning it. +// +// If the format specifier includes a %w verb with an error operand in a +// position other than at the end, the returned error will still implement an +// Unwrap method returning the operand, but the error's Format method will not +// return the wrapped error. +// +// It is invalid to include more than one %w verb or to supply it with an +// operand that does not implement the error interface. The %w verb is otherwise +// a synonym for %v. +func Errorf(format string, a ...interface{}) error { + format = formatPlusW(format) + // Support a ": %[wsv]" suffix, which works well with xerrors.Formatter. + wrap := strings.HasSuffix(format, ": %w") + idx, format2, ok := parsePercentW(format) + percentWElsewhere := !wrap && idx >= 0 + if !percentWElsewhere && (wrap || strings.HasSuffix(format, ": %s") || strings.HasSuffix(format, ": %v")) { + err := errorAt(a, len(a)-1) + if err == nil { + return &noWrapError{fmt.Sprintf(format, a...), nil, Caller(1)} + } + // TODO: this is not entirely correct. The error value could be + // printed elsewhere in format if it mixes numbered with unnumbered + // substitutions. With relatively small changes to doPrintf we can + // have it optionally ignore extra arguments and pass the argument + // list in its entirety. + msg := fmt.Sprintf(format[:len(format)-len(": %s")], a[:len(a)-1]...) + frame := Frame{} + if internal.EnableTrace { + frame = Caller(1) + } + if wrap { + return &wrapError{msg, err, frame} + } + return &noWrapError{msg, err, frame} + } + // Support %w anywhere. + // TODO: don't repeat the wrapped error's message when %w occurs in the middle. + msg := fmt.Sprintf(format2, a...) + if idx < 0 { + return &noWrapError{msg, nil, Caller(1)} + } + err := errorAt(a, idx) + if !ok || err == nil { + // Too many %ws or argument of %w is not an error. Approximate the Go + // 1.13 fmt.Errorf message. + return &noWrapError{fmt.Sprintf("%sw(%s)", percentBangString, msg), nil, Caller(1)} + } + frame := Frame{} + if internal.EnableTrace { + frame = Caller(1) + } + return &wrapError{msg, err, frame} +} + +func errorAt(args []interface{}, i int) error { + if i < 0 || i >= len(args) { + return nil + } + err, ok := args[i].(error) + if !ok { + return nil + } + return err +} + +// formatPlusW is used to avoid the vet check that will barf at %w. +func formatPlusW(s string) string { + return s +} + +// Return the index of the only %w in format, or -1 if none. +// Also return a rewritten format string with %w replaced by %v, and +// false if there is more than one %w. +// TODO: handle "%[N]w". +func parsePercentW(format string) (idx int, newFormat string, ok bool) { + // Loosely copied from golang.org/x/tools/go/analysis/passes/printf/printf.go. + idx = -1 + ok = true + n := 0 + sz := 0 + var isW bool + for i := 0; i < len(format); i += sz { + if format[i] != '%' { + sz = 1 + continue + } + // "%%" is not a format directive. + if i+1 < len(format) && format[i+1] == '%' { + sz = 2 + continue + } + sz, isW = parsePrintfVerb(format[i:]) + if isW { + if idx >= 0 { + ok = false + } else { + idx = n + } + // "Replace" the last character, the 'w', with a 'v'. + p := i + sz - 1 + format = format[:p] + "v" + format[p+1:] + } + n++ + } + return idx, format, ok +} + +// Parse the printf verb starting with a % at s[0]. +// Return how many bytes it occupies and whether the verb is 'w'. +func parsePrintfVerb(s string) (int, bool) { + // Assume only that the directive is a sequence of non-letters followed by a single letter. + sz := 0 + var r rune + for i := 1; i < len(s); i += sz { + r, sz = utf8.DecodeRuneInString(s[i:]) + if unicode.IsLetter(r) { + return i + sz, r == 'w' + } + } + return len(s), false +} + +type noWrapError struct { + msg string + err error + frame Frame +} + +func (e *noWrapError) Error() string { + return fmt.Sprint(e) +} + +func (e *noWrapError) Format(s fmt.State, v rune) { FormatError(e, s, v) } + +func (e *noWrapError) FormatError(p Printer) (next error) { + p.Print(e.msg) + e.frame.Format(p) + return e.err +} + +type wrapError struct { + msg string + err error + frame Frame +} + +func (e *wrapError) Error() string { + return fmt.Sprint(e) +} + +func (e *wrapError) Format(s fmt.State, v rune) { FormatError(e, s, v) } + +func (e *wrapError) FormatError(p Printer) (next error) { + p.Print(e.msg) + e.frame.Format(p) + return e.err +} + +func (e *wrapError) Unwrap() error { + return e.err +} diff --git a/vendor/golang.org/x/xerrors/format.go b/vendor/golang.org/x/xerrors/format.go new file mode 100644 index 0000000000..1bc9c26b97 --- /dev/null +++ b/vendor/golang.org/x/xerrors/format.go @@ -0,0 +1,34 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xerrors + +// A Formatter formats error messages. +type Formatter interface { + error + + // FormatError prints the receiver's first error and returns the next error in + // the error chain, if any. + FormatError(p Printer) (next error) +} + +// A Printer formats error messages. +// +// The most common implementation of Printer is the one provided by package fmt +// during Printf (as of Go 1.13). Localization packages such as golang.org/x/text/message +// typically provide their own implementations. +type Printer interface { + // Print appends args to the message output. + Print(args ...interface{}) + + // Printf writes a formatted string. + Printf(format string, args ...interface{}) + + // Detail reports whether error detail is requested. + // After the first call to Detail, all text written to the Printer + // is formatted as additional detail, or ignored when + // detail has not been requested. + // If Detail returns false, the caller can avoid printing the detail at all. + Detail() bool +} diff --git a/vendor/golang.org/x/xerrors/frame.go b/vendor/golang.org/x/xerrors/frame.go new file mode 100644 index 0000000000..0de628ec50 --- /dev/null +++ b/vendor/golang.org/x/xerrors/frame.go @@ -0,0 +1,56 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xerrors + +import ( + "runtime" +) + +// A Frame contains part of a call stack. +type Frame struct { + // Make room for three PCs: the one we were asked for, what it called, + // and possibly a PC for skipPleaseUseCallersFrames. See: + // https://go.googlesource.com/go/+/032678e0fb/src/runtime/extern.go#169 + frames [3]uintptr +} + +// Caller returns a Frame that describes a frame on the caller's stack. +// The argument skip is the number of frames to skip over. +// Caller(0) returns the frame for the caller of Caller. +func Caller(skip int) Frame { + var s Frame + runtime.Callers(skip+1, s.frames[:]) + return s +} + +// location reports the file, line, and function of a frame. +// +// The returned function may be "" even if file and line are not. +func (f Frame) location() (function, file string, line int) { + frames := runtime.CallersFrames(f.frames[:]) + if _, ok := frames.Next(); !ok { + return "", "", 0 + } + fr, ok := frames.Next() + if !ok { + return "", "", 0 + } + return fr.Function, fr.File, fr.Line +} + +// Format prints the stack as error detail. +// It should be called from an error's Format implementation +// after printing any other error detail. +func (f Frame) Format(p Printer) { + if p.Detail() { + function, file, line := f.location() + if function != "" { + p.Printf("%s\n ", function) + } + if file != "" { + p.Printf("%s:%d\n", file, line) + } + } +} diff --git a/vendor/golang.org/x/xerrors/go.mod b/vendor/golang.org/x/xerrors/go.mod new file mode 100644 index 0000000000..870d4f612d --- /dev/null +++ b/vendor/golang.org/x/xerrors/go.mod @@ -0,0 +1,3 @@ +module golang.org/x/xerrors + +go 1.11 diff --git a/vendor/golang.org/x/xerrors/internal/internal.go b/vendor/golang.org/x/xerrors/internal/internal.go new file mode 100644 index 0000000000..89f4eca5df --- /dev/null +++ b/vendor/golang.org/x/xerrors/internal/internal.go @@ -0,0 +1,8 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package internal + +// EnableTrace indicates whether stack information should be recorded in errors. +var EnableTrace = true diff --git a/vendor/golang.org/x/xerrors/wrap.go b/vendor/golang.org/x/xerrors/wrap.go new file mode 100644 index 0000000000..9a3b510374 --- /dev/null +++ b/vendor/golang.org/x/xerrors/wrap.go @@ -0,0 +1,106 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package xerrors + +import ( + "reflect" +) + +// A Wrapper provides context around another error. +type Wrapper interface { + // Unwrap returns the next error in the error chain. + // If there is no next error, Unwrap returns nil. + Unwrap() error +} + +// Opaque returns an error with the same error formatting as err +// but that does not match err and cannot be unwrapped. +func Opaque(err error) error { + return noWrapper{err} +} + +type noWrapper struct { + error +} + +func (e noWrapper) FormatError(p Printer) (next error) { + if f, ok := e.error.(Formatter); ok { + return f.FormatError(p) + } + p.Print(e.error) + return nil +} + +// Unwrap returns the result of calling the Unwrap method on err, if err implements +// Unwrap. Otherwise, Unwrap returns nil. +func Unwrap(err error) error { + u, ok := err.(Wrapper) + if !ok { + return nil + } + return u.Unwrap() +} + +// Is reports whether any error in err's chain matches target. +// +// An error is considered to match a target if it is equal to that target or if +// it implements a method Is(error) bool such that Is(target) returns true. +func Is(err, target error) bool { + if target == nil { + return err == target + } + + isComparable := reflect.TypeOf(target).Comparable() + for { + if isComparable && err == target { + return true + } + if x, ok := err.(interface{ Is(error) bool }); ok && x.Is(target) { + return true + } + // TODO: consider supporing target.Is(err). This would allow + // user-definable predicates, but also may allow for coping with sloppy + // APIs, thereby making it easier to get away with them. + if err = Unwrap(err); err == nil { + return false + } + } +} + +// As finds the first error in err's chain that matches the type to which target +// points, and if so, sets the target to its value and returns true. An error +// matches a type if it is assignable to the target type, or if it has a method +// As(interface{}) bool such that As(target) returns true. As will panic if target +// is not a non-nil pointer to a type which implements error or is of interface type. +// +// The As method should set the target to its value and return true if err +// matches the type to which target points. +func As(err error, target interface{}) bool { + if target == nil { + panic("errors: target cannot be nil") + } + val := reflect.ValueOf(target) + typ := val.Type() + if typ.Kind() != reflect.Ptr || val.IsNil() { + panic("errors: target must be a non-nil pointer") + } + if e := typ.Elem(); e.Kind() != reflect.Interface && !e.Implements(errorType) { + panic("errors: *target must be interface or implement error") + } + targetType := typ.Elem() + for err != nil { + if reflect.TypeOf(err).AssignableTo(targetType) { + val.Elem().Set(reflect.ValueOf(err)) + return true + } + if x, ok := err.(interface{ As(interface{}) bool }); ok && x.As(target) { + return true + } + err = Unwrap(err) + } + return false +} + +var errorType = reflect.TypeOf((*error)(nil)).Elem() diff --git a/vendor/modules.txt b/vendor/modules.txt index 71dabf2039..d3d541ded7 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -112,6 +112,7 @@ github.com/golang/protobuf/ptypes/wrappers # github.com/google/go-cmp v0.5.6 ## explicit github.com/google/go-cmp/cmp +github.com/google/go-cmp/cmp/cmpopts github.com/google/go-cmp/cmp/internal/diff github.com/google/go-cmp/cmp/internal/flags github.com/google/go-cmp/cmp/internal/function @@ -225,6 +226,9 @@ golang.org/x/text/unicode/norm golang.org/x/text/width # golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac golang.org/x/time/rate +# golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 +golang.org/x/xerrors +golang.org/x/xerrors/internal # google.golang.org/api v0.60.0 ## explicit google.golang.org/api/compute/v0.alpha