From 1b2319bf1168582d47ffdb6c8359b36d6165b62b Mon Sep 17 00:00:00 2001 From: John Murret Date: Wed, 16 Aug 2023 19:53:06 -0600 Subject: [PATCH] NET-4853 - xds v2 - implement base connect proxy functionality for clusters --- agent/xds/clusters.go | 25 +- agent/xds/clusters_test.go | 219 ++- agent/xds/delta_envoy_extender_oss_test.go | 6 +- agent/xds/endpoints.go | 21 +- agent/xds/endpoints_test.go | 30 +- agent/xds/listeners.go | 7 +- agent/xds/listeners_test.go | 63 +- agent/xds/proxystateconverter/clusters.go | 1183 ++++++++++++++++- agent/xds/proxystateconverter/converter.go | 6 +- agent/xds/proxystateconverter/endpoints.go | 85 ++ .../proxystateconverter/failover_policy.go | 158 +++ .../failover_policy_oss.go | 15 + agent/xds/proxystateconverter/listeners.go | 11 +- agent/xds/rbac.go | 11 +- agent/xds/resources_test.go | 3 +- agent/xds/{ => response}/response.go | 14 +- agent/xds/routes.go | 33 +- agent/xds/routes_test.go | 3 +- agent/xds/xds_protocol_helpers_test.go | 23 +- agent/xdsv2/cluster_resources.go | 327 ++++- agent/xdsv2/endpoint_resources.go | 41 + agent/xdsv2/listener_resources.go | 7 +- agent/xdsv2/resources.go | 8 +- agent/xdsv2/route_resources.go | 3 + .../v1alpha1/pbproxystate/cluster.pb.go | 26 +- .../v1alpha1/pbproxystate/cluster.proto | 4 + .../pbproxystate/transport_socket.pb.go | 414 +++--- .../pbproxystate/transport_socket.proto | 14 +- 28 files changed, 2410 insertions(+), 350 deletions(-) create mode 100644 agent/xds/proxystateconverter/endpoints.go create mode 100644 agent/xds/proxystateconverter/failover_policy.go create mode 100644 agent/xds/proxystateconverter/failover_policy_oss.go rename agent/xds/{ => response}/response.go (82%) create mode 100644 agent/xdsv2/endpoint_resources.go diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index f56700c02a91..908aa746bc9c 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -32,6 +32,7 @@ import ( "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/proto/private/pbpeering" ) @@ -1008,11 +1009,11 @@ func (s *ResourceGenerator) configIngressUpstreamCluster(c *envoy_cluster_v3.Clu switch limitType { case "max_connections": - threshold.MaxConnections = makeUint32Value(limit) + threshold.MaxConnections = response.MakeUint32Value(limit) case "max_pending_requests": - threshold.MaxPendingRequests = makeUint32Value(limit) + threshold.MaxPendingRequests = response.MakeUint32Value(limit) case "max_requests": - threshold.MaxRequests = makeUint32Value(limit) + threshold.MaxRequests = response.MakeUint32Value(limit) } } @@ -1103,7 +1104,7 @@ func (s *ResourceGenerator) makeAppCluster(cfgSnap *proxycfg.ConfigSnapshot, nam c.CircuitBreakers = &envoy_cluster_v3.CircuitBreakers{ Thresholds: []*envoy_cluster_v3.CircuitBreakers_Thresholds{ { - MaxConnections: makeUint32Value(cfg.MaxInboundConnections), + MaxConnections: response.MakeUint32Value(cfg.MaxInboundConnections), }, }, } @@ -1708,13 +1709,13 @@ func (s *ResourceGenerator) makeGatewayCluster(snap *proxycfg.ConfigSnapshot, op TcpKeepalive: &envoy_core_v3.TcpKeepalive{}, } if cfg.TcpKeepaliveTime != 0 { - cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveTime = makeUint32Value(cfg.TcpKeepaliveTime) + cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveTime = response.MakeUint32Value(cfg.TcpKeepaliveTime) } if cfg.TcpKeepaliveInterval != 0 { - cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveInterval = makeUint32Value(cfg.TcpKeepaliveInterval) + cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveInterval = response.MakeUint32Value(cfg.TcpKeepaliveInterval) } if cfg.TcpKeepaliveProbes != 0 { - cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveProbes = makeUint32Value(cfg.TcpKeepaliveProbes) + cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveProbes = response.MakeUint32Value(cfg.TcpKeepaliveProbes) } } @@ -1884,7 +1885,7 @@ func (s *ResourceGenerator) makeExternalHostnameCluster(snap *proxycfg.ConfigSna endpoints := make([]*envoy_endpoint_v3.LbEndpoint, 0, len(opts.addresses)) for _, pair := range opts.addresses { - address := makeAddress(pair.Address, pair.Port) + address := response.MakeAddress(pair.Address, pair.Port) endpoint := &envoy_endpoint_v3.LbEndpoint{ HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ @@ -1917,13 +1918,13 @@ func makeThresholdsIfNeeded(limits *structs.UpstreamLimits) []*envoy_cluster_v3. // Likewise, make sure to not set any threshold values on the zero-value in // order to rely on Envoy defaults if limits.MaxConnections != nil { - threshold.MaxConnections = makeUint32Value(*limits.MaxConnections) + threshold.MaxConnections = response.MakeUint32Value(*limits.MaxConnections) } if limits.MaxPendingRequests != nil { - threshold.MaxPendingRequests = makeUint32Value(*limits.MaxPendingRequests) + threshold.MaxPendingRequests = response.MakeUint32Value(*limits.MaxPendingRequests) } if limits.MaxConcurrentRequests != nil { - threshold.MaxRequests = makeUint32Value(*limits.MaxConcurrentRequests) + threshold.MaxRequests = response.MakeUint32Value(*limits.MaxConcurrentRequests) } return []*envoy_cluster_v3.CircuitBreakers_Thresholds{threshold} @@ -1946,7 +1947,7 @@ func makeLbEndpoint(addr string, port int, health envoy_core_v3.HealthStatus, we }, }, HealthStatus: health, - LoadBalancingWeight: makeUint32Value(weight), + LoadBalancingWeight: response.MakeUint32Value(weight), } } diff --git a/agent/xds/clusters_test.go b/agent/xds/clusters_test.go index 32723bf48143..2651d77c2844 100644 --- a/agent/xds/clusters_test.go +++ b/agent/xds/clusters_test.go @@ -20,16 +20,28 @@ import ( "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/proxystateconverter" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/agent/xds/testcommon" + "github.com/hashicorp/consul/agent/xdsv2" "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/sdk/testutil" "github.com/hashicorp/consul/types" ) +type mockCfgFetcher struct { + addressLan string +} + +func (s *mockCfgFetcher) AdvertiseAddrLAN() string { + return s.addressLan +} + type clusterTestCase struct { name string create func(t testinf.T) *proxycfg.ConfigSnapshot overrideGoldenName string + alsoRunTestForV2 bool } func uint32ptr(i uint32) *uint32 { @@ -52,12 +64,15 @@ func makeClusterDiscoChainTests(enterprise bool) []clusterTestCase { }) }, nil) }, + // TODO(proxystate): requires custom cluster work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-chain", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-chain-http2", @@ -66,84 +81,111 @@ func makeClusterDiscoChainTests(enterprise bool) []clusterTestCase { ns.Proxy.Upstreams[0].Config["protocol"] = "http2" }, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-chain-external-sni", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "external-sni", enterprise, nil, nil) }, + //TODO(proxystate): this requires terminating gateway work + alsoRunTestForV2: true, }, { name: "connect-proxy-with-chain-and-overrides", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple-with-overrides", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-chain-and-failover", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway-triggered", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-double-failover-through-remote-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-remote-gateway", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-remote-gateway-triggered", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-failover-through-local-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway-triggered", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-double-failover-through-local-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-local-gateway", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-local-gateway-triggered", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "splitter-with-resolver-redirect", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "splitter-with-resolver-redirect-multidc", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-lb-in-resolver", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "lb-resolver", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, } } @@ -175,6 +217,7 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tls-outgoing-min-version", @@ -194,6 +237,7 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tls-outgoing-max-version", @@ -213,6 +257,7 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tls-outgoing-cipher-suites", @@ -235,6 +280,7 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-jwt-config-entry-with-local", @@ -258,6 +304,8 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + // TODO(proxystate): jwt work will come at a later time + alsoRunTestForV2: false, }, { name: "connect-proxy-with-jwt-config-entry-with-remote-jwks", @@ -292,6 +340,8 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + // TODO(proxystate): jwt work will come at a later time + alsoRunTestForV2: false, }, { name: "custom-local-app", @@ -303,6 +353,8 @@ func TestClustersFromSnapshot(t *testing.T) { }) }, nil) }, + // TODO(proxystate): requires custom cluster work + alsoRunTestForV2: false, }, { name: "custom-upstream", @@ -314,6 +366,8 @@ func TestClustersFromSnapshot(t *testing.T) { }) }, nil) }, + // TODO(proxystate): requires custom cluster work + alsoRunTestForV2: false, }, { name: "custom-upstream-ignores-tls", @@ -328,6 +382,8 @@ func TestClustersFromSnapshot(t *testing.T) { }) }, nil) }, + // TODO(proxystate): requires custom cluster work + alsoRunTestForV2: false, }, { name: "custom-upstream-with-prepared-query", @@ -364,6 +420,8 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + // TODO(proxystate): requires custom cluster work + alsoRunTestForV2: false, }, { name: "custom-timeouts", @@ -373,6 +431,7 @@ func TestClustersFromSnapshot(t *testing.T) { ns.Proxy.Upstreams[0].Config["connect_timeout_ms"] = 2345 }, nil) }, + alsoRunTestForV2: true, }, { name: "custom-passive-healthcheck", @@ -387,6 +446,7 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + alsoRunTestForV2: true, }, { name: "custom-passive-healthcheck-zero-consecutive_5xx", @@ -401,6 +461,7 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + alsoRunTestForV2: true, }, { name: "custom-max-inbound-connections", @@ -409,6 +470,7 @@ func TestClustersFromSnapshot(t *testing.T) { ns.Proxy.Config["max_inbound_connections"] = 3456 }, nil) }, + alsoRunTestForV2: true, }, { name: "custom-limits-max-connections-only", @@ -429,6 +491,7 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + alsoRunTestForV2: true, }, { name: "custom-limits-set-to-zero", @@ -447,6 +510,7 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + alsoRunTestForV2: true, }, { name: "custom-limits", @@ -465,12 +529,14 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + alsoRunTestForV2: true, }, { name: "expose-paths-local-app-paths", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotExposeConfig(t, nil) }, + alsoRunTestForV2: true, }, { name: "downstream-service-with-unix-sockets", @@ -483,6 +549,7 @@ func TestClustersFromSnapshot(t *testing.T) { ns.Proxy.LocalServiceSocketPath = "/tmp/downstream_proxy.sock" }, nil) }, + alsoRunTestForV2: true, }, { name: "expose-paths-new-cluster-http2", @@ -496,68 +563,89 @@ func TestClustersFromSnapshot(t *testing.T) { } }) }, + alsoRunTestForV2: true, }, { - name: "expose-checks", - create: proxycfg.TestConfigSnapshotExposeChecks, + name: "expose-checks", + create: proxycfg.TestConfigSnapshotExposeChecks, + alsoRunTestForV2: true, }, { - name: "expose-paths-grpc-new-cluster-http1", - create: proxycfg.TestConfigSnapshotGRPCExposeHTTP1, + name: "expose-paths-grpc-new-cluster-http1", + create: proxycfg.TestConfigSnapshotGRPCExposeHTTP1, + alsoRunTestForV2: true, }, { name: "mesh-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "default", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-using-federation-states", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "federation-states", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-using-federation-control-plane", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "mesh-gateway-federation", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-no-services", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "no-services", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-service-subsets", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "service-subsets", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-ignore-extra-resolvers", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "ignore-extra-resolvers", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-service-timeouts", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "service-timeouts", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-non-hash-lb-injected", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "non-hash-lb-injected", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-hash-lb-ignored", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "hash-lb-ignored", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-tcp-keepalives", @@ -569,6 +657,8 @@ func TestClustersFromSnapshot(t *testing.T) { ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_probes"] = 7 }, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-gateway", @@ -576,12 +666,16 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "default", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-gateway-nil-config-entry", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotIngressGateway_NilConfigEntry(t) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-gateway-with-tls-outgoing-min-version", @@ -601,6 +695,8 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-gateway-with-tls-outgoing-max-version", @@ -620,6 +716,8 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-gateway-with-tls-outgoing-cipher-suites", @@ -642,6 +740,8 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-gateway-no-services", @@ -649,6 +749,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, false, "tcp", "default", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-chain", @@ -656,6 +758,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "simple", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-service-max-connections", @@ -666,6 +770,8 @@ func TestClustersFromSnapshot(t *testing.T) { entry.Listeners[0].Services[0].MaxConnections = 4096 }, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-defaults-service-max-connections", @@ -680,6 +786,8 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-overwrite-defaults-service-max-connections", @@ -695,6 +803,8 @@ func TestClustersFromSnapshot(t *testing.T) { entry.Listeners[0].Services[0].MaxPendingRequests = 2048 }, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-service-passive-health-check", @@ -710,6 +820,8 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-defaults-passive-health-check", @@ -731,6 +843,8 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-overwrite-defaults-passive-health-check", @@ -759,6 +873,8 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-chain-external-sni", @@ -766,6 +882,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "external-sni", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-chain-and-failover", @@ -773,6 +891,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-chain-and-failover-to-cluster-peer", @@ -780,6 +900,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-to-cluster-peer", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-failover-through-remote-gateway", @@ -787,6 +909,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-remote-gateway", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-failover-through-remote-gateway-triggered", @@ -794,6 +918,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-remote-gateway-triggered", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-double-failover-through-remote-gateway", @@ -801,6 +927,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-remote-gateway", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered", @@ -808,6 +936,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-remote-gateway-triggered", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-failover-through-local-gateway", @@ -815,6 +945,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-local-gateway", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-failover-through-local-gateway-triggered", @@ -822,6 +954,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-local-gateway-triggered", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-double-failover-through-local-gateway", @@ -829,6 +963,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-local-gateway", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-double-failover-through-local-gateway-triggered", @@ -836,6 +972,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-local-gateway-triggered", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-splitter-with-resolver-redirect", @@ -843,6 +981,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "http", "splitter-with-resolver-redirect-multidc", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-lb-in-resolver", @@ -850,46 +990,66 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "http", "lb-resolver", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotTerminatingGateway(t, true, nil, nil) }, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-no-services", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotTerminatingGateway(t, false, nil, nil) }, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-service-subsets", create: proxycfg.TestConfigSnapshotTerminatingGatewayServiceSubsetsWebAndCache, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-hostname-service-subsets", create: proxycfg.TestConfigSnapshotTerminatingGatewayHostnameSubsets, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-sni", create: proxycfg.TestConfigSnapshotTerminatingGatewaySNI, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-http2-upstream", create: proxycfg.TestConfigSnapshotTerminatingGatewayHTTP2, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-http2-upstream-subsets", create: proxycfg.TestConfigSnapshotTerminatingGatewaySubsetsHTTP2, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-ignore-extra-resolvers", create: proxycfg.TestConfigSnapshotTerminatingGatewayIgnoreExtraResolvers, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-lb-config", create: proxycfg.TestConfigSnapshotTerminatingGatewayLBConfigNoHashPolicies, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-tcp-keepalives", @@ -904,18 +1064,24 @@ func TestClustersFromSnapshot(t *testing.T) { ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_probes"] = 5 }, nil) }, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-multiple-listeners-duplicate-service", create: proxycfg.TestConfigSnapshotIngress_MultipleListenersDuplicateService, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { - name: "transparent-proxy-catalog-destinations-only", - create: proxycfg.TestConfigSnapshotTransparentProxyCatalogDestinationsOnly, + name: "transparent-proxy-catalog-destinations-only", + create: proxycfg.TestConfigSnapshotTransparentProxyCatalogDestinationsOnly, + alsoRunTestForV2: true, }, { - name: "transparent-proxy-dial-instances-directly", - create: proxycfg.TestConfigSnapshotTransparentProxyDialDirectly, + name: "transparent-proxy-dial-instances-directly", + create: proxycfg.TestConfigSnapshotTransparentProxyDialDirectly, + alsoRunTestForV2: true, }, } @@ -947,10 +1113,10 @@ func TestClustersFromSnapshot(t *testing.T) { return clusters[i].(*envoy_cluster_v3.Cluster).Name < clusters[j].(*envoy_cluster_v3.Cluster).Name }) - r, err := createResponse(xdscommon.ClusterType, "00000001", "00000001", clusters) + r, err := response.CreateResponse(xdscommon.ClusterType, "00000001", "00000001", clusters) require.NoError(t, err) - t.Run("current", func(t *testing.T) { + t.Run("current-xdsv1", func(t *testing.T) { gotJSON := protoToJSON(t, r) gName := tt.name @@ -960,6 +1126,39 @@ func TestClustersFromSnapshot(t *testing.T) { require.JSONEq(t, goldenEnvoy(t, filepath.Join("clusters", gName), envoyVersion, latestEnvoyVersion, gotJSON), gotJSON) }) + + if tt.alsoRunTestForV2 { + generator := xdsv2.NewResourceGenerator(testutil.Logger(t)) + + converter := proxystateconverter.NewConverter(testutil.Logger(t), &mockCfgFetcher{addressLan: "10.10.10.10"}) + proxyState, err := converter.ProxyStateFromSnapshot(snap) + require.NoError(t, err) + + res, err := generator.AllResourcesFromIR(proxyState) + require.NoError(t, err) + + clusters = res[xdscommon.ClusterType] + // The order of clusters returned via CDS isn't relevant, so it's safe + // to sort these for the purposes of test comparisons. + sort.Slice(clusters, func(i, j int) bool { + return clusters[i].(*envoy_cluster_v3.Cluster).Name < clusters[j].(*envoy_cluster_v3.Cluster).Name + }) + + r, err := response.CreateResponse(xdscommon.ClusterType, "00000001", "00000001", clusters) + require.NoError(t, err) + + t.Run("current-xdsv2", func(t *testing.T) { + gotJSON := protoToJSON(t, r) + + gName := tt.name + if tt.overrideGoldenName != "" { + gName = tt.overrideGoldenName + } + + expectedJSON := goldenEnvoy(t, filepath.Join("clusters", gName), envoyVersion, latestEnvoyVersion, gotJSON) + require.JSONEq(t, expectedJSON, gotJSON) + }) + } }) } }) diff --git a/agent/xds/delta_envoy_extender_oss_test.go b/agent/xds/delta_envoy_extender_oss_test.go index 5f4b563b5992..0607919ef5a5 100644 --- a/agent/xds/delta_envoy_extender_oss_test.go +++ b/agent/xds/delta_envoy_extender_oss_test.go @@ -21,12 +21,12 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" - "github.com/hashicorp/consul/agent/xds/testcommon" - propertyoverride "github.com/hashicorp/consul/agent/envoyextensions/builtin/property-override" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/xds/extensionruntime" + "github.com/hashicorp/consul/agent/xds/response" + "github.com/hashicorp/consul/agent/xds/testcommon" "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/envoyextensions/extensioncommon" "github.com/hashicorp/consul/envoyextensions/xdscommon" @@ -803,7 +803,7 @@ end`, } sort.Slice(msgs, entity.sorter(msgs)) - r, err := createResponse(entity.key, "00000001", "00000001", msgs) + r, err := response.CreateResponse(entity.key, "00000001", "00000001", msgs) require.NoError(t, err) t.Run(entity.name, func(t *testing.T) { diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index 8903ba3e7cc6..a2c36f06bdc2 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -6,21 +6,22 @@ package xds import ( "errors" "fmt" - "github.com/hashicorp/go-hclog" "strconv" + "github.com/hashicorp/go-hclog" + envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" "github.com/hashicorp/go-bexpr" "google.golang.org/protobuf/proto" - "github.com/hashicorp/consul/envoyextensions/xdscommon" - "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/envoyextensions/xdscommon" ) const ( @@ -278,7 +279,7 @@ func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.C lbEndpoint := &envoy_endpoint_v3.LbEndpoint{ HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress(addr, port), + Address: response.MakeAddress(addr, port), }, }, HealthStatus: envoy_core_v3.HealthStatus_UNKNOWN, @@ -335,7 +336,7 @@ func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.C serverEndpoints = append(serverEndpoints, &envoy_endpoint_v3.LbEndpoint{ HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress(addr, port), + Address: response.MakeAddress(addr, port), }, }, }) @@ -584,7 +585,7 @@ func makeEndpoint(host string, port int) *envoy_endpoint_v3.LbEndpoint { return &envoy_endpoint_v3.LbEndpoint{ HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress(host, port), + Address: response.MakeAddress(host, port), }, }, } @@ -594,7 +595,7 @@ func makePipeEndpoint(path string) *envoy_endpoint_v3.LbEndpoint { return &envoy_endpoint_v3.LbEndpoint{ HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makePipeAddress(path, 0), + Address: response.MakePipeAddress(path, 0), }, }, } @@ -881,7 +882,7 @@ func makeLoadAssignment(logger hclog.Logger, cfgSnap *proxycfg.ConfigSnapshot, c cla.Policy = &envoy_endpoint_v3.ClusterLoadAssignment_Policy{ // We choose such a large value here that the failover math should // in effect not happen until zero instances are healthy. - OverprovisioningFactor: makeUint32Value(100000), + OverprovisioningFactor: response.MakeUint32Value(100000), } } @@ -907,14 +908,14 @@ func makeLoadAssignment(logger hclog.Logger, cfgSnap *proxycfg.ConfigSnapshot, c } endpoint := &envoy_endpoint_v3.Endpoint{ - Address: makeAddress(addr, port), + Address: response.MakeAddress(addr, port), } es = append(es, &envoy_endpoint_v3.LbEndpoint{ HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: endpoint, }, HealthStatus: healthStatus, - LoadBalancingWeight: makeUint32Value(weight), + LoadBalancingWeight: response.MakeUint32Value(weight), }) } diff --git a/agent/xds/endpoints_test.go b/agent/xds/endpoints_test.go index a4757f00c534..3156685934d5 100644 --- a/agent/xds/endpoints_test.go +++ b/agent/xds/endpoints_test.go @@ -4,11 +4,12 @@ package xds import ( - "github.com/hashicorp/go-hclog" "path/filepath" "sort" "testing" + "github.com/hashicorp/go-hclog" + envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" @@ -20,6 +21,7 @@ import ( "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/sdk/testutil" ) @@ -133,18 +135,18 @@ func Test_makeLoadAssignment(t *testing.T) { { HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress("10.10.10.10", 1234), + Address: response.MakeAddress("10.10.10.10", 1234), }}, HealthStatus: envoy_core_v3.HealthStatus_HEALTHY, - LoadBalancingWeight: makeUint32Value(1), + LoadBalancingWeight: response.MakeUint32Value(1), }, { HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress("10.10.10.20", 1234), + Address: response.MakeAddress("10.10.10.20", 1234), }}, HealthStatus: envoy_core_v3.HealthStatus_HEALTHY, - LoadBalancingWeight: makeUint32Value(1), + LoadBalancingWeight: response.MakeUint32Value(1), }, }, }}, @@ -163,18 +165,18 @@ func Test_makeLoadAssignment(t *testing.T) { { HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress("10.10.10.10", 1234), + Address: response.MakeAddress("10.10.10.10", 1234), }}, HealthStatus: envoy_core_v3.HealthStatus_HEALTHY, - LoadBalancingWeight: makeUint32Value(10), + LoadBalancingWeight: response.MakeUint32Value(10), }, { HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress("10.10.10.20", 1234), + Address: response.MakeAddress("10.10.10.20", 1234), }}, HealthStatus: envoy_core_v3.HealthStatus_HEALTHY, - LoadBalancingWeight: makeUint32Value(5), + LoadBalancingWeight: response.MakeUint32Value(5), }, }, }}, @@ -193,18 +195,18 @@ func Test_makeLoadAssignment(t *testing.T) { { HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress("10.10.10.10", 1234), + Address: response.MakeAddress("10.10.10.10", 1234), }}, HealthStatus: envoy_core_v3.HealthStatus_HEALTHY, - LoadBalancingWeight: makeUint32Value(1), + LoadBalancingWeight: response.MakeUint32Value(1), }, { HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress("10.10.10.20", 1234), + Address: response.MakeAddress("10.10.10.20", 1234), }}, HealthStatus: envoy_core_v3.HealthStatus_UNHEALTHY, - LoadBalancingWeight: makeUint32Value(1), + LoadBalancingWeight: response.MakeUint32Value(1), }, }, }}, @@ -559,7 +561,7 @@ func TestEndpointsFromSnapshot(t *testing.T) { sort.Slice(endpoints, func(i, j int) bool { return endpoints[i].(*envoy_endpoint_v3.ClusterLoadAssignment).ClusterName < endpoints[j].(*envoy_endpoint_v3.ClusterLoadAssignment).ClusterName }) - r, err := createResponse(xdscommon.EndpointType, "00000001", "00000001", endpoints) + r, err := response.CreateResponse(xdscommon.EndpointType, "00000001", "00000001", endpoints) require.NoError(t, err) t.Run("current", func(t *testing.T) { diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 68c7ebdfadab..846cbf6d536f 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -45,6 +45,7 @@ import ( "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/xds/accesslogs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/lib" "github.com/hashicorp/consul/lib/stringslice" @@ -936,7 +937,7 @@ func makeListenerWithDefault(opts makeListenerOpts) *envoy_listener_v3.Listener return &envoy_listener_v3.Listener{ Name: fmt.Sprintf("%s:%s:%d", opts.name, opts.addr, opts.port), AccessLog: accessLog, - Address: makeAddress(opts.addr, opts.port), + Address: response.MakeAddress(opts.addr, opts.port), TrafficDirection: opts.direction, } } @@ -955,7 +956,7 @@ func makePipeListener(opts makeListenerOpts) *envoy_listener_v3.Listener { return &envoy_listener_v3.Listener{ Name: fmt.Sprintf("%s:%s", opts.name, opts.path), AccessLog: accessLog, - Address: makePipeAddress(opts.path, uint32(modeInt)), + Address: response.MakePipeAddress(opts.path, uint32(modeInt)), TrafficDirection: opts.direction, } } @@ -2592,7 +2593,7 @@ func makeHTTPFilter(opts listenerFilterOpts) (*envoy_listener_v3.Filter, error) "envoy.filters.http.grpc_stats", &envoy_grpc_stats_v3.FilterConfig{ PerMethodStatSpecifier: &envoy_grpc_stats_v3.FilterConfig_StatsForAllMethods{ - StatsForAllMethods: makeBoolValue(true), + StatsForAllMethods: response.MakeBoolValue(true), }, }, ) diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go index b5d6c66f0008..130f2234517e 100644 --- a/agent/xds/listeners_test.go +++ b/agent/xds/listeners_test.go @@ -21,6 +21,7 @@ import ( "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/xds/configfetcher" "github.com/hashicorp/consul/agent/xds/proxystateconverter" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/agent/xds/testcommon" "github.com/hashicorp/consul/agent/xdsv2" "github.com/hashicorp/consul/envoyextensions/xdscommon" @@ -1301,24 +1302,24 @@ func TestListenersFromSnapshot(t *testing.T) { var listeners []proto.Message - t.Run("current-xdsv1", func(t *testing.T) { - // Need server just for logger dependency - g := NewResourceGenerator(testutil.Logger(t), nil, false) - g.ProxyFeatures = sf - if tt.generatorSetup != nil { - tt.generatorSetup(g) - } - listeners, err = g.listenersFromSnapshot(snap) - require.NoError(t, err) - // The order of listeners returned via LDS isn't relevant, so it's safe - // to sort these for the purposes of test comparisons. - sort.Slice(listeners, func(i, j int) bool { - return listeners[i].(*envoy_listener_v3.Listener).Name < listeners[j].(*envoy_listener_v3.Listener).Name - }) + // Need server just for logger dependency + g := NewResourceGenerator(testutil.Logger(t), nil, false) + g.ProxyFeatures = sf + if tt.generatorSetup != nil { + tt.generatorSetup(g) + } + listeners, err = g.listenersFromSnapshot(snap) + require.NoError(t, err) + // The order of listeners returned via LDS isn't relevant, so it's safe + // to sort these for the purposes of test comparisons. + sort.Slice(listeners, func(i, j int) bool { + return listeners[i].(*envoy_listener_v3.Listener).Name < listeners[j].(*envoy_listener_v3.Listener).Name + }) - r, err := createResponse(xdscommon.ListenerType, "00000001", "00000001", listeners) - require.NoError(t, err) + r, err := response.CreateResponse(xdscommon.ListenerType, "00000001", "00000001", listeners) + require.NoError(t, err) + t.Run("current-xdsv1", func(t *testing.T) { gotJSON := protoToJSON(t, r) gName := tt.name @@ -1331,25 +1332,25 @@ func TestListenersFromSnapshot(t *testing.T) { }) if tt.alsoRunTestForV2 { - t.Run("current-xdsv2", func(t *testing.T) { - generator := xdsv2.NewResourceGenerator(testutil.Logger(t)) - converter := proxystateconverter.NewConverter(testutil.Logger(t), nil) - proxyState, err := converter.ProxyStateFromSnapshot(snap) - require.NoError(t, err) + generator := xdsv2.NewResourceGenerator(testutil.Logger(t)) + converter := proxystateconverter.NewConverter(testutil.Logger(t), nil) + proxyState, err := converter.ProxyStateFromSnapshot(snap) + require.NoError(t, err) - res, err := generator.AllResourcesFromIR(proxyState) - require.NoError(t, err) + res, err := generator.AllResourcesFromIR(proxyState) + require.NoError(t, err) - listeners = res[xdscommon.ListenerType] - // The order of listeners returned via LDS isn't relevant, so it's safe - // to sort these for the purposes of test comparisons. - sort.Slice(listeners, func(i, j int) bool { - return listeners[i].(*envoy_listener_v3.Listener).Name < listeners[j].(*envoy_listener_v3.Listener).Name - }) + listeners = res[xdscommon.ListenerType] + // The order of listeners returned via LDS isn't relevant, so it's safe + // to sort these for the purposes of test comparisons. + sort.Slice(listeners, func(i, j int) bool { + return listeners[i].(*envoy_listener_v3.Listener).Name < listeners[j].(*envoy_listener_v3.Listener).Name + }) - r, err := createResponse(xdscommon.ListenerType, "00000001", "00000001", listeners) - require.NoError(t, err) + r, err := response.CreateResponse(xdscommon.ListenerType, "00000001", "00000001", listeners) + require.NoError(t, err) + t.Run("current-xdsv2", func(t *testing.T) { gotJSON := protoToJSON(t, r) gName := tt.name diff --git a/agent/xds/proxystateconverter/clusters.go b/agent/xds/proxystateconverter/clusters.go index b5f10bf32e03..2a43d6eb049c 100644 --- a/agent/xds/proxystateconverter/clusters.go +++ b/agent/xds/proxystateconverter/clusters.go @@ -1,16 +1,28 @@ // Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 +// SPDX-License-Identifier: BUSL-1.1 package proxystateconverter import ( + "errors" "fmt" + envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-uuid" "strings" + "time" + + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/config" "github.com/hashicorp/consul/agent/xds/naming" + "github.com/hashicorp/consul/agent/xds/response" + "github.com/hashicorp/consul/envoyextensions/xdscommon" + "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" "github.com/hashicorp/consul/proto/private/pbpeering" ) @@ -18,15 +30,333 @@ const ( meshGatewayExportedClusterNamePrefix = "exported~" ) +type namedCluster struct { + name string + cluster *pbproxystate.Cluster +} + +// clustersFromSnapshot returns the xDS API representation of the "clusters" in the snapshot. +func (s *Converter) clustersFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot) error { + if cfgSnap == nil { + return errors.New("nil config given") + } + + switch cfgSnap.Kind { + case structs.ServiceKindConnectProxy: + return s.clustersFromSnapshotConnectProxy(cfgSnap) + // TODO(proxystate): Terminating Gateways will be added in the future. + //case structs.ServiceKindTerminatingGateway: + // err := s.clustersFromSnapshotTerminatingGateway(cfgSnap) + // if err != nil { + // return err + // } + // return nil + // TODO(proxystate): Mesh Gateways will be added in the future. + //case structs.ServiceKindMeshGateway: + // err := s.clustersFromSnapshotMeshGateway(cfgSnap) + // if err != nil { + // return err + // } + // return nil + // TODO(proxystate): Ingress Gateways will be added in the future. + //case structs.ServiceKindIngressGateway: + // err := s.clustersFromSnapshotIngressGateway(cfgSnap) + // if err != nil { + // return err + // } + // return nil + // TODO(proxystate): API Gateways will be added in the future. + //case structs.ServiceKindAPIGateway: + // res, err := s.clustersFromSnapshotAPIGateway(cfgSnap) + // if err != nil { + // return err + // } + // return nil + default: + return fmt.Errorf("Invalid service kind: %v", cfgSnap.Kind) + } +} + +// clustersFromSnapshot returns the xDS API representation of the "clusters" +// (upstreams) in the snapshot. +func (s *Converter) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapshot) error { + // This is the list of listeners we add to. It will be empty to start. + clusters := s.proxyState.Clusters + var err error + + // Include the "app" cluster for the public listener + appCluster, err := s.makeAppCluster(cfgSnap, xdscommon.LocalAppClusterName, "", cfgSnap.Proxy.LocalServicePort) + if err != nil { + return err + } + clusters[appCluster.name] = appCluster.cluster + + if cfgSnap.Proxy.Mode == structs.ProxyModeTransparent { + passthroughs, err := s.makePassthroughClusters(cfgSnap) + if err != nil { + return fmt.Errorf("failed to make passthrough clusters for transparent proxy: %v", err) + } + for clusterName, cluster := range passthroughs { + clusters[clusterName] = cluster + } + } + + // NOTE: Any time we skip a chain below we MUST also skip that discovery chain in endpoints.go + // so that the sets of endpoints generated matches the sets of clusters. + for uid, chain := range cfgSnap.ConnectProxy.DiscoveryChain { + upstream, skip := cfgSnap.ConnectProxy.GetUpstream(uid, &cfgSnap.ProxyID.EnterpriseMeta) + if skip { + continue + } + + upstreamClusters, err := s.makeUpstreamClustersForDiscoveryChain( + uid, + upstream, + chain, + cfgSnap, + false, + ) + if err != nil { + return err + } + + for name, cluster := range upstreamClusters { + clusters[name] = cluster + } + } + + // TODO(proxystate): peering will be added in the future. + //// NOTE: Any time we skip an upstream below we MUST also skip that same + //// upstream in endpoints.go so that the sets of endpoints generated matches + //// the sets of clusters. + //for _, uid := range cfgSnap.ConnectProxy.PeeredUpstreamIDs() { + // upstream, skip := cfgSnap.ConnectProxy.GetUpstream(uid, &cfgSnap.ProxyID.EnterpriseMeta) + // if skip { + // continue + // } + // + // peerMeta, found := cfgSnap.ConnectProxy.UpstreamPeerMeta(uid) + // if !found { + // s.Logger.Warn("failed to fetch upstream peering metadata for cluster", "uid", uid) + // } + // cfg := s.getAndModifyUpstreamConfigForPeeredListener(uid, upstream, peerMeta) + // + // upstreamCluster, err := s.makeUpstreamClusterForPeerService(uid, cfg, peerMeta, cfgSnap) + // if err != nil { + // return nil, err + // } + // clusters = append(clusters, upstreamCluster) + //} + + // TODO(proxystate): L7 Intentions and JWT Auth will be added in the future. + //// add clusters for jwt-providers + //for _, prov := range cfgSnap.JWTProviders { + // //skip cluster creation for local providers + // if prov.JSONWebKeySet == nil || prov.JSONWebKeySet.Remote == nil { + // continue + // } + // + // cluster, err := makeJWTProviderCluster(prov) + // if err != nil { + // s.Logger.Warn("failed to make jwt-provider cluster", "provider name", prov.Name, "error", err) + // continue + // } + // + // clusters[cluster.GetName()] = cluster + //} + + for _, u := range cfgSnap.Proxy.Upstreams { + if u.DestinationType != structs.UpstreamDestTypePreparedQuery { + continue + } + + upstreamCluster, err := s.makeUpstreamClusterForPreparedQuery(u, cfgSnap) + if err != nil { + return err + } + clusters[upstreamCluster.name] = upstreamCluster.cluster + } + + cfgSnap.Proxy.Expose.Finalize() + paths := cfgSnap.Proxy.Expose.Paths + + // Add service health checks to the list of paths to create clusters for if needed + if cfgSnap.Proxy.Expose.Checks { + psid := structs.NewServiceID(cfgSnap.Proxy.DestinationServiceID, &cfgSnap.ProxyID.EnterpriseMeta) + for _, check := range cfgSnap.ConnectProxy.WatchedServiceChecks[psid] { + p, err := parseCheckPath(check) + if err != nil { + s.Logger.Warn("failed to create cluster for", "check", check.CheckID, "error", err) + continue + } + paths = append(paths, p) + } + } + + // Create a new cluster if we need to expose a port that is different from the service port + for _, path := range paths { + if path.LocalPathPort == cfgSnap.Proxy.LocalServicePort { + continue + } + c, err := s.makeAppCluster(cfgSnap, makeExposeClusterName(path.LocalPathPort), path.Protocol, path.LocalPathPort) + if err != nil { + s.Logger.Warn("failed to make local cluster", "path", path.Path, "error", err) + continue + } + clusters[c.name] = c.cluster + } + + return nil +} + +// TODO(proxystate): L7 Intentions and JWT Auth will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeJWTProviderCluster +// func makeJWKSDiscoveryClusterType +// func makeJWTCertValidationContext +// func parseJWTRemoteURL + func makeExposeClusterName(destinationPort int) string { return fmt.Sprintf("exposed_cluster_%d", destinationPort) } -func clusterNameForDestination(cfgSnap *proxycfg.ConfigSnapshot, name string, address string, namespace string, partition string) string { +// In transparent proxy mode there are potentially multiple passthrough clusters added. +// The first is for destinations outside of Consul's catalog. This is for a plain TCP proxy. +// All of these use Envoy's ORIGINAL_DST listener filter, which forwards to the original +// destination address (before the iptables redirection). +// The rest are for destinations inside the mesh, which require certificates for mTLS. +func (s *Converter) makePassthroughClusters(cfgSnap *proxycfg.ConfigSnapshot) (map[string]*pbproxystate.Cluster, error) { + // This size is an upper bound. + clusters := make(map[string]*pbproxystate.Cluster, 0) + if meshConf := cfgSnap.MeshConfig(); meshConf == nil || + !meshConf.TransparentProxy.MeshDestinationsOnly { + + clusters[naming.OriginalDestinationClusterName] = &pbproxystate.Cluster{ + Group: &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Passthrough{ + Passthrough: &pbproxystate.PassthroughEndpointGroup{ + Config: &pbproxystate.PassthroughEndpointGroupConfig{ + ConnectTimeout: durationpb.New(5 * time.Second), + }, + }, + }, + }, + }, + } + } + + for uid, chain := range cfgSnap.ConnectProxy.DiscoveryChain { + targetMap, ok := cfgSnap.ConnectProxy.PassthroughUpstreams[uid] + if !ok { + continue + } + + for targetID := range targetMap { + uid := proxycfg.NewUpstreamIDFromTargetID(targetID) + + sni := connect.ServiceSNI( + uid.Name, "", uid.NamespaceOrDefault(), + uid.PartitionOrDefault(), cfgSnap.Datacenter, + cfgSnap.Roots.TrustDomain) + + // Prefixed with passthrough to distinguish from non-passthrough clusters for the same upstream. + name := "passthrough~" + sni + + c := pbproxystate.Cluster{ + Group: &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Passthrough{ + Passthrough: &pbproxystate.PassthroughEndpointGroup{ + Config: &pbproxystate.PassthroughEndpointGroupConfig{ + ConnectTimeout: durationpb.New(5 * time.Second), + }, + }, + }, + }, + }, + } + + if discoTarget, ok := chain.Targets[targetID]; ok && discoTarget.ConnectTimeout > 0 { + c.GetEndpointGroup().GetPassthrough().GetConfig(). + ConnectTimeout = durationpb.New(discoTarget.ConnectTimeout) + } + + transportSocket, err := s.createOutboundMeshMTLS(cfgSnap, []string{getSpiffeID(cfgSnap, uid)}, sni) + if err != nil { + return nil, err + } + c.GetEndpointGroup().GetPassthrough().OutboundTls = transportSocket + + clusters[name] = &c + } + } + + err := cfgSnap.ConnectProxy.DestinationsUpstream.ForEachKeyE(func(uid proxycfg.UpstreamID) error { + svcConfig, ok := cfgSnap.ConnectProxy.DestinationsUpstream.Get(uid) + if !ok || svcConfig.Destination == nil { + return nil + } + + // One Cluster per Destination Address + for _, address := range svcConfig.Destination.Addresses { + name := clusterNameForDestination(cfgSnap, uid.Name, address, uid.NamespaceOrDefault(), uid.PartitionOrDefault()) + + c := &pbproxystate.Cluster{ + AltStatName: name, + Group: &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Dynamic{ + Dynamic: &pbproxystate.DynamicEndpointGroup{ + Config: &pbproxystate.DynamicEndpointGroupConfig{ + ConnectTimeout: durationpb.New(5 * time.Second), + // Endpoints are managed separately by EDS + // Having an empty config enables outlier detection with default config. + OutlierDetection: &pbproxystate.OutlierDetection{}, + }, + }, + }, + }, + }, + } + sni := connect.ServiceSNI( + uid.Name, "", uid.NamespaceOrDefault(), + uid.PartitionOrDefault(), cfgSnap.Datacenter, + cfgSnap.Roots.TrustDomain) + transportSocket, err := s.createOutboundMeshMTLS(cfgSnap, []string{getSpiffeID(cfgSnap, uid)}, sni) + if err != nil { + return err + } + c.GetEndpointGroup().GetDynamic().OutboundTls = transportSocket + clusters[name] = c + } + return nil + }) + if err != nil { + return nil, err + } + + return clusters, nil +} + +func getSpiffeID(cfgSnap *proxycfg.ConfigSnapshot, uid proxycfg.UpstreamID) string { + spiffeIDService := &connect.SpiffeIDService{ + Host: cfgSnap.Roots.TrustDomain, + Partition: uid.PartitionOrDefault(), + Namespace: uid.NamespaceOrDefault(), + Datacenter: cfgSnap.Datacenter, + Service: uid.Name, + } + return spiffeIDService.URI().String() +} +func clusterNameForDestination(cfgSnap *proxycfg.ConfigSnapshot, name string, + address string, namespace string, partition string) string { name = destinationSpecificServiceName(name, address) - sni := connect.ServiceSNI(name, "", namespace, partition, cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain) + sni := connect.ServiceSNI(name, "", namespace, partition, + cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain) - // Prefixed with destination to distinguish from non-passthrough clusters for the same upstream. + // Prefixed with destination to distinguish from non-passthrough clusters + // for the same upstream. return "destination." + sni } @@ -36,6 +366,785 @@ func destinationSpecificServiceName(name string, address string) string { return fmt.Sprintf("%s.%s", address, name) } +// TODO(proxystate): Mesh Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func clustersFromSnapshotMeshGateway +// func haveVoters + +// TODO(proxystate): Peering will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makePeerServerClusters + +// TODO(proxystate): Terminating Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func clustersFromSnapshotTerminatingGateway + +// TODO(proxystate): Mesh Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeGatewayServiceClusters + +// TODO(proxystate): Cluster Peering will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeGatewayOutgoingClusterPeeringServiceClusters + +// TODO(proxystate): Terminating Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeDestinationClusters + +// TODO(proxystate): Mesh Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func injectGatewayServiceAddons + +// TODO(proxystate): Terminating Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func injectGatewayDestinationAddons + +// TODO(proxystate): Ingress Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func clustersFromSnapshotIngressGateway + +// TODO(proxystate): API Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func clustersFromSnapshotAPIGateway + +// TODO(proxystate): Ingress Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func configIngressUpstreamCluster + +func (s *Converter) makeAppCluster(cfgSnap *proxycfg.ConfigSnapshot, name, pathProtocol string, port int) (*namedCluster, error) { + var err error + namedCluster := &namedCluster{} + + cfg, err := config.ParseProxyConfig(cfgSnap.Proxy.Config) + if err != nil { + // Don't hard fail on a config typo, just warn. The parse func returns + // default config if there is an error so it's safe to continue. + s.Logger.Warn("failed to parse Connect.Proxy.Config", "error", err) + } + + //// If we have overridden local cluster config try to parse it into an Envoy cluster + //if cfg.LocalClusterJSON != "" { + // return makeClusterFromUserConfig(cfg.LocalClusterJSON) + //} + + var endpoint *pbproxystate.Endpoint + if cfgSnap.Proxy.LocalServiceSocketPath != "" { + endpoint = makeUnixSocketEndpoint(cfgSnap.Proxy.LocalServiceSocketPath) + } else { + addr := cfgSnap.Proxy.LocalServiceAddress + if addr == "" { + addr = "127.0.0.1" + } + endpoint = makeHostPortEndpoint(addr, port) + } + s.proxyState.Endpoints[name] = &pbproxystate.Endpoints{ + Endpoints: []*pbproxystate.Endpoint{endpoint}, + } + + namedCluster.name = name + namedCluster.cluster = &pbproxystate.Cluster{ + Group: &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Static{ + Static: &pbproxystate.StaticEndpointGroup{ + Config: &pbproxystate.StaticEndpointGroupConfig{ + ConnectTimeout: durationpb.New(time.Duration(cfg.LocalConnectTimeoutMs) * time.Millisecond), + }, + }, + }, + }, + }, + } + protocol := pathProtocol + if protocol == "" { + protocol = cfg.Protocol + } + namedCluster.cluster.Protocol = protocol + if cfg.MaxInboundConnections > 0 { + namedCluster.cluster.GetEndpointGroup().GetStatic().GetConfig(). + CircuitBreakers = &pbproxystate.CircuitBreakers{ + UpstreamLimits: &pbproxystate.UpstreamLimits{ + MaxConnections: response.MakeUint32Value(cfg.MaxInboundConnections), + }, + } + } + + return namedCluster, err +} + +func (s *Converter) makeUpstreamClusterForPeerService( + uid proxycfg.UpstreamID, + upstreamConfig structs.UpstreamConfig, + peerMeta structs.PeeringServiceMeta, + cfgSnap *proxycfg.ConfigSnapshot, +) (string, *pbproxystate.Cluster, *pbproxystate.Endpoints, error) { + var ( + c *pbproxystate.Cluster + e *pbproxystate.Endpoints + err error + ) + + // TODO(proxystate): escapeHatches will be implemented in the future + //if upstreamConfig.EnvoyClusterJSON != "" { + // c, err = makeClusterFromUserConfig(upstreamConfig.EnvoyClusterJSON) + // if err != nil { + // return "", c, e, err + // } + // // In the happy path don't return yet as we need to inject TLS config still. + //} + + upstreamsSnapshot, err := cfgSnap.ToConfigSnapshotUpstreams() + + if err != nil { + return "", c, e, err + } + + tbs, ok := upstreamsSnapshot.UpstreamPeerTrustBundles.Get(uid.Peer) + if !ok { + // this should never happen since we loop through upstreams with + // set trust bundles + return "", c, e, fmt.Errorf("trust bundle not ready for peer %s", uid.Peer) + } + + clusterName := generatePeeredClusterName(uid, tbs) + + outlierDetection := makeOutlierDetection(upstreamConfig.PassiveHealthCheck, nil, true) + // We can't rely on health checks for services on cluster peers because they + // don't take into account service resolvers, splitters and routers. Setting + // MaxEjectionPercent too 100% gives outlier detection the power to eject the + // entire cluster. + outlierDetection.MaxEjectionPercent = &wrapperspb.UInt32Value{Value: 100} + + s.Logger.Trace("generating cluster for", "cluster", clusterName) + if c == nil { + c = &pbproxystate.Cluster{} + + useEDS := true + if _, ok := cfgSnap.ConnectProxy.PeerUpstreamEndpointsUseHostnames[uid]; ok { + // If we're using local mesh gw, the fact that upstreams use hostnames don't matter. + // If we're not using local mesh gw, then resort to CDS. + if upstreamConfig.MeshGateway.Mode != structs.MeshGatewayModeLocal { + useEDS = false + } + } + + // If none of the service instances are addressed by a hostname we + // provide the endpoint IP addresses via EDS + if useEDS { + d := &pbproxystate.DynamicEndpointGroup{ + Config: &pbproxystate.DynamicEndpointGroupConfig{ + UseAltStatName: false, + ConnectTimeout: durationpb.New(time.Duration(upstreamConfig.ConnectTimeoutMs) * time.Millisecond), + DisablePanicThreshold: true, + CircuitBreakers: &pbproxystate.CircuitBreakers{ + UpstreamLimits: makeUpstreamLimitsIfNeeded(upstreamConfig.Limits), + }, + OutlierDetection: outlierDetection, + }, + } + c.Group = &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Dynamic{ + Dynamic: d, + }, + }, + } + transportSocket := &pbproxystate.TransportSocket{ + ConnectionTls: &pbproxystate.TransportSocket_OutboundMesh{ + OutboundMesh: &pbproxystate.OutboundMeshMTLS{ + ValidationContext: &pbproxystate.MeshOutboundValidationContext{ + SpiffeIds: peerMeta.SpiffeID, + TrustBundlePeerNameKey: uid.Peer, + }, + Sni: peerMeta.PrimarySNI(), + }, + }, + } + d.OutboundTls = transportSocket + } else { + d := &pbproxystate.DNSEndpointGroup{ + Config: &pbproxystate.DNSEndpointGroupConfig{ + UseAltStatName: false, + ConnectTimeout: durationpb.New(time.Duration(upstreamConfig.ConnectTimeoutMs) * time.Millisecond), + DisablePanicThreshold: true, + CircuitBreakers: &pbproxystate.CircuitBreakers{ + UpstreamLimits: makeUpstreamLimitsIfNeeded(upstreamConfig.Limits), + }, + OutlierDetection: outlierDetection, + }, + } + c.Group = &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Dns{ + Dns: d, + }, + }, + } + e = &pbproxystate.Endpoints{ + Endpoints: make([]*pbproxystate.Endpoint, 0), + } + + ep, _ := cfgSnap.ConnectProxy.PeerUpstreamEndpoints.Get(uid) + configureClusterWithHostnames( + s.Logger, + d, + e, + "", /*TODO:make configurable?*/ + ep, + true, /*isRemote*/ + false, /*onlyPassing*/ + ) + transportSocket := &pbproxystate.TransportSocket{ + ConnectionTls: &pbproxystate.TransportSocket_OutboundMesh{ + OutboundMesh: &pbproxystate.OutboundMeshMTLS{ + ValidationContext: &pbproxystate.MeshOutboundValidationContext{ + SpiffeIds: peerMeta.SpiffeID, + TrustBundlePeerNameKey: uid.Peer, + }, + Sni: peerMeta.PrimarySNI(), + }, + }, + } + d.OutboundTls = transportSocket + } + } + + return clusterName, c, e, nil +} + +func (s *Converter) makeUpstreamClusterForPreparedQuery(upstream structs.Upstream, cfgSnap *proxycfg.ConfigSnapshot) (*namedCluster, error) { + var c *pbproxystate.Cluster + var err error + + uid := proxycfg.NewUpstreamID(&upstream) + + dc := upstream.Datacenter + if dc == "" { + dc = cfgSnap.Datacenter + } + sni := connect.UpstreamSNI(&upstream, "", dc, cfgSnap.Roots.TrustDomain) + + cfg, _ := structs.ParseUpstreamConfig(upstream.Config) + // TODO(proxystate): add logger and enable this + //if err != nil { + // Don't hard fail on a config typo, just warn. The parse func returns + // default config if there is an error so it's safe to continue. + //s.Logger.Warn("failed to parse", "upstream", uid, "error", err) + //} + + // TODO(proxystate): escapeHatches will be implemented in the future + //if cfg.EnvoyClusterJSON != "" { + // c, err = makeClusterFromUserConfig(cfg.EnvoyClusterJSON) + // if err != nil { + // return c, err + // } + // // In the happy path don't return yet as we need to inject TLS config still. + //} + + if c == nil { + c = &pbproxystate.Cluster{ + Protocol: cfg.Protocol, + Group: &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Dynamic{ + Dynamic: &pbproxystate.DynamicEndpointGroup{ + Config: &pbproxystate.DynamicEndpointGroupConfig{ + ConnectTimeout: durationpb.New(time.Duration(cfg.ConnectTimeoutMs) * time.Millisecond), + // Endpoints are managed separately by EDS + // Having an empty config enables outlier detection with default config. + OutlierDetection: makeOutlierDetection(cfg.PassiveHealthCheck, nil, true), + DisablePanicThreshold: true, + CircuitBreakers: &pbproxystate.CircuitBreakers{ + UpstreamLimits: makeUpstreamLimitsIfNeeded(cfg.Limits), + }, + }, + }, + }, + }, + }, + } + } + + endpoints := cfgSnap.ConnectProxy.PreparedQueryEndpoints[uid] + var ( + spiffeIDs = make([]string, 0) + seen = make(map[string]struct{}) + ) + for _, e := range endpoints { + id := fmt.Sprintf("%s/%s", e.Node.Datacenter, e.Service.CompoundServiceName()) + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + + name := e.Service.Proxy.DestinationServiceName + if e.Service.Connect.Native { + name = e.Service.Service + } + + spiffeIDs = append(spiffeIDs, connect.SpiffeIDService{ + Host: cfgSnap.Roots.TrustDomain, + Namespace: e.Service.NamespaceOrDefault(), + Partition: e.Service.PartitionOrDefault(), + Datacenter: e.Node.Datacenter, + Service: name, + }.URI().String()) + } + + transportSocket, err := s.createOutboundMeshMTLS(cfgSnap, spiffeIDs, sni) + if err != nil { + return nil, err + } + c.GetEndpointGroup().GetDynamic().OutboundTls = transportSocket + + return &namedCluster{name: sni, cluster: c}, nil +} + +func finalizeUpstreamConfig(cfg structs.UpstreamConfig, chain *structs.CompiledDiscoveryChain, connectTimeout time.Duration) structs.UpstreamConfig { + if cfg.Protocol == "" { + cfg.Protocol = chain.Protocol + } + + if cfg.Protocol == "" { + cfg.Protocol = "tcp" + } + + if cfg.ConnectTimeoutMs == 0 { + cfg.ConnectTimeoutMs = int(connectTimeout / time.Millisecond) + } + return cfg +} + +func (s *Converter) createOutboundMeshMTLS(cfgSnap *proxycfg.ConfigSnapshot, spiffeIDs []string, sni string) (*pbproxystate.TransportSocket, error) { + switch cfgSnap.Kind { + case structs.ServiceKindConnectProxy: + case structs.ServiceKindMeshGateway: + default: + return nil, fmt.Errorf("cannot inject peering trust bundles for kind %q", cfgSnap.Kind) + } + + cfg, err := config.ParseProxyConfig(cfgSnap.Proxy.Config) + if err != nil { + // Don't hard fail on a config typo, just warn. The parse func returns + // default config if there is an error so it's safe to continue. + s.Logger.Warn("failed to parse Connect.Proxy.Config", "error", err) + } + + // Add all trust bundle peer names, including local. + trustBundlePeerNames := []string{"local"} + for _, tb := range cfgSnap.PeeringTrustBundles() { + trustBundlePeerNames = append(trustBundlePeerNames, tb.PeerName) + } + // Arbitrary UUID to reference the identity by. + uuid, err := uuid.GenerateUUID() + if err != nil { + return nil, err + } + // Create the transport socket + ts := &pbproxystate.TransportSocket{} + + ts.ConnectionTls = &pbproxystate.TransportSocket_OutboundMesh{ + OutboundMesh: &pbproxystate.OutboundMeshMTLS{ + IdentityKey: uuid, + ValidationContext: &pbproxystate.MeshOutboundValidationContext{ + TrustBundlePeerNameKey: trustBundlePeerNames[0], + SpiffeIds: spiffeIDs, + }, + Sni: sni, + }, + } + s.proxyState.LeafCertificates[uuid] = &pbproxystate.LeafCertificate{ + Cert: cfgSnap.Leaf().CertPEM, + Key: cfgSnap.Leaf().PrivateKeyPEM, + } + ts.TlsParameters = makeTLSParametersFromProxyTLSConfig(cfgSnap.MeshConfigTLSOutgoing()) + ts.AlpnProtocols = getAlpnProtocols(cfg.Protocol) + + return ts, nil +} +func (s *Converter) makeUpstreamClustersForDiscoveryChain( + uid proxycfg.UpstreamID, + upstream *structs.Upstream, + chain *structs.CompiledDiscoveryChain, + cfgSnap *proxycfg.ConfigSnapshot, + forMeshGateway bool, +) (map[string]*pbproxystate.Cluster, error) { + if chain == nil { + return nil, fmt.Errorf("cannot create upstream cluster without discovery chain for %s", uid) + } + + if uid.Peer != "" && forMeshGateway { + return nil, fmt.Errorf("impossible to get a peer discovery chain in a mesh gateway") + } + + upstreamConfigMap := make(map[string]interface{}) + if upstream != nil { + upstreamConfigMap = upstream.Config + } + + upstreamsSnapshot, err := cfgSnap.ToConfigSnapshotUpstreams() + + // Mesh gateways are exempt because upstreamsSnapshot is only used for + // cluster peering targets and transative failover/redirects are unsupported. + if err != nil && !forMeshGateway { + return nil, err + } + + rawUpstreamConfig, err := structs.ParseUpstreamConfigNoDefaults(upstreamConfigMap) + if err != nil { + // Don't hard fail on a config typo, just warn. The parse func returns + // default config if there is an error so it's safe to continue. + s.Logger.Warn("failed to parse", "upstream", uid, + "error", err) + } + + // TODO(proxystate): escapeHatches will be implemented in the future + //var escapeHatchCluster *pbproxystate.Cluster + //if !forMeshGateway { + // if rawUpstreamConfig.EnvoyClusterJSON != "" { + // if chain.Default { + // // If you haven't done anything to setup the discovery chain, then + // // you can use the envoy_cluster_json escape hatch. + // escapeHatchCluster = &pbproxystate.Cluster{ + // EscapeHatchClusterJson: rawUpstreamConfig.EnvoyClusterJSON, + // } + // } else { + // s.Logger.Warn("ignoring escape hatch setting, because a discovery chain is configured for", + // "discovery chain", chain.ServiceName, "upstream", uid, + // "envoy_cluster_json", chain.ServiceName) + // } + // } + //} + + out := make(map[string]*pbproxystate.Cluster) + for _, node := range chain.Nodes { + switch { + case node == nil: + return nil, fmt.Errorf("impossible to process a nil node") + case node.Type != structs.DiscoveryGraphNodeTypeResolver: + continue + case node.Resolver == nil: + return nil, fmt.Errorf("impossible to process a non-resolver node") + } + // These variables are prefixed with primary to avoid shaddowing bugs. + primaryTargetID := node.Resolver.Target + primaryTarget := chain.Targets[primaryTargetID] + primaryTargetClusterName := s.getTargetClusterName(upstreamsSnapshot, chain, primaryTargetID, forMeshGateway) + if primaryTargetClusterName == "" { + continue + } + if forMeshGateway && !cfgSnap.Locality.Matches(primaryTarget.Datacenter, primaryTarget.Partition) { + s.Logger.Warn("ignoring discovery chain target that crosses a datacenter or partition boundary in a mesh gateway", + "target", primaryTarget, + "gatewayLocality", cfgSnap.Locality, + ) + continue + } + + upstreamConfig := finalizeUpstreamConfig(rawUpstreamConfig, chain, node.Resolver.ConnectTimeout) + + mappedTargets, err := s.mapDiscoChainTargets(cfgSnap, chain, node, upstreamConfig, forMeshGateway) + if err != nil { + return nil, err + } + + targetGroups, err := mappedTargets.groupedTargets() + if err != nil { + return nil, err + } + + var failoverGroup *pbproxystate.FailoverGroup + endpointGroups := make([]*pbproxystate.EndpointGroup, 0) + if mappedTargets.failover { + // Create a failover group. The endpoint groups that are part of this failover group are created by the loop + // below. + failoverGroup = &pbproxystate.FailoverGroup{ + Config: &pbproxystate.FailoverGroupConfig{ + ConnectTimeout: durationpb.New(node.Resolver.ConnectTimeout), + }, + } + } + + // Construct the target dynamic endpoint groups. If these are not part of a failover group, they will get added + // directly to the map of pbproxystate.Cluster, if they are a part of a failover group, they will be added to + // the failover group. + for _, groupedTarget := range targetGroups { + s.Logger.Debug("generating cluster for", "cluster", groupedTarget.ClusterName) + dynamic := &pbproxystate.DynamicEndpointGroup{ + Config: &pbproxystate.DynamicEndpointGroupConfig{ + UseAltStatName: true, + ConnectTimeout: durationpb.New(node.Resolver.ConnectTimeout), + // TODO(peering): make circuit breakers or outlier detection work? + CircuitBreakers: &pbproxystate.CircuitBreakers{ + UpstreamLimits: makeUpstreamLimitsIfNeeded(upstreamConfig.Limits), + }, + OutlierDetection: makeOutlierDetection(upstreamConfig.PassiveHealthCheck, nil, true), + }, + } + ti := groupedTarget.Targets[0] + transportSocket, err := s.createOutboundMeshMTLS(cfgSnap, ti.SpiffeIDs, ti.SNI) + if err != nil { + return nil, err + } + dynamic.OutboundTls = transportSocket + + var lb *structs.LoadBalancer + if node.LoadBalancer != nil { + lb = node.LoadBalancer + } + if err := injectLBToCluster(lb, dynamic.Config); err != nil { + return nil, fmt.Errorf("failed to apply load balancer configuration to cluster %q: %v", groupedTarget.ClusterName, err) + } + + // TODO: IR: http2 options not currently supported + //if upstreamConfig.Protocol == "http2" || upstreamConfig.Protocol == "grpc" { + // if err := s.setHttp2ProtocolOptions(c); err != nil { + // return nil, err + // } + //} + + switch len(groupedTarget.Targets) { + case 0: + continue + case 1: + // We expect one target so this passes through to continue setting the cluster up. + default: + return nil, fmt.Errorf("cannot have more than one target") + } + + if targetInfo := groupedTarget.Targets[0]; targetInfo.TransportSocket != nil { + dynamic.OutboundTls = targetInfo.TransportSocket + } + + // If the endpoint group is part of a failover group, add it to the failover group. Otherwise add it + // directly to the clusters. + if failoverGroup != nil { + eg := &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Dynamic{ + Dynamic: dynamic, + }, + } + endpointGroups = append(endpointGroups, eg) + } else { + cluster := &pbproxystate.Cluster{ + AltStatName: mappedTargets.baseClusterName, + Protocol: upstreamConfig.Protocol, + Group: &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Dynamic{ + Dynamic: dynamic, + }, + }, + }, + } + + out[mappedTargets.baseClusterName] = cluster + } + } + + // If there's a failover group, we only add the failover group to the top level list of clusters. Its endpoint + // groups are inlined. + if failoverGroup != nil { + failoverGroup.EndpointGroups = endpointGroups + cluster := &pbproxystate.Cluster{ + AltStatName: mappedTargets.baseClusterName, + Protocol: upstreamConfig.Protocol, + Group: &pbproxystate.Cluster_FailoverGroup{ + FailoverGroup: failoverGroup, + }, + } + out[mappedTargets.baseClusterName] = cluster + } + } + + //if escapeHatchCluster != nil { + // if len(out) != 1 { + // return nil, fmt.Errorf("cannot inject escape hatch cluster when discovery chain had no nodes") + // } + // var defaultCluster *pbproxystate.Cluster + // for _, k := range out { + // defaultCluster = k + // break + // } + // + // // Overlay what the user provided. + // escapeHatchCluster.GetEndpointGroup().GetDynamic().OutboundTls.ConnectionTls = + // defaultCluster.GetEndpointGroup().GetDynamic().OutboundTls.ConnectionTls + // + // out = append(out, escapeHatchCluster) + //} + + return out, nil +} + +// TODO(proxystate): Mesh Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeExportedUpstreamClustersForMeshGateway + +// makeClusterFromUserConfig returns the listener config decoded from an +// arbitrary proto3 json format string or an error if it's invalid. +// +// For now we only support embedding in JSON strings because of the hcl parsing +// pain (see Background section in the comment for decode.HookWeakDecodeFromSlice). +// This may be fixed in decode.HookWeakDecodeFromSlice in the future. +// +// When we do that we can support just nesting the config directly into the +// JSON/hcl naturally but this is a stop-gap that gets us an escape hatch +// immediately. It's also probably not a bad thing to support long-term since +// any config generated by other systems will likely be in canonical protobuf +// from rather than our slight variant in JSON/hcl. + +// TODO(proxystate): Mesh and Terminating Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeGatewayCluster + +func configureClusterWithHostnames( + logger hclog.Logger, + dnsEndpointGroup *pbproxystate.DNSEndpointGroup, + endpointList *pbproxystate.Endpoints, + dnsDiscoveryType string, + // hostnameEndpoints is a list of endpoints with a hostname as their address + hostnameEndpoints structs.CheckServiceNodes, + // isRemote determines whether the cluster is in a remote DC or partition and we should prefer a WAN address + isRemote bool, + // onlyPassing determines whether endpoints that do not have a passing status should be considered unhealthy + onlyPassing bool, +) { + // When a service instance is addressed by a hostname we have Envoy do the DNS resolution + // by setting a DNS cluster type and passing the hostname endpoints via CDS. + if dnsEndpointGroup.Config == nil { + dnsEndpointGroup.Config = &pbproxystate.DNSEndpointGroupConfig{} + } + dnsEndpointGroup.Config.DiscoveryType = pbproxystate.DiscoveryType_DISCOVERY_TYPE_LOGICAL + if dnsDiscoveryType == "strict_dns" { + dnsEndpointGroup.Config.DiscoveryType = pbproxystate.DiscoveryType_DISCOVERY_TYPE_STRICT + } + + endpoints := make([]*envoy_endpoint_v3.LbEndpoint, 0, 1) + uniqueHostnames := make(map[string]bool) + + var ( + hostname string + idx int + fallback *pbproxystate.Endpoint + ) + for i, e := range hostnameEndpoints { + _, addr, port := e.BestAddress(isRemote) + uniqueHostnames[addr] = true + + health, weight := calculateEndpointHealthAndWeight(e, onlyPassing) + if health == pbproxystate.HealthStatus_HEALTH_STATUS_UNHEALTHY { + fallback = makeLbEndpoint(addr, port, health, weight) + continue + } + + if len(endpoints) == 0 { + endpointList.Endpoints = append(endpointList.Endpoints, makeLbEndpoint(addr, port, health, weight)) + + hostname = addr + idx = i + break + } + } + + dc := hostnameEndpoints[idx].Node.Datacenter + service := hostnameEndpoints[idx].Service.CompoundServiceName() + + // Fall back to last unhealthy endpoint if none were healthy + if len(endpoints) == 0 { + logger.Warn("upstream service does not contain any healthy instances", + "dc", dc, "service", service.String()) + + //endpoints = append(endpoints, fallback) + endpointList.Endpoints = append(endpointList.Endpoints, fallback) + } + if len(uniqueHostnames) > 1 { + logger.Warn(fmt.Sprintf("service contains instances with more than one unique hostname; only %q be resolved by Envoy", hostname), + "dc", dc, "service", service.String()) + } + +} + +// TODO(proxystate): Terminating Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeExternalIPCluster + +// TODO(proxystate): Terminating Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeExternalHostnameCluster + +func makeUpstreamLimitsIfNeeded(limits *structs.UpstreamLimits) *pbproxystate.UpstreamLimits { + if limits == nil { + return nil + } + + upstreamLimits := &pbproxystate.UpstreamLimits{} + + // Likewise, make sure to not set any threshold values on the zero-value in + // order to rely on Envoy defaults + if limits.MaxConnections != nil { + upstreamLimits.MaxConnections = response.MakeUint32Value(*limits.MaxConnections) + } + if limits.MaxPendingRequests != nil { + upstreamLimits.MaxPendingRequests = response.MakeUint32Value(*limits.MaxPendingRequests) + } + if limits.MaxConcurrentRequests != nil { + upstreamLimits.MaxConcurrentRequests = response.MakeUint32Value(*limits.MaxConcurrentRequests) + } + + return upstreamLimits +} + +func injectLBToCluster(ec *structs.LoadBalancer, dc *pbproxystate.DynamicEndpointGroupConfig) error { + if ec == nil { + return nil + } + + switch ec.Policy { + case "": + return nil + case structs.LBPolicyLeastRequest: + lr := &pbproxystate.DynamicEndpointGroupConfig_LeastRequest{ + LeastRequest: &pbproxystate.LBPolicyLeastRequest{}, + } + + dc.LbPolicy = lr + + if ec.LeastRequestConfig != nil { + lr.LeastRequest.ChoiceCount = &wrapperspb.UInt32Value{Value: ec.LeastRequestConfig.ChoiceCount} + } + case structs.LBPolicyRoundRobin: + dc.LbPolicy = &pbproxystate.DynamicEndpointGroupConfig_RoundRobin{ + RoundRobin: &pbproxystate.LBPolicyRoundRobin{}, + } + + case structs.LBPolicyRandom: + dc.LbPolicy = &pbproxystate.DynamicEndpointGroupConfig_Random{ + Random: &pbproxystate.LBPolicyRandom{}, + } + + case structs.LBPolicyRingHash: + rh := &pbproxystate.DynamicEndpointGroupConfig_RingHash{ + RingHash: &pbproxystate.LBPolicyRingHash{}, + } + + dc.LbPolicy = rh + + if ec.RingHashConfig != nil { + rh.RingHash.MinimumRingSize = &wrapperspb.UInt64Value{Value: ec.RingHashConfig.MinimumRingSize} + rh.RingHash.MaximumRingSize = &wrapperspb.UInt64Value{Value: ec.RingHashConfig.MaximumRingSize} + } + case structs.LBPolicyMaglev: + dc.LbPolicy = &pbproxystate.DynamicEndpointGroupConfig_Maglev{ + Maglev: &pbproxystate.LBPolicyMaglev{}, + } + + default: + return fmt.Errorf("unsupported load balancer policy %q", ec.Policy) + } + return nil +} + // generatePeeredClusterName returns an SNI-like cluster name which mimics PeeredServiceSNI // but excludes partition information which could be ambiguous (local vs remote partition). func generatePeeredClusterName(uid proxycfg.UpstreamID, tb *pbpeering.PeeringTrustBundle) string { @@ -72,3 +1181,69 @@ func (s *Converter) getTargetClusterName(upstreamsSnapshot *proxycfg.ConfigSnaps } return clusterName } + +// Return an pbproxystate.OutlierDetection populated by the values from structs.PassiveHealthCheck. +// If all values are zero a default empty OutlierDetection will be returned to +// enable outlier detection with default values. +// - If override is not nil, it will overwrite the values from p, e.g., ingress gateway defaults +// - allowZero is added to handle the legacy case where connect-proxy and mesh gateway can set 0 +// for EnforcingConsecutive5xx. Due to the definition of proto of PassiveHealthCheck, ingress +// gateway's EnforcingConsecutive5xx must be > 0. +func makeOutlierDetection(p *structs.PassiveHealthCheck, override *structs.PassiveHealthCheck, allowZero bool) *pbproxystate.OutlierDetection { + od := &pbproxystate.OutlierDetection{} + if p != nil { + + if p.Interval != 0 { + od.Interval = durationpb.New(p.Interval) + } + if p.MaxFailures != 0 { + od.Consecutive_5Xx = &wrapperspb.UInt32Value{Value: p.MaxFailures} + } + + if p.EnforcingConsecutive5xx != nil { + // NOTE: EnforcingConsecutive5xx must be greater than 0 for ingress-gateway + if *p.EnforcingConsecutive5xx != 0 { + od.EnforcingConsecutive_5Xx = &wrapperspb.UInt32Value{Value: *p.EnforcingConsecutive5xx} + } else if allowZero { + od.EnforcingConsecutive_5Xx = &wrapperspb.UInt32Value{Value: *p.EnforcingConsecutive5xx} + } + } + + if p.MaxEjectionPercent != nil { + od.MaxEjectionPercent = &wrapperspb.UInt32Value{Value: *p.MaxEjectionPercent} + } + if p.BaseEjectionTime != nil { + od.BaseEjectionTime = durationpb.New(*p.BaseEjectionTime) + } + } + + if override == nil { + return od + } + + // override the default outlier detection value + if override.Interval != 0 { + od.Interval = durationpb.New(override.Interval) + } + if override.MaxFailures != 0 { + od.Consecutive_5Xx = &wrapperspb.UInt32Value{Value: override.MaxFailures} + } + + if override.EnforcingConsecutive5xx != nil { + // NOTE: EnforcingConsecutive5xx must be great than 0 for ingress-gateway + if *override.EnforcingConsecutive5xx != 0 { + od.EnforcingConsecutive_5Xx = &wrapperspb.UInt32Value{Value: *override.EnforcingConsecutive5xx} + } + // Because only ingress gateways have overrides and they cannot have a value of 0, there is no allowZero + // override case to handle + } + + if override.MaxEjectionPercent != nil { + od.MaxEjectionPercent = &wrapperspb.UInt32Value{Value: *override.MaxEjectionPercent} + } + if override.BaseEjectionTime != nil { + od.BaseEjectionTime = durationpb.New(*override.BaseEjectionTime) + } + + return od +} diff --git a/agent/xds/proxystateconverter/converter.go b/agent/xds/proxystateconverter/converter.go index 3a8037b44b1c..061a59d6e93e 100644 --- a/agent/xds/proxystateconverter/converter.go +++ b/agent/xds/proxystateconverter/converter.go @@ -1,5 +1,5 @@ // Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 +// SPDX-License-Identifier: BUSL-1.1 package proxystateconverter @@ -57,6 +57,10 @@ func (g *Converter) resourcesFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot) erro if err != nil { return err } + //g.routesFromSnapshot(cfgSnap) + g.clustersFromSnapshot(cfgSnap) + //g.endpointsFromSnapshot(cfgSnap) + //g.secretsFromSnapshot(cfgSnap) return nil } diff --git a/agent/xds/proxystateconverter/endpoints.go b/agent/xds/proxystateconverter/endpoints.go new file mode 100644 index 000000000000..5a5dc3708098 --- /dev/null +++ b/agent/xds/proxystateconverter/endpoints.go @@ -0,0 +1,85 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package proxystateconverter + +import ( + "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func makeLbEndpoint(addr string, port int, health pbproxystate.HealthStatus, weight int) *pbproxystate.Endpoint { + ep := &pbproxystate.Endpoint{ + Address: &pbproxystate.Endpoint_HostPort{ + HostPort: &pbproxystate.HostPortAddress{ + Host: addr, + Port: uint32(port), + }, + }, + } + ep.HealthStatus = health + ep.LoadBalancingWeight = &wrapperspb.UInt32Value{Value: uint32(weight)} + return ep +} + +// used in clusters.go +func makeHostPortEndpoint(host string, port int) *pbproxystate.Endpoint { + return &pbproxystate.Endpoint{ + Address: &pbproxystate.Endpoint_HostPort{ + HostPort: &pbproxystate.HostPortAddress{ + Host: host, + Port: uint32(port), + }, + }, + } +} + +func makeUnixSocketEndpoint(path string) *pbproxystate.Endpoint { + return &pbproxystate.Endpoint{ + Address: &pbproxystate.Endpoint_UnixSocket{ + UnixSocket: &pbproxystate.UnixSocketAddress{ + Path: path, + // envoy's mode is particular to a pipe address and is uint32. + // it also says "The mode for the Pipe. Not applicable for abstract sockets." + // https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#config-core-v3-pipe + Mode: "0", + }, + }, + } +} + +func calculateEndpointHealthAndWeight( + ep structs.CheckServiceNode, + onlyPassing bool, +) (pbproxystate.HealthStatus, int) { + healthStatus := pbproxystate.HealthStatus_HEALTH_STATUS_HEALTHY + weight := 1 + if ep.Service.Weights != nil { + weight = ep.Service.Weights.Passing + } + + for _, chk := range ep.Checks { + if chk.Status == api.HealthCritical { + healthStatus = pbproxystate.HealthStatus_HEALTH_STATUS_UNHEALTHY + } + if onlyPassing && chk.Status != api.HealthPassing { + healthStatus = pbproxystate.HealthStatus_HEALTH_STATUS_UNHEALTHY + } + if chk.Status == api.HealthWarning && ep.Service.Weights != nil { + weight = ep.Service.Weights.Warning + } + } + // Make weights fit Envoy's limits. A zero weight means that either Warning + // (likely) or Passing (weirdly) weight has been set to 0 effectively making + // this instance unhealthy and should not be sent traffic. + if weight < 1 { + healthStatus = pbproxystate.HealthStatus_HEALTH_STATUS_UNHEALTHY + weight = 1 + } + if weight > 128 { + weight = 128 + } + return healthStatus, weight +} diff --git a/agent/xds/proxystateconverter/failover_policy.go b/agent/xds/proxystateconverter/failover_policy.go new file mode 100644 index 000000000000..6ae120eaf941 --- /dev/null +++ b/agent/xds/proxystateconverter/failover_policy.go @@ -0,0 +1,158 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package proxystateconverter + +import ( + "fmt" + + "github.com/hashicorp/consul/agent/connect" + "github.com/hashicorp/consul/agent/proxycfg" + "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/envoyextensions/xdscommon" + "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" +) + +type discoChainTargets struct { + baseClusterName string + targets []targetInfo + failover bool + failoverPolicy structs.ServiceResolverFailoverPolicy +} + +type targetInfo struct { + TargetID string + TransportSocket *pbproxystate.TransportSocket + SNI string + RootPEMs string + SpiffeIDs []string + Region *string + + PrioritizeByLocality *structs.DiscoveryPrioritizeByLocality +} + +type discoChainTargetGroup struct { + Targets []targetInfo + ClusterName string +} + +func (ft discoChainTargets) groupedTargets() ([]discoChainTargetGroup, error) { + var targetGroups []discoChainTargetGroup + + if !ft.failover { + targetGroups = append(targetGroups, discoChainTargetGroup{ + ClusterName: ft.baseClusterName, + Targets: ft.targets, + }) + return targetGroups, nil + } + + switch ft.failoverPolicy.Mode { + case "sequential", "": + return ft.sequential() + case "order-by-locality": + return ft.orderByLocality() + default: + return targetGroups, fmt.Errorf("unexpected failover policy") + } +} + +func (s *Converter) mapDiscoChainTargets(cfgSnap *proxycfg.ConfigSnapshot, chain *structs.CompiledDiscoveryChain, node *structs.DiscoveryGraphNode, upstreamConfig structs.UpstreamConfig, forMeshGateway bool) (discoChainTargets, error) { + failoverTargets := discoChainTargets{} + + if node.Resolver == nil { + return discoChainTargets{}, fmt.Errorf("impossible to process a non-resolver node") + } + + primaryTargetID := node.Resolver.Target + upstreamsSnapshot, err := cfgSnap.ToConfigSnapshotUpstreams() + if err != nil && !forMeshGateway { + return discoChainTargets{}, err + } + + failoverTargets.baseClusterName = s.getTargetClusterName(upstreamsSnapshot, chain, primaryTargetID, forMeshGateway) + + tids := []string{primaryTargetID} + failover := node.Resolver.Failover + if failover != nil && !forMeshGateway { + tids = append(tids, failover.Targets...) + failoverTargets.failover = true + if failover.Policy == nil { + failoverTargets.failoverPolicy = structs.ServiceResolverFailoverPolicy{} + } else { + failoverTargets.failoverPolicy = *failover.Policy + } + } + + for _, tid := range tids { + target := chain.Targets[tid] + targetUID := proxycfg.NewUpstreamIDFromTargetID(tid) + ti := targetInfo{TargetID: tid, PrioritizeByLocality: target.PrioritizeByLocality} + + configureTLS := true + if forMeshGateway { + // We only initiate TLS if we're doing an L7 proxy. + configureTLS = structs.IsProtocolHTTPLike(upstreamConfig.Protocol) + } + + if !configureTLS { + failoverTargets.targets = append(failoverTargets.targets, ti) + continue + } + + if targetUID.Peer != "" { + tbs, _ := upstreamsSnapshot.UpstreamPeerTrustBundles.Get(targetUID.Peer) + + peerMeta, found := upstreamsSnapshot.UpstreamPeerMeta(targetUID) + if !found { + s.Logger.Warn("failed to fetch upstream peering metadata", "target", targetUID) + continue + } + ti.SNI = peerMeta.PrimarySNI() + ti.SpiffeIDs = peerMeta.SpiffeID + region := target.Locality.GetRegion() + ti.Region = ®ion + ti.RootPEMs = tbs.ConcatenatedRootPEMs() + } else { + ti.SNI = target.SNI + ti.RootPEMs = cfgSnap.RootPEMs() + ti.SpiffeIDs = []string{connect.SpiffeIDService{ + Host: cfgSnap.Roots.TrustDomain, + Namespace: target.Namespace, + Partition: target.Partition, + Datacenter: target.Datacenter, + Service: target.Service, + }.URI().String()} + } + //commonTLSContext := makeCommonTLSContext( + // cfgSnap.Leaf(), + // rootPEMs, + // makeTLSParametersFromProxyTLSConfig(cfgSnap.MeshConfigTLSOutgoing()), + //) + // + //err := injectSANMatcher(commonTLSContext, spiffeIDs...) + //if err != nil { + // return failoverTargets, fmt.Errorf("failed to inject SAN matcher rules for cluster %q: %v", sni, err) + //} + + //tlsContext := &envoy_tls_v3.UpstreamTlsContext{ + // CommonTlsContext: commonTLSContext, + // Sni: sni, + //} + //ti.TLSContext = tlsContext + failoverTargets.targets = append(failoverTargets.targets, ti) + } + + return failoverTargets, nil +} + +func (ft discoChainTargets) sequential() ([]discoChainTargetGroup, error) { + var targetGroups []discoChainTargetGroup + for i, t := range ft.targets { + targetGroups = append(targetGroups, discoChainTargetGroup{ + ClusterName: fmt.Sprintf("%s%d~%s", xdscommon.FailoverClusterNamePrefix, i, ft.baseClusterName), + Targets: []targetInfo{t}, + }) + } + return targetGroups, nil +} diff --git a/agent/xds/proxystateconverter/failover_policy_oss.go b/agent/xds/proxystateconverter/failover_policy_oss.go new file mode 100644 index 000000000000..a9b2ec24843c --- /dev/null +++ b/agent/xds/proxystateconverter/failover_policy_oss.go @@ -0,0 +1,15 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +//go:build !consulent +// +build !consulent + +package proxystateconverter + +import ( + "fmt" +) + +func (ft discoChainTargets) orderByLocality() ([]discoChainTargetGroup, error) { + return nil, fmt.Errorf("order-by-locality is a Consul Enterprise feature") +} diff --git a/agent/xds/proxystateconverter/listeners.go b/agent/xds/proxystateconverter/listeners.go index a530fbf4fe96..7e21819260ee 100644 --- a/agent/xds/proxystateconverter/listeners.go +++ b/agent/xds/proxystateconverter/listeners.go @@ -1,5 +1,5 @@ // Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 +// SPDX-License-Identifier: BUSL-1.1 package proxystateconverter @@ -16,11 +16,6 @@ import ( envoy_http_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" "github.com/hashicorp/go-uuid" - "github.com/hashicorp/consul/agent/xds/config" - "github.com/hashicorp/consul/agent/xds/naming" - "github.com/hashicorp/consul/agent/xds/platform" - "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" - "github.com/hashicorp/go-hclog" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/wrapperspb" @@ -29,7 +24,11 @@ import ( "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/config" + "github.com/hashicorp/consul/agent/xds/naming" + "github.com/hashicorp/consul/agent/xds/platform" "github.com/hashicorp/consul/envoyextensions/xdscommon" + "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" "github.com/hashicorp/consul/sdk/iptables" "github.com/hashicorp/consul/types" ) diff --git a/agent/xds/rbac.go b/agent/xds/rbac.go index b3f9267eefb4..0c00cb92cb0c 100644 --- a/agent/xds/rbac.go +++ b/agent/xds/rbac.go @@ -19,6 +19,7 @@ import ( "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/proto/private/pbpeering" ) @@ -987,7 +988,7 @@ func authenticatedPatternPrincipal(pattern string) *envoy_rbac_v3.Principal { Authenticated: &envoy_rbac_v3.Principal_Authenticated{ PrincipalName: &envoy_matcher_v3.StringMatcher{ MatchPattern: &envoy_matcher_v3.StringMatcher_SafeRegex{ - SafeRegex: makeEnvoyRegexMatch(pattern), + SafeRegex: response.MakeEnvoyRegexMatch(pattern), }, }, }, @@ -1019,7 +1020,7 @@ func xfccPrincipal(src rbacService) *envoy_rbac_v3.Principal { HeaderMatchSpecifier: &envoy_route_v3.HeaderMatcher_StringMatch{ StringMatch: &envoy_matcher_v3.StringMatcher{ MatchPattern: &envoy_matcher_v3.StringMatcher_SafeRegex{ - SafeRegex: makeEnvoyRegexMatch(pattern), + SafeRegex: response.MakeEnvoyRegexMatch(pattern), }, }, }, @@ -1224,7 +1225,7 @@ func convertPermission(perm *structs.IntentionPermission) *envoy_rbac_v3.Permiss Rule: &envoy_matcher_v3.PathMatcher_Path{ Path: &envoy_matcher_v3.StringMatcher{ MatchPattern: &envoy_matcher_v3.StringMatcher_SafeRegex{ - SafeRegex: makeEnvoyRegexMatch(perm.HTTP.PathRegex), + SafeRegex: response.MakeEnvoyRegexMatch(perm.HTTP.PathRegex), }, }, }, @@ -1245,7 +1246,7 @@ func convertPermission(perm *structs.IntentionPermission) *envoy_rbac_v3.Permiss } case hdr.Regex != "": eh.HeaderMatchSpecifier = &envoy_route_v3.HeaderMatcher_SafeRegexMatch{ - SafeRegexMatch: makeEnvoyRegexMatch(hdr.Regex), + SafeRegexMatch: response.MakeEnvoyRegexMatch(hdr.Regex), } case hdr.Prefix != "": eh.HeaderMatchSpecifier = &envoy_route_v3.HeaderMatcher_PrefixMatch{ @@ -1280,7 +1281,7 @@ func convertPermission(perm *structs.IntentionPermission) *envoy_rbac_v3.Permiss eh := &envoy_route_v3.HeaderMatcher{ Name: ":method", HeaderMatchSpecifier: &envoy_route_v3.HeaderMatcher_SafeRegexMatch{ - SafeRegexMatch: makeEnvoyRegexMatch(methodHeaderRegex), + SafeRegexMatch: response.MakeEnvoyRegexMatch(methodHeaderRegex), }, } diff --git a/agent/xds/resources_test.go b/agent/xds/resources_test.go index 22ef50bc876d..926c44fab86e 100644 --- a/agent/xds/resources_test.go +++ b/agent/xds/resources_test.go @@ -27,6 +27,7 @@ import ( "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/sdk/testutil" ) @@ -120,7 +121,7 @@ func TestAllResourcesFromSnapshot(t *testing.T) { } }) - r, err := createResponse(typeUrl, "00000001", "00000001", items) + r, err := response.CreateResponse(typeUrl, "00000001", "00000001", items) require.NoError(t, err) gotJSON := protoToJSON(t, r) diff --git a/agent/xds/response.go b/agent/xds/response/response.go similarity index 82% rename from agent/xds/response.go rename to agent/xds/response/response.go index 92e0f9d0a890..91ce6d739738 100644 --- a/agent/xds/response.go +++ b/agent/xds/response/response.go @@ -1,7 +1,7 @@ // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: BUSL-1.1 -package xds +package response import ( envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" @@ -13,7 +13,7 @@ import ( "google.golang.org/protobuf/types/known/wrapperspb" ) -func createResponse(typeURL string, version, nonce string, resources []proto.Message) (*envoy_discovery_v3.DiscoveryResponse, error) { +func CreateResponse(typeURL string, version, nonce string, resources []proto.Message) (*envoy_discovery_v3.DiscoveryResponse, error) { anys := make([]*anypb.Any, 0, len(resources)) for _, r := range resources { if r == nil { @@ -41,7 +41,7 @@ func createResponse(typeURL string, version, nonce string, resources []proto.Mes return resp, nil } -func makePipeAddress(path string, mode uint32) *envoy_core_v3.Address { +func MakePipeAddress(path string, mode uint32) *envoy_core_v3.Address { return &envoy_core_v3.Address{ Address: &envoy_core_v3.Address_Pipe{ Pipe: &envoy_core_v3.Pipe{ @@ -52,7 +52,7 @@ func makePipeAddress(path string, mode uint32) *envoy_core_v3.Address { } } -func makeAddress(ip string, port int) *envoy_core_v3.Address { +func MakeAddress(ip string, port int) *envoy_core_v3.Address { return &envoy_core_v3.Address{ Address: &envoy_core_v3.Address_SocketAddress{ SocketAddress: &envoy_core_v3.SocketAddress{ @@ -65,15 +65,15 @@ func makeAddress(ip string, port int) *envoy_core_v3.Address { } } -func makeUint32Value(n int) *wrapperspb.UInt32Value { +func MakeUint32Value(n int) *wrapperspb.UInt32Value { return &wrapperspb.UInt32Value{Value: uint32(n)} } -func makeBoolValue(n bool) *wrapperspb.BoolValue { +func MakeBoolValue(n bool) *wrapperspb.BoolValue { return &wrapperspb.BoolValue{Value: n} } -func makeEnvoyRegexMatch(patt string) *envoy_matcher_v3.RegexMatcher { +func MakeEnvoyRegexMatch(patt string) *envoy_matcher_v3.RegexMatcher { return &envoy_matcher_v3.RegexMatcher{ EngineType: &envoy_matcher_v3.RegexMatcher_GoogleRe2{ GoogleRe2: &envoy_matcher_v3.RegexMatcher_GoogleRE2{}, diff --git a/agent/xds/routes.go b/agent/xds/routes.go index 31b1c4a93241..6f0d18b05dcb 100644 --- a/agent/xds/routes.go +++ b/agent/xds/routes.go @@ -23,6 +23,7 @@ import ( "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/xds/config" + "github.com/hashicorp/consul/agent/xds/response" ) // routesFromSnapshot returns the xDS API representation of the "routes" in the @@ -71,7 +72,7 @@ func (s *ResourceGenerator) routesForConnectProxy(cfgSnap *proxycfg.ConfigSnapsh // ValidateClusters defaults to true when defined statically and false // when done via RDS. Re-set the reasonable value of true to prevent // null-routing traffic. - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), } resources = append(resources, route) } @@ -268,7 +269,7 @@ func (s *ResourceGenerator) routesForMeshGateway(cfgSnap *proxycfg.ConfigSnapsho // ValidateClusters defaults to true when defined statically and false // when done via RDS. Re-set the reasonable value of true to prevent // null-routing traffic. - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), } resources = append(resources, route) } @@ -286,7 +287,7 @@ func makeNamedDefaultRouteWithLB(clusterName string, lb *structs.LoadBalancer, t // Configure Envoy to rewrite Host header if autoHostRewrite { action.Route.HostRewriteSpecifier = &envoy_route_v3.RouteAction_AutoHostRewrite{ - AutoHostRewrite: makeBoolValue(true), + AutoHostRewrite: response.MakeBoolValue(true), } } @@ -311,7 +312,7 @@ func makeNamedDefaultRouteWithLB(clusterName string, lb *structs.LoadBalancer, t // ValidateClusters defaults to true when defined statically and false // when done via RDS. Re-set the reasonable value of true to prevent // null-routing traffic. - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), }, nil } @@ -321,7 +322,7 @@ func makeNamedAddressesRoute(routeName string, addresses map[string]string) (*en // ValidateClusters defaults to true when defined statically and false // when done via RDS. Re-set the reasonable value of true to prevent // null-routing traffic. - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), } for clusterName, address := range addresses { action := makeRouteActionFromName(clusterName) @@ -363,7 +364,7 @@ func (s *ResourceGenerator) routesForIngressGateway(cfgSnap *proxycfg.ConfigSnap // ValidateClusters defaults to true when defined statically and false // when done via RDS. Re-set the reasonable value of true to prevent // null-routing traffic. - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), } for _, u := range upstreams { @@ -412,7 +413,7 @@ func (s *ResourceGenerator) routesForIngressGateway(cfgSnap *proxycfg.ConfigSnap } else { svcRoute := &envoy_route_v3.RouteConfiguration{ Name: svcRouteName, - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), VirtualHosts: []*envoy_route_v3.VirtualHost{virtualHost}, } result = append(result, svcRoute) @@ -448,7 +449,7 @@ func (s *ResourceGenerator) routesForAPIGateway(cfgSnap *proxycfg.ConfigSnapshot // ValidateClusters defaults to true when defined statically and false // when done via RDS. Re-set the reasonable value of true to prevent // null-routing traffic. - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), } route, ok := cfgSnap.APIGateway.HTTPRoutes.Get(routeRef) @@ -515,7 +516,7 @@ func makeHeadersValueOptions(vals map[string]string, add bool) []*envoy_core_v3. Key: k, Value: v, }, - Append: makeBoolValue(add), + Append: response.MakeBoolValue(add), } opts = append(opts, o) } @@ -752,7 +753,7 @@ func (s *ResourceGenerator) makeUpstreamRouteForDiscoveryChain( func getRetryPolicyForDestination(destination *structs.ServiceRouteDestination) *envoy_route_v3.RetryPolicy { retryPolicy := &envoy_route_v3.RetryPolicy{} if destination.NumRetries > 0 { - retryPolicy.NumRetries = makeUint32Value(int(destination.NumRetries)) + retryPolicy.NumRetries = response.MakeUint32Value(int(destination.NumRetries)) } // The RetryOn magic values come from: https://www.envoyproxy.io/docs/envoy/v1.10.0/configuration/http_filters/router_filter#config-http-filters-router-x-envoy-retry-on @@ -805,7 +806,7 @@ func makeRouteMatchForDiscoveryRoute(discoveryRoute *structs.DiscoveryRoute) *en } case match.HTTP.PathRegex != "": em.PathSpecifier = &envoy_route_v3.RouteMatch_SafeRegex{ - SafeRegex: makeEnvoyRegexMatch(match.HTTP.PathRegex), + SafeRegex: response.MakeEnvoyRegexMatch(match.HTTP.PathRegex), } default: em.PathSpecifier = &envoy_route_v3.RouteMatch_Prefix{ @@ -827,7 +828,7 @@ func makeRouteMatchForDiscoveryRoute(discoveryRoute *structs.DiscoveryRoute) *en } case hdr.Regex != "": eh.HeaderMatchSpecifier = &envoy_route_v3.HeaderMatcher_SafeRegexMatch{ - SafeRegexMatch: makeEnvoyRegexMatch(hdr.Regex), + SafeRegexMatch: response.MakeEnvoyRegexMatch(hdr.Regex), } case hdr.Prefix != "": eh.HeaderMatchSpecifier = &envoy_route_v3.HeaderMatcher_PrefixMatch{ @@ -859,7 +860,7 @@ func makeRouteMatchForDiscoveryRoute(discoveryRoute *structs.DiscoveryRoute) *en eh := &envoy_route_v3.HeaderMatcher{ Name: ":method", HeaderMatchSpecifier: &envoy_route_v3.HeaderMatcher_SafeRegexMatch{ - SafeRegexMatch: makeEnvoyRegexMatch(methodHeaderRegex), + SafeRegexMatch: response.MakeEnvoyRegexMatch(methodHeaderRegex), }, } @@ -886,7 +887,7 @@ func makeRouteMatchForDiscoveryRoute(discoveryRoute *structs.DiscoveryRoute) *en eq.QueryParameterMatchSpecifier = &envoy_route_v3.QueryParameterMatcher_StringMatch{ StringMatch: &envoy_matcher_v3.StringMatcher{ MatchPattern: &envoy_matcher_v3.StringMatcher_SafeRegex{ - SafeRegex: makeEnvoyRegexMatch(qm.Regex), + SafeRegex: response.MakeEnvoyRegexMatch(qm.Regex), }, }, } @@ -967,7 +968,7 @@ func (s *ResourceGenerator) makeRouteActionForSplitter( weight := int(split.Weight * 100) totalWeight += weight cw := &envoy_route_v3.WeightedCluster_ClusterWeight{ - Weight: makeUint32Value(weight), + Weight: response.MakeUint32Value(weight), Name: clusterName, } if err := injectHeaderManipToWeightedCluster(split.Definition, cw); err != nil { @@ -983,7 +984,7 @@ func (s *ResourceGenerator) makeRouteActionForSplitter( var envoyWeightScale *wrapperspb.UInt32Value if totalWeight == 10000 { - envoyWeightScale = makeUint32Value(10000) + envoyWeightScale = response.MakeUint32Value(10000) } return &envoy_route_v3.Route_Route{ diff --git a/agent/xds/routes_test.go b/agent/xds/routes_test.go index c691114b5e3a..8b7345d88a52 100644 --- a/agent/xds/routes_test.go +++ b/agent/xds/routes_test.go @@ -16,6 +16,7 @@ import ( "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/agent/xds/testcommon" "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/sdk/testutil" @@ -241,7 +242,7 @@ func TestRoutesFromSnapshot(t *testing.T) { sort.Slice(routes, func(i, j int) bool { return routes[i].(*envoy_route_v3.RouteConfiguration).Name < routes[j].(*envoy_route_v3.RouteConfiguration).Name }) - r, err := createResponse(xdscommon.RouteType, "00000001", "00000001", routes) + r, err := response.CreateResponse(xdscommon.RouteType, "00000001", "00000001", routes) require.NoError(t, err) t.Run("current", func(t *testing.T) { diff --git a/agent/xds/xds_protocol_helpers_test.go b/agent/xds/xds_protocol_helpers_test.go index d609268a206b..bf632217d7ee 100644 --- a/agent/xds/xds_protocol_helpers_test.go +++ b/agent/xds/xds_protocol_helpers_test.go @@ -4,6 +4,7 @@ package xds import ( + "github.com/hashicorp/consul/agent/xds/response" "sort" "sync" "testing" @@ -228,7 +229,7 @@ func xdsNewEndpoint(ip string, port int) *envoy_endpoint_v3.LbEndpoint { return &envoy_endpoint_v3.LbEndpoint{ HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress(ip, port), + Address: response.MakeAddress(ip, port), }, }, } @@ -237,7 +238,7 @@ func xdsNewEndpoint(ip string, port int) *envoy_endpoint_v3.LbEndpoint { func xdsNewEndpointWithHealth(ip string, port int, health envoy_core_v3.HealthStatus, weight int) *envoy_endpoint_v3.LbEndpoint { ep := xdsNewEndpoint(ip, port) ep.HealthStatus = health - ep.LoadBalancingWeight = makeUint32Value(weight) + ep.LoadBalancingWeight = response.MakeUint32Value(weight) return ep } @@ -297,7 +298,7 @@ func xdsNewTransportSocket( if downstream { var requireClientCertPB *wrapperspb.BoolValue if requireClientCert { - requireClientCertPB = makeBoolValue(true) + requireClientCertPB = response.MakeBoolValue(true) } tlsContext = &envoy_tls_v3.DownstreamTlsContext{ @@ -597,7 +598,7 @@ func makeTestListener(t *testing.T, snap *proxycfg.ConfigSnapshot, fixtureName s return &envoy_listener_v3.Listener{ // Envoy can't bind to port 1 Name: "public_listener:0.0.0.0:1", - Address: makeAddress("0.0.0.0", 1), + Address: response.MakeAddress("0.0.0.0", 1), TrafficDirection: envoy_core_v3.TrafficDirection_INBOUND, FilterChains: []*envoy_listener_v3.FilterChain{ { @@ -620,7 +621,7 @@ func makeTestListener(t *testing.T, snap *proxycfg.ConfigSnapshot, fixtureName s case "tcp:public_listener": return &envoy_listener_v3.Listener{ Name: "public_listener:0.0.0.0:9999", - Address: makeAddress("0.0.0.0", 9999), + Address: response.MakeAddress("0.0.0.0", 9999), TrafficDirection: envoy_core_v3.TrafficDirection_INBOUND, FilterChains: []*envoy_listener_v3.FilterChain{ { @@ -643,7 +644,7 @@ func makeTestListener(t *testing.T, snap *proxycfg.ConfigSnapshot, fixtureName s case "tcp:db": return &envoy_listener_v3.Listener{ Name: "db:127.0.0.1:9191", - Address: makeAddress("127.0.0.1", 9191), + Address: response.MakeAddress("127.0.0.1", 9191), TrafficDirection: envoy_core_v3.TrafficDirection_OUTBOUND, FilterChains: []*envoy_listener_v3.FilterChain{ { @@ -661,7 +662,7 @@ func makeTestListener(t *testing.T, snap *proxycfg.ConfigSnapshot, fixtureName s case "http2:db": return &envoy_listener_v3.Listener{ Name: "db:127.0.0.1:9191", - Address: makeAddress("127.0.0.1", 9191), + Address: response.MakeAddress("127.0.0.1", 9191), TrafficDirection: envoy_core_v3.TrafficDirection_OUTBOUND, FilterChains: []*envoy_listener_v3.FilterChain{ { @@ -689,7 +690,7 @@ func makeTestListener(t *testing.T, snap *proxycfg.ConfigSnapshot, fixtureName s case "http2:db:rds": return &envoy_listener_v3.Listener{ Name: "db:127.0.0.1:9191", - Address: makeAddress("127.0.0.1", 9191), + Address: response.MakeAddress("127.0.0.1", 9191), TrafficDirection: envoy_core_v3.TrafficDirection_OUTBOUND, FilterChains: []*envoy_listener_v3.FilterChain{ { @@ -720,7 +721,7 @@ func makeTestListener(t *testing.T, snap *proxycfg.ConfigSnapshot, fixtureName s case "http:db:rds": return &envoy_listener_v3.Listener{ Name: "db:127.0.0.1:9191", - Address: makeAddress("127.0.0.1", 9191), + Address: response.MakeAddress("127.0.0.1", 9191), TrafficDirection: envoy_core_v3.TrafficDirection_OUTBOUND, FilterChains: []*envoy_listener_v3.FilterChain{ { @@ -751,7 +752,7 @@ func makeTestListener(t *testing.T, snap *proxycfg.ConfigSnapshot, fixtureName s case "tcp:geo-cache": return &envoy_listener_v3.Listener{ Name: "prepared_query:geo-cache:127.10.10.10:8181", - Address: makeAddress("127.10.10.10", 8181), + Address: response.MakeAddress("127.10.10.10", 8181), TrafficDirection: envoy_core_v3.TrafficDirection_OUTBOUND, FilterChains: []*envoy_listener_v3.FilterChain{ { @@ -777,7 +778,7 @@ func makeTestRoute(t *testing.T, fixtureName string) *envoy_route_v3.RouteConfig case "http2:db", "http:db": return &envoy_route_v3.RouteConfiguration{ Name: "db", - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), VirtualHosts: []*envoy_route_v3.VirtualHost{ { Name: "db", diff --git a/agent/xdsv2/cluster_resources.go b/agent/xdsv2/cluster_resources.go index c162d05c8d84..4c1573d5db3d 100644 --- a/agent/xdsv2/cluster_resources.go +++ b/agent/xdsv2/cluster_resources.go @@ -1,6 +1,331 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + package xdsv2 +import ( + "errors" + "fmt" + + envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + envoy_aggregate_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3" + envoy_upstreams_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" + envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +func (pr *ProxyResources) makeXDSClusters() ([]proto.Message, error) { + clusters := make([]proto.Message, 0) + + for clusterName := range pr.proxyState.Clusters { + protoCluster, err := pr.makeClusters(clusterName) + // TODO: aggregate errors for clusters and still return any properly formed clusters. + if err != nil { + return nil, err + } + clusters = append(clusters, protoCluster...) + } + + return clusters, nil +} + +func (pr *ProxyResources) makeClusters(name string) ([]proto.Message, error) { + clusters := make([]proto.Message, 0) + proxyStateCluster, ok := pr.proxyState.Clusters[name] + if !ok { + return nil, fmt.Errorf("cluster %q not found", name) + } + + switch proxyStateCluster.Group.(type) { + case *pbproxystate.Cluster_FailoverGroup: + fg := proxyStateCluster.GetFailoverGroup() + clusters, err := pr.makeEnvoyAggregateCluster(name, proxyStateCluster.Protocol, fg) + if err != nil { + return nil, err + } + for _, c := range clusters { + clusters = append(clusters, c) + } + + case *pbproxystate.Cluster_EndpointGroup: + eg := proxyStateCluster.GetEndpointGroup() + cluster, err := pr.makeEnvoyCluster(name, proxyStateCluster.Protocol, eg) + if err != nil { + return nil, err + } + clusters = append(clusters, cluster) + + default: + return nil, errors.New("cluster group type should be Endpoint Group or Failover Group") + } + return clusters, nil +} + +func (pr *ProxyResources) makeEnvoyCluster(name string, protocol string, eg *pbproxystate.EndpointGroup) (*envoy_cluster_v3.Cluster, error) { + if eg != nil { + switch t := eg.Group.(type) { + case *pbproxystate.EndpointGroup_Dynamic: + dynamic := eg.GetDynamic() + return pr.makeEnvoyDynamicCluster(name, protocol, dynamic) + case *pbproxystate.EndpointGroup_Static: + static := eg.GetStatic() + return pr.makeEnvoyStaticCluster(name, protocol, static) + case *pbproxystate.EndpointGroup_Dns: + dns := eg.GetDns() + return pr.makeEnvoyDnsCluster(name, protocol, dns) + case *pbproxystate.EndpointGroup_Passthrough: + passthrough := eg.GetPassthrough() + return pr.makeEnvoyPassthroughCluster(name, protocol, passthrough) + default: + return nil, fmt.Errorf("unsupported endpoint group type: %s", t) + } + } + return nil, fmt.Errorf("no endpoint group") +} + +func (pr *ProxyResources) makeEnvoyDynamicCluster(name string, protocol string, dynamic *pbproxystate.DynamicEndpointGroup) (*envoy_cluster_v3.Cluster, error) { + cluster := &envoy_cluster_v3.Cluster{ + Name: name, + ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_EDS}, + EdsClusterConfig: &envoy_cluster_v3.Cluster_EdsClusterConfig{ + EdsConfig: &envoy_core_v3.ConfigSource{ + ResourceApiVersion: envoy_core_v3.ApiVersion_V3, + ConfigSourceSpecifier: &envoy_core_v3.ConfigSource_Ads{ + Ads: &envoy_core_v3.AggregatedConfigSource{}, + }, + }, + }, + } + err := addHttpProtocolOptions(protocol, cluster) + if err != nil { + return nil, err + } + if dynamic.Config != nil { + if dynamic.Config.UseAltStatName { + cluster.AltStatName = name + } + cluster.ConnectTimeout = dynamic.Config.ConnectTimeout + if !dynamic.Config.DisablePanicThreshold { + cluster.CommonLbConfig = &envoy_cluster_v3.Cluster_CommonLbConfig{ + HealthyPanicThreshold: &envoy_type_v3.Percent{ + Value: 0, // disable panic threshold + }, + } + } + addEnvoyCircuitBreakers(dynamic.Config.CircuitBreakers, cluster) + addEnvoyOutlierDetection(dynamic.Config.OutlierDetection, cluster) + + err := addEnvoyLBToCluster(dynamic.Config, cluster) + if err != nil { + return nil, err + } + } + + if dynamic.OutboundTls != nil { + envoyTransportSocket, err := pr.makeEnvoyTransportSocket(dynamic.OutboundTls) + if err != nil { + return nil, err + } + cluster.TransportSocket = envoyTransportSocket + } + + return cluster, nil + +} + +func (pr *ProxyResources) makeEnvoyStaticCluster(name string, protocol string, static *pbproxystate.StaticEndpointGroup) (*envoy_cluster_v3.Cluster, error) { + endpointList, ok := pr.proxyState.Endpoints[name] + if !ok || endpointList == nil { + return nil, fmt.Errorf("static cluster %q is missing endpoints", name) + } + cluster := &envoy_cluster_v3.Cluster{ + Name: name, + ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_STATIC}, + LoadAssignment: makeEnvoyClusterLoadAssignment(name, endpointList.Endpoints), + } + err := addHttpProtocolOptions(protocol, cluster) + if err != nil { + return nil, err + } + + if static.Config != nil { + cluster.ConnectTimeout = static.Config.ConnectTimeout + addEnvoyCircuitBreakers(static.GetConfig().CircuitBreakers, cluster) + } + return cluster, nil +} +func (pr *ProxyResources) makeEnvoyDnsCluster(name string, protocol string, dns *pbproxystate.DNSEndpointGroup) (*envoy_cluster_v3.Cluster, error) { + return nil, nil +} +func (pr *ProxyResources) makeEnvoyPassthroughCluster(name string, protocol string, passthrough *pbproxystate.PassthroughEndpointGroup) (*envoy_cluster_v3.Cluster, error) { + cluster := &envoy_cluster_v3.Cluster{ + Name: name, + ConnectTimeout: passthrough.Config.ConnectTimeout, + LbPolicy: envoy_cluster_v3.Cluster_CLUSTER_PROVIDED, + ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_ORIGINAL_DST}, + } + if passthrough.OutboundTls != nil { + envoyTransportSocket, err := pr.makeEnvoyTransportSocket(passthrough.OutboundTls) + if err != nil { + return nil, err + } + cluster.TransportSocket = envoyTransportSocket + } + err := addHttpProtocolOptions(protocol, cluster) + if err != nil { + return nil, err + } + return cluster, nil +} + +func (pr *ProxyResources) makeEnvoyAggregateCluster(name string, protocol string, fg *pbproxystate.FailoverGroup) ([]*envoy_cluster_v3.Cluster, error) { + var clusters []*envoy_cluster_v3.Cluster + if fg != nil { + + var egNames []string + for _, eg := range fg.EndpointGroups { + cluster, err := pr.makeEnvoyCluster(name, protocol, eg) + if err != nil { + return nil, err + } + egNames = append(egNames, cluster.Name) + clusters = append(clusters, cluster) + } + aggregateClusterConfig, err := anypb.New(&envoy_aggregate_cluster_v3.ClusterConfig{ + Clusters: egNames, + }) + + if err != nil { + return nil, err + } + + c := &envoy_cluster_v3.Cluster{ + Name: name, + AltStatName: name, + ConnectTimeout: fg.Config.ConnectTimeout, + LbPolicy: envoy_cluster_v3.Cluster_CLUSTER_PROVIDED, + ClusterDiscoveryType: &envoy_cluster_v3.Cluster_ClusterType{ + ClusterType: &envoy_cluster_v3.Cluster_CustomClusterType{ + Name: "envoy.clusters.aggregate", + TypedConfig: aggregateClusterConfig, + }, + }, + } + err = addHttpProtocolOptions(protocol, c) + if err != nil { + return nil, err + } + clusters = append(clusters, c) + } + return clusters, nil +} + +func addHttpProtocolOptions(protocol string, c *envoy_cluster_v3.Cluster) error { + if !(protocol == "http2" || protocol == "grpc") { + // do not error. returning nil means it won't get set. + return nil + } + cfg := &envoy_upstreams_v3.HttpProtocolOptions{ + UpstreamProtocolOptions: &envoy_upstreams_v3.HttpProtocolOptions_ExplicitHttpConfig_{ + ExplicitHttpConfig: &envoy_upstreams_v3.HttpProtocolOptions_ExplicitHttpConfig{ + ProtocolConfig: &envoy_upstreams_v3.HttpProtocolOptions_ExplicitHttpConfig_Http2ProtocolOptions{ + Http2ProtocolOptions: &envoy_core_v3.Http2ProtocolOptions{}, + }, + }, + }, + } + any, err := anypb.New(cfg) + if err != nil { + return err + } + c.TypedExtensionProtocolOptions = map[string]*anypb.Any{ + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": any, + } + return nil +} + +// addEnvoyOutlierDetection will add outlier detection config to the cluster, and if nil, add empty OutlierDetection to +// enable it with default values +func addEnvoyOutlierDetection(outlierDetection *pbproxystate.OutlierDetection, c *envoy_cluster_v3.Cluster) { + if outlierDetection == nil { + return + } + od := &envoy_cluster_v3.OutlierDetection{ + BaseEjectionTime: outlierDetection.GetBaseEjectionTime(), + Consecutive_5Xx: outlierDetection.GetConsecutive_5Xx(), + EnforcingConsecutive_5Xx: outlierDetection.GetEnforcingConsecutive_5Xx(), + Interval: outlierDetection.GetInterval(), + MaxEjectionPercent: outlierDetection.GetMaxEjectionPercent(), + } + c.OutlierDetection = od + +} + +func addEnvoyCircuitBreakers(circuitBreakers *pbproxystate.CircuitBreakers, c *envoy_cluster_v3.Cluster) { + if circuitBreakers != nil { + if circuitBreakers.UpstreamLimits == nil { + c.CircuitBreakers = &envoy_cluster_v3.CircuitBreakers{} + return + } + threshold := &envoy_cluster_v3.CircuitBreakers_Thresholds{} + threshold.MaxConnections = circuitBreakers.UpstreamLimits.MaxConnections + threshold.MaxPendingRequests = circuitBreakers.UpstreamLimits.MaxPendingRequests + threshold.MaxRequests = circuitBreakers.UpstreamLimits.MaxConcurrentRequests + + c.CircuitBreakers = &envoy_cluster_v3.CircuitBreakers{ + Thresholds: []*envoy_cluster_v3.CircuitBreakers_Thresholds{threshold}, + } + } +} + +func addEnvoyLBToCluster(dynamicConfig *pbproxystate.DynamicEndpointGroupConfig, c *envoy_cluster_v3.Cluster) error { + if dynamicConfig == nil || dynamicConfig.LbPolicy == nil { + return nil + } + + switch d := dynamicConfig.LbPolicy.(type) { + case *pbproxystate.DynamicEndpointGroupConfig_LeastRequest: + c.LbPolicy = envoy_cluster_v3.Cluster_LEAST_REQUEST + + lb := dynamicConfig.LbPolicy.(*pbproxystate.DynamicEndpointGroupConfig_LeastRequest) + if lb.LeastRequest != nil { + c.LbConfig = &envoy_cluster_v3.Cluster_LeastRequestLbConfig_{ + LeastRequestLbConfig: &envoy_cluster_v3.Cluster_LeastRequestLbConfig{ + ChoiceCount: lb.LeastRequest.ChoiceCount, + }, + } + } + case *pbproxystate.DynamicEndpointGroupConfig_RoundRobin: + c.LbPolicy = envoy_cluster_v3.Cluster_ROUND_ROBIN + + case *pbproxystate.DynamicEndpointGroupConfig_Random: + c.LbPolicy = envoy_cluster_v3.Cluster_RANDOM + + case *pbproxystate.DynamicEndpointGroupConfig_RingHash: + c.LbPolicy = envoy_cluster_v3.Cluster_RING_HASH + + lb := dynamicConfig.LbPolicy.(*pbproxystate.DynamicEndpointGroupConfig_RingHash) + if lb.RingHash != nil { + c.LbConfig = &envoy_cluster_v3.Cluster_RingHashLbConfig_{ + RingHashLbConfig: &envoy_cluster_v3.Cluster_RingHashLbConfig{ + MinimumRingSize: lb.RingHash.MinimumRingSize, + MaximumRingSize: lb.RingHash.MaximumRingSize, + }, + } + } + case *pbproxystate.DynamicEndpointGroupConfig_Maglev: + c.LbPolicy = envoy_cluster_v3.Cluster_MAGLEV + + default: + return fmt.Errorf("unsupported load balancer policy %q for cluster %q", d, c.Name) + } + return nil +} + // TODO(proxystate): In a future PR this will create clusters and add it to ProxyResources.proxyState -func (pr *ProxyResources) makeCluster(name string) error { +func (pr *ProxyResources) makeEnvoyClusterFromL4Destination(name string) error { return nil } diff --git a/agent/xdsv2/endpoint_resources.go b/agent/xdsv2/endpoint_resources.go new file mode 100644 index 000000000000..f5fd1b9c5d09 --- /dev/null +++ b/agent/xdsv2/endpoint_resources.go @@ -0,0 +1,41 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package xdsv2 + +import ( + envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + "github.com/hashicorp/consul/agent/xds/response" + "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" +) + +func makeEnvoyEndpoint(endpoint *pbproxystate.Endpoint) *envoy_endpoint_v3.LbEndpoint { + var address *envoy_core_v3.Address + if endpoint.GetUnixSocket() != nil { + address = response.MakePipeAddress(endpoint.GetUnixSocket().GetPath(), 0) + } else { + address = response.MakeAddress(endpoint.GetHostPort().GetHost(), int(endpoint.GetHostPort().Port)) + } + return &envoy_endpoint_v3.LbEndpoint{ + HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ + Endpoint: &envoy_endpoint_v3.Endpoint{ + Address: address, + }, + }, + } +} + +func makeEnvoyClusterLoadAssignment(clusterName string, endpoints []*pbproxystate.Endpoint) *envoy_endpoint_v3.ClusterLoadAssignment { + localityLbEndpoints := make([]*envoy_endpoint_v3.LocalityLbEndpoints, 0, len(endpoints)) + for _, endpoint := range endpoints { + localityLbEndpoints = append(localityLbEndpoints, &envoy_endpoint_v3.LocalityLbEndpoints{ + LbEndpoints: []*envoy_endpoint_v3.LbEndpoint{makeEnvoyEndpoint(endpoint)}, + }) + } + return &envoy_endpoint_v3.ClusterLoadAssignment{ + ClusterName: clusterName, + Endpoints: localityLbEndpoints, + } + +} diff --git a/agent/xdsv2/listener_resources.go b/agent/xdsv2/listener_resources.go index 6d6ce7c8002e..2d3bf5d60da5 100644 --- a/agent/xdsv2/listener_resources.go +++ b/agent/xdsv2/listener_resources.go @@ -1,3 +1,6 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + package xdsv2 import ( @@ -281,7 +284,7 @@ func (pr *ProxyResources) makeEnvoyResourcesForSNIDestination(sni *pbproxystate. } func (pr *ProxyResources) makeEnvoyResourcesForL4Destination(l4 *pbproxystate.Router_L4) ([]*envoy_listener_v3.Filter, error) { - err := pr.makeCluster(l4.L4.Name) + err := pr.makeEnvoyClusterFromL4Destination(l4.L4.Name) if err != nil { return nil, err } @@ -632,7 +635,7 @@ func (pr *ProxyResources) makeEnvoyTransportSocket(ts *pbproxystate.TransportSoc // if tls is nil but connection tls is provided, then the proxy state is misconfigured return nil, fmt.Errorf("proxyState.Tls is required to generate router's transport socket") } - om := ts.ConnectionTls.(*pbproxystate.TransportSocket_OutboundMesh).OutboundMesh + om := ts.GetOutboundMesh() leaf, ok := pr.proxyState.LeafCertificates[om.IdentityKey] if !ok { return nil, fmt.Errorf("leaf %s not found in proxyState", om.IdentityKey) diff --git a/agent/xdsv2/resources.go b/agent/xdsv2/resources.go index 6bc1df58c664..6082f7dd56b5 100644 --- a/agent/xdsv2/resources.go +++ b/agent/xdsv2/resources.go @@ -1,5 +1,5 @@ // Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 +// SPDX-License-Identifier: BUSL-1.1 package xdsv2 @@ -47,7 +47,6 @@ func (g *ResourceGenerator) AllResourcesFromIR(proxyState *pbmesh.ProxyState) (m func (pr *ProxyResources) generateXDSResources() error { listeners := make([]proto.Message, 0) - clusters := make([]proto.Message, 0) routes := make([]proto.Message, 0) endpoints := make([]proto.Message, 0) @@ -60,6 +59,11 @@ func (pr *ProxyResources) generateXDSResources() error { listeners = append(listeners, protoListener) } + clusters, err := pr.makeXDSClusters() + if err != nil { + return err + } + pr.envoyResources[xdscommon.ListenerType] = listeners pr.envoyResources[xdscommon.ClusterType] = clusters pr.envoyResources[xdscommon.RouteType] = routes diff --git a/agent/xdsv2/route_resources.go b/agent/xdsv2/route_resources.go index 1433534e8f0a..86d82a5d306f 100644 --- a/agent/xdsv2/route_resources.go +++ b/agent/xdsv2/route_resources.go @@ -1,3 +1,6 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + package xdsv2 import ( diff --git a/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.pb.go b/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.pb.go index 9cbfe4ad6943..671e25f4ca04 100644 --- a/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.pb.go +++ b/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.pb.go @@ -86,6 +86,10 @@ type Cluster struct { Group isCluster_Group `protobuf_oneof:"group"` // escape_hatch_cluster_json configures a user configured escape hatch cluster. EscapeHatchClusterJson string `protobuf:"bytes,3,opt,name=escape_hatch_cluster_json,json=escapeHatchClusterJson,proto3" json:"escape_hatch_cluster_json,omitempty"` + // alt_stat_name is the name used for observability in place of cluster name if provided. + AltStatName string `protobuf:"bytes,4,opt,name=alt_stat_name,json=altStatName,proto3" json:"alt_stat_name,omitempty"` + // protocol is the local path protocol or the service protocol. + Protocol string `protobuf:"bytes,5,opt,name=protocol,proto3" json:"protocol,omitempty"` } func (x *Cluster) Reset() { @@ -148,6 +152,20 @@ func (x *Cluster) GetEscapeHatchClusterJson() string { return "" } +func (x *Cluster) GetAltStatName() string { + if x != nil { + return x.AltStatName + } + return "" +} + +func (x *Cluster) GetProtocol() string { + if x != nil { + return x.Protocol + } + return "" +} + type isCluster_Group interface { isCluster_Group() } @@ -1679,7 +1697,7 @@ var file_pbmesh_v1alpha1_pbproxystate_cluster_proto_rawDesc = []byte{ 0x70, 0x62, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x22, 0x97, 0x02, 0x0a, 0x07, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, + 0x6f, 0x74, 0x6f, 0x22, 0xd7, 0x02, 0x0a, 0x07, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x63, 0x0a, 0x0e, 0x66, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, @@ -1696,7 +1714,11 @@ var file_pbmesh_v1alpha1_pbproxystate_cluster_proto_rawDesc = []byte{ 0x61, 0x70, 0x65, 0x5f, 0x68, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x48, 0x61, 0x74, 0x63, 0x68, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x4a, 0x73, 0x6f, 0x6e, 0x42, 0x07, 0x0a, 0x05, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x22, 0xce, 0x01, + 0x4a, 0x73, 0x6f, 0x6e, 0x12, 0x22, 0x0a, 0x0d, 0x61, 0x6c, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x61, 0x6c, 0x74, + 0x53, 0x74, 0x61, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x42, 0x07, 0x0a, 0x05, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x22, 0xce, 0x01, 0x0a, 0x0d, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x63, 0x0a, 0x0f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, diff --git a/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.proto b/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.proto index 9b4194d981f0..860ebf2a5cbf 100644 --- a/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.proto +++ b/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.proto @@ -18,6 +18,10 @@ message Cluster { } // escape_hatch_cluster_json configures a user configured escape hatch cluster. string escape_hatch_cluster_json = 3; + // alt_stat_name is the name used for observability in place of cluster name if provided. + string alt_stat_name = 4; + // protocol is the local path protocol or the service protocol. + string protocol = 5; } message FailoverGroup { diff --git a/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.pb.go b/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.pb.go index 9801e29c3c70..59317396be4a 100644 --- a/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.pb.go +++ b/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.pb.go @@ -231,6 +231,8 @@ type TransportSocket struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + // name of the transport socket + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // Types that are assignable to ConnectionTls: // // *TransportSocket_InboundMesh @@ -239,8 +241,8 @@ type TransportSocket struct { // *TransportSocket_OutboundNonMesh ConnectionTls isTransportSocket_ConnectionTls `protobuf_oneof:"connection_tls"` // tls_parameters can override any top level tls parameters that are configured. - TlsParameters *TLSParameters `protobuf:"bytes,5,opt,name=tls_parameters,json=tlsParameters,proto3" json:"tls_parameters,omitempty"` - AlpnProtocols []string `protobuf:"bytes,6,rep,name=alpn_protocols,json=alpnProtocols,proto3" json:"alpn_protocols,omitempty"` + TlsParameters *TLSParameters `protobuf:"bytes,6,opt,name=tls_parameters,json=tlsParameters,proto3" json:"tls_parameters,omitempty"` + AlpnProtocols []string `protobuf:"bytes,7,rep,name=alpn_protocols,json=alpnProtocols,proto3" json:"alpn_protocols,omitempty"` } func (x *TransportSocket) Reset() { @@ -275,6 +277,13 @@ func (*TransportSocket) Descriptor() ([]byte, []int) { return file_pbmesh_v1alpha1_pbproxystate_transport_socket_proto_rawDescGZIP(), []int{1} } +func (x *TransportSocket) GetName() string { + if x != nil { + return x.Name + } + return "" +} + func (m *TransportSocket) GetConnectionTls() isTransportSocket_ConnectionTls { if m != nil { return m.ConnectionTls @@ -330,22 +339,22 @@ type isTransportSocket_ConnectionTls interface { type TransportSocket_InboundMesh struct { // inbound_mesh is for incoming connections FROM the mesh. - InboundMesh *InboundMeshMTLS `protobuf:"bytes,1,opt,name=inbound_mesh,json=inboundMesh,proto3,oneof"` + InboundMesh *InboundMeshMTLS `protobuf:"bytes,2,opt,name=inbound_mesh,json=inboundMesh,proto3,oneof"` } type TransportSocket_OutboundMesh struct { // outbound_mesh is for outbound connections TO mesh destinations. - OutboundMesh *OutboundMeshMTLS `protobuf:"bytes,2,opt,name=outbound_mesh,json=outboundMesh,proto3,oneof"` + OutboundMesh *OutboundMeshMTLS `protobuf:"bytes,3,opt,name=outbound_mesh,json=outboundMesh,proto3,oneof"` } type TransportSocket_InboundNonMesh struct { // inbound_non_mesh is for incoming connections FROM non mesh. - InboundNonMesh *InboundNonMeshTLS `protobuf:"bytes,3,opt,name=inbound_non_mesh,json=inboundNonMesh,proto3,oneof"` + InboundNonMesh *InboundNonMeshTLS `protobuf:"bytes,4,opt,name=inbound_non_mesh,json=inboundNonMesh,proto3,oneof"` } type TransportSocket_OutboundNonMesh struct { // outbound_non_mesh is for outbound connections TO non mesh destinations. - OutboundNonMesh *OutboundNonMeshTLS `protobuf:"bytes,4,opt,name=outbound_non_mesh,json=outboundNonMesh,proto3,oneof"` + OutboundNonMesh *OutboundNonMeshTLS `protobuf:"bytes,5,opt,name=outbound_non_mesh,json=outboundNonMesh,proto3,oneof"` } func (*TransportSocket_InboundMesh) isTransportSocket_ConnectionTls() {} @@ -1038,213 +1047,214 @@ var file_pbmesh_v1alpha1_pbproxystate_transport_socket_proto_rawDesc = []byte{ 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x4c, 0x53, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x52, 0x15, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x54, 0x6c, 0x73, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, - 0x22, 0xd1, 0x04, 0x0a, 0x0f, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x6f, - 0x63, 0x6b, 0x65, 0x74, 0x12, 0x61, 0x0a, 0x0c, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, - 0x6d, 0x65, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x68, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, - 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, - 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, - 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, 0x53, 0x48, 0x00, 0x52, 0x0b, 0x69, 0x6e, 0x62, 0x6f, - 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x64, 0x0a, 0x0d, 0x6f, 0x75, 0x74, 0x62, 0x6f, - 0x75, 0x6e, 0x64, 0x5f, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, + 0x22, 0xe5, 0x04, 0x0a, 0x0f, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x6f, + 0x63, 0x6b, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x61, 0x0a, 0x0c, 0x69, 0x6e, 0x62, 0x6f, + 0x75, 0x6e, 0x64, 0x5f, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, - 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4f, 0x75, 0x74, - 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, 0x53, 0x48, 0x00, 0x52, - 0x0c, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x6a, 0x0a, - 0x10, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x6e, 0x6f, 0x6e, 0x5f, 0x6d, 0x65, 0x73, - 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, - 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, - 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, - 0x4d, 0x65, 0x73, 0x68, 0x54, 0x4c, 0x53, 0x48, 0x00, 0x52, 0x0e, 0x69, 0x6e, 0x62, 0x6f, 0x75, - 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x6d, 0x0a, 0x11, 0x6f, 0x75, 0x74, - 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x6e, 0x6f, 0x6e, 0x5f, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x49, 0x6e, 0x62, + 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, 0x53, 0x48, 0x00, 0x52, 0x0b, + 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x64, 0x0a, 0x0d, 0x6f, + 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, + 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x2e, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, + 0x53, 0x48, 0x00, 0x52, 0x0c, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, + 0x68, 0x12, 0x6a, 0x0a, 0x10, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x6e, 0x6f, 0x6e, + 0x5f, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, 0x61, + 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, + 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, + 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, + 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x54, 0x4c, 0x53, 0x48, 0x00, 0x52, 0x0e, 0x69, + 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x6d, 0x0a, + 0x11, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x6e, 0x6f, 0x6e, 0x5f, 0x6d, 0x65, + 0x73, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, + 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, + 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, + 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, + 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x54, 0x4c, 0x53, 0x48, 0x00, 0x52, 0x0f, 0x6f, 0x75, 0x74, + 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x61, 0x0a, 0x0e, + 0x74, 0x6c, 0x73, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, - 0x74, 0x65, 0x2e, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, - 0x73, 0x68, 0x54, 0x4c, 0x53, 0x48, 0x00, 0x52, 0x0f, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, - 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x61, 0x0a, 0x0e, 0x74, 0x6c, 0x73, 0x5f, - 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, - 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, - 0x4c, 0x53, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x52, 0x0d, 0x74, 0x6c, - 0x73, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x61, - 0x6c, 0x70, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x73, 0x18, 0x06, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x0d, 0x61, 0x6c, 0x70, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, - 0x6c, 0x73, 0x42, 0x10, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x74, 0x6c, 0x73, 0x22, 0xae, 0x01, 0x0a, 0x0f, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, - 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, 0x53, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x64, 0x65, 0x6e, - 0x74, 0x69, 0x74, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x4b, 0x65, 0x79, 0x12, 0x78, 0x0a, 0x12, 0x76, - 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, - 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x49, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, - 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, - 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, - 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, - 0x78, 0x74, 0x52, 0x11, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x22, 0xc2, 0x01, 0x0a, 0x10, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, - 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, 0x53, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x64, - 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0b, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x4b, 0x65, 0x79, 0x12, 0x79, 0x0a, + 0x74, 0x65, 0x2e, 0x54, 0x4c, 0x53, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, + 0x52, 0x0d, 0x74, 0x6c, 0x73, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, + 0x25, 0x0a, 0x0e, 0x61, 0x6c, 0x70, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, + 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x61, 0x6c, 0x70, 0x6e, 0x50, 0x72, 0x6f, + 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x73, 0x42, 0x10, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6c, 0x73, 0x22, 0xae, 0x01, 0x0a, 0x0f, 0x49, 0x6e, 0x62, + 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, 0x53, 0x12, 0x21, 0x0a, 0x0c, + 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x4b, 0x65, 0x79, 0x12, + 0x78, 0x0a, 0x12, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x49, 0x2e, 0x68, 0x61, + 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, + 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, + 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x49, 0x6e, + 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x11, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x22, 0xc2, 0x01, 0x0a, 0x10, 0x4f, 0x75, + 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, 0x53, 0x12, 0x21, + 0x0a, 0x0c, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x4b, 0x65, + 0x79, 0x12, 0x79, 0x0a, 0x12, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4a, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, + 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x73, 0x68, + 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x11, 0x76, 0x61, 0x6c, 0x69, 0x64, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x10, 0x0a, 0x03, + 0x73, 0x6e, 0x69, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x6e, 0x69, 0x22, 0x8d, + 0x01, 0x0a, 0x11, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, + 0x68, 0x54, 0x4c, 0x53, 0x12, 0x1b, 0x0a, 0x08, 0x6c, 0x65, 0x61, 0x66, 0x5f, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x07, 0x6c, 0x65, 0x61, 0x66, 0x4b, 0x65, + 0x79, 0x12, 0x4f, 0x0a, 0x03, 0x73, 0x64, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3b, + 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, + 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, + 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x53, 0x44, 0x53, + 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x48, 0x00, 0x52, 0x03, 0x73, + 0x64, 0x73, 0x42, 0x0a, 0x0a, 0x08, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x22, 0xca, + 0x01, 0x0a, 0x12, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, + 0x73, 0x68, 0x54, 0x4c, 0x53, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x66, 0x69, + 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x65, 0x72, 0x74, 0x46, 0x69, + 0x6c, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6b, 0x65, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x7c, 0x0a, 0x12, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x74, - 0x65, 0x78, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4a, 0x2e, 0x68, 0x61, 0x73, 0x68, + 0x65, 0x78, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, - 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x4f, 0x75, 0x74, 0x62, - 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x11, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x6e, 0x69, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x6e, 0x69, 0x22, 0x8d, 0x01, 0x0a, 0x11, 0x49, - 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x54, 0x4c, 0x53, - 0x12, 0x1b, 0x0a, 0x08, 0x6c, 0x65, 0x61, 0x66, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x48, 0x00, 0x52, 0x07, 0x6c, 0x65, 0x61, 0x66, 0x4b, 0x65, 0x79, 0x12, 0x4f, 0x0a, - 0x03, 0x73, 0x64, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, - 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, - 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x53, 0x44, 0x53, 0x43, 0x65, 0x72, 0x74, - 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x48, 0x00, 0x52, 0x03, 0x73, 0x64, 0x73, 0x42, 0x0a, - 0x0a, 0x08, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x22, 0xca, 0x01, 0x0a, 0x12, 0x4f, - 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x54, 0x4c, - 0x53, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x65, 0x72, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x19, - 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x6b, 0x65, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x7c, 0x0a, 0x12, 0x76, 0x61, 0x6c, - 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, - 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, - 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, - 0x61, 0x74, 0x65, 0x2e, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x4f, 0x75, 0x74, 0x62, 0x6f, - 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x52, 0x11, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x22, 0x5c, 0x0a, 0x1c, 0x4d, 0x65, 0x73, 0x68, 0x49, - 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x3c, 0x0a, 0x1b, 0x74, 0x72, 0x75, 0x73, 0x74, - 0x5f, 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, - 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x17, 0x74, 0x72, - 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x50, 0x65, 0x65, 0x72, 0x4e, 0x61, 0x6d, - 0x65, 0x4b, 0x65, 0x79, 0x73, 0x22, 0x7a, 0x0a, 0x1d, 0x4d, 0x65, 0x73, 0x68, 0x4f, 0x75, 0x74, - 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, - 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x3a, 0x0a, 0x1a, 0x74, 0x72, 0x75, 0x73, 0x74, 0x5f, - 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, - 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x74, 0x72, 0x75, 0x73, - 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x50, 0x65, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x4b, - 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x70, 0x69, 0x66, 0x66, 0x65, 0x5f, 0x69, 0x64, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x73, 0x70, 0x69, 0x66, 0x66, 0x65, 0x49, 0x64, - 0x73, 0x22, 0x3b, 0x0a, 0x20, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x4f, 0x75, 0x74, 0x62, - 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x63, 0x61, 0x5f, 0x66, 0x69, 0x6c, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x61, 0x46, 0x69, 0x6c, 0x65, 0x22, 0x58, - 0x0a, 0x0e, 0x53, 0x44, 0x53, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, - 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, - 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x72, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x63, 0x65, 0x72, 0x74, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0xa5, 0x02, 0x0a, 0x0d, 0x54, 0x4c, 0x53, - 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x58, 0x0a, 0x0b, 0x6d, 0x69, - 0x6e, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x37, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, - 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, - 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x4c, - 0x53, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x6d, 0x69, 0x6e, 0x56, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x58, 0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x37, 0x2e, 0x68, 0x61, 0x73, 0x68, + 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x4f, + 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x11, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x22, 0x5c, 0x0a, 0x1c, 0x4d, + 0x65, 0x73, 0x68, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x3c, 0x0a, 0x1b, 0x74, + 0x72, 0x75, 0x73, 0x74, 0x5f, 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x70, 0x65, 0x65, 0x72, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x17, 0x74, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x50, 0x65, 0x65, + 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x4b, 0x65, 0x79, 0x73, 0x22, 0x7a, 0x0a, 0x1d, 0x4d, 0x65, 0x73, + 0x68, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x3a, 0x0a, 0x1a, 0x74, 0x72, + 0x75, 0x73, 0x74, 0x5f, 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x70, 0x65, 0x65, 0x72, 0x5f, + 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, + 0x74, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x50, 0x65, 0x65, 0x72, 0x4e, + 0x61, 0x6d, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x70, 0x69, 0x66, 0x66, 0x65, + 0x5f, 0x69, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x73, 0x70, 0x69, 0x66, + 0x66, 0x65, 0x49, 0x64, 0x73, 0x22, 0x3b, 0x0a, 0x20, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, + 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x63, 0x61, 0x5f, + 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x61, 0x46, 0x69, + 0x6c, 0x65, 0x22, 0x58, 0x0a, 0x0e, 0x53, 0x44, 0x53, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, + 0x63, 0x61, 0x74, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73, + 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x65, 0x72, 0x74, 0x5f, + 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, + 0x63, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0xa5, 0x02, 0x0a, + 0x0d, 0x54, 0x4c, 0x53, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x58, + 0x0a, 0x0b, 0x6d, 0x69, 0x6e, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x37, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, + 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, + 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, + 0x65, 0x2e, 0x54, 0x4c, 0x53, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x6d, 0x69, + 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x58, 0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, + 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x37, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, + 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x4c, 0x53, 0x56, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x60, 0x0a, 0x0d, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x5f, 0x73, 0x75, 0x69, + 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, - 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x4c, 0x53, 0x56, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x60, - 0x0a, 0x0d, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x5f, 0x73, 0x75, 0x69, 0x74, 0x65, 0x73, 0x18, - 0x03, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, - 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, - 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, - 0x61, 0x74, 0x65, 0x2e, 0x54, 0x4c, 0x53, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, - 0x74, 0x65, 0x52, 0x0c, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x73, - 0x22, 0x37, 0x0a, 0x0f, 0x4c, 0x65, 0x61, 0x66, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, - 0x61, 0x74, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x65, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x63, 0x65, 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x46, 0x0a, 0x0b, 0x54, 0x72, 0x75, - 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x75, 0x73, - 0x74, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x74, 0x72, 0x75, 0x73, 0x74, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x72, - 0x6f, 0x6f, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x72, 0x6f, 0x6f, 0x74, - 0x73, 0x2a, 0xac, 0x01, 0x0a, 0x0a, 0x54, 0x4c, 0x53, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x14, 0x0a, 0x10, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, - 0x41, 0x55, 0x54, 0x4f, 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, - 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x31, 0x5f, 0x30, 0x10, 0x01, 0x12, 0x13, 0x0a, 0x0f, 0x54, - 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x31, 0x5f, 0x31, 0x10, 0x02, - 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, - 0x31, 0x5f, 0x32, 0x10, 0x03, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, - 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x31, 0x5f, 0x33, 0x10, 0x04, 0x12, 0x17, 0x0a, 0x13, 0x54, 0x4c, - 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, - 0x44, 0x10, 0x05, 0x12, 0x1b, 0x0a, 0x17, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, - 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x06, - 0x2a, 0x84, 0x05, 0x0a, 0x0e, 0x54, 0x4c, 0x53, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, - 0x69, 0x74, 0x65, 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, - 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x45, 0x43, - 0x44, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, - 0x48, 0x41, 0x32, 0x35, 0x36, 0x10, 0x00, 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x4c, 0x53, 0x5f, 0x43, + 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x4c, 0x53, 0x43, 0x69, 0x70, 0x68, 0x65, + 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x52, 0x0c, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, + 0x69, 0x74, 0x65, 0x73, 0x22, 0x37, 0x0a, 0x0f, 0x4c, 0x65, 0x61, 0x66, 0x43, 0x65, 0x72, 0x74, + 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x65, 0x72, 0x74, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x65, 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x46, 0x0a, + 0x0b, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x21, 0x0a, 0x0c, + 0x74, 0x72, 0x75, 0x73, 0x74, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x75, 0x73, 0x74, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, + 0x14, 0x0a, 0x05, 0x72, 0x6f, 0x6f, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, + 0x72, 0x6f, 0x6f, 0x74, 0x73, 0x2a, 0xac, 0x01, 0x0a, 0x0a, 0x54, 0x4c, 0x53, 0x56, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x10, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, + 0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x55, 0x54, 0x4f, 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x4c, + 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x31, 0x5f, 0x30, 0x10, 0x01, 0x12, + 0x13, 0x0a, 0x0f, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x31, + 0x5f, 0x31, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, + 0x49, 0x4f, 0x4e, 0x5f, 0x31, 0x5f, 0x32, 0x10, 0x03, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x4c, 0x53, + 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x31, 0x5f, 0x33, 0x10, 0x04, 0x12, 0x17, + 0x0a, 0x13, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x49, 0x4e, + 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x05, 0x12, 0x1b, 0x0a, 0x17, 0x54, 0x4c, 0x53, 0x5f, 0x56, + 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, + 0x45, 0x44, 0x10, 0x06, 0x2a, 0x84, 0x05, 0x0a, 0x0e, 0x54, 0x4c, 0x53, 0x43, 0x69, 0x70, 0x68, + 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, - 0x45, 0x5f, 0x45, 0x43, 0x44, 0x53, 0x41, 0x5f, 0x43, 0x48, 0x41, 0x43, 0x48, 0x41, 0x32, 0x30, - 0x5f, 0x50, 0x4f, 0x4c, 0x59, 0x31, 0x33, 0x30, 0x35, 0x10, 0x01, 0x12, 0x30, 0x0a, 0x2c, 0x54, + 0x45, 0x5f, 0x45, 0x43, 0x44, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x47, + 0x43, 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36, 0x10, 0x00, 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, - 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, - 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36, 0x10, 0x02, 0x12, 0x30, 0x0a, - 0x2c, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, - 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, 0x43, 0x48, 0x41, 0x43, - 0x48, 0x41, 0x32, 0x30, 0x5f, 0x50, 0x4f, 0x4c, 0x59, 0x31, 0x33, 0x30, 0x35, 0x10, 0x03, 0x12, - 0x2b, 0x0a, 0x27, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, - 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x53, 0x41, 0x5f, - 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x04, 0x12, 0x29, 0x0a, 0x25, - 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, - 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x31, 0x32, - 0x38, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x05, 0x12, 0x26, 0x0a, 0x22, 0x54, 0x4c, 0x53, 0x5f, 0x43, - 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x41, 0x45, 0x53, 0x31, - 0x32, 0x38, 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36, 0x10, 0x06, 0x12, - 0x1f, 0x0a, 0x1b, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, - 0x49, 0x54, 0x45, 0x5f, 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x07, - 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, - 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x53, 0x41, - 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x33, - 0x38, 0x34, 0x10, 0x08, 0x12, 0x30, 0x0a, 0x2c, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, - 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, - 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, 0x48, - 0x41, 0x33, 0x38, 0x34, 0x10, 0x09, 0x12, 0x2b, 0x0a, 0x27, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, - 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, - 0x5f, 0x45, 0x43, 0x44, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x53, 0x48, - 0x41, 0x10, 0x0a, 0x12, 0x29, 0x0a, 0x25, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, - 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, 0x53, - 0x41, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x0b, 0x12, 0x26, - 0x0a, 0x22, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, - 0x54, 0x45, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, 0x48, - 0x41, 0x33, 0x38, 0x34, 0x10, 0x0c, 0x12, 0x1f, 0x0a, 0x1b, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, - 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, - 0x36, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x0d, 0x42, 0xe0, 0x02, 0x0a, 0x2f, 0x63, 0x6f, 0x6d, 0x2e, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, - 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x42, 0x14, 0x54, 0x72, 0x61, - 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x74, - 0x6f, 0x50, 0x01, 0x5a, 0x45, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2d, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x2f, 0x70, 0x62, - 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x70, 0x62, - 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0xa2, 0x02, 0x05, 0x48, 0x43, 0x4d, - 0x56, 0x50, 0xaa, 0x02, 0x2b, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x43, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x2e, 0x56, 0x31, 0x61, 0x6c, 0x70, - 0x68, 0x61, 0x31, 0x2e, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, - 0xca, 0x02, 0x2b, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x5c, 0x4d, 0x65, 0x73, 0x68, 0x5c, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, - 0x31, 0x5c, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0xe2, 0x02, - 0x37, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, 0x73, 0x75, - 0x6c, 0x5c, 0x4d, 0x65, 0x73, 0x68, 0x5c, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x5c, - 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5c, 0x47, 0x50, 0x42, - 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x2f, 0x48, 0x61, 0x73, 0x68, 0x69, - 0x63, 0x6f, 0x72, 0x70, 0x3a, 0x3a, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x3a, 0x3a, 0x4d, 0x65, - 0x73, 0x68, 0x3a, 0x3a, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x3a, 0x3a, 0x50, 0x62, - 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x53, 0x41, 0x5f, 0x43, 0x48, 0x41, 0x43, + 0x48, 0x41, 0x32, 0x30, 0x5f, 0x50, 0x4f, 0x4c, 0x59, 0x31, 0x33, 0x30, 0x35, 0x10, 0x01, 0x12, + 0x30, 0x0a, 0x2c, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, + 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, 0x41, 0x45, + 0x53, 0x31, 0x32, 0x38, 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36, 0x10, + 0x02, 0x12, 0x30, 0x0a, 0x2c, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, + 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, + 0x43, 0x48, 0x41, 0x43, 0x48, 0x41, 0x32, 0x30, 0x5f, 0x50, 0x4f, 0x4c, 0x59, 0x31, 0x33, 0x30, + 0x35, 0x10, 0x03, 0x12, 0x2b, 0x0a, 0x27, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, + 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x45, 0x43, + 0x44, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x04, + 0x12, 0x29, 0x0a, 0x25, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, + 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, 0x41, + 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x05, 0x12, 0x26, 0x0a, 0x22, 0x54, + 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, + 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x32, 0x35, + 0x36, 0x10, 0x06, 0x12, 0x1f, 0x0a, 0x1b, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, + 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x53, + 0x48, 0x41, 0x10, 0x07, 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, + 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x45, + 0x43, 0x44, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x47, 0x43, 0x4d, 0x5f, + 0x53, 0x48, 0x41, 0x33, 0x38, 0x34, 0x10, 0x08, 0x12, 0x30, 0x0a, 0x2c, 0x54, 0x4c, 0x53, 0x5f, + 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, + 0x48, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x47, 0x43, + 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x33, 0x38, 0x34, 0x10, 0x09, 0x12, 0x2b, 0x0a, 0x27, 0x54, 0x4c, + 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, + 0x43, 0x44, 0x48, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, + 0x36, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x0a, 0x12, 0x29, 0x0a, 0x25, 0x54, 0x4c, 0x53, 0x5f, 0x43, + 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, + 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x53, 0x48, 0x41, + 0x10, 0x0b, 0x12, 0x26, 0x0a, 0x22, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, + 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x47, 0x43, + 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x33, 0x38, 0x34, 0x10, 0x0c, 0x12, 0x1f, 0x0a, 0x1b, 0x54, 0x4c, + 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x41, + 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x0d, 0x42, 0xe0, 0x02, 0x0a, 0x2f, + 0x63, 0x6f, 0x6d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, + 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x42, + 0x14, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x45, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2d, 0x70, 0x75, 0x62, 0x6c, 0x69, + 0x63, 0x2f, 0x70, 0x62, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, + 0x31, 0x2f, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0xa2, 0x02, + 0x05, 0x48, 0x43, 0x4d, 0x56, 0x50, 0xaa, 0x02, 0x2b, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x2e, 0x56, + 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, + 0x74, 0x61, 0x74, 0x65, 0xca, 0x02, 0x2b, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x5c, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x4d, 0x65, 0x73, 0x68, 0x5c, 0x56, 0x31, 0x61, + 0x6c, 0x70, 0x68, 0x61, 0x31, 0x5c, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, + 0x74, 0x65, 0xe2, 0x02, 0x37, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x4d, 0x65, 0x73, 0x68, 0x5c, 0x56, 0x31, 0x61, 0x6c, 0x70, + 0x68, 0x61, 0x31, 0x5c, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x2f, 0x48, + 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x3a, 0x3a, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x3a, 0x3a, 0x4d, 0x65, 0x73, 0x68, 0x3a, 0x3a, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, + 0x3a, 0x3a, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.proto b/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.proto index 4e53326183f9..d5173f59a5e8 100644 --- a/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.proto +++ b/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.proto @@ -14,19 +14,21 @@ message TLS { } message TransportSocket { + // name of the transport socket + string name = 1; oneof connection_tls { // inbound_mesh is for incoming connections FROM the mesh. - InboundMeshMTLS inbound_mesh = 1; + InboundMeshMTLS inbound_mesh = 2; // outbound_mesh is for outbound connections TO mesh destinations. - OutboundMeshMTLS outbound_mesh = 2; + OutboundMeshMTLS outbound_mesh = 3; // inbound_non_mesh is for incoming connections FROM non mesh. - InboundNonMeshTLS inbound_non_mesh = 3; + InboundNonMeshTLS inbound_non_mesh = 4; // outbound_non_mesh is for outbound connections TO non mesh destinations. - OutboundNonMeshTLS outbound_non_mesh = 4; + OutboundNonMeshTLS outbound_non_mesh = 5; } // tls_parameters can override any top level tls parameters that are configured. - TLSParameters tls_parameters = 5; - repeated string alpn_protocols = 6; + TLSParameters tls_parameters = 6; + repeated string alpn_protocols = 7; } message InboundMeshMTLS {