From 735ccc5891f187627b71db987fd0d1858cb1ae0c Mon Sep 17 00:00:00 2001 From: John Murret Date: Wed, 16 Aug 2023 19:53:06 -0600 Subject: [PATCH 01/12] NET-4853 - xds v2 - implement base connect proxy functionality for clusters --- agent/xds/clusters.go | 25 +- agent/xds/clusters_test.go | 219 ++- agent/xds/delta_envoy_extender_oss_test.go | 6 +- agent/xds/endpoints.go | 21 +- agent/xds/endpoints_test.go | 30 +- agent/xds/listeners.go | 7 +- agent/xds/listeners_test.go | 63 +- agent/xds/proxystateconverter/clusters.go | 1183 ++++++++++++++++- agent/xds/proxystateconverter/converter.go | 6 +- agent/xds/proxystateconverter/endpoints.go | 85 ++ .../proxystateconverter/failover_policy.go | 158 +++ .../failover_policy_oss.go | 15 + agent/xds/proxystateconverter/listeners.go | 11 +- agent/xds/rbac.go | 11 +- agent/xds/resources_test.go | 3 +- agent/xds/{ => response}/response.go | 14 +- agent/xds/routes.go | 33 +- agent/xds/routes_test.go | 3 +- agent/xds/xds_protocol_helpers_test.go | 23 +- agent/xdsv2/cluster_resources.go | 327 ++++- agent/xdsv2/endpoint_resources.go | 41 + agent/xdsv2/listener_resources.go | 7 +- agent/xdsv2/resources.go | 8 +- agent/xdsv2/route_resources.go | 3 + .../v1alpha1/pbproxystate/cluster.pb.go | 26 +- .../v1alpha1/pbproxystate/cluster.proto | 4 + .../pbproxystate/transport_socket.pb.go | 414 +++--- .../pbproxystate/transport_socket.proto | 14 +- 28 files changed, 2410 insertions(+), 350 deletions(-) create mode 100644 agent/xds/proxystateconverter/endpoints.go create mode 100644 agent/xds/proxystateconverter/failover_policy.go create mode 100644 agent/xds/proxystateconverter/failover_policy_oss.go rename agent/xds/{ => response}/response.go (82%) create mode 100644 agent/xdsv2/endpoint_resources.go diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index f56700c02a91..908aa746bc9c 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -32,6 +32,7 @@ import ( "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/proto/private/pbpeering" ) @@ -1008,11 +1009,11 @@ func (s *ResourceGenerator) configIngressUpstreamCluster(c *envoy_cluster_v3.Clu switch limitType { case "max_connections": - threshold.MaxConnections = makeUint32Value(limit) + threshold.MaxConnections = response.MakeUint32Value(limit) case "max_pending_requests": - threshold.MaxPendingRequests = makeUint32Value(limit) + threshold.MaxPendingRequests = response.MakeUint32Value(limit) case "max_requests": - threshold.MaxRequests = makeUint32Value(limit) + threshold.MaxRequests = response.MakeUint32Value(limit) } } @@ -1103,7 +1104,7 @@ func (s *ResourceGenerator) makeAppCluster(cfgSnap *proxycfg.ConfigSnapshot, nam c.CircuitBreakers = &envoy_cluster_v3.CircuitBreakers{ Thresholds: []*envoy_cluster_v3.CircuitBreakers_Thresholds{ { - MaxConnections: makeUint32Value(cfg.MaxInboundConnections), + MaxConnections: response.MakeUint32Value(cfg.MaxInboundConnections), }, }, } @@ -1708,13 +1709,13 @@ func (s *ResourceGenerator) makeGatewayCluster(snap *proxycfg.ConfigSnapshot, op TcpKeepalive: &envoy_core_v3.TcpKeepalive{}, } if cfg.TcpKeepaliveTime != 0 { - cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveTime = makeUint32Value(cfg.TcpKeepaliveTime) + cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveTime = response.MakeUint32Value(cfg.TcpKeepaliveTime) } if cfg.TcpKeepaliveInterval != 0 { - cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveInterval = makeUint32Value(cfg.TcpKeepaliveInterval) + cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveInterval = response.MakeUint32Value(cfg.TcpKeepaliveInterval) } if cfg.TcpKeepaliveProbes != 0 { - cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveProbes = makeUint32Value(cfg.TcpKeepaliveProbes) + cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveProbes = response.MakeUint32Value(cfg.TcpKeepaliveProbes) } } @@ -1884,7 +1885,7 @@ func (s *ResourceGenerator) makeExternalHostnameCluster(snap *proxycfg.ConfigSna endpoints := make([]*envoy_endpoint_v3.LbEndpoint, 0, len(opts.addresses)) for _, pair := range opts.addresses { - address := makeAddress(pair.Address, pair.Port) + address := response.MakeAddress(pair.Address, pair.Port) endpoint := &envoy_endpoint_v3.LbEndpoint{ HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ @@ -1917,13 +1918,13 @@ func makeThresholdsIfNeeded(limits *structs.UpstreamLimits) []*envoy_cluster_v3. // Likewise, make sure to not set any threshold values on the zero-value in // order to rely on Envoy defaults if limits.MaxConnections != nil { - threshold.MaxConnections = makeUint32Value(*limits.MaxConnections) + threshold.MaxConnections = response.MakeUint32Value(*limits.MaxConnections) } if limits.MaxPendingRequests != nil { - threshold.MaxPendingRequests = makeUint32Value(*limits.MaxPendingRequests) + threshold.MaxPendingRequests = response.MakeUint32Value(*limits.MaxPendingRequests) } if limits.MaxConcurrentRequests != nil { - threshold.MaxRequests = makeUint32Value(*limits.MaxConcurrentRequests) + threshold.MaxRequests = response.MakeUint32Value(*limits.MaxConcurrentRequests) } return []*envoy_cluster_v3.CircuitBreakers_Thresholds{threshold} @@ -1946,7 +1947,7 @@ func makeLbEndpoint(addr string, port int, health envoy_core_v3.HealthStatus, we }, }, HealthStatus: health, - LoadBalancingWeight: makeUint32Value(weight), + LoadBalancingWeight: response.MakeUint32Value(weight), } } diff --git a/agent/xds/clusters_test.go b/agent/xds/clusters_test.go index 32723bf48143..2651d77c2844 100644 --- a/agent/xds/clusters_test.go +++ b/agent/xds/clusters_test.go @@ -20,16 +20,28 @@ import ( "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/proxystateconverter" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/agent/xds/testcommon" + "github.com/hashicorp/consul/agent/xdsv2" "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/sdk/testutil" "github.com/hashicorp/consul/types" ) +type mockCfgFetcher struct { + addressLan string +} + +func (s *mockCfgFetcher) AdvertiseAddrLAN() string { + return s.addressLan +} + type clusterTestCase struct { name string create func(t testinf.T) *proxycfg.ConfigSnapshot overrideGoldenName string + alsoRunTestForV2 bool } func uint32ptr(i uint32) *uint32 { @@ -52,12 +64,15 @@ func makeClusterDiscoChainTests(enterprise bool) []clusterTestCase { }) }, nil) }, + // TODO(proxystate): requires custom cluster work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-chain", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-chain-http2", @@ -66,84 +81,111 @@ func makeClusterDiscoChainTests(enterprise bool) []clusterTestCase { ns.Proxy.Upstreams[0].Config["protocol"] = "http2" }, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-chain-external-sni", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "external-sni", enterprise, nil, nil) }, + //TODO(proxystate): this requires terminating gateway work + alsoRunTestForV2: true, }, { name: "connect-proxy-with-chain-and-overrides", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple-with-overrides", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-chain-and-failover", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway-triggered", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-double-failover-through-remote-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-remote-gateway", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-remote-gateway-triggered", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-failover-through-local-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway-triggered", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-double-failover-through-local-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-local-gateway", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-local-gateway-triggered", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "splitter-with-resolver-redirect", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "splitter-with-resolver-redirect-multidc", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-lb-in-resolver", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "lb-resolver", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, } } @@ -175,6 +217,7 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tls-outgoing-min-version", @@ -194,6 +237,7 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tls-outgoing-max-version", @@ -213,6 +257,7 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tls-outgoing-cipher-suites", @@ -235,6 +280,7 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-jwt-config-entry-with-local", @@ -258,6 +304,8 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + // TODO(proxystate): jwt work will come at a later time + alsoRunTestForV2: false, }, { name: "connect-proxy-with-jwt-config-entry-with-remote-jwks", @@ -292,6 +340,8 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + // TODO(proxystate): jwt work will come at a later time + alsoRunTestForV2: false, }, { name: "custom-local-app", @@ -303,6 +353,8 @@ func TestClustersFromSnapshot(t *testing.T) { }) }, nil) }, + // TODO(proxystate): requires custom cluster work + alsoRunTestForV2: false, }, { name: "custom-upstream", @@ -314,6 +366,8 @@ func TestClustersFromSnapshot(t *testing.T) { }) }, nil) }, + // TODO(proxystate): requires custom cluster work + alsoRunTestForV2: false, }, { name: "custom-upstream-ignores-tls", @@ -328,6 +382,8 @@ func TestClustersFromSnapshot(t *testing.T) { }) }, nil) }, + // TODO(proxystate): requires custom cluster work + alsoRunTestForV2: false, }, { name: "custom-upstream-with-prepared-query", @@ -364,6 +420,8 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + // TODO(proxystate): requires custom cluster work + alsoRunTestForV2: false, }, { name: "custom-timeouts", @@ -373,6 +431,7 @@ func TestClustersFromSnapshot(t *testing.T) { ns.Proxy.Upstreams[0].Config["connect_timeout_ms"] = 2345 }, nil) }, + alsoRunTestForV2: true, }, { name: "custom-passive-healthcheck", @@ -387,6 +446,7 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + alsoRunTestForV2: true, }, { name: "custom-passive-healthcheck-zero-consecutive_5xx", @@ -401,6 +461,7 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + alsoRunTestForV2: true, }, { name: "custom-max-inbound-connections", @@ -409,6 +470,7 @@ func TestClustersFromSnapshot(t *testing.T) { ns.Proxy.Config["max_inbound_connections"] = 3456 }, nil) }, + alsoRunTestForV2: true, }, { name: "custom-limits-max-connections-only", @@ -429,6 +491,7 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + alsoRunTestForV2: true, }, { name: "custom-limits-set-to-zero", @@ -447,6 +510,7 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + alsoRunTestForV2: true, }, { name: "custom-limits", @@ -465,12 +529,14 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + alsoRunTestForV2: true, }, { name: "expose-paths-local-app-paths", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotExposeConfig(t, nil) }, + alsoRunTestForV2: true, }, { name: "downstream-service-with-unix-sockets", @@ -483,6 +549,7 @@ func TestClustersFromSnapshot(t *testing.T) { ns.Proxy.LocalServiceSocketPath = "/tmp/downstream_proxy.sock" }, nil) }, + alsoRunTestForV2: true, }, { name: "expose-paths-new-cluster-http2", @@ -496,68 +563,89 @@ func TestClustersFromSnapshot(t *testing.T) { } }) }, + alsoRunTestForV2: true, }, { - name: "expose-checks", - create: proxycfg.TestConfigSnapshotExposeChecks, + name: "expose-checks", + create: proxycfg.TestConfigSnapshotExposeChecks, + alsoRunTestForV2: true, }, { - name: "expose-paths-grpc-new-cluster-http1", - create: proxycfg.TestConfigSnapshotGRPCExposeHTTP1, + name: "expose-paths-grpc-new-cluster-http1", + create: proxycfg.TestConfigSnapshotGRPCExposeHTTP1, + alsoRunTestForV2: true, }, { name: "mesh-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "default", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-using-federation-states", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "federation-states", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-using-federation-control-plane", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "mesh-gateway-federation", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-no-services", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "no-services", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-service-subsets", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "service-subsets", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-ignore-extra-resolvers", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "ignore-extra-resolvers", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-service-timeouts", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "service-timeouts", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-non-hash-lb-injected", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "non-hash-lb-injected", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-hash-lb-ignored", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "hash-lb-ignored", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-tcp-keepalives", @@ -569,6 +657,8 @@ func TestClustersFromSnapshot(t *testing.T) { ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_probes"] = 7 }, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-gateway", @@ -576,12 +666,16 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "default", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-gateway-nil-config-entry", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotIngressGateway_NilConfigEntry(t) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-gateway-with-tls-outgoing-min-version", @@ -601,6 +695,8 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-gateway-with-tls-outgoing-max-version", @@ -620,6 +716,8 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-gateway-with-tls-outgoing-cipher-suites", @@ -642,6 +740,8 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-gateway-no-services", @@ -649,6 +749,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, false, "tcp", "default", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-chain", @@ -656,6 +758,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "simple", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-service-max-connections", @@ -666,6 +770,8 @@ func TestClustersFromSnapshot(t *testing.T) { entry.Listeners[0].Services[0].MaxConnections = 4096 }, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-defaults-service-max-connections", @@ -680,6 +786,8 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-overwrite-defaults-service-max-connections", @@ -695,6 +803,8 @@ func TestClustersFromSnapshot(t *testing.T) { entry.Listeners[0].Services[0].MaxPendingRequests = 2048 }, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-service-passive-health-check", @@ -710,6 +820,8 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-defaults-passive-health-check", @@ -731,6 +843,8 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-overwrite-defaults-passive-health-check", @@ -759,6 +873,8 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-chain-external-sni", @@ -766,6 +882,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "external-sni", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-chain-and-failover", @@ -773,6 +891,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-chain-and-failover-to-cluster-peer", @@ -780,6 +900,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-to-cluster-peer", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-failover-through-remote-gateway", @@ -787,6 +909,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-remote-gateway", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-failover-through-remote-gateway-triggered", @@ -794,6 +918,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-remote-gateway-triggered", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-double-failover-through-remote-gateway", @@ -801,6 +927,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-remote-gateway", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered", @@ -808,6 +936,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-remote-gateway-triggered", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-failover-through-local-gateway", @@ -815,6 +945,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-local-gateway", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-failover-through-local-gateway-triggered", @@ -822,6 +954,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-local-gateway-triggered", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-double-failover-through-local-gateway", @@ -829,6 +963,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-local-gateway", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-double-failover-through-local-gateway-triggered", @@ -836,6 +972,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-local-gateway-triggered", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-splitter-with-resolver-redirect", @@ -843,6 +981,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "http", "splitter-with-resolver-redirect-multidc", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-lb-in-resolver", @@ -850,46 +990,66 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "http", "lb-resolver", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotTerminatingGateway(t, true, nil, nil) }, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-no-services", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotTerminatingGateway(t, false, nil, nil) }, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-service-subsets", create: proxycfg.TestConfigSnapshotTerminatingGatewayServiceSubsetsWebAndCache, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-hostname-service-subsets", create: proxycfg.TestConfigSnapshotTerminatingGatewayHostnameSubsets, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-sni", create: proxycfg.TestConfigSnapshotTerminatingGatewaySNI, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-http2-upstream", create: proxycfg.TestConfigSnapshotTerminatingGatewayHTTP2, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-http2-upstream-subsets", create: proxycfg.TestConfigSnapshotTerminatingGatewaySubsetsHTTP2, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-ignore-extra-resolvers", create: proxycfg.TestConfigSnapshotTerminatingGatewayIgnoreExtraResolvers, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-lb-config", create: proxycfg.TestConfigSnapshotTerminatingGatewayLBConfigNoHashPolicies, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-tcp-keepalives", @@ -904,18 +1064,24 @@ func TestClustersFromSnapshot(t *testing.T) { ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_probes"] = 5 }, nil) }, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-multiple-listeners-duplicate-service", create: proxycfg.TestConfigSnapshotIngress_MultipleListenersDuplicateService, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { - name: "transparent-proxy-catalog-destinations-only", - create: proxycfg.TestConfigSnapshotTransparentProxyCatalogDestinationsOnly, + name: "transparent-proxy-catalog-destinations-only", + create: proxycfg.TestConfigSnapshotTransparentProxyCatalogDestinationsOnly, + alsoRunTestForV2: true, }, { - name: "transparent-proxy-dial-instances-directly", - create: proxycfg.TestConfigSnapshotTransparentProxyDialDirectly, + name: "transparent-proxy-dial-instances-directly", + create: proxycfg.TestConfigSnapshotTransparentProxyDialDirectly, + alsoRunTestForV2: true, }, } @@ -947,10 +1113,10 @@ func TestClustersFromSnapshot(t *testing.T) { return clusters[i].(*envoy_cluster_v3.Cluster).Name < clusters[j].(*envoy_cluster_v3.Cluster).Name }) - r, err := createResponse(xdscommon.ClusterType, "00000001", "00000001", clusters) + r, err := response.CreateResponse(xdscommon.ClusterType, "00000001", "00000001", clusters) require.NoError(t, err) - t.Run("current", func(t *testing.T) { + t.Run("current-xdsv1", func(t *testing.T) { gotJSON := protoToJSON(t, r) gName := tt.name @@ -960,6 +1126,39 @@ func TestClustersFromSnapshot(t *testing.T) { require.JSONEq(t, goldenEnvoy(t, filepath.Join("clusters", gName), envoyVersion, latestEnvoyVersion, gotJSON), gotJSON) }) + + if tt.alsoRunTestForV2 { + generator := xdsv2.NewResourceGenerator(testutil.Logger(t)) + + converter := proxystateconverter.NewConverter(testutil.Logger(t), &mockCfgFetcher{addressLan: "10.10.10.10"}) + proxyState, err := converter.ProxyStateFromSnapshot(snap) + require.NoError(t, err) + + res, err := generator.AllResourcesFromIR(proxyState) + require.NoError(t, err) + + clusters = res[xdscommon.ClusterType] + // The order of clusters returned via CDS isn't relevant, so it's safe + // to sort these for the purposes of test comparisons. + sort.Slice(clusters, func(i, j int) bool { + return clusters[i].(*envoy_cluster_v3.Cluster).Name < clusters[j].(*envoy_cluster_v3.Cluster).Name + }) + + r, err := response.CreateResponse(xdscommon.ClusterType, "00000001", "00000001", clusters) + require.NoError(t, err) + + t.Run("current-xdsv2", func(t *testing.T) { + gotJSON := protoToJSON(t, r) + + gName := tt.name + if tt.overrideGoldenName != "" { + gName = tt.overrideGoldenName + } + + expectedJSON := goldenEnvoy(t, filepath.Join("clusters", gName), envoyVersion, latestEnvoyVersion, gotJSON) + require.JSONEq(t, expectedJSON, gotJSON) + }) + } }) } }) diff --git a/agent/xds/delta_envoy_extender_oss_test.go b/agent/xds/delta_envoy_extender_oss_test.go index 5f4b563b5992..0607919ef5a5 100644 --- a/agent/xds/delta_envoy_extender_oss_test.go +++ b/agent/xds/delta_envoy_extender_oss_test.go @@ -21,12 +21,12 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" - "github.com/hashicorp/consul/agent/xds/testcommon" - propertyoverride "github.com/hashicorp/consul/agent/envoyextensions/builtin/property-override" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/xds/extensionruntime" + "github.com/hashicorp/consul/agent/xds/response" + "github.com/hashicorp/consul/agent/xds/testcommon" "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/envoyextensions/extensioncommon" "github.com/hashicorp/consul/envoyextensions/xdscommon" @@ -803,7 +803,7 @@ end`, } sort.Slice(msgs, entity.sorter(msgs)) - r, err := createResponse(entity.key, "00000001", "00000001", msgs) + r, err := response.CreateResponse(entity.key, "00000001", "00000001", msgs) require.NoError(t, err) t.Run(entity.name, func(t *testing.T) { diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index 8903ba3e7cc6..a2c36f06bdc2 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -6,21 +6,22 @@ package xds import ( "errors" "fmt" - "github.com/hashicorp/go-hclog" "strconv" + "github.com/hashicorp/go-hclog" + envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" "github.com/hashicorp/go-bexpr" "google.golang.org/protobuf/proto" - "github.com/hashicorp/consul/envoyextensions/xdscommon" - "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/envoyextensions/xdscommon" ) const ( @@ -278,7 +279,7 @@ func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.C lbEndpoint := &envoy_endpoint_v3.LbEndpoint{ HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress(addr, port), + Address: response.MakeAddress(addr, port), }, }, HealthStatus: envoy_core_v3.HealthStatus_UNKNOWN, @@ -335,7 +336,7 @@ func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.C serverEndpoints = append(serverEndpoints, &envoy_endpoint_v3.LbEndpoint{ HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress(addr, port), + Address: response.MakeAddress(addr, port), }, }, }) @@ -584,7 +585,7 @@ func makeEndpoint(host string, port int) *envoy_endpoint_v3.LbEndpoint { return &envoy_endpoint_v3.LbEndpoint{ HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress(host, port), + Address: response.MakeAddress(host, port), }, }, } @@ -594,7 +595,7 @@ func makePipeEndpoint(path string) *envoy_endpoint_v3.LbEndpoint { return &envoy_endpoint_v3.LbEndpoint{ HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makePipeAddress(path, 0), + Address: response.MakePipeAddress(path, 0), }, }, } @@ -881,7 +882,7 @@ func makeLoadAssignment(logger hclog.Logger, cfgSnap *proxycfg.ConfigSnapshot, c cla.Policy = &envoy_endpoint_v3.ClusterLoadAssignment_Policy{ // We choose such a large value here that the failover math should // in effect not happen until zero instances are healthy. - OverprovisioningFactor: makeUint32Value(100000), + OverprovisioningFactor: response.MakeUint32Value(100000), } } @@ -907,14 +908,14 @@ func makeLoadAssignment(logger hclog.Logger, cfgSnap *proxycfg.ConfigSnapshot, c } endpoint := &envoy_endpoint_v3.Endpoint{ - Address: makeAddress(addr, port), + Address: response.MakeAddress(addr, port), } es = append(es, &envoy_endpoint_v3.LbEndpoint{ HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: endpoint, }, HealthStatus: healthStatus, - LoadBalancingWeight: makeUint32Value(weight), + LoadBalancingWeight: response.MakeUint32Value(weight), }) } diff --git a/agent/xds/endpoints_test.go b/agent/xds/endpoints_test.go index a4757f00c534..3156685934d5 100644 --- a/agent/xds/endpoints_test.go +++ b/agent/xds/endpoints_test.go @@ -4,11 +4,12 @@ package xds import ( - "github.com/hashicorp/go-hclog" "path/filepath" "sort" "testing" + "github.com/hashicorp/go-hclog" + envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" @@ -20,6 +21,7 @@ import ( "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/sdk/testutil" ) @@ -133,18 +135,18 @@ func Test_makeLoadAssignment(t *testing.T) { { HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress("10.10.10.10", 1234), + Address: response.MakeAddress("10.10.10.10", 1234), }}, HealthStatus: envoy_core_v3.HealthStatus_HEALTHY, - LoadBalancingWeight: makeUint32Value(1), + LoadBalancingWeight: response.MakeUint32Value(1), }, { HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress("10.10.10.20", 1234), + Address: response.MakeAddress("10.10.10.20", 1234), }}, HealthStatus: envoy_core_v3.HealthStatus_HEALTHY, - LoadBalancingWeight: makeUint32Value(1), + LoadBalancingWeight: response.MakeUint32Value(1), }, }, }}, @@ -163,18 +165,18 @@ func Test_makeLoadAssignment(t *testing.T) { { HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress("10.10.10.10", 1234), + Address: response.MakeAddress("10.10.10.10", 1234), }}, HealthStatus: envoy_core_v3.HealthStatus_HEALTHY, - LoadBalancingWeight: makeUint32Value(10), + LoadBalancingWeight: response.MakeUint32Value(10), }, { HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress("10.10.10.20", 1234), + Address: response.MakeAddress("10.10.10.20", 1234), }}, HealthStatus: envoy_core_v3.HealthStatus_HEALTHY, - LoadBalancingWeight: makeUint32Value(5), + LoadBalancingWeight: response.MakeUint32Value(5), }, }, }}, @@ -193,18 +195,18 @@ func Test_makeLoadAssignment(t *testing.T) { { HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress("10.10.10.10", 1234), + Address: response.MakeAddress("10.10.10.10", 1234), }}, HealthStatus: envoy_core_v3.HealthStatus_HEALTHY, - LoadBalancingWeight: makeUint32Value(1), + LoadBalancingWeight: response.MakeUint32Value(1), }, { HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress("10.10.10.20", 1234), + Address: response.MakeAddress("10.10.10.20", 1234), }}, HealthStatus: envoy_core_v3.HealthStatus_UNHEALTHY, - LoadBalancingWeight: makeUint32Value(1), + LoadBalancingWeight: response.MakeUint32Value(1), }, }, }}, @@ -559,7 +561,7 @@ func TestEndpointsFromSnapshot(t *testing.T) { sort.Slice(endpoints, func(i, j int) bool { return endpoints[i].(*envoy_endpoint_v3.ClusterLoadAssignment).ClusterName < endpoints[j].(*envoy_endpoint_v3.ClusterLoadAssignment).ClusterName }) - r, err := createResponse(xdscommon.EndpointType, "00000001", "00000001", endpoints) + r, err := response.CreateResponse(xdscommon.EndpointType, "00000001", "00000001", endpoints) require.NoError(t, err) t.Run("current", func(t *testing.T) { diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 68c7ebdfadab..846cbf6d536f 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -45,6 +45,7 @@ import ( "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/xds/accesslogs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/lib" "github.com/hashicorp/consul/lib/stringslice" @@ -936,7 +937,7 @@ func makeListenerWithDefault(opts makeListenerOpts) *envoy_listener_v3.Listener return &envoy_listener_v3.Listener{ Name: fmt.Sprintf("%s:%s:%d", opts.name, opts.addr, opts.port), AccessLog: accessLog, - Address: makeAddress(opts.addr, opts.port), + Address: response.MakeAddress(opts.addr, opts.port), TrafficDirection: opts.direction, } } @@ -955,7 +956,7 @@ func makePipeListener(opts makeListenerOpts) *envoy_listener_v3.Listener { return &envoy_listener_v3.Listener{ Name: fmt.Sprintf("%s:%s", opts.name, opts.path), AccessLog: accessLog, - Address: makePipeAddress(opts.path, uint32(modeInt)), + Address: response.MakePipeAddress(opts.path, uint32(modeInt)), TrafficDirection: opts.direction, } } @@ -2592,7 +2593,7 @@ func makeHTTPFilter(opts listenerFilterOpts) (*envoy_listener_v3.Filter, error) "envoy.filters.http.grpc_stats", &envoy_grpc_stats_v3.FilterConfig{ PerMethodStatSpecifier: &envoy_grpc_stats_v3.FilterConfig_StatsForAllMethods{ - StatsForAllMethods: makeBoolValue(true), + StatsForAllMethods: response.MakeBoolValue(true), }, }, ) diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go index b5d6c66f0008..130f2234517e 100644 --- a/agent/xds/listeners_test.go +++ b/agent/xds/listeners_test.go @@ -21,6 +21,7 @@ import ( "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/xds/configfetcher" "github.com/hashicorp/consul/agent/xds/proxystateconverter" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/agent/xds/testcommon" "github.com/hashicorp/consul/agent/xdsv2" "github.com/hashicorp/consul/envoyextensions/xdscommon" @@ -1301,24 +1302,24 @@ func TestListenersFromSnapshot(t *testing.T) { var listeners []proto.Message - t.Run("current-xdsv1", func(t *testing.T) { - // Need server just for logger dependency - g := NewResourceGenerator(testutil.Logger(t), nil, false) - g.ProxyFeatures = sf - if tt.generatorSetup != nil { - tt.generatorSetup(g) - } - listeners, err = g.listenersFromSnapshot(snap) - require.NoError(t, err) - // The order of listeners returned via LDS isn't relevant, so it's safe - // to sort these for the purposes of test comparisons. - sort.Slice(listeners, func(i, j int) bool { - return listeners[i].(*envoy_listener_v3.Listener).Name < listeners[j].(*envoy_listener_v3.Listener).Name - }) + // Need server just for logger dependency + g := NewResourceGenerator(testutil.Logger(t), nil, false) + g.ProxyFeatures = sf + if tt.generatorSetup != nil { + tt.generatorSetup(g) + } + listeners, err = g.listenersFromSnapshot(snap) + require.NoError(t, err) + // The order of listeners returned via LDS isn't relevant, so it's safe + // to sort these for the purposes of test comparisons. + sort.Slice(listeners, func(i, j int) bool { + return listeners[i].(*envoy_listener_v3.Listener).Name < listeners[j].(*envoy_listener_v3.Listener).Name + }) - r, err := createResponse(xdscommon.ListenerType, "00000001", "00000001", listeners) - require.NoError(t, err) + r, err := response.CreateResponse(xdscommon.ListenerType, "00000001", "00000001", listeners) + require.NoError(t, err) + t.Run("current-xdsv1", func(t *testing.T) { gotJSON := protoToJSON(t, r) gName := tt.name @@ -1331,25 +1332,25 @@ func TestListenersFromSnapshot(t *testing.T) { }) if tt.alsoRunTestForV2 { - t.Run("current-xdsv2", func(t *testing.T) { - generator := xdsv2.NewResourceGenerator(testutil.Logger(t)) - converter := proxystateconverter.NewConverter(testutil.Logger(t), nil) - proxyState, err := converter.ProxyStateFromSnapshot(snap) - require.NoError(t, err) + generator := xdsv2.NewResourceGenerator(testutil.Logger(t)) + converter := proxystateconverter.NewConverter(testutil.Logger(t), nil) + proxyState, err := converter.ProxyStateFromSnapshot(snap) + require.NoError(t, err) - res, err := generator.AllResourcesFromIR(proxyState) - require.NoError(t, err) + res, err := generator.AllResourcesFromIR(proxyState) + require.NoError(t, err) - listeners = res[xdscommon.ListenerType] - // The order of listeners returned via LDS isn't relevant, so it's safe - // to sort these for the purposes of test comparisons. - sort.Slice(listeners, func(i, j int) bool { - return listeners[i].(*envoy_listener_v3.Listener).Name < listeners[j].(*envoy_listener_v3.Listener).Name - }) + listeners = res[xdscommon.ListenerType] + // The order of listeners returned via LDS isn't relevant, so it's safe + // to sort these for the purposes of test comparisons. + sort.Slice(listeners, func(i, j int) bool { + return listeners[i].(*envoy_listener_v3.Listener).Name < listeners[j].(*envoy_listener_v3.Listener).Name + }) - r, err := createResponse(xdscommon.ListenerType, "00000001", "00000001", listeners) - require.NoError(t, err) + r, err := response.CreateResponse(xdscommon.ListenerType, "00000001", "00000001", listeners) + require.NoError(t, err) + t.Run("current-xdsv2", func(t *testing.T) { gotJSON := protoToJSON(t, r) gName := tt.name diff --git a/agent/xds/proxystateconverter/clusters.go b/agent/xds/proxystateconverter/clusters.go index b5f10bf32e03..2a43d6eb049c 100644 --- a/agent/xds/proxystateconverter/clusters.go +++ b/agent/xds/proxystateconverter/clusters.go @@ -1,16 +1,28 @@ // Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 +// SPDX-License-Identifier: BUSL-1.1 package proxystateconverter import ( + "errors" "fmt" + envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-uuid" "strings" + "time" + + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/config" "github.com/hashicorp/consul/agent/xds/naming" + "github.com/hashicorp/consul/agent/xds/response" + "github.com/hashicorp/consul/envoyextensions/xdscommon" + "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" "github.com/hashicorp/consul/proto/private/pbpeering" ) @@ -18,15 +30,333 @@ const ( meshGatewayExportedClusterNamePrefix = "exported~" ) +type namedCluster struct { + name string + cluster *pbproxystate.Cluster +} + +// clustersFromSnapshot returns the xDS API representation of the "clusters" in the snapshot. +func (s *Converter) clustersFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot) error { + if cfgSnap == nil { + return errors.New("nil config given") + } + + switch cfgSnap.Kind { + case structs.ServiceKindConnectProxy: + return s.clustersFromSnapshotConnectProxy(cfgSnap) + // TODO(proxystate): Terminating Gateways will be added in the future. + //case structs.ServiceKindTerminatingGateway: + // err := s.clustersFromSnapshotTerminatingGateway(cfgSnap) + // if err != nil { + // return err + // } + // return nil + // TODO(proxystate): Mesh Gateways will be added in the future. + //case structs.ServiceKindMeshGateway: + // err := s.clustersFromSnapshotMeshGateway(cfgSnap) + // if err != nil { + // return err + // } + // return nil + // TODO(proxystate): Ingress Gateways will be added in the future. + //case structs.ServiceKindIngressGateway: + // err := s.clustersFromSnapshotIngressGateway(cfgSnap) + // if err != nil { + // return err + // } + // return nil + // TODO(proxystate): API Gateways will be added in the future. + //case structs.ServiceKindAPIGateway: + // res, err := s.clustersFromSnapshotAPIGateway(cfgSnap) + // if err != nil { + // return err + // } + // return nil + default: + return fmt.Errorf("Invalid service kind: %v", cfgSnap.Kind) + } +} + +// clustersFromSnapshot returns the xDS API representation of the "clusters" +// (upstreams) in the snapshot. +func (s *Converter) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapshot) error { + // This is the list of listeners we add to. It will be empty to start. + clusters := s.proxyState.Clusters + var err error + + // Include the "app" cluster for the public listener + appCluster, err := s.makeAppCluster(cfgSnap, xdscommon.LocalAppClusterName, "", cfgSnap.Proxy.LocalServicePort) + if err != nil { + return err + } + clusters[appCluster.name] = appCluster.cluster + + if cfgSnap.Proxy.Mode == structs.ProxyModeTransparent { + passthroughs, err := s.makePassthroughClusters(cfgSnap) + if err != nil { + return fmt.Errorf("failed to make passthrough clusters for transparent proxy: %v", err) + } + for clusterName, cluster := range passthroughs { + clusters[clusterName] = cluster + } + } + + // NOTE: Any time we skip a chain below we MUST also skip that discovery chain in endpoints.go + // so that the sets of endpoints generated matches the sets of clusters. + for uid, chain := range cfgSnap.ConnectProxy.DiscoveryChain { + upstream, skip := cfgSnap.ConnectProxy.GetUpstream(uid, &cfgSnap.ProxyID.EnterpriseMeta) + if skip { + continue + } + + upstreamClusters, err := s.makeUpstreamClustersForDiscoveryChain( + uid, + upstream, + chain, + cfgSnap, + false, + ) + if err != nil { + return err + } + + for name, cluster := range upstreamClusters { + clusters[name] = cluster + } + } + + // TODO(proxystate): peering will be added in the future. + //// NOTE: Any time we skip an upstream below we MUST also skip that same + //// upstream in endpoints.go so that the sets of endpoints generated matches + //// the sets of clusters. + //for _, uid := range cfgSnap.ConnectProxy.PeeredUpstreamIDs() { + // upstream, skip := cfgSnap.ConnectProxy.GetUpstream(uid, &cfgSnap.ProxyID.EnterpriseMeta) + // if skip { + // continue + // } + // + // peerMeta, found := cfgSnap.ConnectProxy.UpstreamPeerMeta(uid) + // if !found { + // s.Logger.Warn("failed to fetch upstream peering metadata for cluster", "uid", uid) + // } + // cfg := s.getAndModifyUpstreamConfigForPeeredListener(uid, upstream, peerMeta) + // + // upstreamCluster, err := s.makeUpstreamClusterForPeerService(uid, cfg, peerMeta, cfgSnap) + // if err != nil { + // return nil, err + // } + // clusters = append(clusters, upstreamCluster) + //} + + // TODO(proxystate): L7 Intentions and JWT Auth will be added in the future. + //// add clusters for jwt-providers + //for _, prov := range cfgSnap.JWTProviders { + // //skip cluster creation for local providers + // if prov.JSONWebKeySet == nil || prov.JSONWebKeySet.Remote == nil { + // continue + // } + // + // cluster, err := makeJWTProviderCluster(prov) + // if err != nil { + // s.Logger.Warn("failed to make jwt-provider cluster", "provider name", prov.Name, "error", err) + // continue + // } + // + // clusters[cluster.GetName()] = cluster + //} + + for _, u := range cfgSnap.Proxy.Upstreams { + if u.DestinationType != structs.UpstreamDestTypePreparedQuery { + continue + } + + upstreamCluster, err := s.makeUpstreamClusterForPreparedQuery(u, cfgSnap) + if err != nil { + return err + } + clusters[upstreamCluster.name] = upstreamCluster.cluster + } + + cfgSnap.Proxy.Expose.Finalize() + paths := cfgSnap.Proxy.Expose.Paths + + // Add service health checks to the list of paths to create clusters for if needed + if cfgSnap.Proxy.Expose.Checks { + psid := structs.NewServiceID(cfgSnap.Proxy.DestinationServiceID, &cfgSnap.ProxyID.EnterpriseMeta) + for _, check := range cfgSnap.ConnectProxy.WatchedServiceChecks[psid] { + p, err := parseCheckPath(check) + if err != nil { + s.Logger.Warn("failed to create cluster for", "check", check.CheckID, "error", err) + continue + } + paths = append(paths, p) + } + } + + // Create a new cluster if we need to expose a port that is different from the service port + for _, path := range paths { + if path.LocalPathPort == cfgSnap.Proxy.LocalServicePort { + continue + } + c, err := s.makeAppCluster(cfgSnap, makeExposeClusterName(path.LocalPathPort), path.Protocol, path.LocalPathPort) + if err != nil { + s.Logger.Warn("failed to make local cluster", "path", path.Path, "error", err) + continue + } + clusters[c.name] = c.cluster + } + + return nil +} + +// TODO(proxystate): L7 Intentions and JWT Auth will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeJWTProviderCluster +// func makeJWKSDiscoveryClusterType +// func makeJWTCertValidationContext +// func parseJWTRemoteURL + func makeExposeClusterName(destinationPort int) string { return fmt.Sprintf("exposed_cluster_%d", destinationPort) } -func clusterNameForDestination(cfgSnap *proxycfg.ConfigSnapshot, name string, address string, namespace string, partition string) string { +// In transparent proxy mode there are potentially multiple passthrough clusters added. +// The first is for destinations outside of Consul's catalog. This is for a plain TCP proxy. +// All of these use Envoy's ORIGINAL_DST listener filter, which forwards to the original +// destination address (before the iptables redirection). +// The rest are for destinations inside the mesh, which require certificates for mTLS. +func (s *Converter) makePassthroughClusters(cfgSnap *proxycfg.ConfigSnapshot) (map[string]*pbproxystate.Cluster, error) { + // This size is an upper bound. + clusters := make(map[string]*pbproxystate.Cluster, 0) + if meshConf := cfgSnap.MeshConfig(); meshConf == nil || + !meshConf.TransparentProxy.MeshDestinationsOnly { + + clusters[naming.OriginalDestinationClusterName] = &pbproxystate.Cluster{ + Group: &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Passthrough{ + Passthrough: &pbproxystate.PassthroughEndpointGroup{ + Config: &pbproxystate.PassthroughEndpointGroupConfig{ + ConnectTimeout: durationpb.New(5 * time.Second), + }, + }, + }, + }, + }, + } + } + + for uid, chain := range cfgSnap.ConnectProxy.DiscoveryChain { + targetMap, ok := cfgSnap.ConnectProxy.PassthroughUpstreams[uid] + if !ok { + continue + } + + for targetID := range targetMap { + uid := proxycfg.NewUpstreamIDFromTargetID(targetID) + + sni := connect.ServiceSNI( + uid.Name, "", uid.NamespaceOrDefault(), + uid.PartitionOrDefault(), cfgSnap.Datacenter, + cfgSnap.Roots.TrustDomain) + + // Prefixed with passthrough to distinguish from non-passthrough clusters for the same upstream. + name := "passthrough~" + sni + + c := pbproxystate.Cluster{ + Group: &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Passthrough{ + Passthrough: &pbproxystate.PassthroughEndpointGroup{ + Config: &pbproxystate.PassthroughEndpointGroupConfig{ + ConnectTimeout: durationpb.New(5 * time.Second), + }, + }, + }, + }, + }, + } + + if discoTarget, ok := chain.Targets[targetID]; ok && discoTarget.ConnectTimeout > 0 { + c.GetEndpointGroup().GetPassthrough().GetConfig(). + ConnectTimeout = durationpb.New(discoTarget.ConnectTimeout) + } + + transportSocket, err := s.createOutboundMeshMTLS(cfgSnap, []string{getSpiffeID(cfgSnap, uid)}, sni) + if err != nil { + return nil, err + } + c.GetEndpointGroup().GetPassthrough().OutboundTls = transportSocket + + clusters[name] = &c + } + } + + err := cfgSnap.ConnectProxy.DestinationsUpstream.ForEachKeyE(func(uid proxycfg.UpstreamID) error { + svcConfig, ok := cfgSnap.ConnectProxy.DestinationsUpstream.Get(uid) + if !ok || svcConfig.Destination == nil { + return nil + } + + // One Cluster per Destination Address + for _, address := range svcConfig.Destination.Addresses { + name := clusterNameForDestination(cfgSnap, uid.Name, address, uid.NamespaceOrDefault(), uid.PartitionOrDefault()) + + c := &pbproxystate.Cluster{ + AltStatName: name, + Group: &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Dynamic{ + Dynamic: &pbproxystate.DynamicEndpointGroup{ + Config: &pbproxystate.DynamicEndpointGroupConfig{ + ConnectTimeout: durationpb.New(5 * time.Second), + // Endpoints are managed separately by EDS + // Having an empty config enables outlier detection with default config. + OutlierDetection: &pbproxystate.OutlierDetection{}, + }, + }, + }, + }, + }, + } + sni := connect.ServiceSNI( + uid.Name, "", uid.NamespaceOrDefault(), + uid.PartitionOrDefault(), cfgSnap.Datacenter, + cfgSnap.Roots.TrustDomain) + transportSocket, err := s.createOutboundMeshMTLS(cfgSnap, []string{getSpiffeID(cfgSnap, uid)}, sni) + if err != nil { + return err + } + c.GetEndpointGroup().GetDynamic().OutboundTls = transportSocket + clusters[name] = c + } + return nil + }) + if err != nil { + return nil, err + } + + return clusters, nil +} + +func getSpiffeID(cfgSnap *proxycfg.ConfigSnapshot, uid proxycfg.UpstreamID) string { + spiffeIDService := &connect.SpiffeIDService{ + Host: cfgSnap.Roots.TrustDomain, + Partition: uid.PartitionOrDefault(), + Namespace: uid.NamespaceOrDefault(), + Datacenter: cfgSnap.Datacenter, + Service: uid.Name, + } + return spiffeIDService.URI().String() +} +func clusterNameForDestination(cfgSnap *proxycfg.ConfigSnapshot, name string, + address string, namespace string, partition string) string { name = destinationSpecificServiceName(name, address) - sni := connect.ServiceSNI(name, "", namespace, partition, cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain) + sni := connect.ServiceSNI(name, "", namespace, partition, + cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain) - // Prefixed with destination to distinguish from non-passthrough clusters for the same upstream. + // Prefixed with destination to distinguish from non-passthrough clusters + // for the same upstream. return "destination." + sni } @@ -36,6 +366,785 @@ func destinationSpecificServiceName(name string, address string) string { return fmt.Sprintf("%s.%s", address, name) } +// TODO(proxystate): Mesh Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func clustersFromSnapshotMeshGateway +// func haveVoters + +// TODO(proxystate): Peering will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makePeerServerClusters + +// TODO(proxystate): Terminating Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func clustersFromSnapshotTerminatingGateway + +// TODO(proxystate): Mesh Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeGatewayServiceClusters + +// TODO(proxystate): Cluster Peering will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeGatewayOutgoingClusterPeeringServiceClusters + +// TODO(proxystate): Terminating Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeDestinationClusters + +// TODO(proxystate): Mesh Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func injectGatewayServiceAddons + +// TODO(proxystate): Terminating Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func injectGatewayDestinationAddons + +// TODO(proxystate): Ingress Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func clustersFromSnapshotIngressGateway + +// TODO(proxystate): API Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func clustersFromSnapshotAPIGateway + +// TODO(proxystate): Ingress Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func configIngressUpstreamCluster + +func (s *Converter) makeAppCluster(cfgSnap *proxycfg.ConfigSnapshot, name, pathProtocol string, port int) (*namedCluster, error) { + var err error + namedCluster := &namedCluster{} + + cfg, err := config.ParseProxyConfig(cfgSnap.Proxy.Config) + if err != nil { + // Don't hard fail on a config typo, just warn. The parse func returns + // default config if there is an error so it's safe to continue. + s.Logger.Warn("failed to parse Connect.Proxy.Config", "error", err) + } + + //// If we have overridden local cluster config try to parse it into an Envoy cluster + //if cfg.LocalClusterJSON != "" { + // return makeClusterFromUserConfig(cfg.LocalClusterJSON) + //} + + var endpoint *pbproxystate.Endpoint + if cfgSnap.Proxy.LocalServiceSocketPath != "" { + endpoint = makeUnixSocketEndpoint(cfgSnap.Proxy.LocalServiceSocketPath) + } else { + addr := cfgSnap.Proxy.LocalServiceAddress + if addr == "" { + addr = "127.0.0.1" + } + endpoint = makeHostPortEndpoint(addr, port) + } + s.proxyState.Endpoints[name] = &pbproxystate.Endpoints{ + Endpoints: []*pbproxystate.Endpoint{endpoint}, + } + + namedCluster.name = name + namedCluster.cluster = &pbproxystate.Cluster{ + Group: &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Static{ + Static: &pbproxystate.StaticEndpointGroup{ + Config: &pbproxystate.StaticEndpointGroupConfig{ + ConnectTimeout: durationpb.New(time.Duration(cfg.LocalConnectTimeoutMs) * time.Millisecond), + }, + }, + }, + }, + }, + } + protocol := pathProtocol + if protocol == "" { + protocol = cfg.Protocol + } + namedCluster.cluster.Protocol = protocol + if cfg.MaxInboundConnections > 0 { + namedCluster.cluster.GetEndpointGroup().GetStatic().GetConfig(). + CircuitBreakers = &pbproxystate.CircuitBreakers{ + UpstreamLimits: &pbproxystate.UpstreamLimits{ + MaxConnections: response.MakeUint32Value(cfg.MaxInboundConnections), + }, + } + } + + return namedCluster, err +} + +func (s *Converter) makeUpstreamClusterForPeerService( + uid proxycfg.UpstreamID, + upstreamConfig structs.UpstreamConfig, + peerMeta structs.PeeringServiceMeta, + cfgSnap *proxycfg.ConfigSnapshot, +) (string, *pbproxystate.Cluster, *pbproxystate.Endpoints, error) { + var ( + c *pbproxystate.Cluster + e *pbproxystate.Endpoints + err error + ) + + // TODO(proxystate): escapeHatches will be implemented in the future + //if upstreamConfig.EnvoyClusterJSON != "" { + // c, err = makeClusterFromUserConfig(upstreamConfig.EnvoyClusterJSON) + // if err != nil { + // return "", c, e, err + // } + // // In the happy path don't return yet as we need to inject TLS config still. + //} + + upstreamsSnapshot, err := cfgSnap.ToConfigSnapshotUpstreams() + + if err != nil { + return "", c, e, err + } + + tbs, ok := upstreamsSnapshot.UpstreamPeerTrustBundles.Get(uid.Peer) + if !ok { + // this should never happen since we loop through upstreams with + // set trust bundles + return "", c, e, fmt.Errorf("trust bundle not ready for peer %s", uid.Peer) + } + + clusterName := generatePeeredClusterName(uid, tbs) + + outlierDetection := makeOutlierDetection(upstreamConfig.PassiveHealthCheck, nil, true) + // We can't rely on health checks for services on cluster peers because they + // don't take into account service resolvers, splitters and routers. Setting + // MaxEjectionPercent too 100% gives outlier detection the power to eject the + // entire cluster. + outlierDetection.MaxEjectionPercent = &wrapperspb.UInt32Value{Value: 100} + + s.Logger.Trace("generating cluster for", "cluster", clusterName) + if c == nil { + c = &pbproxystate.Cluster{} + + useEDS := true + if _, ok := cfgSnap.ConnectProxy.PeerUpstreamEndpointsUseHostnames[uid]; ok { + // If we're using local mesh gw, the fact that upstreams use hostnames don't matter. + // If we're not using local mesh gw, then resort to CDS. + if upstreamConfig.MeshGateway.Mode != structs.MeshGatewayModeLocal { + useEDS = false + } + } + + // If none of the service instances are addressed by a hostname we + // provide the endpoint IP addresses via EDS + if useEDS { + d := &pbproxystate.DynamicEndpointGroup{ + Config: &pbproxystate.DynamicEndpointGroupConfig{ + UseAltStatName: false, + ConnectTimeout: durationpb.New(time.Duration(upstreamConfig.ConnectTimeoutMs) * time.Millisecond), + DisablePanicThreshold: true, + CircuitBreakers: &pbproxystate.CircuitBreakers{ + UpstreamLimits: makeUpstreamLimitsIfNeeded(upstreamConfig.Limits), + }, + OutlierDetection: outlierDetection, + }, + } + c.Group = &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Dynamic{ + Dynamic: d, + }, + }, + } + transportSocket := &pbproxystate.TransportSocket{ + ConnectionTls: &pbproxystate.TransportSocket_OutboundMesh{ + OutboundMesh: &pbproxystate.OutboundMeshMTLS{ + ValidationContext: &pbproxystate.MeshOutboundValidationContext{ + SpiffeIds: peerMeta.SpiffeID, + TrustBundlePeerNameKey: uid.Peer, + }, + Sni: peerMeta.PrimarySNI(), + }, + }, + } + d.OutboundTls = transportSocket + } else { + d := &pbproxystate.DNSEndpointGroup{ + Config: &pbproxystate.DNSEndpointGroupConfig{ + UseAltStatName: false, + ConnectTimeout: durationpb.New(time.Duration(upstreamConfig.ConnectTimeoutMs) * time.Millisecond), + DisablePanicThreshold: true, + CircuitBreakers: &pbproxystate.CircuitBreakers{ + UpstreamLimits: makeUpstreamLimitsIfNeeded(upstreamConfig.Limits), + }, + OutlierDetection: outlierDetection, + }, + } + c.Group = &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Dns{ + Dns: d, + }, + }, + } + e = &pbproxystate.Endpoints{ + Endpoints: make([]*pbproxystate.Endpoint, 0), + } + + ep, _ := cfgSnap.ConnectProxy.PeerUpstreamEndpoints.Get(uid) + configureClusterWithHostnames( + s.Logger, + d, + e, + "", /*TODO:make configurable?*/ + ep, + true, /*isRemote*/ + false, /*onlyPassing*/ + ) + transportSocket := &pbproxystate.TransportSocket{ + ConnectionTls: &pbproxystate.TransportSocket_OutboundMesh{ + OutboundMesh: &pbproxystate.OutboundMeshMTLS{ + ValidationContext: &pbproxystate.MeshOutboundValidationContext{ + SpiffeIds: peerMeta.SpiffeID, + TrustBundlePeerNameKey: uid.Peer, + }, + Sni: peerMeta.PrimarySNI(), + }, + }, + } + d.OutboundTls = transportSocket + } + } + + return clusterName, c, e, nil +} + +func (s *Converter) makeUpstreamClusterForPreparedQuery(upstream structs.Upstream, cfgSnap *proxycfg.ConfigSnapshot) (*namedCluster, error) { + var c *pbproxystate.Cluster + var err error + + uid := proxycfg.NewUpstreamID(&upstream) + + dc := upstream.Datacenter + if dc == "" { + dc = cfgSnap.Datacenter + } + sni := connect.UpstreamSNI(&upstream, "", dc, cfgSnap.Roots.TrustDomain) + + cfg, _ := structs.ParseUpstreamConfig(upstream.Config) + // TODO(proxystate): add logger and enable this + //if err != nil { + // Don't hard fail on a config typo, just warn. The parse func returns + // default config if there is an error so it's safe to continue. + //s.Logger.Warn("failed to parse", "upstream", uid, "error", err) + //} + + // TODO(proxystate): escapeHatches will be implemented in the future + //if cfg.EnvoyClusterJSON != "" { + // c, err = makeClusterFromUserConfig(cfg.EnvoyClusterJSON) + // if err != nil { + // return c, err + // } + // // In the happy path don't return yet as we need to inject TLS config still. + //} + + if c == nil { + c = &pbproxystate.Cluster{ + Protocol: cfg.Protocol, + Group: &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Dynamic{ + Dynamic: &pbproxystate.DynamicEndpointGroup{ + Config: &pbproxystate.DynamicEndpointGroupConfig{ + ConnectTimeout: durationpb.New(time.Duration(cfg.ConnectTimeoutMs) * time.Millisecond), + // Endpoints are managed separately by EDS + // Having an empty config enables outlier detection with default config. + OutlierDetection: makeOutlierDetection(cfg.PassiveHealthCheck, nil, true), + DisablePanicThreshold: true, + CircuitBreakers: &pbproxystate.CircuitBreakers{ + UpstreamLimits: makeUpstreamLimitsIfNeeded(cfg.Limits), + }, + }, + }, + }, + }, + }, + } + } + + endpoints := cfgSnap.ConnectProxy.PreparedQueryEndpoints[uid] + var ( + spiffeIDs = make([]string, 0) + seen = make(map[string]struct{}) + ) + for _, e := range endpoints { + id := fmt.Sprintf("%s/%s", e.Node.Datacenter, e.Service.CompoundServiceName()) + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + + name := e.Service.Proxy.DestinationServiceName + if e.Service.Connect.Native { + name = e.Service.Service + } + + spiffeIDs = append(spiffeIDs, connect.SpiffeIDService{ + Host: cfgSnap.Roots.TrustDomain, + Namespace: e.Service.NamespaceOrDefault(), + Partition: e.Service.PartitionOrDefault(), + Datacenter: e.Node.Datacenter, + Service: name, + }.URI().String()) + } + + transportSocket, err := s.createOutboundMeshMTLS(cfgSnap, spiffeIDs, sni) + if err != nil { + return nil, err + } + c.GetEndpointGroup().GetDynamic().OutboundTls = transportSocket + + return &namedCluster{name: sni, cluster: c}, nil +} + +func finalizeUpstreamConfig(cfg structs.UpstreamConfig, chain *structs.CompiledDiscoveryChain, connectTimeout time.Duration) structs.UpstreamConfig { + if cfg.Protocol == "" { + cfg.Protocol = chain.Protocol + } + + if cfg.Protocol == "" { + cfg.Protocol = "tcp" + } + + if cfg.ConnectTimeoutMs == 0 { + cfg.ConnectTimeoutMs = int(connectTimeout / time.Millisecond) + } + return cfg +} + +func (s *Converter) createOutboundMeshMTLS(cfgSnap *proxycfg.ConfigSnapshot, spiffeIDs []string, sni string) (*pbproxystate.TransportSocket, error) { + switch cfgSnap.Kind { + case structs.ServiceKindConnectProxy: + case structs.ServiceKindMeshGateway: + default: + return nil, fmt.Errorf("cannot inject peering trust bundles for kind %q", cfgSnap.Kind) + } + + cfg, err := config.ParseProxyConfig(cfgSnap.Proxy.Config) + if err != nil { + // Don't hard fail on a config typo, just warn. The parse func returns + // default config if there is an error so it's safe to continue. + s.Logger.Warn("failed to parse Connect.Proxy.Config", "error", err) + } + + // Add all trust bundle peer names, including local. + trustBundlePeerNames := []string{"local"} + for _, tb := range cfgSnap.PeeringTrustBundles() { + trustBundlePeerNames = append(trustBundlePeerNames, tb.PeerName) + } + // Arbitrary UUID to reference the identity by. + uuid, err := uuid.GenerateUUID() + if err != nil { + return nil, err + } + // Create the transport socket + ts := &pbproxystate.TransportSocket{} + + ts.ConnectionTls = &pbproxystate.TransportSocket_OutboundMesh{ + OutboundMesh: &pbproxystate.OutboundMeshMTLS{ + IdentityKey: uuid, + ValidationContext: &pbproxystate.MeshOutboundValidationContext{ + TrustBundlePeerNameKey: trustBundlePeerNames[0], + SpiffeIds: spiffeIDs, + }, + Sni: sni, + }, + } + s.proxyState.LeafCertificates[uuid] = &pbproxystate.LeafCertificate{ + Cert: cfgSnap.Leaf().CertPEM, + Key: cfgSnap.Leaf().PrivateKeyPEM, + } + ts.TlsParameters = makeTLSParametersFromProxyTLSConfig(cfgSnap.MeshConfigTLSOutgoing()) + ts.AlpnProtocols = getAlpnProtocols(cfg.Protocol) + + return ts, nil +} +func (s *Converter) makeUpstreamClustersForDiscoveryChain( + uid proxycfg.UpstreamID, + upstream *structs.Upstream, + chain *structs.CompiledDiscoveryChain, + cfgSnap *proxycfg.ConfigSnapshot, + forMeshGateway bool, +) (map[string]*pbproxystate.Cluster, error) { + if chain == nil { + return nil, fmt.Errorf("cannot create upstream cluster without discovery chain for %s", uid) + } + + if uid.Peer != "" && forMeshGateway { + return nil, fmt.Errorf("impossible to get a peer discovery chain in a mesh gateway") + } + + upstreamConfigMap := make(map[string]interface{}) + if upstream != nil { + upstreamConfigMap = upstream.Config + } + + upstreamsSnapshot, err := cfgSnap.ToConfigSnapshotUpstreams() + + // Mesh gateways are exempt because upstreamsSnapshot is only used for + // cluster peering targets and transative failover/redirects are unsupported. + if err != nil && !forMeshGateway { + return nil, err + } + + rawUpstreamConfig, err := structs.ParseUpstreamConfigNoDefaults(upstreamConfigMap) + if err != nil { + // Don't hard fail on a config typo, just warn. The parse func returns + // default config if there is an error so it's safe to continue. + s.Logger.Warn("failed to parse", "upstream", uid, + "error", err) + } + + // TODO(proxystate): escapeHatches will be implemented in the future + //var escapeHatchCluster *pbproxystate.Cluster + //if !forMeshGateway { + // if rawUpstreamConfig.EnvoyClusterJSON != "" { + // if chain.Default { + // // If you haven't done anything to setup the discovery chain, then + // // you can use the envoy_cluster_json escape hatch. + // escapeHatchCluster = &pbproxystate.Cluster{ + // EscapeHatchClusterJson: rawUpstreamConfig.EnvoyClusterJSON, + // } + // } else { + // s.Logger.Warn("ignoring escape hatch setting, because a discovery chain is configured for", + // "discovery chain", chain.ServiceName, "upstream", uid, + // "envoy_cluster_json", chain.ServiceName) + // } + // } + //} + + out := make(map[string]*pbproxystate.Cluster) + for _, node := range chain.Nodes { + switch { + case node == nil: + return nil, fmt.Errorf("impossible to process a nil node") + case node.Type != structs.DiscoveryGraphNodeTypeResolver: + continue + case node.Resolver == nil: + return nil, fmt.Errorf("impossible to process a non-resolver node") + } + // These variables are prefixed with primary to avoid shaddowing bugs. + primaryTargetID := node.Resolver.Target + primaryTarget := chain.Targets[primaryTargetID] + primaryTargetClusterName := s.getTargetClusterName(upstreamsSnapshot, chain, primaryTargetID, forMeshGateway) + if primaryTargetClusterName == "" { + continue + } + if forMeshGateway && !cfgSnap.Locality.Matches(primaryTarget.Datacenter, primaryTarget.Partition) { + s.Logger.Warn("ignoring discovery chain target that crosses a datacenter or partition boundary in a mesh gateway", + "target", primaryTarget, + "gatewayLocality", cfgSnap.Locality, + ) + continue + } + + upstreamConfig := finalizeUpstreamConfig(rawUpstreamConfig, chain, node.Resolver.ConnectTimeout) + + mappedTargets, err := s.mapDiscoChainTargets(cfgSnap, chain, node, upstreamConfig, forMeshGateway) + if err != nil { + return nil, err + } + + targetGroups, err := mappedTargets.groupedTargets() + if err != nil { + return nil, err + } + + var failoverGroup *pbproxystate.FailoverGroup + endpointGroups := make([]*pbproxystate.EndpointGroup, 0) + if mappedTargets.failover { + // Create a failover group. The endpoint groups that are part of this failover group are created by the loop + // below. + failoverGroup = &pbproxystate.FailoverGroup{ + Config: &pbproxystate.FailoverGroupConfig{ + ConnectTimeout: durationpb.New(node.Resolver.ConnectTimeout), + }, + } + } + + // Construct the target dynamic endpoint groups. If these are not part of a failover group, they will get added + // directly to the map of pbproxystate.Cluster, if they are a part of a failover group, they will be added to + // the failover group. + for _, groupedTarget := range targetGroups { + s.Logger.Debug("generating cluster for", "cluster", groupedTarget.ClusterName) + dynamic := &pbproxystate.DynamicEndpointGroup{ + Config: &pbproxystate.DynamicEndpointGroupConfig{ + UseAltStatName: true, + ConnectTimeout: durationpb.New(node.Resolver.ConnectTimeout), + // TODO(peering): make circuit breakers or outlier detection work? + CircuitBreakers: &pbproxystate.CircuitBreakers{ + UpstreamLimits: makeUpstreamLimitsIfNeeded(upstreamConfig.Limits), + }, + OutlierDetection: makeOutlierDetection(upstreamConfig.PassiveHealthCheck, nil, true), + }, + } + ti := groupedTarget.Targets[0] + transportSocket, err := s.createOutboundMeshMTLS(cfgSnap, ti.SpiffeIDs, ti.SNI) + if err != nil { + return nil, err + } + dynamic.OutboundTls = transportSocket + + var lb *structs.LoadBalancer + if node.LoadBalancer != nil { + lb = node.LoadBalancer + } + if err := injectLBToCluster(lb, dynamic.Config); err != nil { + return nil, fmt.Errorf("failed to apply load balancer configuration to cluster %q: %v", groupedTarget.ClusterName, err) + } + + // TODO: IR: http2 options not currently supported + //if upstreamConfig.Protocol == "http2" || upstreamConfig.Protocol == "grpc" { + // if err := s.setHttp2ProtocolOptions(c); err != nil { + // return nil, err + // } + //} + + switch len(groupedTarget.Targets) { + case 0: + continue + case 1: + // We expect one target so this passes through to continue setting the cluster up. + default: + return nil, fmt.Errorf("cannot have more than one target") + } + + if targetInfo := groupedTarget.Targets[0]; targetInfo.TransportSocket != nil { + dynamic.OutboundTls = targetInfo.TransportSocket + } + + // If the endpoint group is part of a failover group, add it to the failover group. Otherwise add it + // directly to the clusters. + if failoverGroup != nil { + eg := &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Dynamic{ + Dynamic: dynamic, + }, + } + endpointGroups = append(endpointGroups, eg) + } else { + cluster := &pbproxystate.Cluster{ + AltStatName: mappedTargets.baseClusterName, + Protocol: upstreamConfig.Protocol, + Group: &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Dynamic{ + Dynamic: dynamic, + }, + }, + }, + } + + out[mappedTargets.baseClusterName] = cluster + } + } + + // If there's a failover group, we only add the failover group to the top level list of clusters. Its endpoint + // groups are inlined. + if failoverGroup != nil { + failoverGroup.EndpointGroups = endpointGroups + cluster := &pbproxystate.Cluster{ + AltStatName: mappedTargets.baseClusterName, + Protocol: upstreamConfig.Protocol, + Group: &pbproxystate.Cluster_FailoverGroup{ + FailoverGroup: failoverGroup, + }, + } + out[mappedTargets.baseClusterName] = cluster + } + } + + //if escapeHatchCluster != nil { + // if len(out) != 1 { + // return nil, fmt.Errorf("cannot inject escape hatch cluster when discovery chain had no nodes") + // } + // var defaultCluster *pbproxystate.Cluster + // for _, k := range out { + // defaultCluster = k + // break + // } + // + // // Overlay what the user provided. + // escapeHatchCluster.GetEndpointGroup().GetDynamic().OutboundTls.ConnectionTls = + // defaultCluster.GetEndpointGroup().GetDynamic().OutboundTls.ConnectionTls + // + // out = append(out, escapeHatchCluster) + //} + + return out, nil +} + +// TODO(proxystate): Mesh Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeExportedUpstreamClustersForMeshGateway + +// makeClusterFromUserConfig returns the listener config decoded from an +// arbitrary proto3 json format string or an error if it's invalid. +// +// For now we only support embedding in JSON strings because of the hcl parsing +// pain (see Background section in the comment for decode.HookWeakDecodeFromSlice). +// This may be fixed in decode.HookWeakDecodeFromSlice in the future. +// +// When we do that we can support just nesting the config directly into the +// JSON/hcl naturally but this is a stop-gap that gets us an escape hatch +// immediately. It's also probably not a bad thing to support long-term since +// any config generated by other systems will likely be in canonical protobuf +// from rather than our slight variant in JSON/hcl. + +// TODO(proxystate): Mesh and Terminating Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeGatewayCluster + +func configureClusterWithHostnames( + logger hclog.Logger, + dnsEndpointGroup *pbproxystate.DNSEndpointGroup, + endpointList *pbproxystate.Endpoints, + dnsDiscoveryType string, + // hostnameEndpoints is a list of endpoints with a hostname as their address + hostnameEndpoints structs.CheckServiceNodes, + // isRemote determines whether the cluster is in a remote DC or partition and we should prefer a WAN address + isRemote bool, + // onlyPassing determines whether endpoints that do not have a passing status should be considered unhealthy + onlyPassing bool, +) { + // When a service instance is addressed by a hostname we have Envoy do the DNS resolution + // by setting a DNS cluster type and passing the hostname endpoints via CDS. + if dnsEndpointGroup.Config == nil { + dnsEndpointGroup.Config = &pbproxystate.DNSEndpointGroupConfig{} + } + dnsEndpointGroup.Config.DiscoveryType = pbproxystate.DiscoveryType_DISCOVERY_TYPE_LOGICAL + if dnsDiscoveryType == "strict_dns" { + dnsEndpointGroup.Config.DiscoveryType = pbproxystate.DiscoveryType_DISCOVERY_TYPE_STRICT + } + + endpoints := make([]*envoy_endpoint_v3.LbEndpoint, 0, 1) + uniqueHostnames := make(map[string]bool) + + var ( + hostname string + idx int + fallback *pbproxystate.Endpoint + ) + for i, e := range hostnameEndpoints { + _, addr, port := e.BestAddress(isRemote) + uniqueHostnames[addr] = true + + health, weight := calculateEndpointHealthAndWeight(e, onlyPassing) + if health == pbproxystate.HealthStatus_HEALTH_STATUS_UNHEALTHY { + fallback = makeLbEndpoint(addr, port, health, weight) + continue + } + + if len(endpoints) == 0 { + endpointList.Endpoints = append(endpointList.Endpoints, makeLbEndpoint(addr, port, health, weight)) + + hostname = addr + idx = i + break + } + } + + dc := hostnameEndpoints[idx].Node.Datacenter + service := hostnameEndpoints[idx].Service.CompoundServiceName() + + // Fall back to last unhealthy endpoint if none were healthy + if len(endpoints) == 0 { + logger.Warn("upstream service does not contain any healthy instances", + "dc", dc, "service", service.String()) + + //endpoints = append(endpoints, fallback) + endpointList.Endpoints = append(endpointList.Endpoints, fallback) + } + if len(uniqueHostnames) > 1 { + logger.Warn(fmt.Sprintf("service contains instances with more than one unique hostname; only %q be resolved by Envoy", hostname), + "dc", dc, "service", service.String()) + } + +} + +// TODO(proxystate): Terminating Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeExternalIPCluster + +// TODO(proxystate): Terminating Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeExternalHostnameCluster + +func makeUpstreamLimitsIfNeeded(limits *structs.UpstreamLimits) *pbproxystate.UpstreamLimits { + if limits == nil { + return nil + } + + upstreamLimits := &pbproxystate.UpstreamLimits{} + + // Likewise, make sure to not set any threshold values on the zero-value in + // order to rely on Envoy defaults + if limits.MaxConnections != nil { + upstreamLimits.MaxConnections = response.MakeUint32Value(*limits.MaxConnections) + } + if limits.MaxPendingRequests != nil { + upstreamLimits.MaxPendingRequests = response.MakeUint32Value(*limits.MaxPendingRequests) + } + if limits.MaxConcurrentRequests != nil { + upstreamLimits.MaxConcurrentRequests = response.MakeUint32Value(*limits.MaxConcurrentRequests) + } + + return upstreamLimits +} + +func injectLBToCluster(ec *structs.LoadBalancer, dc *pbproxystate.DynamicEndpointGroupConfig) error { + if ec == nil { + return nil + } + + switch ec.Policy { + case "": + return nil + case structs.LBPolicyLeastRequest: + lr := &pbproxystate.DynamicEndpointGroupConfig_LeastRequest{ + LeastRequest: &pbproxystate.LBPolicyLeastRequest{}, + } + + dc.LbPolicy = lr + + if ec.LeastRequestConfig != nil { + lr.LeastRequest.ChoiceCount = &wrapperspb.UInt32Value{Value: ec.LeastRequestConfig.ChoiceCount} + } + case structs.LBPolicyRoundRobin: + dc.LbPolicy = &pbproxystate.DynamicEndpointGroupConfig_RoundRobin{ + RoundRobin: &pbproxystate.LBPolicyRoundRobin{}, + } + + case structs.LBPolicyRandom: + dc.LbPolicy = &pbproxystate.DynamicEndpointGroupConfig_Random{ + Random: &pbproxystate.LBPolicyRandom{}, + } + + case structs.LBPolicyRingHash: + rh := &pbproxystate.DynamicEndpointGroupConfig_RingHash{ + RingHash: &pbproxystate.LBPolicyRingHash{}, + } + + dc.LbPolicy = rh + + if ec.RingHashConfig != nil { + rh.RingHash.MinimumRingSize = &wrapperspb.UInt64Value{Value: ec.RingHashConfig.MinimumRingSize} + rh.RingHash.MaximumRingSize = &wrapperspb.UInt64Value{Value: ec.RingHashConfig.MaximumRingSize} + } + case structs.LBPolicyMaglev: + dc.LbPolicy = &pbproxystate.DynamicEndpointGroupConfig_Maglev{ + Maglev: &pbproxystate.LBPolicyMaglev{}, + } + + default: + return fmt.Errorf("unsupported load balancer policy %q", ec.Policy) + } + return nil +} + // generatePeeredClusterName returns an SNI-like cluster name which mimics PeeredServiceSNI // but excludes partition information which could be ambiguous (local vs remote partition). func generatePeeredClusterName(uid proxycfg.UpstreamID, tb *pbpeering.PeeringTrustBundle) string { @@ -72,3 +1181,69 @@ func (s *Converter) getTargetClusterName(upstreamsSnapshot *proxycfg.ConfigSnaps } return clusterName } + +// Return an pbproxystate.OutlierDetection populated by the values from structs.PassiveHealthCheck. +// If all values are zero a default empty OutlierDetection will be returned to +// enable outlier detection with default values. +// - If override is not nil, it will overwrite the values from p, e.g., ingress gateway defaults +// - allowZero is added to handle the legacy case where connect-proxy and mesh gateway can set 0 +// for EnforcingConsecutive5xx. Due to the definition of proto of PassiveHealthCheck, ingress +// gateway's EnforcingConsecutive5xx must be > 0. +func makeOutlierDetection(p *structs.PassiveHealthCheck, override *structs.PassiveHealthCheck, allowZero bool) *pbproxystate.OutlierDetection { + od := &pbproxystate.OutlierDetection{} + if p != nil { + + if p.Interval != 0 { + od.Interval = durationpb.New(p.Interval) + } + if p.MaxFailures != 0 { + od.Consecutive_5Xx = &wrapperspb.UInt32Value{Value: p.MaxFailures} + } + + if p.EnforcingConsecutive5xx != nil { + // NOTE: EnforcingConsecutive5xx must be greater than 0 for ingress-gateway + if *p.EnforcingConsecutive5xx != 0 { + od.EnforcingConsecutive_5Xx = &wrapperspb.UInt32Value{Value: *p.EnforcingConsecutive5xx} + } else if allowZero { + od.EnforcingConsecutive_5Xx = &wrapperspb.UInt32Value{Value: *p.EnforcingConsecutive5xx} + } + } + + if p.MaxEjectionPercent != nil { + od.MaxEjectionPercent = &wrapperspb.UInt32Value{Value: *p.MaxEjectionPercent} + } + if p.BaseEjectionTime != nil { + od.BaseEjectionTime = durationpb.New(*p.BaseEjectionTime) + } + } + + if override == nil { + return od + } + + // override the default outlier detection value + if override.Interval != 0 { + od.Interval = durationpb.New(override.Interval) + } + if override.MaxFailures != 0 { + od.Consecutive_5Xx = &wrapperspb.UInt32Value{Value: override.MaxFailures} + } + + if override.EnforcingConsecutive5xx != nil { + // NOTE: EnforcingConsecutive5xx must be great than 0 for ingress-gateway + if *override.EnforcingConsecutive5xx != 0 { + od.EnforcingConsecutive_5Xx = &wrapperspb.UInt32Value{Value: *override.EnforcingConsecutive5xx} + } + // Because only ingress gateways have overrides and they cannot have a value of 0, there is no allowZero + // override case to handle + } + + if override.MaxEjectionPercent != nil { + od.MaxEjectionPercent = &wrapperspb.UInt32Value{Value: *override.MaxEjectionPercent} + } + if override.BaseEjectionTime != nil { + od.BaseEjectionTime = durationpb.New(*override.BaseEjectionTime) + } + + return od +} diff --git a/agent/xds/proxystateconverter/converter.go b/agent/xds/proxystateconverter/converter.go index 3a8037b44b1c..061a59d6e93e 100644 --- a/agent/xds/proxystateconverter/converter.go +++ b/agent/xds/proxystateconverter/converter.go @@ -1,5 +1,5 @@ // Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 +// SPDX-License-Identifier: BUSL-1.1 package proxystateconverter @@ -57,6 +57,10 @@ func (g *Converter) resourcesFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot) erro if err != nil { return err } + //g.routesFromSnapshot(cfgSnap) + g.clustersFromSnapshot(cfgSnap) + //g.endpointsFromSnapshot(cfgSnap) + //g.secretsFromSnapshot(cfgSnap) return nil } diff --git a/agent/xds/proxystateconverter/endpoints.go b/agent/xds/proxystateconverter/endpoints.go new file mode 100644 index 000000000000..5a5dc3708098 --- /dev/null +++ b/agent/xds/proxystateconverter/endpoints.go @@ -0,0 +1,85 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package proxystateconverter + +import ( + "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func makeLbEndpoint(addr string, port int, health pbproxystate.HealthStatus, weight int) *pbproxystate.Endpoint { + ep := &pbproxystate.Endpoint{ + Address: &pbproxystate.Endpoint_HostPort{ + HostPort: &pbproxystate.HostPortAddress{ + Host: addr, + Port: uint32(port), + }, + }, + } + ep.HealthStatus = health + ep.LoadBalancingWeight = &wrapperspb.UInt32Value{Value: uint32(weight)} + return ep +} + +// used in clusters.go +func makeHostPortEndpoint(host string, port int) *pbproxystate.Endpoint { + return &pbproxystate.Endpoint{ + Address: &pbproxystate.Endpoint_HostPort{ + HostPort: &pbproxystate.HostPortAddress{ + Host: host, + Port: uint32(port), + }, + }, + } +} + +func makeUnixSocketEndpoint(path string) *pbproxystate.Endpoint { + return &pbproxystate.Endpoint{ + Address: &pbproxystate.Endpoint_UnixSocket{ + UnixSocket: &pbproxystate.UnixSocketAddress{ + Path: path, + // envoy's mode is particular to a pipe address and is uint32. + // it also says "The mode for the Pipe. Not applicable for abstract sockets." + // https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#config-core-v3-pipe + Mode: "0", + }, + }, + } +} + +func calculateEndpointHealthAndWeight( + ep structs.CheckServiceNode, + onlyPassing bool, +) (pbproxystate.HealthStatus, int) { + healthStatus := pbproxystate.HealthStatus_HEALTH_STATUS_HEALTHY + weight := 1 + if ep.Service.Weights != nil { + weight = ep.Service.Weights.Passing + } + + for _, chk := range ep.Checks { + if chk.Status == api.HealthCritical { + healthStatus = pbproxystate.HealthStatus_HEALTH_STATUS_UNHEALTHY + } + if onlyPassing && chk.Status != api.HealthPassing { + healthStatus = pbproxystate.HealthStatus_HEALTH_STATUS_UNHEALTHY + } + if chk.Status == api.HealthWarning && ep.Service.Weights != nil { + weight = ep.Service.Weights.Warning + } + } + // Make weights fit Envoy's limits. A zero weight means that either Warning + // (likely) or Passing (weirdly) weight has been set to 0 effectively making + // this instance unhealthy and should not be sent traffic. + if weight < 1 { + healthStatus = pbproxystate.HealthStatus_HEALTH_STATUS_UNHEALTHY + weight = 1 + } + if weight > 128 { + weight = 128 + } + return healthStatus, weight +} diff --git a/agent/xds/proxystateconverter/failover_policy.go b/agent/xds/proxystateconverter/failover_policy.go new file mode 100644 index 000000000000..6ae120eaf941 --- /dev/null +++ b/agent/xds/proxystateconverter/failover_policy.go @@ -0,0 +1,158 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package proxystateconverter + +import ( + "fmt" + + "github.com/hashicorp/consul/agent/connect" + "github.com/hashicorp/consul/agent/proxycfg" + "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/envoyextensions/xdscommon" + "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" +) + +type discoChainTargets struct { + baseClusterName string + targets []targetInfo + failover bool + failoverPolicy structs.ServiceResolverFailoverPolicy +} + +type targetInfo struct { + TargetID string + TransportSocket *pbproxystate.TransportSocket + SNI string + RootPEMs string + SpiffeIDs []string + Region *string + + PrioritizeByLocality *structs.DiscoveryPrioritizeByLocality +} + +type discoChainTargetGroup struct { + Targets []targetInfo + ClusterName string +} + +func (ft discoChainTargets) groupedTargets() ([]discoChainTargetGroup, error) { + var targetGroups []discoChainTargetGroup + + if !ft.failover { + targetGroups = append(targetGroups, discoChainTargetGroup{ + ClusterName: ft.baseClusterName, + Targets: ft.targets, + }) + return targetGroups, nil + } + + switch ft.failoverPolicy.Mode { + case "sequential", "": + return ft.sequential() + case "order-by-locality": + return ft.orderByLocality() + default: + return targetGroups, fmt.Errorf("unexpected failover policy") + } +} + +func (s *Converter) mapDiscoChainTargets(cfgSnap *proxycfg.ConfigSnapshot, chain *structs.CompiledDiscoveryChain, node *structs.DiscoveryGraphNode, upstreamConfig structs.UpstreamConfig, forMeshGateway bool) (discoChainTargets, error) { + failoverTargets := discoChainTargets{} + + if node.Resolver == nil { + return discoChainTargets{}, fmt.Errorf("impossible to process a non-resolver node") + } + + primaryTargetID := node.Resolver.Target + upstreamsSnapshot, err := cfgSnap.ToConfigSnapshotUpstreams() + if err != nil && !forMeshGateway { + return discoChainTargets{}, err + } + + failoverTargets.baseClusterName = s.getTargetClusterName(upstreamsSnapshot, chain, primaryTargetID, forMeshGateway) + + tids := []string{primaryTargetID} + failover := node.Resolver.Failover + if failover != nil && !forMeshGateway { + tids = append(tids, failover.Targets...) + failoverTargets.failover = true + if failover.Policy == nil { + failoverTargets.failoverPolicy = structs.ServiceResolverFailoverPolicy{} + } else { + failoverTargets.failoverPolicy = *failover.Policy + } + } + + for _, tid := range tids { + target := chain.Targets[tid] + targetUID := proxycfg.NewUpstreamIDFromTargetID(tid) + ti := targetInfo{TargetID: tid, PrioritizeByLocality: target.PrioritizeByLocality} + + configureTLS := true + if forMeshGateway { + // We only initiate TLS if we're doing an L7 proxy. + configureTLS = structs.IsProtocolHTTPLike(upstreamConfig.Protocol) + } + + if !configureTLS { + failoverTargets.targets = append(failoverTargets.targets, ti) + continue + } + + if targetUID.Peer != "" { + tbs, _ := upstreamsSnapshot.UpstreamPeerTrustBundles.Get(targetUID.Peer) + + peerMeta, found := upstreamsSnapshot.UpstreamPeerMeta(targetUID) + if !found { + s.Logger.Warn("failed to fetch upstream peering metadata", "target", targetUID) + continue + } + ti.SNI = peerMeta.PrimarySNI() + ti.SpiffeIDs = peerMeta.SpiffeID + region := target.Locality.GetRegion() + ti.Region = ®ion + ti.RootPEMs = tbs.ConcatenatedRootPEMs() + } else { + ti.SNI = target.SNI + ti.RootPEMs = cfgSnap.RootPEMs() + ti.SpiffeIDs = []string{connect.SpiffeIDService{ + Host: cfgSnap.Roots.TrustDomain, + Namespace: target.Namespace, + Partition: target.Partition, + Datacenter: target.Datacenter, + Service: target.Service, + }.URI().String()} + } + //commonTLSContext := makeCommonTLSContext( + // cfgSnap.Leaf(), + // rootPEMs, + // makeTLSParametersFromProxyTLSConfig(cfgSnap.MeshConfigTLSOutgoing()), + //) + // + //err := injectSANMatcher(commonTLSContext, spiffeIDs...) + //if err != nil { + // return failoverTargets, fmt.Errorf("failed to inject SAN matcher rules for cluster %q: %v", sni, err) + //} + + //tlsContext := &envoy_tls_v3.UpstreamTlsContext{ + // CommonTlsContext: commonTLSContext, + // Sni: sni, + //} + //ti.TLSContext = tlsContext + failoverTargets.targets = append(failoverTargets.targets, ti) + } + + return failoverTargets, nil +} + +func (ft discoChainTargets) sequential() ([]discoChainTargetGroup, error) { + var targetGroups []discoChainTargetGroup + for i, t := range ft.targets { + targetGroups = append(targetGroups, discoChainTargetGroup{ + ClusterName: fmt.Sprintf("%s%d~%s", xdscommon.FailoverClusterNamePrefix, i, ft.baseClusterName), + Targets: []targetInfo{t}, + }) + } + return targetGroups, nil +} diff --git a/agent/xds/proxystateconverter/failover_policy_oss.go b/agent/xds/proxystateconverter/failover_policy_oss.go new file mode 100644 index 000000000000..a9b2ec24843c --- /dev/null +++ b/agent/xds/proxystateconverter/failover_policy_oss.go @@ -0,0 +1,15 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +//go:build !consulent +// +build !consulent + +package proxystateconverter + +import ( + "fmt" +) + +func (ft discoChainTargets) orderByLocality() ([]discoChainTargetGroup, error) { + return nil, fmt.Errorf("order-by-locality is a Consul Enterprise feature") +} diff --git a/agent/xds/proxystateconverter/listeners.go b/agent/xds/proxystateconverter/listeners.go index a530fbf4fe96..7e21819260ee 100644 --- a/agent/xds/proxystateconverter/listeners.go +++ b/agent/xds/proxystateconverter/listeners.go @@ -1,5 +1,5 @@ // Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 +// SPDX-License-Identifier: BUSL-1.1 package proxystateconverter @@ -16,11 +16,6 @@ import ( envoy_http_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" "github.com/hashicorp/go-uuid" - "github.com/hashicorp/consul/agent/xds/config" - "github.com/hashicorp/consul/agent/xds/naming" - "github.com/hashicorp/consul/agent/xds/platform" - "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" - "github.com/hashicorp/go-hclog" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/wrapperspb" @@ -29,7 +24,11 @@ import ( "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/config" + "github.com/hashicorp/consul/agent/xds/naming" + "github.com/hashicorp/consul/agent/xds/platform" "github.com/hashicorp/consul/envoyextensions/xdscommon" + "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" "github.com/hashicorp/consul/sdk/iptables" "github.com/hashicorp/consul/types" ) diff --git a/agent/xds/rbac.go b/agent/xds/rbac.go index b3f9267eefb4..0c00cb92cb0c 100644 --- a/agent/xds/rbac.go +++ b/agent/xds/rbac.go @@ -19,6 +19,7 @@ import ( "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/proto/private/pbpeering" ) @@ -987,7 +988,7 @@ func authenticatedPatternPrincipal(pattern string) *envoy_rbac_v3.Principal { Authenticated: &envoy_rbac_v3.Principal_Authenticated{ PrincipalName: &envoy_matcher_v3.StringMatcher{ MatchPattern: &envoy_matcher_v3.StringMatcher_SafeRegex{ - SafeRegex: makeEnvoyRegexMatch(pattern), + SafeRegex: response.MakeEnvoyRegexMatch(pattern), }, }, }, @@ -1019,7 +1020,7 @@ func xfccPrincipal(src rbacService) *envoy_rbac_v3.Principal { HeaderMatchSpecifier: &envoy_route_v3.HeaderMatcher_StringMatch{ StringMatch: &envoy_matcher_v3.StringMatcher{ MatchPattern: &envoy_matcher_v3.StringMatcher_SafeRegex{ - SafeRegex: makeEnvoyRegexMatch(pattern), + SafeRegex: response.MakeEnvoyRegexMatch(pattern), }, }, }, @@ -1224,7 +1225,7 @@ func convertPermission(perm *structs.IntentionPermission) *envoy_rbac_v3.Permiss Rule: &envoy_matcher_v3.PathMatcher_Path{ Path: &envoy_matcher_v3.StringMatcher{ MatchPattern: &envoy_matcher_v3.StringMatcher_SafeRegex{ - SafeRegex: makeEnvoyRegexMatch(perm.HTTP.PathRegex), + SafeRegex: response.MakeEnvoyRegexMatch(perm.HTTP.PathRegex), }, }, }, @@ -1245,7 +1246,7 @@ func convertPermission(perm *structs.IntentionPermission) *envoy_rbac_v3.Permiss } case hdr.Regex != "": eh.HeaderMatchSpecifier = &envoy_route_v3.HeaderMatcher_SafeRegexMatch{ - SafeRegexMatch: makeEnvoyRegexMatch(hdr.Regex), + SafeRegexMatch: response.MakeEnvoyRegexMatch(hdr.Regex), } case hdr.Prefix != "": eh.HeaderMatchSpecifier = &envoy_route_v3.HeaderMatcher_PrefixMatch{ @@ -1280,7 +1281,7 @@ func convertPermission(perm *structs.IntentionPermission) *envoy_rbac_v3.Permiss eh := &envoy_route_v3.HeaderMatcher{ Name: ":method", HeaderMatchSpecifier: &envoy_route_v3.HeaderMatcher_SafeRegexMatch{ - SafeRegexMatch: makeEnvoyRegexMatch(methodHeaderRegex), + SafeRegexMatch: response.MakeEnvoyRegexMatch(methodHeaderRegex), }, } diff --git a/agent/xds/resources_test.go b/agent/xds/resources_test.go index 22ef50bc876d..926c44fab86e 100644 --- a/agent/xds/resources_test.go +++ b/agent/xds/resources_test.go @@ -27,6 +27,7 @@ import ( "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/sdk/testutil" ) @@ -120,7 +121,7 @@ func TestAllResourcesFromSnapshot(t *testing.T) { } }) - r, err := createResponse(typeUrl, "00000001", "00000001", items) + r, err := response.CreateResponse(typeUrl, "00000001", "00000001", items) require.NoError(t, err) gotJSON := protoToJSON(t, r) diff --git a/agent/xds/response.go b/agent/xds/response/response.go similarity index 82% rename from agent/xds/response.go rename to agent/xds/response/response.go index 92e0f9d0a890..91ce6d739738 100644 --- a/agent/xds/response.go +++ b/agent/xds/response/response.go @@ -1,7 +1,7 @@ // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: BUSL-1.1 -package xds +package response import ( envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" @@ -13,7 +13,7 @@ import ( "google.golang.org/protobuf/types/known/wrapperspb" ) -func createResponse(typeURL string, version, nonce string, resources []proto.Message) (*envoy_discovery_v3.DiscoveryResponse, error) { +func CreateResponse(typeURL string, version, nonce string, resources []proto.Message) (*envoy_discovery_v3.DiscoveryResponse, error) { anys := make([]*anypb.Any, 0, len(resources)) for _, r := range resources { if r == nil { @@ -41,7 +41,7 @@ func createResponse(typeURL string, version, nonce string, resources []proto.Mes return resp, nil } -func makePipeAddress(path string, mode uint32) *envoy_core_v3.Address { +func MakePipeAddress(path string, mode uint32) *envoy_core_v3.Address { return &envoy_core_v3.Address{ Address: &envoy_core_v3.Address_Pipe{ Pipe: &envoy_core_v3.Pipe{ @@ -52,7 +52,7 @@ func makePipeAddress(path string, mode uint32) *envoy_core_v3.Address { } } -func makeAddress(ip string, port int) *envoy_core_v3.Address { +func MakeAddress(ip string, port int) *envoy_core_v3.Address { return &envoy_core_v3.Address{ Address: &envoy_core_v3.Address_SocketAddress{ SocketAddress: &envoy_core_v3.SocketAddress{ @@ -65,15 +65,15 @@ func makeAddress(ip string, port int) *envoy_core_v3.Address { } } -func makeUint32Value(n int) *wrapperspb.UInt32Value { +func MakeUint32Value(n int) *wrapperspb.UInt32Value { return &wrapperspb.UInt32Value{Value: uint32(n)} } -func makeBoolValue(n bool) *wrapperspb.BoolValue { +func MakeBoolValue(n bool) *wrapperspb.BoolValue { return &wrapperspb.BoolValue{Value: n} } -func makeEnvoyRegexMatch(patt string) *envoy_matcher_v3.RegexMatcher { +func MakeEnvoyRegexMatch(patt string) *envoy_matcher_v3.RegexMatcher { return &envoy_matcher_v3.RegexMatcher{ EngineType: &envoy_matcher_v3.RegexMatcher_GoogleRe2{ GoogleRe2: &envoy_matcher_v3.RegexMatcher_GoogleRE2{}, diff --git a/agent/xds/routes.go b/agent/xds/routes.go index 31b1c4a93241..6f0d18b05dcb 100644 --- a/agent/xds/routes.go +++ b/agent/xds/routes.go @@ -23,6 +23,7 @@ import ( "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/xds/config" + "github.com/hashicorp/consul/agent/xds/response" ) // routesFromSnapshot returns the xDS API representation of the "routes" in the @@ -71,7 +72,7 @@ func (s *ResourceGenerator) routesForConnectProxy(cfgSnap *proxycfg.ConfigSnapsh // ValidateClusters defaults to true when defined statically and false // when done via RDS. Re-set the reasonable value of true to prevent // null-routing traffic. - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), } resources = append(resources, route) } @@ -268,7 +269,7 @@ func (s *ResourceGenerator) routesForMeshGateway(cfgSnap *proxycfg.ConfigSnapsho // ValidateClusters defaults to true when defined statically and false // when done via RDS. Re-set the reasonable value of true to prevent // null-routing traffic. - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), } resources = append(resources, route) } @@ -286,7 +287,7 @@ func makeNamedDefaultRouteWithLB(clusterName string, lb *structs.LoadBalancer, t // Configure Envoy to rewrite Host header if autoHostRewrite { action.Route.HostRewriteSpecifier = &envoy_route_v3.RouteAction_AutoHostRewrite{ - AutoHostRewrite: makeBoolValue(true), + AutoHostRewrite: response.MakeBoolValue(true), } } @@ -311,7 +312,7 @@ func makeNamedDefaultRouteWithLB(clusterName string, lb *structs.LoadBalancer, t // ValidateClusters defaults to true when defined statically and false // when done via RDS. Re-set the reasonable value of true to prevent // null-routing traffic. - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), }, nil } @@ -321,7 +322,7 @@ func makeNamedAddressesRoute(routeName string, addresses map[string]string) (*en // ValidateClusters defaults to true when defined statically and false // when done via RDS. Re-set the reasonable value of true to prevent // null-routing traffic. - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), } for clusterName, address := range addresses { action := makeRouteActionFromName(clusterName) @@ -363,7 +364,7 @@ func (s *ResourceGenerator) routesForIngressGateway(cfgSnap *proxycfg.ConfigSnap // ValidateClusters defaults to true when defined statically and false // when done via RDS. Re-set the reasonable value of true to prevent // null-routing traffic. - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), } for _, u := range upstreams { @@ -412,7 +413,7 @@ func (s *ResourceGenerator) routesForIngressGateway(cfgSnap *proxycfg.ConfigSnap } else { svcRoute := &envoy_route_v3.RouteConfiguration{ Name: svcRouteName, - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), VirtualHosts: []*envoy_route_v3.VirtualHost{virtualHost}, } result = append(result, svcRoute) @@ -448,7 +449,7 @@ func (s *ResourceGenerator) routesForAPIGateway(cfgSnap *proxycfg.ConfigSnapshot // ValidateClusters defaults to true when defined statically and false // when done via RDS. Re-set the reasonable value of true to prevent // null-routing traffic. - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), } route, ok := cfgSnap.APIGateway.HTTPRoutes.Get(routeRef) @@ -515,7 +516,7 @@ func makeHeadersValueOptions(vals map[string]string, add bool) []*envoy_core_v3. Key: k, Value: v, }, - Append: makeBoolValue(add), + Append: response.MakeBoolValue(add), } opts = append(opts, o) } @@ -752,7 +753,7 @@ func (s *ResourceGenerator) makeUpstreamRouteForDiscoveryChain( func getRetryPolicyForDestination(destination *structs.ServiceRouteDestination) *envoy_route_v3.RetryPolicy { retryPolicy := &envoy_route_v3.RetryPolicy{} if destination.NumRetries > 0 { - retryPolicy.NumRetries = makeUint32Value(int(destination.NumRetries)) + retryPolicy.NumRetries = response.MakeUint32Value(int(destination.NumRetries)) } // The RetryOn magic values come from: https://www.envoyproxy.io/docs/envoy/v1.10.0/configuration/http_filters/router_filter#config-http-filters-router-x-envoy-retry-on @@ -805,7 +806,7 @@ func makeRouteMatchForDiscoveryRoute(discoveryRoute *structs.DiscoveryRoute) *en } case match.HTTP.PathRegex != "": em.PathSpecifier = &envoy_route_v3.RouteMatch_SafeRegex{ - SafeRegex: makeEnvoyRegexMatch(match.HTTP.PathRegex), + SafeRegex: response.MakeEnvoyRegexMatch(match.HTTP.PathRegex), } default: em.PathSpecifier = &envoy_route_v3.RouteMatch_Prefix{ @@ -827,7 +828,7 @@ func makeRouteMatchForDiscoveryRoute(discoveryRoute *structs.DiscoveryRoute) *en } case hdr.Regex != "": eh.HeaderMatchSpecifier = &envoy_route_v3.HeaderMatcher_SafeRegexMatch{ - SafeRegexMatch: makeEnvoyRegexMatch(hdr.Regex), + SafeRegexMatch: response.MakeEnvoyRegexMatch(hdr.Regex), } case hdr.Prefix != "": eh.HeaderMatchSpecifier = &envoy_route_v3.HeaderMatcher_PrefixMatch{ @@ -859,7 +860,7 @@ func makeRouteMatchForDiscoveryRoute(discoveryRoute *structs.DiscoveryRoute) *en eh := &envoy_route_v3.HeaderMatcher{ Name: ":method", HeaderMatchSpecifier: &envoy_route_v3.HeaderMatcher_SafeRegexMatch{ - SafeRegexMatch: makeEnvoyRegexMatch(methodHeaderRegex), + SafeRegexMatch: response.MakeEnvoyRegexMatch(methodHeaderRegex), }, } @@ -886,7 +887,7 @@ func makeRouteMatchForDiscoveryRoute(discoveryRoute *structs.DiscoveryRoute) *en eq.QueryParameterMatchSpecifier = &envoy_route_v3.QueryParameterMatcher_StringMatch{ StringMatch: &envoy_matcher_v3.StringMatcher{ MatchPattern: &envoy_matcher_v3.StringMatcher_SafeRegex{ - SafeRegex: makeEnvoyRegexMatch(qm.Regex), + SafeRegex: response.MakeEnvoyRegexMatch(qm.Regex), }, }, } @@ -967,7 +968,7 @@ func (s *ResourceGenerator) makeRouteActionForSplitter( weight := int(split.Weight * 100) totalWeight += weight cw := &envoy_route_v3.WeightedCluster_ClusterWeight{ - Weight: makeUint32Value(weight), + Weight: response.MakeUint32Value(weight), Name: clusterName, } if err := injectHeaderManipToWeightedCluster(split.Definition, cw); err != nil { @@ -983,7 +984,7 @@ func (s *ResourceGenerator) makeRouteActionForSplitter( var envoyWeightScale *wrapperspb.UInt32Value if totalWeight == 10000 { - envoyWeightScale = makeUint32Value(10000) + envoyWeightScale = response.MakeUint32Value(10000) } return &envoy_route_v3.Route_Route{ diff --git a/agent/xds/routes_test.go b/agent/xds/routes_test.go index c691114b5e3a..8b7345d88a52 100644 --- a/agent/xds/routes_test.go +++ b/agent/xds/routes_test.go @@ -16,6 +16,7 @@ import ( "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/agent/xds/testcommon" "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/sdk/testutil" @@ -241,7 +242,7 @@ func TestRoutesFromSnapshot(t *testing.T) { sort.Slice(routes, func(i, j int) bool { return routes[i].(*envoy_route_v3.RouteConfiguration).Name < routes[j].(*envoy_route_v3.RouteConfiguration).Name }) - r, err := createResponse(xdscommon.RouteType, "00000001", "00000001", routes) + r, err := response.CreateResponse(xdscommon.RouteType, "00000001", "00000001", routes) require.NoError(t, err) t.Run("current", func(t *testing.T) { diff --git a/agent/xds/xds_protocol_helpers_test.go b/agent/xds/xds_protocol_helpers_test.go index d609268a206b..bf632217d7ee 100644 --- a/agent/xds/xds_protocol_helpers_test.go +++ b/agent/xds/xds_protocol_helpers_test.go @@ -4,6 +4,7 @@ package xds import ( + "github.com/hashicorp/consul/agent/xds/response" "sort" "sync" "testing" @@ -228,7 +229,7 @@ func xdsNewEndpoint(ip string, port int) *envoy_endpoint_v3.LbEndpoint { return &envoy_endpoint_v3.LbEndpoint{ HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress(ip, port), + Address: response.MakeAddress(ip, port), }, }, } @@ -237,7 +238,7 @@ func xdsNewEndpoint(ip string, port int) *envoy_endpoint_v3.LbEndpoint { func xdsNewEndpointWithHealth(ip string, port int, health envoy_core_v3.HealthStatus, weight int) *envoy_endpoint_v3.LbEndpoint { ep := xdsNewEndpoint(ip, port) ep.HealthStatus = health - ep.LoadBalancingWeight = makeUint32Value(weight) + ep.LoadBalancingWeight = response.MakeUint32Value(weight) return ep } @@ -297,7 +298,7 @@ func xdsNewTransportSocket( if downstream { var requireClientCertPB *wrapperspb.BoolValue if requireClientCert { - requireClientCertPB = makeBoolValue(true) + requireClientCertPB = response.MakeBoolValue(true) } tlsContext = &envoy_tls_v3.DownstreamTlsContext{ @@ -597,7 +598,7 @@ func makeTestListener(t *testing.T, snap *proxycfg.ConfigSnapshot, fixtureName s return &envoy_listener_v3.Listener{ // Envoy can't bind to port 1 Name: "public_listener:0.0.0.0:1", - Address: makeAddress("0.0.0.0", 1), + Address: response.MakeAddress("0.0.0.0", 1), TrafficDirection: envoy_core_v3.TrafficDirection_INBOUND, FilterChains: []*envoy_listener_v3.FilterChain{ { @@ -620,7 +621,7 @@ func makeTestListener(t *testing.T, snap *proxycfg.ConfigSnapshot, fixtureName s case "tcp:public_listener": return &envoy_listener_v3.Listener{ Name: "public_listener:0.0.0.0:9999", - Address: makeAddress("0.0.0.0", 9999), + Address: response.MakeAddress("0.0.0.0", 9999), TrafficDirection: envoy_core_v3.TrafficDirection_INBOUND, FilterChains: []*envoy_listener_v3.FilterChain{ { @@ -643,7 +644,7 @@ func makeTestListener(t *testing.T, snap *proxycfg.ConfigSnapshot, fixtureName s case "tcp:db": return &envoy_listener_v3.Listener{ Name: "db:127.0.0.1:9191", - Address: makeAddress("127.0.0.1", 9191), + Address: response.MakeAddress("127.0.0.1", 9191), TrafficDirection: envoy_core_v3.TrafficDirection_OUTBOUND, FilterChains: []*envoy_listener_v3.FilterChain{ { @@ -661,7 +662,7 @@ func makeTestListener(t *testing.T, snap *proxycfg.ConfigSnapshot, fixtureName s case "http2:db": return &envoy_listener_v3.Listener{ Name: "db:127.0.0.1:9191", - Address: makeAddress("127.0.0.1", 9191), + Address: response.MakeAddress("127.0.0.1", 9191), TrafficDirection: envoy_core_v3.TrafficDirection_OUTBOUND, FilterChains: []*envoy_listener_v3.FilterChain{ { @@ -689,7 +690,7 @@ func makeTestListener(t *testing.T, snap *proxycfg.ConfigSnapshot, fixtureName s case "http2:db:rds": return &envoy_listener_v3.Listener{ Name: "db:127.0.0.1:9191", - Address: makeAddress("127.0.0.1", 9191), + Address: response.MakeAddress("127.0.0.1", 9191), TrafficDirection: envoy_core_v3.TrafficDirection_OUTBOUND, FilterChains: []*envoy_listener_v3.FilterChain{ { @@ -720,7 +721,7 @@ func makeTestListener(t *testing.T, snap *proxycfg.ConfigSnapshot, fixtureName s case "http:db:rds": return &envoy_listener_v3.Listener{ Name: "db:127.0.0.1:9191", - Address: makeAddress("127.0.0.1", 9191), + Address: response.MakeAddress("127.0.0.1", 9191), TrafficDirection: envoy_core_v3.TrafficDirection_OUTBOUND, FilterChains: []*envoy_listener_v3.FilterChain{ { @@ -751,7 +752,7 @@ func makeTestListener(t *testing.T, snap *proxycfg.ConfigSnapshot, fixtureName s case "tcp:geo-cache": return &envoy_listener_v3.Listener{ Name: "prepared_query:geo-cache:127.10.10.10:8181", - Address: makeAddress("127.10.10.10", 8181), + Address: response.MakeAddress("127.10.10.10", 8181), TrafficDirection: envoy_core_v3.TrafficDirection_OUTBOUND, FilterChains: []*envoy_listener_v3.FilterChain{ { @@ -777,7 +778,7 @@ func makeTestRoute(t *testing.T, fixtureName string) *envoy_route_v3.RouteConfig case "http2:db", "http:db": return &envoy_route_v3.RouteConfiguration{ Name: "db", - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), VirtualHosts: []*envoy_route_v3.VirtualHost{ { Name: "db", diff --git a/agent/xdsv2/cluster_resources.go b/agent/xdsv2/cluster_resources.go index c162d05c8d84..4c1573d5db3d 100644 --- a/agent/xdsv2/cluster_resources.go +++ b/agent/xdsv2/cluster_resources.go @@ -1,6 +1,331 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + package xdsv2 +import ( + "errors" + "fmt" + + envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + envoy_aggregate_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3" + envoy_upstreams_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" + envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +func (pr *ProxyResources) makeXDSClusters() ([]proto.Message, error) { + clusters := make([]proto.Message, 0) + + for clusterName := range pr.proxyState.Clusters { + protoCluster, err := pr.makeClusters(clusterName) + // TODO: aggregate errors for clusters and still return any properly formed clusters. + if err != nil { + return nil, err + } + clusters = append(clusters, protoCluster...) + } + + return clusters, nil +} + +func (pr *ProxyResources) makeClusters(name string) ([]proto.Message, error) { + clusters := make([]proto.Message, 0) + proxyStateCluster, ok := pr.proxyState.Clusters[name] + if !ok { + return nil, fmt.Errorf("cluster %q not found", name) + } + + switch proxyStateCluster.Group.(type) { + case *pbproxystate.Cluster_FailoverGroup: + fg := proxyStateCluster.GetFailoverGroup() + clusters, err := pr.makeEnvoyAggregateCluster(name, proxyStateCluster.Protocol, fg) + if err != nil { + return nil, err + } + for _, c := range clusters { + clusters = append(clusters, c) + } + + case *pbproxystate.Cluster_EndpointGroup: + eg := proxyStateCluster.GetEndpointGroup() + cluster, err := pr.makeEnvoyCluster(name, proxyStateCluster.Protocol, eg) + if err != nil { + return nil, err + } + clusters = append(clusters, cluster) + + default: + return nil, errors.New("cluster group type should be Endpoint Group or Failover Group") + } + return clusters, nil +} + +func (pr *ProxyResources) makeEnvoyCluster(name string, protocol string, eg *pbproxystate.EndpointGroup) (*envoy_cluster_v3.Cluster, error) { + if eg != nil { + switch t := eg.Group.(type) { + case *pbproxystate.EndpointGroup_Dynamic: + dynamic := eg.GetDynamic() + return pr.makeEnvoyDynamicCluster(name, protocol, dynamic) + case *pbproxystate.EndpointGroup_Static: + static := eg.GetStatic() + return pr.makeEnvoyStaticCluster(name, protocol, static) + case *pbproxystate.EndpointGroup_Dns: + dns := eg.GetDns() + return pr.makeEnvoyDnsCluster(name, protocol, dns) + case *pbproxystate.EndpointGroup_Passthrough: + passthrough := eg.GetPassthrough() + return pr.makeEnvoyPassthroughCluster(name, protocol, passthrough) + default: + return nil, fmt.Errorf("unsupported endpoint group type: %s", t) + } + } + return nil, fmt.Errorf("no endpoint group") +} + +func (pr *ProxyResources) makeEnvoyDynamicCluster(name string, protocol string, dynamic *pbproxystate.DynamicEndpointGroup) (*envoy_cluster_v3.Cluster, error) { + cluster := &envoy_cluster_v3.Cluster{ + Name: name, + ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_EDS}, + EdsClusterConfig: &envoy_cluster_v3.Cluster_EdsClusterConfig{ + EdsConfig: &envoy_core_v3.ConfigSource{ + ResourceApiVersion: envoy_core_v3.ApiVersion_V3, + ConfigSourceSpecifier: &envoy_core_v3.ConfigSource_Ads{ + Ads: &envoy_core_v3.AggregatedConfigSource{}, + }, + }, + }, + } + err := addHttpProtocolOptions(protocol, cluster) + if err != nil { + return nil, err + } + if dynamic.Config != nil { + if dynamic.Config.UseAltStatName { + cluster.AltStatName = name + } + cluster.ConnectTimeout = dynamic.Config.ConnectTimeout + if !dynamic.Config.DisablePanicThreshold { + cluster.CommonLbConfig = &envoy_cluster_v3.Cluster_CommonLbConfig{ + HealthyPanicThreshold: &envoy_type_v3.Percent{ + Value: 0, // disable panic threshold + }, + } + } + addEnvoyCircuitBreakers(dynamic.Config.CircuitBreakers, cluster) + addEnvoyOutlierDetection(dynamic.Config.OutlierDetection, cluster) + + err := addEnvoyLBToCluster(dynamic.Config, cluster) + if err != nil { + return nil, err + } + } + + if dynamic.OutboundTls != nil { + envoyTransportSocket, err := pr.makeEnvoyTransportSocket(dynamic.OutboundTls) + if err != nil { + return nil, err + } + cluster.TransportSocket = envoyTransportSocket + } + + return cluster, nil + +} + +func (pr *ProxyResources) makeEnvoyStaticCluster(name string, protocol string, static *pbproxystate.StaticEndpointGroup) (*envoy_cluster_v3.Cluster, error) { + endpointList, ok := pr.proxyState.Endpoints[name] + if !ok || endpointList == nil { + return nil, fmt.Errorf("static cluster %q is missing endpoints", name) + } + cluster := &envoy_cluster_v3.Cluster{ + Name: name, + ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_STATIC}, + LoadAssignment: makeEnvoyClusterLoadAssignment(name, endpointList.Endpoints), + } + err := addHttpProtocolOptions(protocol, cluster) + if err != nil { + return nil, err + } + + if static.Config != nil { + cluster.ConnectTimeout = static.Config.ConnectTimeout + addEnvoyCircuitBreakers(static.GetConfig().CircuitBreakers, cluster) + } + return cluster, nil +} +func (pr *ProxyResources) makeEnvoyDnsCluster(name string, protocol string, dns *pbproxystate.DNSEndpointGroup) (*envoy_cluster_v3.Cluster, error) { + return nil, nil +} +func (pr *ProxyResources) makeEnvoyPassthroughCluster(name string, protocol string, passthrough *pbproxystate.PassthroughEndpointGroup) (*envoy_cluster_v3.Cluster, error) { + cluster := &envoy_cluster_v3.Cluster{ + Name: name, + ConnectTimeout: passthrough.Config.ConnectTimeout, + LbPolicy: envoy_cluster_v3.Cluster_CLUSTER_PROVIDED, + ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_ORIGINAL_DST}, + } + if passthrough.OutboundTls != nil { + envoyTransportSocket, err := pr.makeEnvoyTransportSocket(passthrough.OutboundTls) + if err != nil { + return nil, err + } + cluster.TransportSocket = envoyTransportSocket + } + err := addHttpProtocolOptions(protocol, cluster) + if err != nil { + return nil, err + } + return cluster, nil +} + +func (pr *ProxyResources) makeEnvoyAggregateCluster(name string, protocol string, fg *pbproxystate.FailoverGroup) ([]*envoy_cluster_v3.Cluster, error) { + var clusters []*envoy_cluster_v3.Cluster + if fg != nil { + + var egNames []string + for _, eg := range fg.EndpointGroups { + cluster, err := pr.makeEnvoyCluster(name, protocol, eg) + if err != nil { + return nil, err + } + egNames = append(egNames, cluster.Name) + clusters = append(clusters, cluster) + } + aggregateClusterConfig, err := anypb.New(&envoy_aggregate_cluster_v3.ClusterConfig{ + Clusters: egNames, + }) + + if err != nil { + return nil, err + } + + c := &envoy_cluster_v3.Cluster{ + Name: name, + AltStatName: name, + ConnectTimeout: fg.Config.ConnectTimeout, + LbPolicy: envoy_cluster_v3.Cluster_CLUSTER_PROVIDED, + ClusterDiscoveryType: &envoy_cluster_v3.Cluster_ClusterType{ + ClusterType: &envoy_cluster_v3.Cluster_CustomClusterType{ + Name: "envoy.clusters.aggregate", + TypedConfig: aggregateClusterConfig, + }, + }, + } + err = addHttpProtocolOptions(protocol, c) + if err != nil { + return nil, err + } + clusters = append(clusters, c) + } + return clusters, nil +} + +func addHttpProtocolOptions(protocol string, c *envoy_cluster_v3.Cluster) error { + if !(protocol == "http2" || protocol == "grpc") { + // do not error. returning nil means it won't get set. + return nil + } + cfg := &envoy_upstreams_v3.HttpProtocolOptions{ + UpstreamProtocolOptions: &envoy_upstreams_v3.HttpProtocolOptions_ExplicitHttpConfig_{ + ExplicitHttpConfig: &envoy_upstreams_v3.HttpProtocolOptions_ExplicitHttpConfig{ + ProtocolConfig: &envoy_upstreams_v3.HttpProtocolOptions_ExplicitHttpConfig_Http2ProtocolOptions{ + Http2ProtocolOptions: &envoy_core_v3.Http2ProtocolOptions{}, + }, + }, + }, + } + any, err := anypb.New(cfg) + if err != nil { + return err + } + c.TypedExtensionProtocolOptions = map[string]*anypb.Any{ + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": any, + } + return nil +} + +// addEnvoyOutlierDetection will add outlier detection config to the cluster, and if nil, add empty OutlierDetection to +// enable it with default values +func addEnvoyOutlierDetection(outlierDetection *pbproxystate.OutlierDetection, c *envoy_cluster_v3.Cluster) { + if outlierDetection == nil { + return + } + od := &envoy_cluster_v3.OutlierDetection{ + BaseEjectionTime: outlierDetection.GetBaseEjectionTime(), + Consecutive_5Xx: outlierDetection.GetConsecutive_5Xx(), + EnforcingConsecutive_5Xx: outlierDetection.GetEnforcingConsecutive_5Xx(), + Interval: outlierDetection.GetInterval(), + MaxEjectionPercent: outlierDetection.GetMaxEjectionPercent(), + } + c.OutlierDetection = od + +} + +func addEnvoyCircuitBreakers(circuitBreakers *pbproxystate.CircuitBreakers, c *envoy_cluster_v3.Cluster) { + if circuitBreakers != nil { + if circuitBreakers.UpstreamLimits == nil { + c.CircuitBreakers = &envoy_cluster_v3.CircuitBreakers{} + return + } + threshold := &envoy_cluster_v3.CircuitBreakers_Thresholds{} + threshold.MaxConnections = circuitBreakers.UpstreamLimits.MaxConnections + threshold.MaxPendingRequests = circuitBreakers.UpstreamLimits.MaxPendingRequests + threshold.MaxRequests = circuitBreakers.UpstreamLimits.MaxConcurrentRequests + + c.CircuitBreakers = &envoy_cluster_v3.CircuitBreakers{ + Thresholds: []*envoy_cluster_v3.CircuitBreakers_Thresholds{threshold}, + } + } +} + +func addEnvoyLBToCluster(dynamicConfig *pbproxystate.DynamicEndpointGroupConfig, c *envoy_cluster_v3.Cluster) error { + if dynamicConfig == nil || dynamicConfig.LbPolicy == nil { + return nil + } + + switch d := dynamicConfig.LbPolicy.(type) { + case *pbproxystate.DynamicEndpointGroupConfig_LeastRequest: + c.LbPolicy = envoy_cluster_v3.Cluster_LEAST_REQUEST + + lb := dynamicConfig.LbPolicy.(*pbproxystate.DynamicEndpointGroupConfig_LeastRequest) + if lb.LeastRequest != nil { + c.LbConfig = &envoy_cluster_v3.Cluster_LeastRequestLbConfig_{ + LeastRequestLbConfig: &envoy_cluster_v3.Cluster_LeastRequestLbConfig{ + ChoiceCount: lb.LeastRequest.ChoiceCount, + }, + } + } + case *pbproxystate.DynamicEndpointGroupConfig_RoundRobin: + c.LbPolicy = envoy_cluster_v3.Cluster_ROUND_ROBIN + + case *pbproxystate.DynamicEndpointGroupConfig_Random: + c.LbPolicy = envoy_cluster_v3.Cluster_RANDOM + + case *pbproxystate.DynamicEndpointGroupConfig_RingHash: + c.LbPolicy = envoy_cluster_v3.Cluster_RING_HASH + + lb := dynamicConfig.LbPolicy.(*pbproxystate.DynamicEndpointGroupConfig_RingHash) + if lb.RingHash != nil { + c.LbConfig = &envoy_cluster_v3.Cluster_RingHashLbConfig_{ + RingHashLbConfig: &envoy_cluster_v3.Cluster_RingHashLbConfig{ + MinimumRingSize: lb.RingHash.MinimumRingSize, + MaximumRingSize: lb.RingHash.MaximumRingSize, + }, + } + } + case *pbproxystate.DynamicEndpointGroupConfig_Maglev: + c.LbPolicy = envoy_cluster_v3.Cluster_MAGLEV + + default: + return fmt.Errorf("unsupported load balancer policy %q for cluster %q", d, c.Name) + } + return nil +} + // TODO(proxystate): In a future PR this will create clusters and add it to ProxyResources.proxyState -func (pr *ProxyResources) makeCluster(name string) error { +func (pr *ProxyResources) makeEnvoyClusterFromL4Destination(name string) error { return nil } diff --git a/agent/xdsv2/endpoint_resources.go b/agent/xdsv2/endpoint_resources.go new file mode 100644 index 000000000000..f5fd1b9c5d09 --- /dev/null +++ b/agent/xdsv2/endpoint_resources.go @@ -0,0 +1,41 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package xdsv2 + +import ( + envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + "github.com/hashicorp/consul/agent/xds/response" + "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" +) + +func makeEnvoyEndpoint(endpoint *pbproxystate.Endpoint) *envoy_endpoint_v3.LbEndpoint { + var address *envoy_core_v3.Address + if endpoint.GetUnixSocket() != nil { + address = response.MakePipeAddress(endpoint.GetUnixSocket().GetPath(), 0) + } else { + address = response.MakeAddress(endpoint.GetHostPort().GetHost(), int(endpoint.GetHostPort().Port)) + } + return &envoy_endpoint_v3.LbEndpoint{ + HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ + Endpoint: &envoy_endpoint_v3.Endpoint{ + Address: address, + }, + }, + } +} + +func makeEnvoyClusterLoadAssignment(clusterName string, endpoints []*pbproxystate.Endpoint) *envoy_endpoint_v3.ClusterLoadAssignment { + localityLbEndpoints := make([]*envoy_endpoint_v3.LocalityLbEndpoints, 0, len(endpoints)) + for _, endpoint := range endpoints { + localityLbEndpoints = append(localityLbEndpoints, &envoy_endpoint_v3.LocalityLbEndpoints{ + LbEndpoints: []*envoy_endpoint_v3.LbEndpoint{makeEnvoyEndpoint(endpoint)}, + }) + } + return &envoy_endpoint_v3.ClusterLoadAssignment{ + ClusterName: clusterName, + Endpoints: localityLbEndpoints, + } + +} diff --git a/agent/xdsv2/listener_resources.go b/agent/xdsv2/listener_resources.go index 6d6ce7c8002e..2d3bf5d60da5 100644 --- a/agent/xdsv2/listener_resources.go +++ b/agent/xdsv2/listener_resources.go @@ -1,3 +1,6 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + package xdsv2 import ( @@ -281,7 +284,7 @@ func (pr *ProxyResources) makeEnvoyResourcesForSNIDestination(sni *pbproxystate. } func (pr *ProxyResources) makeEnvoyResourcesForL4Destination(l4 *pbproxystate.Router_L4) ([]*envoy_listener_v3.Filter, error) { - err := pr.makeCluster(l4.L4.Name) + err := pr.makeEnvoyClusterFromL4Destination(l4.L4.Name) if err != nil { return nil, err } @@ -632,7 +635,7 @@ func (pr *ProxyResources) makeEnvoyTransportSocket(ts *pbproxystate.TransportSoc // if tls is nil but connection tls is provided, then the proxy state is misconfigured return nil, fmt.Errorf("proxyState.Tls is required to generate router's transport socket") } - om := ts.ConnectionTls.(*pbproxystate.TransportSocket_OutboundMesh).OutboundMesh + om := ts.GetOutboundMesh() leaf, ok := pr.proxyState.LeafCertificates[om.IdentityKey] if !ok { return nil, fmt.Errorf("leaf %s not found in proxyState", om.IdentityKey) diff --git a/agent/xdsv2/resources.go b/agent/xdsv2/resources.go index 6bc1df58c664..6082f7dd56b5 100644 --- a/agent/xdsv2/resources.go +++ b/agent/xdsv2/resources.go @@ -1,5 +1,5 @@ // Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 +// SPDX-License-Identifier: BUSL-1.1 package xdsv2 @@ -47,7 +47,6 @@ func (g *ResourceGenerator) AllResourcesFromIR(proxyState *pbmesh.ProxyState) (m func (pr *ProxyResources) generateXDSResources() error { listeners := make([]proto.Message, 0) - clusters := make([]proto.Message, 0) routes := make([]proto.Message, 0) endpoints := make([]proto.Message, 0) @@ -60,6 +59,11 @@ func (pr *ProxyResources) generateXDSResources() error { listeners = append(listeners, protoListener) } + clusters, err := pr.makeXDSClusters() + if err != nil { + return err + } + pr.envoyResources[xdscommon.ListenerType] = listeners pr.envoyResources[xdscommon.ClusterType] = clusters pr.envoyResources[xdscommon.RouteType] = routes diff --git a/agent/xdsv2/route_resources.go b/agent/xdsv2/route_resources.go index 1433534e8f0a..86d82a5d306f 100644 --- a/agent/xdsv2/route_resources.go +++ b/agent/xdsv2/route_resources.go @@ -1,3 +1,6 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + package xdsv2 import ( diff --git a/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.pb.go b/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.pb.go index 9cbfe4ad6943..671e25f4ca04 100644 --- a/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.pb.go +++ b/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.pb.go @@ -86,6 +86,10 @@ type Cluster struct { Group isCluster_Group `protobuf_oneof:"group"` // escape_hatch_cluster_json configures a user configured escape hatch cluster. EscapeHatchClusterJson string `protobuf:"bytes,3,opt,name=escape_hatch_cluster_json,json=escapeHatchClusterJson,proto3" json:"escape_hatch_cluster_json,omitempty"` + // alt_stat_name is the name used for observability in place of cluster name if provided. + AltStatName string `protobuf:"bytes,4,opt,name=alt_stat_name,json=altStatName,proto3" json:"alt_stat_name,omitempty"` + // protocol is the local path protocol or the service protocol. + Protocol string `protobuf:"bytes,5,opt,name=protocol,proto3" json:"protocol,omitempty"` } func (x *Cluster) Reset() { @@ -148,6 +152,20 @@ func (x *Cluster) GetEscapeHatchClusterJson() string { return "" } +func (x *Cluster) GetAltStatName() string { + if x != nil { + return x.AltStatName + } + return "" +} + +func (x *Cluster) GetProtocol() string { + if x != nil { + return x.Protocol + } + return "" +} + type isCluster_Group interface { isCluster_Group() } @@ -1679,7 +1697,7 @@ var file_pbmesh_v1alpha1_pbproxystate_cluster_proto_rawDesc = []byte{ 0x70, 0x62, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x22, 0x97, 0x02, 0x0a, 0x07, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, + 0x6f, 0x74, 0x6f, 0x22, 0xd7, 0x02, 0x0a, 0x07, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x63, 0x0a, 0x0e, 0x66, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, @@ -1696,7 +1714,11 @@ var file_pbmesh_v1alpha1_pbproxystate_cluster_proto_rawDesc = []byte{ 0x61, 0x70, 0x65, 0x5f, 0x68, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x48, 0x61, 0x74, 0x63, 0x68, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x4a, 0x73, 0x6f, 0x6e, 0x42, 0x07, 0x0a, 0x05, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x22, 0xce, 0x01, + 0x4a, 0x73, 0x6f, 0x6e, 0x12, 0x22, 0x0a, 0x0d, 0x61, 0x6c, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x61, 0x6c, 0x74, + 0x53, 0x74, 0x61, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x42, 0x07, 0x0a, 0x05, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x22, 0xce, 0x01, 0x0a, 0x0d, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x63, 0x0a, 0x0f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, diff --git a/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.proto b/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.proto index 9b4194d981f0..860ebf2a5cbf 100644 --- a/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.proto +++ b/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.proto @@ -18,6 +18,10 @@ message Cluster { } // escape_hatch_cluster_json configures a user configured escape hatch cluster. string escape_hatch_cluster_json = 3; + // alt_stat_name is the name used for observability in place of cluster name if provided. + string alt_stat_name = 4; + // protocol is the local path protocol or the service protocol. + string protocol = 5; } message FailoverGroup { diff --git a/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.pb.go b/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.pb.go index 9801e29c3c70..59317396be4a 100644 --- a/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.pb.go +++ b/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.pb.go @@ -231,6 +231,8 @@ type TransportSocket struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + // name of the transport socket + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // Types that are assignable to ConnectionTls: // // *TransportSocket_InboundMesh @@ -239,8 +241,8 @@ type TransportSocket struct { // *TransportSocket_OutboundNonMesh ConnectionTls isTransportSocket_ConnectionTls `protobuf_oneof:"connection_tls"` // tls_parameters can override any top level tls parameters that are configured. - TlsParameters *TLSParameters `protobuf:"bytes,5,opt,name=tls_parameters,json=tlsParameters,proto3" json:"tls_parameters,omitempty"` - AlpnProtocols []string `protobuf:"bytes,6,rep,name=alpn_protocols,json=alpnProtocols,proto3" json:"alpn_protocols,omitempty"` + TlsParameters *TLSParameters `protobuf:"bytes,6,opt,name=tls_parameters,json=tlsParameters,proto3" json:"tls_parameters,omitempty"` + AlpnProtocols []string `protobuf:"bytes,7,rep,name=alpn_protocols,json=alpnProtocols,proto3" json:"alpn_protocols,omitempty"` } func (x *TransportSocket) Reset() { @@ -275,6 +277,13 @@ func (*TransportSocket) Descriptor() ([]byte, []int) { return file_pbmesh_v1alpha1_pbproxystate_transport_socket_proto_rawDescGZIP(), []int{1} } +func (x *TransportSocket) GetName() string { + if x != nil { + return x.Name + } + return "" +} + func (m *TransportSocket) GetConnectionTls() isTransportSocket_ConnectionTls { if m != nil { return m.ConnectionTls @@ -330,22 +339,22 @@ type isTransportSocket_ConnectionTls interface { type TransportSocket_InboundMesh struct { // inbound_mesh is for incoming connections FROM the mesh. - InboundMesh *InboundMeshMTLS `protobuf:"bytes,1,opt,name=inbound_mesh,json=inboundMesh,proto3,oneof"` + InboundMesh *InboundMeshMTLS `protobuf:"bytes,2,opt,name=inbound_mesh,json=inboundMesh,proto3,oneof"` } type TransportSocket_OutboundMesh struct { // outbound_mesh is for outbound connections TO mesh destinations. - OutboundMesh *OutboundMeshMTLS `protobuf:"bytes,2,opt,name=outbound_mesh,json=outboundMesh,proto3,oneof"` + OutboundMesh *OutboundMeshMTLS `protobuf:"bytes,3,opt,name=outbound_mesh,json=outboundMesh,proto3,oneof"` } type TransportSocket_InboundNonMesh struct { // inbound_non_mesh is for incoming connections FROM non mesh. - InboundNonMesh *InboundNonMeshTLS `protobuf:"bytes,3,opt,name=inbound_non_mesh,json=inboundNonMesh,proto3,oneof"` + InboundNonMesh *InboundNonMeshTLS `protobuf:"bytes,4,opt,name=inbound_non_mesh,json=inboundNonMesh,proto3,oneof"` } type TransportSocket_OutboundNonMesh struct { // outbound_non_mesh is for outbound connections TO non mesh destinations. - OutboundNonMesh *OutboundNonMeshTLS `protobuf:"bytes,4,opt,name=outbound_non_mesh,json=outboundNonMesh,proto3,oneof"` + OutboundNonMesh *OutboundNonMeshTLS `protobuf:"bytes,5,opt,name=outbound_non_mesh,json=outboundNonMesh,proto3,oneof"` } func (*TransportSocket_InboundMesh) isTransportSocket_ConnectionTls() {} @@ -1038,213 +1047,214 @@ var file_pbmesh_v1alpha1_pbproxystate_transport_socket_proto_rawDesc = []byte{ 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x4c, 0x53, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x52, 0x15, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x54, 0x6c, 0x73, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, - 0x22, 0xd1, 0x04, 0x0a, 0x0f, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x6f, - 0x63, 0x6b, 0x65, 0x74, 0x12, 0x61, 0x0a, 0x0c, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, - 0x6d, 0x65, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x68, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, - 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, - 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, - 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, 0x53, 0x48, 0x00, 0x52, 0x0b, 0x69, 0x6e, 0x62, 0x6f, - 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x64, 0x0a, 0x0d, 0x6f, 0x75, 0x74, 0x62, 0x6f, - 0x75, 0x6e, 0x64, 0x5f, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, + 0x22, 0xe5, 0x04, 0x0a, 0x0f, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x6f, + 0x63, 0x6b, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x61, 0x0a, 0x0c, 0x69, 0x6e, 0x62, 0x6f, + 0x75, 0x6e, 0x64, 0x5f, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, - 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4f, 0x75, 0x74, - 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, 0x53, 0x48, 0x00, 0x52, - 0x0c, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x6a, 0x0a, - 0x10, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x6e, 0x6f, 0x6e, 0x5f, 0x6d, 0x65, 0x73, - 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, - 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, - 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, - 0x4d, 0x65, 0x73, 0x68, 0x54, 0x4c, 0x53, 0x48, 0x00, 0x52, 0x0e, 0x69, 0x6e, 0x62, 0x6f, 0x75, - 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x6d, 0x0a, 0x11, 0x6f, 0x75, 0x74, - 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x6e, 0x6f, 0x6e, 0x5f, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x49, 0x6e, 0x62, + 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, 0x53, 0x48, 0x00, 0x52, 0x0b, + 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x64, 0x0a, 0x0d, 0x6f, + 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, + 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x2e, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, + 0x53, 0x48, 0x00, 0x52, 0x0c, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, + 0x68, 0x12, 0x6a, 0x0a, 0x10, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x6e, 0x6f, 0x6e, + 0x5f, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, 0x61, + 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, + 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, + 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, + 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x54, 0x4c, 0x53, 0x48, 0x00, 0x52, 0x0e, 0x69, + 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x6d, 0x0a, + 0x11, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x6e, 0x6f, 0x6e, 0x5f, 0x6d, 0x65, + 0x73, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, + 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, + 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, + 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, + 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x54, 0x4c, 0x53, 0x48, 0x00, 0x52, 0x0f, 0x6f, 0x75, 0x74, + 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x61, 0x0a, 0x0e, + 0x74, 0x6c, 0x73, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, - 0x74, 0x65, 0x2e, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, - 0x73, 0x68, 0x54, 0x4c, 0x53, 0x48, 0x00, 0x52, 0x0f, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, - 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x61, 0x0a, 0x0e, 0x74, 0x6c, 0x73, 0x5f, - 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, - 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, - 0x4c, 0x53, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x52, 0x0d, 0x74, 0x6c, - 0x73, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x61, - 0x6c, 0x70, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x73, 0x18, 0x06, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x0d, 0x61, 0x6c, 0x70, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, - 0x6c, 0x73, 0x42, 0x10, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x74, 0x6c, 0x73, 0x22, 0xae, 0x01, 0x0a, 0x0f, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, - 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, 0x53, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x64, 0x65, 0x6e, - 0x74, 0x69, 0x74, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x4b, 0x65, 0x79, 0x12, 0x78, 0x0a, 0x12, 0x76, - 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, - 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x49, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, - 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, - 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, - 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, - 0x78, 0x74, 0x52, 0x11, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x22, 0xc2, 0x01, 0x0a, 0x10, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, - 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, 0x53, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x64, - 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0b, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x4b, 0x65, 0x79, 0x12, 0x79, 0x0a, + 0x74, 0x65, 0x2e, 0x54, 0x4c, 0x53, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, + 0x52, 0x0d, 0x74, 0x6c, 0x73, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, + 0x25, 0x0a, 0x0e, 0x61, 0x6c, 0x70, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, + 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x61, 0x6c, 0x70, 0x6e, 0x50, 0x72, 0x6f, + 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x73, 0x42, 0x10, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6c, 0x73, 0x22, 0xae, 0x01, 0x0a, 0x0f, 0x49, 0x6e, 0x62, + 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, 0x53, 0x12, 0x21, 0x0a, 0x0c, + 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x4b, 0x65, 0x79, 0x12, + 0x78, 0x0a, 0x12, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x49, 0x2e, 0x68, 0x61, + 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, + 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, + 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x49, 0x6e, + 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x11, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x22, 0xc2, 0x01, 0x0a, 0x10, 0x4f, 0x75, + 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, 0x53, 0x12, 0x21, + 0x0a, 0x0c, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x4b, 0x65, + 0x79, 0x12, 0x79, 0x0a, 0x12, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4a, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, + 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x73, 0x68, + 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x11, 0x76, 0x61, 0x6c, 0x69, 0x64, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x10, 0x0a, 0x03, + 0x73, 0x6e, 0x69, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x6e, 0x69, 0x22, 0x8d, + 0x01, 0x0a, 0x11, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, + 0x68, 0x54, 0x4c, 0x53, 0x12, 0x1b, 0x0a, 0x08, 0x6c, 0x65, 0x61, 0x66, 0x5f, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x07, 0x6c, 0x65, 0x61, 0x66, 0x4b, 0x65, + 0x79, 0x12, 0x4f, 0x0a, 0x03, 0x73, 0x64, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3b, + 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, + 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, + 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x53, 0x44, 0x53, + 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x48, 0x00, 0x52, 0x03, 0x73, + 0x64, 0x73, 0x42, 0x0a, 0x0a, 0x08, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x22, 0xca, + 0x01, 0x0a, 0x12, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, + 0x73, 0x68, 0x54, 0x4c, 0x53, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x66, 0x69, + 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x65, 0x72, 0x74, 0x46, 0x69, + 0x6c, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6b, 0x65, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x7c, 0x0a, 0x12, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x74, - 0x65, 0x78, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4a, 0x2e, 0x68, 0x61, 0x73, 0x68, + 0x65, 0x78, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, - 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x4f, 0x75, 0x74, 0x62, - 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x11, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x6e, 0x69, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x6e, 0x69, 0x22, 0x8d, 0x01, 0x0a, 0x11, 0x49, - 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x54, 0x4c, 0x53, - 0x12, 0x1b, 0x0a, 0x08, 0x6c, 0x65, 0x61, 0x66, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x48, 0x00, 0x52, 0x07, 0x6c, 0x65, 0x61, 0x66, 0x4b, 0x65, 0x79, 0x12, 0x4f, 0x0a, - 0x03, 0x73, 0x64, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, - 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, - 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x53, 0x44, 0x53, 0x43, 0x65, 0x72, 0x74, - 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x48, 0x00, 0x52, 0x03, 0x73, 0x64, 0x73, 0x42, 0x0a, - 0x0a, 0x08, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x22, 0xca, 0x01, 0x0a, 0x12, 0x4f, - 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x54, 0x4c, - 0x53, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x65, 0x72, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x19, - 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x6b, 0x65, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x7c, 0x0a, 0x12, 0x76, 0x61, 0x6c, - 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, - 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, - 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, - 0x61, 0x74, 0x65, 0x2e, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x4f, 0x75, 0x74, 0x62, 0x6f, - 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x52, 0x11, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x22, 0x5c, 0x0a, 0x1c, 0x4d, 0x65, 0x73, 0x68, 0x49, - 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x3c, 0x0a, 0x1b, 0x74, 0x72, 0x75, 0x73, 0x74, - 0x5f, 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, - 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x17, 0x74, 0x72, - 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x50, 0x65, 0x65, 0x72, 0x4e, 0x61, 0x6d, - 0x65, 0x4b, 0x65, 0x79, 0x73, 0x22, 0x7a, 0x0a, 0x1d, 0x4d, 0x65, 0x73, 0x68, 0x4f, 0x75, 0x74, - 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, - 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x3a, 0x0a, 0x1a, 0x74, 0x72, 0x75, 0x73, 0x74, 0x5f, - 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, - 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x74, 0x72, 0x75, 0x73, - 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x50, 0x65, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x4b, - 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x70, 0x69, 0x66, 0x66, 0x65, 0x5f, 0x69, 0x64, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x73, 0x70, 0x69, 0x66, 0x66, 0x65, 0x49, 0x64, - 0x73, 0x22, 0x3b, 0x0a, 0x20, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x4f, 0x75, 0x74, 0x62, - 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x63, 0x61, 0x5f, 0x66, 0x69, 0x6c, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x61, 0x46, 0x69, 0x6c, 0x65, 0x22, 0x58, - 0x0a, 0x0e, 0x53, 0x44, 0x53, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, - 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, - 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x72, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x63, 0x65, 0x72, 0x74, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0xa5, 0x02, 0x0a, 0x0d, 0x54, 0x4c, 0x53, - 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x58, 0x0a, 0x0b, 0x6d, 0x69, - 0x6e, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x37, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, - 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, - 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x4c, - 0x53, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x6d, 0x69, 0x6e, 0x56, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x58, 0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x37, 0x2e, 0x68, 0x61, 0x73, 0x68, + 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x4f, + 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x11, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x22, 0x5c, 0x0a, 0x1c, 0x4d, + 0x65, 0x73, 0x68, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x3c, 0x0a, 0x1b, 0x74, + 0x72, 0x75, 0x73, 0x74, 0x5f, 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x70, 0x65, 0x65, 0x72, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x17, 0x74, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x50, 0x65, 0x65, + 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x4b, 0x65, 0x79, 0x73, 0x22, 0x7a, 0x0a, 0x1d, 0x4d, 0x65, 0x73, + 0x68, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x3a, 0x0a, 0x1a, 0x74, 0x72, + 0x75, 0x73, 0x74, 0x5f, 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x70, 0x65, 0x65, 0x72, 0x5f, + 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, + 0x74, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x50, 0x65, 0x65, 0x72, 0x4e, + 0x61, 0x6d, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x70, 0x69, 0x66, 0x66, 0x65, + 0x5f, 0x69, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x73, 0x70, 0x69, 0x66, + 0x66, 0x65, 0x49, 0x64, 0x73, 0x22, 0x3b, 0x0a, 0x20, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, + 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x63, 0x61, 0x5f, + 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x61, 0x46, 0x69, + 0x6c, 0x65, 0x22, 0x58, 0x0a, 0x0e, 0x53, 0x44, 0x53, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, + 0x63, 0x61, 0x74, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73, + 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x65, 0x72, 0x74, 0x5f, + 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, + 0x63, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0xa5, 0x02, 0x0a, + 0x0d, 0x54, 0x4c, 0x53, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x58, + 0x0a, 0x0b, 0x6d, 0x69, 0x6e, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x37, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, + 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, + 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, + 0x65, 0x2e, 0x54, 0x4c, 0x53, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x6d, 0x69, + 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x58, 0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, + 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x37, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, + 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x4c, 0x53, 0x56, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x60, 0x0a, 0x0d, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x5f, 0x73, 0x75, 0x69, + 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, - 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x4c, 0x53, 0x56, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x60, - 0x0a, 0x0d, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x5f, 0x73, 0x75, 0x69, 0x74, 0x65, 0x73, 0x18, - 0x03, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, - 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, - 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, - 0x61, 0x74, 0x65, 0x2e, 0x54, 0x4c, 0x53, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, - 0x74, 0x65, 0x52, 0x0c, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x73, - 0x22, 0x37, 0x0a, 0x0f, 0x4c, 0x65, 0x61, 0x66, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, - 0x61, 0x74, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x65, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x63, 0x65, 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x46, 0x0a, 0x0b, 0x54, 0x72, 0x75, - 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x75, 0x73, - 0x74, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x74, 0x72, 0x75, 0x73, 0x74, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x72, - 0x6f, 0x6f, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x72, 0x6f, 0x6f, 0x74, - 0x73, 0x2a, 0xac, 0x01, 0x0a, 0x0a, 0x54, 0x4c, 0x53, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x14, 0x0a, 0x10, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, - 0x41, 0x55, 0x54, 0x4f, 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, - 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x31, 0x5f, 0x30, 0x10, 0x01, 0x12, 0x13, 0x0a, 0x0f, 0x54, - 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x31, 0x5f, 0x31, 0x10, 0x02, - 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, - 0x31, 0x5f, 0x32, 0x10, 0x03, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, - 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x31, 0x5f, 0x33, 0x10, 0x04, 0x12, 0x17, 0x0a, 0x13, 0x54, 0x4c, - 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, - 0x44, 0x10, 0x05, 0x12, 0x1b, 0x0a, 0x17, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, - 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x06, - 0x2a, 0x84, 0x05, 0x0a, 0x0e, 0x54, 0x4c, 0x53, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, - 0x69, 0x74, 0x65, 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, - 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x45, 0x43, - 0x44, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, - 0x48, 0x41, 0x32, 0x35, 0x36, 0x10, 0x00, 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x4c, 0x53, 0x5f, 0x43, + 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x4c, 0x53, 0x43, 0x69, 0x70, 0x68, 0x65, + 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x52, 0x0c, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, + 0x69, 0x74, 0x65, 0x73, 0x22, 0x37, 0x0a, 0x0f, 0x4c, 0x65, 0x61, 0x66, 0x43, 0x65, 0x72, 0x74, + 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x65, 0x72, 0x74, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x65, 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x46, 0x0a, + 0x0b, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x21, 0x0a, 0x0c, + 0x74, 0x72, 0x75, 0x73, 0x74, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x75, 0x73, 0x74, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, + 0x14, 0x0a, 0x05, 0x72, 0x6f, 0x6f, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, + 0x72, 0x6f, 0x6f, 0x74, 0x73, 0x2a, 0xac, 0x01, 0x0a, 0x0a, 0x54, 0x4c, 0x53, 0x56, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x10, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, + 0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x55, 0x54, 0x4f, 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x4c, + 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x31, 0x5f, 0x30, 0x10, 0x01, 0x12, + 0x13, 0x0a, 0x0f, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x31, + 0x5f, 0x31, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, + 0x49, 0x4f, 0x4e, 0x5f, 0x31, 0x5f, 0x32, 0x10, 0x03, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x4c, 0x53, + 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x31, 0x5f, 0x33, 0x10, 0x04, 0x12, 0x17, + 0x0a, 0x13, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x49, 0x4e, + 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x05, 0x12, 0x1b, 0x0a, 0x17, 0x54, 0x4c, 0x53, 0x5f, 0x56, + 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, + 0x45, 0x44, 0x10, 0x06, 0x2a, 0x84, 0x05, 0x0a, 0x0e, 0x54, 0x4c, 0x53, 0x43, 0x69, 0x70, 0x68, + 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, - 0x45, 0x5f, 0x45, 0x43, 0x44, 0x53, 0x41, 0x5f, 0x43, 0x48, 0x41, 0x43, 0x48, 0x41, 0x32, 0x30, - 0x5f, 0x50, 0x4f, 0x4c, 0x59, 0x31, 0x33, 0x30, 0x35, 0x10, 0x01, 0x12, 0x30, 0x0a, 0x2c, 0x54, + 0x45, 0x5f, 0x45, 0x43, 0x44, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x47, + 0x43, 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36, 0x10, 0x00, 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, - 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, - 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36, 0x10, 0x02, 0x12, 0x30, 0x0a, - 0x2c, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, - 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, 0x43, 0x48, 0x41, 0x43, - 0x48, 0x41, 0x32, 0x30, 0x5f, 0x50, 0x4f, 0x4c, 0x59, 0x31, 0x33, 0x30, 0x35, 0x10, 0x03, 0x12, - 0x2b, 0x0a, 0x27, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, - 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x53, 0x41, 0x5f, - 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x04, 0x12, 0x29, 0x0a, 0x25, - 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, - 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x31, 0x32, - 0x38, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x05, 0x12, 0x26, 0x0a, 0x22, 0x54, 0x4c, 0x53, 0x5f, 0x43, - 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x41, 0x45, 0x53, 0x31, - 0x32, 0x38, 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36, 0x10, 0x06, 0x12, - 0x1f, 0x0a, 0x1b, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, - 0x49, 0x54, 0x45, 0x5f, 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x07, - 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, - 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x53, 0x41, - 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x33, - 0x38, 0x34, 0x10, 0x08, 0x12, 0x30, 0x0a, 0x2c, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, - 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, - 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, 0x48, - 0x41, 0x33, 0x38, 0x34, 0x10, 0x09, 0x12, 0x2b, 0x0a, 0x27, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, - 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, - 0x5f, 0x45, 0x43, 0x44, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x53, 0x48, - 0x41, 0x10, 0x0a, 0x12, 0x29, 0x0a, 0x25, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, - 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, 0x53, - 0x41, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x0b, 0x12, 0x26, - 0x0a, 0x22, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, - 0x54, 0x45, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, 0x48, - 0x41, 0x33, 0x38, 0x34, 0x10, 0x0c, 0x12, 0x1f, 0x0a, 0x1b, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, - 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, - 0x36, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x0d, 0x42, 0xe0, 0x02, 0x0a, 0x2f, 0x63, 0x6f, 0x6d, 0x2e, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, - 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x42, 0x14, 0x54, 0x72, 0x61, - 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x74, - 0x6f, 0x50, 0x01, 0x5a, 0x45, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2d, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x2f, 0x70, 0x62, - 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x70, 0x62, - 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0xa2, 0x02, 0x05, 0x48, 0x43, 0x4d, - 0x56, 0x50, 0xaa, 0x02, 0x2b, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x43, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x2e, 0x56, 0x31, 0x61, 0x6c, 0x70, - 0x68, 0x61, 0x31, 0x2e, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, - 0xca, 0x02, 0x2b, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x5c, 0x4d, 0x65, 0x73, 0x68, 0x5c, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, - 0x31, 0x5c, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0xe2, 0x02, - 0x37, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, 0x73, 0x75, - 0x6c, 0x5c, 0x4d, 0x65, 0x73, 0x68, 0x5c, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x5c, - 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5c, 0x47, 0x50, 0x42, - 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x2f, 0x48, 0x61, 0x73, 0x68, 0x69, - 0x63, 0x6f, 0x72, 0x70, 0x3a, 0x3a, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x3a, 0x3a, 0x4d, 0x65, - 0x73, 0x68, 0x3a, 0x3a, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x3a, 0x3a, 0x50, 0x62, - 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x53, 0x41, 0x5f, 0x43, 0x48, 0x41, 0x43, + 0x48, 0x41, 0x32, 0x30, 0x5f, 0x50, 0x4f, 0x4c, 0x59, 0x31, 0x33, 0x30, 0x35, 0x10, 0x01, 0x12, + 0x30, 0x0a, 0x2c, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, + 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, 0x41, 0x45, + 0x53, 0x31, 0x32, 0x38, 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36, 0x10, + 0x02, 0x12, 0x30, 0x0a, 0x2c, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, + 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, + 0x43, 0x48, 0x41, 0x43, 0x48, 0x41, 0x32, 0x30, 0x5f, 0x50, 0x4f, 0x4c, 0x59, 0x31, 0x33, 0x30, + 0x35, 0x10, 0x03, 0x12, 0x2b, 0x0a, 0x27, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, + 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x45, 0x43, + 0x44, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x04, + 0x12, 0x29, 0x0a, 0x25, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, + 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, 0x41, + 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x05, 0x12, 0x26, 0x0a, 0x22, 0x54, + 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, + 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x32, 0x35, + 0x36, 0x10, 0x06, 0x12, 0x1f, 0x0a, 0x1b, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, + 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x53, + 0x48, 0x41, 0x10, 0x07, 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, + 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x45, + 0x43, 0x44, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x47, 0x43, 0x4d, 0x5f, + 0x53, 0x48, 0x41, 0x33, 0x38, 0x34, 0x10, 0x08, 0x12, 0x30, 0x0a, 0x2c, 0x54, 0x4c, 0x53, 0x5f, + 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, + 0x48, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x47, 0x43, + 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x33, 0x38, 0x34, 0x10, 0x09, 0x12, 0x2b, 0x0a, 0x27, 0x54, 0x4c, + 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, + 0x43, 0x44, 0x48, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, + 0x36, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x0a, 0x12, 0x29, 0x0a, 0x25, 0x54, 0x4c, 0x53, 0x5f, 0x43, + 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, + 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x53, 0x48, 0x41, + 0x10, 0x0b, 0x12, 0x26, 0x0a, 0x22, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, + 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x47, 0x43, + 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x33, 0x38, 0x34, 0x10, 0x0c, 0x12, 0x1f, 0x0a, 0x1b, 0x54, 0x4c, + 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x41, + 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x0d, 0x42, 0xe0, 0x02, 0x0a, 0x2f, + 0x63, 0x6f, 0x6d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, + 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x42, + 0x14, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x45, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2d, 0x70, 0x75, 0x62, 0x6c, 0x69, + 0x63, 0x2f, 0x70, 0x62, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, + 0x31, 0x2f, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0xa2, 0x02, + 0x05, 0x48, 0x43, 0x4d, 0x56, 0x50, 0xaa, 0x02, 0x2b, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x2e, 0x56, + 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, + 0x74, 0x61, 0x74, 0x65, 0xca, 0x02, 0x2b, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x5c, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x4d, 0x65, 0x73, 0x68, 0x5c, 0x56, 0x31, 0x61, + 0x6c, 0x70, 0x68, 0x61, 0x31, 0x5c, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, + 0x74, 0x65, 0xe2, 0x02, 0x37, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x4d, 0x65, 0x73, 0x68, 0x5c, 0x56, 0x31, 0x61, 0x6c, 0x70, + 0x68, 0x61, 0x31, 0x5c, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x2f, 0x48, + 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x3a, 0x3a, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x3a, 0x3a, 0x4d, 0x65, 0x73, 0x68, 0x3a, 0x3a, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, + 0x3a, 0x3a, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.proto b/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.proto index 4e53326183f9..d5173f59a5e8 100644 --- a/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.proto +++ b/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.proto @@ -14,19 +14,21 @@ message TLS { } message TransportSocket { + // name of the transport socket + string name = 1; oneof connection_tls { // inbound_mesh is for incoming connections FROM the mesh. - InboundMeshMTLS inbound_mesh = 1; + InboundMeshMTLS inbound_mesh = 2; // outbound_mesh is for outbound connections TO mesh destinations. - OutboundMeshMTLS outbound_mesh = 2; + OutboundMeshMTLS outbound_mesh = 3; // inbound_non_mesh is for incoming connections FROM non mesh. - InboundNonMeshTLS inbound_non_mesh = 3; + InboundNonMeshTLS inbound_non_mesh = 4; // outbound_non_mesh is for outbound connections TO non mesh destinations. - OutboundNonMeshTLS outbound_non_mesh = 4; + OutboundNonMeshTLS outbound_non_mesh = 5; } // tls_parameters can override any top level tls parameters that are configured. - TLSParameters tls_parameters = 5; - repeated string alpn_protocols = 6; + TLSParameters tls_parameters = 6; + repeated string alpn_protocols = 7; } message InboundMeshMTLS { From 1b2319bf1168582d47ffdb6c8359b36d6165b62b Mon Sep 17 00:00:00 2001 From: John Murret Date: Wed, 16 Aug 2023 19:53:06 -0600 Subject: [PATCH 02/12] NET-4853 - xds v2 - implement base connect proxy functionality for clusters --- agent/xds/clusters.go | 25 +- agent/xds/clusters_test.go | 219 ++- agent/xds/delta_envoy_extender_oss_test.go | 6 +- agent/xds/endpoints.go | 21 +- agent/xds/endpoints_test.go | 30 +- agent/xds/listeners.go | 7 +- agent/xds/listeners_test.go | 63 +- agent/xds/proxystateconverter/clusters.go | 1183 ++++++++++++++++- agent/xds/proxystateconverter/converter.go | 6 +- agent/xds/proxystateconverter/endpoints.go | 85 ++ .../proxystateconverter/failover_policy.go | 158 +++ .../failover_policy_oss.go | 15 + agent/xds/proxystateconverter/listeners.go | 11 +- agent/xds/rbac.go | 11 +- agent/xds/resources_test.go | 3 +- agent/xds/{ => response}/response.go | 14 +- agent/xds/routes.go | 33 +- agent/xds/routes_test.go | 3 +- agent/xds/xds_protocol_helpers_test.go | 23 +- agent/xdsv2/cluster_resources.go | 327 ++++- agent/xdsv2/endpoint_resources.go | 41 + agent/xdsv2/listener_resources.go | 7 +- agent/xdsv2/resources.go | 8 +- agent/xdsv2/route_resources.go | 3 + .../v1alpha1/pbproxystate/cluster.pb.go | 26 +- .../v1alpha1/pbproxystate/cluster.proto | 4 + .../pbproxystate/transport_socket.pb.go | 414 +++--- .../pbproxystate/transport_socket.proto | 14 +- 28 files changed, 2410 insertions(+), 350 deletions(-) create mode 100644 agent/xds/proxystateconverter/endpoints.go create mode 100644 agent/xds/proxystateconverter/failover_policy.go create mode 100644 agent/xds/proxystateconverter/failover_policy_oss.go rename agent/xds/{ => response}/response.go (82%) create mode 100644 agent/xdsv2/endpoint_resources.go diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index f56700c02a91..908aa746bc9c 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -32,6 +32,7 @@ import ( "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/proto/private/pbpeering" ) @@ -1008,11 +1009,11 @@ func (s *ResourceGenerator) configIngressUpstreamCluster(c *envoy_cluster_v3.Clu switch limitType { case "max_connections": - threshold.MaxConnections = makeUint32Value(limit) + threshold.MaxConnections = response.MakeUint32Value(limit) case "max_pending_requests": - threshold.MaxPendingRequests = makeUint32Value(limit) + threshold.MaxPendingRequests = response.MakeUint32Value(limit) case "max_requests": - threshold.MaxRequests = makeUint32Value(limit) + threshold.MaxRequests = response.MakeUint32Value(limit) } } @@ -1103,7 +1104,7 @@ func (s *ResourceGenerator) makeAppCluster(cfgSnap *proxycfg.ConfigSnapshot, nam c.CircuitBreakers = &envoy_cluster_v3.CircuitBreakers{ Thresholds: []*envoy_cluster_v3.CircuitBreakers_Thresholds{ { - MaxConnections: makeUint32Value(cfg.MaxInboundConnections), + MaxConnections: response.MakeUint32Value(cfg.MaxInboundConnections), }, }, } @@ -1708,13 +1709,13 @@ func (s *ResourceGenerator) makeGatewayCluster(snap *proxycfg.ConfigSnapshot, op TcpKeepalive: &envoy_core_v3.TcpKeepalive{}, } if cfg.TcpKeepaliveTime != 0 { - cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveTime = makeUint32Value(cfg.TcpKeepaliveTime) + cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveTime = response.MakeUint32Value(cfg.TcpKeepaliveTime) } if cfg.TcpKeepaliveInterval != 0 { - cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveInterval = makeUint32Value(cfg.TcpKeepaliveInterval) + cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveInterval = response.MakeUint32Value(cfg.TcpKeepaliveInterval) } if cfg.TcpKeepaliveProbes != 0 { - cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveProbes = makeUint32Value(cfg.TcpKeepaliveProbes) + cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveProbes = response.MakeUint32Value(cfg.TcpKeepaliveProbes) } } @@ -1884,7 +1885,7 @@ func (s *ResourceGenerator) makeExternalHostnameCluster(snap *proxycfg.ConfigSna endpoints := make([]*envoy_endpoint_v3.LbEndpoint, 0, len(opts.addresses)) for _, pair := range opts.addresses { - address := makeAddress(pair.Address, pair.Port) + address := response.MakeAddress(pair.Address, pair.Port) endpoint := &envoy_endpoint_v3.LbEndpoint{ HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ @@ -1917,13 +1918,13 @@ func makeThresholdsIfNeeded(limits *structs.UpstreamLimits) []*envoy_cluster_v3. // Likewise, make sure to not set any threshold values on the zero-value in // order to rely on Envoy defaults if limits.MaxConnections != nil { - threshold.MaxConnections = makeUint32Value(*limits.MaxConnections) + threshold.MaxConnections = response.MakeUint32Value(*limits.MaxConnections) } if limits.MaxPendingRequests != nil { - threshold.MaxPendingRequests = makeUint32Value(*limits.MaxPendingRequests) + threshold.MaxPendingRequests = response.MakeUint32Value(*limits.MaxPendingRequests) } if limits.MaxConcurrentRequests != nil { - threshold.MaxRequests = makeUint32Value(*limits.MaxConcurrentRequests) + threshold.MaxRequests = response.MakeUint32Value(*limits.MaxConcurrentRequests) } return []*envoy_cluster_v3.CircuitBreakers_Thresholds{threshold} @@ -1946,7 +1947,7 @@ func makeLbEndpoint(addr string, port int, health envoy_core_v3.HealthStatus, we }, }, HealthStatus: health, - LoadBalancingWeight: makeUint32Value(weight), + LoadBalancingWeight: response.MakeUint32Value(weight), } } diff --git a/agent/xds/clusters_test.go b/agent/xds/clusters_test.go index 32723bf48143..2651d77c2844 100644 --- a/agent/xds/clusters_test.go +++ b/agent/xds/clusters_test.go @@ -20,16 +20,28 @@ import ( "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/proxystateconverter" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/agent/xds/testcommon" + "github.com/hashicorp/consul/agent/xdsv2" "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/sdk/testutil" "github.com/hashicorp/consul/types" ) +type mockCfgFetcher struct { + addressLan string +} + +func (s *mockCfgFetcher) AdvertiseAddrLAN() string { + return s.addressLan +} + type clusterTestCase struct { name string create func(t testinf.T) *proxycfg.ConfigSnapshot overrideGoldenName string + alsoRunTestForV2 bool } func uint32ptr(i uint32) *uint32 { @@ -52,12 +64,15 @@ func makeClusterDiscoChainTests(enterprise bool) []clusterTestCase { }) }, nil) }, + // TODO(proxystate): requires custom cluster work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-chain", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-chain-http2", @@ -66,84 +81,111 @@ func makeClusterDiscoChainTests(enterprise bool) []clusterTestCase { ns.Proxy.Upstreams[0].Config["protocol"] = "http2" }, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-chain-external-sni", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "external-sni", enterprise, nil, nil) }, + //TODO(proxystate): this requires terminating gateway work + alsoRunTestForV2: true, }, { name: "connect-proxy-with-chain-and-overrides", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple-with-overrides", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-chain-and-failover", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway-triggered", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-double-failover-through-remote-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-remote-gateway", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-remote-gateway-triggered", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-failover-through-local-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway-triggered", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-double-failover-through-local-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-local-gateway", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-local-gateway-triggered", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "splitter-with-resolver-redirect", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "splitter-with-resolver-redirect-multidc", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-lb-in-resolver", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "lb-resolver", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, } } @@ -175,6 +217,7 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tls-outgoing-min-version", @@ -194,6 +237,7 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tls-outgoing-max-version", @@ -213,6 +257,7 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tls-outgoing-cipher-suites", @@ -235,6 +280,7 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-jwt-config-entry-with-local", @@ -258,6 +304,8 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + // TODO(proxystate): jwt work will come at a later time + alsoRunTestForV2: false, }, { name: "connect-proxy-with-jwt-config-entry-with-remote-jwks", @@ -292,6 +340,8 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + // TODO(proxystate): jwt work will come at a later time + alsoRunTestForV2: false, }, { name: "custom-local-app", @@ -303,6 +353,8 @@ func TestClustersFromSnapshot(t *testing.T) { }) }, nil) }, + // TODO(proxystate): requires custom cluster work + alsoRunTestForV2: false, }, { name: "custom-upstream", @@ -314,6 +366,8 @@ func TestClustersFromSnapshot(t *testing.T) { }) }, nil) }, + // TODO(proxystate): requires custom cluster work + alsoRunTestForV2: false, }, { name: "custom-upstream-ignores-tls", @@ -328,6 +382,8 @@ func TestClustersFromSnapshot(t *testing.T) { }) }, nil) }, + // TODO(proxystate): requires custom cluster work + alsoRunTestForV2: false, }, { name: "custom-upstream-with-prepared-query", @@ -364,6 +420,8 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + // TODO(proxystate): requires custom cluster work + alsoRunTestForV2: false, }, { name: "custom-timeouts", @@ -373,6 +431,7 @@ func TestClustersFromSnapshot(t *testing.T) { ns.Proxy.Upstreams[0].Config["connect_timeout_ms"] = 2345 }, nil) }, + alsoRunTestForV2: true, }, { name: "custom-passive-healthcheck", @@ -387,6 +446,7 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + alsoRunTestForV2: true, }, { name: "custom-passive-healthcheck-zero-consecutive_5xx", @@ -401,6 +461,7 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + alsoRunTestForV2: true, }, { name: "custom-max-inbound-connections", @@ -409,6 +470,7 @@ func TestClustersFromSnapshot(t *testing.T) { ns.Proxy.Config["max_inbound_connections"] = 3456 }, nil) }, + alsoRunTestForV2: true, }, { name: "custom-limits-max-connections-only", @@ -429,6 +491,7 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + alsoRunTestForV2: true, }, { name: "custom-limits-set-to-zero", @@ -447,6 +510,7 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + alsoRunTestForV2: true, }, { name: "custom-limits", @@ -465,12 +529,14 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + alsoRunTestForV2: true, }, { name: "expose-paths-local-app-paths", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotExposeConfig(t, nil) }, + alsoRunTestForV2: true, }, { name: "downstream-service-with-unix-sockets", @@ -483,6 +549,7 @@ func TestClustersFromSnapshot(t *testing.T) { ns.Proxy.LocalServiceSocketPath = "/tmp/downstream_proxy.sock" }, nil) }, + alsoRunTestForV2: true, }, { name: "expose-paths-new-cluster-http2", @@ -496,68 +563,89 @@ func TestClustersFromSnapshot(t *testing.T) { } }) }, + alsoRunTestForV2: true, }, { - name: "expose-checks", - create: proxycfg.TestConfigSnapshotExposeChecks, + name: "expose-checks", + create: proxycfg.TestConfigSnapshotExposeChecks, + alsoRunTestForV2: true, }, { - name: "expose-paths-grpc-new-cluster-http1", - create: proxycfg.TestConfigSnapshotGRPCExposeHTTP1, + name: "expose-paths-grpc-new-cluster-http1", + create: proxycfg.TestConfigSnapshotGRPCExposeHTTP1, + alsoRunTestForV2: true, }, { name: "mesh-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "default", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-using-federation-states", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "federation-states", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-using-federation-control-plane", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "mesh-gateway-federation", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-no-services", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "no-services", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-service-subsets", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "service-subsets", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-ignore-extra-resolvers", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "ignore-extra-resolvers", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-service-timeouts", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "service-timeouts", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-non-hash-lb-injected", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "non-hash-lb-injected", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-hash-lb-ignored", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "hash-lb-ignored", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-tcp-keepalives", @@ -569,6 +657,8 @@ func TestClustersFromSnapshot(t *testing.T) { ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_probes"] = 7 }, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-gateway", @@ -576,12 +666,16 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "default", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-gateway-nil-config-entry", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotIngressGateway_NilConfigEntry(t) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-gateway-with-tls-outgoing-min-version", @@ -601,6 +695,8 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-gateway-with-tls-outgoing-max-version", @@ -620,6 +716,8 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-gateway-with-tls-outgoing-cipher-suites", @@ -642,6 +740,8 @@ func TestClustersFromSnapshot(t *testing.T) { }, }) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-gateway-no-services", @@ -649,6 +749,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, false, "tcp", "default", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-chain", @@ -656,6 +758,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "simple", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-service-max-connections", @@ -666,6 +770,8 @@ func TestClustersFromSnapshot(t *testing.T) { entry.Listeners[0].Services[0].MaxConnections = 4096 }, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-defaults-service-max-connections", @@ -680,6 +786,8 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-overwrite-defaults-service-max-connections", @@ -695,6 +803,8 @@ func TestClustersFromSnapshot(t *testing.T) { entry.Listeners[0].Services[0].MaxPendingRequests = 2048 }, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-service-passive-health-check", @@ -710,6 +820,8 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-defaults-passive-health-check", @@ -731,6 +843,8 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-overwrite-defaults-passive-health-check", @@ -759,6 +873,8 @@ func TestClustersFromSnapshot(t *testing.T) { } }, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-chain-external-sni", @@ -766,6 +882,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "external-sni", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-chain-and-failover", @@ -773,6 +891,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-chain-and-failover-to-cluster-peer", @@ -780,6 +900,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-to-cluster-peer", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-failover-through-remote-gateway", @@ -787,6 +909,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-remote-gateway", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-failover-through-remote-gateway-triggered", @@ -794,6 +918,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-remote-gateway-triggered", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-double-failover-through-remote-gateway", @@ -801,6 +927,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-remote-gateway", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered", @@ -808,6 +936,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-remote-gateway-triggered", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-failover-through-local-gateway", @@ -815,6 +945,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-local-gateway", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-failover-through-local-gateway-triggered", @@ -822,6 +954,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-local-gateway-triggered", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-double-failover-through-local-gateway", @@ -829,6 +963,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-local-gateway", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-double-failover-through-local-gateway-triggered", @@ -836,6 +972,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-local-gateway-triggered", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-splitter-with-resolver-redirect", @@ -843,6 +981,8 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "http", "splitter-with-resolver-redirect-multidc", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-lb-in-resolver", @@ -850,46 +990,66 @@ func TestClustersFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "http", "lb-resolver", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotTerminatingGateway(t, true, nil, nil) }, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-no-services", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotTerminatingGateway(t, false, nil, nil) }, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-service-subsets", create: proxycfg.TestConfigSnapshotTerminatingGatewayServiceSubsetsWebAndCache, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-hostname-service-subsets", create: proxycfg.TestConfigSnapshotTerminatingGatewayHostnameSubsets, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-sni", create: proxycfg.TestConfigSnapshotTerminatingGatewaySNI, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-http2-upstream", create: proxycfg.TestConfigSnapshotTerminatingGatewayHTTP2, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-http2-upstream-subsets", create: proxycfg.TestConfigSnapshotTerminatingGatewaySubsetsHTTP2, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-ignore-extra-resolvers", create: proxycfg.TestConfigSnapshotTerminatingGatewayIgnoreExtraResolvers, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-lb-config", create: proxycfg.TestConfigSnapshotTerminatingGatewayLBConfigNoHashPolicies, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-tcp-keepalives", @@ -904,18 +1064,24 @@ func TestClustersFromSnapshot(t *testing.T) { ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_probes"] = 5 }, nil) }, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-multiple-listeners-duplicate-service", create: proxycfg.TestConfigSnapshotIngress_MultipleListenersDuplicateService, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { - name: "transparent-proxy-catalog-destinations-only", - create: proxycfg.TestConfigSnapshotTransparentProxyCatalogDestinationsOnly, + name: "transparent-proxy-catalog-destinations-only", + create: proxycfg.TestConfigSnapshotTransparentProxyCatalogDestinationsOnly, + alsoRunTestForV2: true, }, { - name: "transparent-proxy-dial-instances-directly", - create: proxycfg.TestConfigSnapshotTransparentProxyDialDirectly, + name: "transparent-proxy-dial-instances-directly", + create: proxycfg.TestConfigSnapshotTransparentProxyDialDirectly, + alsoRunTestForV2: true, }, } @@ -947,10 +1113,10 @@ func TestClustersFromSnapshot(t *testing.T) { return clusters[i].(*envoy_cluster_v3.Cluster).Name < clusters[j].(*envoy_cluster_v3.Cluster).Name }) - r, err := createResponse(xdscommon.ClusterType, "00000001", "00000001", clusters) + r, err := response.CreateResponse(xdscommon.ClusterType, "00000001", "00000001", clusters) require.NoError(t, err) - t.Run("current", func(t *testing.T) { + t.Run("current-xdsv1", func(t *testing.T) { gotJSON := protoToJSON(t, r) gName := tt.name @@ -960,6 +1126,39 @@ func TestClustersFromSnapshot(t *testing.T) { require.JSONEq(t, goldenEnvoy(t, filepath.Join("clusters", gName), envoyVersion, latestEnvoyVersion, gotJSON), gotJSON) }) + + if tt.alsoRunTestForV2 { + generator := xdsv2.NewResourceGenerator(testutil.Logger(t)) + + converter := proxystateconverter.NewConverter(testutil.Logger(t), &mockCfgFetcher{addressLan: "10.10.10.10"}) + proxyState, err := converter.ProxyStateFromSnapshot(snap) + require.NoError(t, err) + + res, err := generator.AllResourcesFromIR(proxyState) + require.NoError(t, err) + + clusters = res[xdscommon.ClusterType] + // The order of clusters returned via CDS isn't relevant, so it's safe + // to sort these for the purposes of test comparisons. + sort.Slice(clusters, func(i, j int) bool { + return clusters[i].(*envoy_cluster_v3.Cluster).Name < clusters[j].(*envoy_cluster_v3.Cluster).Name + }) + + r, err := response.CreateResponse(xdscommon.ClusterType, "00000001", "00000001", clusters) + require.NoError(t, err) + + t.Run("current-xdsv2", func(t *testing.T) { + gotJSON := protoToJSON(t, r) + + gName := tt.name + if tt.overrideGoldenName != "" { + gName = tt.overrideGoldenName + } + + expectedJSON := goldenEnvoy(t, filepath.Join("clusters", gName), envoyVersion, latestEnvoyVersion, gotJSON) + require.JSONEq(t, expectedJSON, gotJSON) + }) + } }) } }) diff --git a/agent/xds/delta_envoy_extender_oss_test.go b/agent/xds/delta_envoy_extender_oss_test.go index 5f4b563b5992..0607919ef5a5 100644 --- a/agent/xds/delta_envoy_extender_oss_test.go +++ b/agent/xds/delta_envoy_extender_oss_test.go @@ -21,12 +21,12 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" - "github.com/hashicorp/consul/agent/xds/testcommon" - propertyoverride "github.com/hashicorp/consul/agent/envoyextensions/builtin/property-override" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/xds/extensionruntime" + "github.com/hashicorp/consul/agent/xds/response" + "github.com/hashicorp/consul/agent/xds/testcommon" "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/envoyextensions/extensioncommon" "github.com/hashicorp/consul/envoyextensions/xdscommon" @@ -803,7 +803,7 @@ end`, } sort.Slice(msgs, entity.sorter(msgs)) - r, err := createResponse(entity.key, "00000001", "00000001", msgs) + r, err := response.CreateResponse(entity.key, "00000001", "00000001", msgs) require.NoError(t, err) t.Run(entity.name, func(t *testing.T) { diff --git a/agent/xds/endpoints.go b/agent/xds/endpoints.go index 8903ba3e7cc6..a2c36f06bdc2 100644 --- a/agent/xds/endpoints.go +++ b/agent/xds/endpoints.go @@ -6,21 +6,22 @@ package xds import ( "errors" "fmt" - "github.com/hashicorp/go-hclog" "strconv" + "github.com/hashicorp/go-hclog" + envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" "github.com/hashicorp/go-bexpr" "google.golang.org/protobuf/proto" - "github.com/hashicorp/consul/envoyextensions/xdscommon" - "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/envoyextensions/xdscommon" ) const ( @@ -278,7 +279,7 @@ func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.C lbEndpoint := &envoy_endpoint_v3.LbEndpoint{ HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress(addr, port), + Address: response.MakeAddress(addr, port), }, }, HealthStatus: envoy_core_v3.HealthStatus_UNKNOWN, @@ -335,7 +336,7 @@ func (s *ResourceGenerator) endpointsFromSnapshotMeshGateway(cfgSnap *proxycfg.C serverEndpoints = append(serverEndpoints, &envoy_endpoint_v3.LbEndpoint{ HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress(addr, port), + Address: response.MakeAddress(addr, port), }, }, }) @@ -584,7 +585,7 @@ func makeEndpoint(host string, port int) *envoy_endpoint_v3.LbEndpoint { return &envoy_endpoint_v3.LbEndpoint{ HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress(host, port), + Address: response.MakeAddress(host, port), }, }, } @@ -594,7 +595,7 @@ func makePipeEndpoint(path string) *envoy_endpoint_v3.LbEndpoint { return &envoy_endpoint_v3.LbEndpoint{ HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makePipeAddress(path, 0), + Address: response.MakePipeAddress(path, 0), }, }, } @@ -881,7 +882,7 @@ func makeLoadAssignment(logger hclog.Logger, cfgSnap *proxycfg.ConfigSnapshot, c cla.Policy = &envoy_endpoint_v3.ClusterLoadAssignment_Policy{ // We choose such a large value here that the failover math should // in effect not happen until zero instances are healthy. - OverprovisioningFactor: makeUint32Value(100000), + OverprovisioningFactor: response.MakeUint32Value(100000), } } @@ -907,14 +908,14 @@ func makeLoadAssignment(logger hclog.Logger, cfgSnap *proxycfg.ConfigSnapshot, c } endpoint := &envoy_endpoint_v3.Endpoint{ - Address: makeAddress(addr, port), + Address: response.MakeAddress(addr, port), } es = append(es, &envoy_endpoint_v3.LbEndpoint{ HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: endpoint, }, HealthStatus: healthStatus, - LoadBalancingWeight: makeUint32Value(weight), + LoadBalancingWeight: response.MakeUint32Value(weight), }) } diff --git a/agent/xds/endpoints_test.go b/agent/xds/endpoints_test.go index a4757f00c534..3156685934d5 100644 --- a/agent/xds/endpoints_test.go +++ b/agent/xds/endpoints_test.go @@ -4,11 +4,12 @@ package xds import ( - "github.com/hashicorp/go-hclog" "path/filepath" "sort" "testing" + "github.com/hashicorp/go-hclog" + envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" @@ -20,6 +21,7 @@ import ( "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/sdk/testutil" ) @@ -133,18 +135,18 @@ func Test_makeLoadAssignment(t *testing.T) { { HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress("10.10.10.10", 1234), + Address: response.MakeAddress("10.10.10.10", 1234), }}, HealthStatus: envoy_core_v3.HealthStatus_HEALTHY, - LoadBalancingWeight: makeUint32Value(1), + LoadBalancingWeight: response.MakeUint32Value(1), }, { HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress("10.10.10.20", 1234), + Address: response.MakeAddress("10.10.10.20", 1234), }}, HealthStatus: envoy_core_v3.HealthStatus_HEALTHY, - LoadBalancingWeight: makeUint32Value(1), + LoadBalancingWeight: response.MakeUint32Value(1), }, }, }}, @@ -163,18 +165,18 @@ func Test_makeLoadAssignment(t *testing.T) { { HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress("10.10.10.10", 1234), + Address: response.MakeAddress("10.10.10.10", 1234), }}, HealthStatus: envoy_core_v3.HealthStatus_HEALTHY, - LoadBalancingWeight: makeUint32Value(10), + LoadBalancingWeight: response.MakeUint32Value(10), }, { HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress("10.10.10.20", 1234), + Address: response.MakeAddress("10.10.10.20", 1234), }}, HealthStatus: envoy_core_v3.HealthStatus_HEALTHY, - LoadBalancingWeight: makeUint32Value(5), + LoadBalancingWeight: response.MakeUint32Value(5), }, }, }}, @@ -193,18 +195,18 @@ func Test_makeLoadAssignment(t *testing.T) { { HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress("10.10.10.10", 1234), + Address: response.MakeAddress("10.10.10.10", 1234), }}, HealthStatus: envoy_core_v3.HealthStatus_HEALTHY, - LoadBalancingWeight: makeUint32Value(1), + LoadBalancingWeight: response.MakeUint32Value(1), }, { HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress("10.10.10.20", 1234), + Address: response.MakeAddress("10.10.10.20", 1234), }}, HealthStatus: envoy_core_v3.HealthStatus_UNHEALTHY, - LoadBalancingWeight: makeUint32Value(1), + LoadBalancingWeight: response.MakeUint32Value(1), }, }, }}, @@ -559,7 +561,7 @@ func TestEndpointsFromSnapshot(t *testing.T) { sort.Slice(endpoints, func(i, j int) bool { return endpoints[i].(*envoy_endpoint_v3.ClusterLoadAssignment).ClusterName < endpoints[j].(*envoy_endpoint_v3.ClusterLoadAssignment).ClusterName }) - r, err := createResponse(xdscommon.EndpointType, "00000001", "00000001", endpoints) + r, err := response.CreateResponse(xdscommon.EndpointType, "00000001", "00000001", endpoints) require.NoError(t, err) t.Run("current", func(t *testing.T) { diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index 68c7ebdfadab..846cbf6d536f 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -45,6 +45,7 @@ import ( "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/xds/accesslogs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/lib" "github.com/hashicorp/consul/lib/stringslice" @@ -936,7 +937,7 @@ func makeListenerWithDefault(opts makeListenerOpts) *envoy_listener_v3.Listener return &envoy_listener_v3.Listener{ Name: fmt.Sprintf("%s:%s:%d", opts.name, opts.addr, opts.port), AccessLog: accessLog, - Address: makeAddress(opts.addr, opts.port), + Address: response.MakeAddress(opts.addr, opts.port), TrafficDirection: opts.direction, } } @@ -955,7 +956,7 @@ func makePipeListener(opts makeListenerOpts) *envoy_listener_v3.Listener { return &envoy_listener_v3.Listener{ Name: fmt.Sprintf("%s:%s", opts.name, opts.path), AccessLog: accessLog, - Address: makePipeAddress(opts.path, uint32(modeInt)), + Address: response.MakePipeAddress(opts.path, uint32(modeInt)), TrafficDirection: opts.direction, } } @@ -2592,7 +2593,7 @@ func makeHTTPFilter(opts listenerFilterOpts) (*envoy_listener_v3.Filter, error) "envoy.filters.http.grpc_stats", &envoy_grpc_stats_v3.FilterConfig{ PerMethodStatSpecifier: &envoy_grpc_stats_v3.FilterConfig_StatsForAllMethods{ - StatsForAllMethods: makeBoolValue(true), + StatsForAllMethods: response.MakeBoolValue(true), }, }, ) diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go index b5d6c66f0008..130f2234517e 100644 --- a/agent/xds/listeners_test.go +++ b/agent/xds/listeners_test.go @@ -21,6 +21,7 @@ import ( "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/xds/configfetcher" "github.com/hashicorp/consul/agent/xds/proxystateconverter" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/agent/xds/testcommon" "github.com/hashicorp/consul/agent/xdsv2" "github.com/hashicorp/consul/envoyextensions/xdscommon" @@ -1301,24 +1302,24 @@ func TestListenersFromSnapshot(t *testing.T) { var listeners []proto.Message - t.Run("current-xdsv1", func(t *testing.T) { - // Need server just for logger dependency - g := NewResourceGenerator(testutil.Logger(t), nil, false) - g.ProxyFeatures = sf - if tt.generatorSetup != nil { - tt.generatorSetup(g) - } - listeners, err = g.listenersFromSnapshot(snap) - require.NoError(t, err) - // The order of listeners returned via LDS isn't relevant, so it's safe - // to sort these for the purposes of test comparisons. - sort.Slice(listeners, func(i, j int) bool { - return listeners[i].(*envoy_listener_v3.Listener).Name < listeners[j].(*envoy_listener_v3.Listener).Name - }) + // Need server just for logger dependency + g := NewResourceGenerator(testutil.Logger(t), nil, false) + g.ProxyFeatures = sf + if tt.generatorSetup != nil { + tt.generatorSetup(g) + } + listeners, err = g.listenersFromSnapshot(snap) + require.NoError(t, err) + // The order of listeners returned via LDS isn't relevant, so it's safe + // to sort these for the purposes of test comparisons. + sort.Slice(listeners, func(i, j int) bool { + return listeners[i].(*envoy_listener_v3.Listener).Name < listeners[j].(*envoy_listener_v3.Listener).Name + }) - r, err := createResponse(xdscommon.ListenerType, "00000001", "00000001", listeners) - require.NoError(t, err) + r, err := response.CreateResponse(xdscommon.ListenerType, "00000001", "00000001", listeners) + require.NoError(t, err) + t.Run("current-xdsv1", func(t *testing.T) { gotJSON := protoToJSON(t, r) gName := tt.name @@ -1331,25 +1332,25 @@ func TestListenersFromSnapshot(t *testing.T) { }) if tt.alsoRunTestForV2 { - t.Run("current-xdsv2", func(t *testing.T) { - generator := xdsv2.NewResourceGenerator(testutil.Logger(t)) - converter := proxystateconverter.NewConverter(testutil.Logger(t), nil) - proxyState, err := converter.ProxyStateFromSnapshot(snap) - require.NoError(t, err) + generator := xdsv2.NewResourceGenerator(testutil.Logger(t)) + converter := proxystateconverter.NewConverter(testutil.Logger(t), nil) + proxyState, err := converter.ProxyStateFromSnapshot(snap) + require.NoError(t, err) - res, err := generator.AllResourcesFromIR(proxyState) - require.NoError(t, err) + res, err := generator.AllResourcesFromIR(proxyState) + require.NoError(t, err) - listeners = res[xdscommon.ListenerType] - // The order of listeners returned via LDS isn't relevant, so it's safe - // to sort these for the purposes of test comparisons. - sort.Slice(listeners, func(i, j int) bool { - return listeners[i].(*envoy_listener_v3.Listener).Name < listeners[j].(*envoy_listener_v3.Listener).Name - }) + listeners = res[xdscommon.ListenerType] + // The order of listeners returned via LDS isn't relevant, so it's safe + // to sort these for the purposes of test comparisons. + sort.Slice(listeners, func(i, j int) bool { + return listeners[i].(*envoy_listener_v3.Listener).Name < listeners[j].(*envoy_listener_v3.Listener).Name + }) - r, err := createResponse(xdscommon.ListenerType, "00000001", "00000001", listeners) - require.NoError(t, err) + r, err := response.CreateResponse(xdscommon.ListenerType, "00000001", "00000001", listeners) + require.NoError(t, err) + t.Run("current-xdsv2", func(t *testing.T) { gotJSON := protoToJSON(t, r) gName := tt.name diff --git a/agent/xds/proxystateconverter/clusters.go b/agent/xds/proxystateconverter/clusters.go index b5f10bf32e03..2a43d6eb049c 100644 --- a/agent/xds/proxystateconverter/clusters.go +++ b/agent/xds/proxystateconverter/clusters.go @@ -1,16 +1,28 @@ // Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 +// SPDX-License-Identifier: BUSL-1.1 package proxystateconverter import ( + "errors" "fmt" + envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + "github.com/hashicorp/go-hclog" + "github.com/hashicorp/go-uuid" "strings" + "time" + + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/config" "github.com/hashicorp/consul/agent/xds/naming" + "github.com/hashicorp/consul/agent/xds/response" + "github.com/hashicorp/consul/envoyextensions/xdscommon" + "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" "github.com/hashicorp/consul/proto/private/pbpeering" ) @@ -18,15 +30,333 @@ const ( meshGatewayExportedClusterNamePrefix = "exported~" ) +type namedCluster struct { + name string + cluster *pbproxystate.Cluster +} + +// clustersFromSnapshot returns the xDS API representation of the "clusters" in the snapshot. +func (s *Converter) clustersFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot) error { + if cfgSnap == nil { + return errors.New("nil config given") + } + + switch cfgSnap.Kind { + case structs.ServiceKindConnectProxy: + return s.clustersFromSnapshotConnectProxy(cfgSnap) + // TODO(proxystate): Terminating Gateways will be added in the future. + //case structs.ServiceKindTerminatingGateway: + // err := s.clustersFromSnapshotTerminatingGateway(cfgSnap) + // if err != nil { + // return err + // } + // return nil + // TODO(proxystate): Mesh Gateways will be added in the future. + //case structs.ServiceKindMeshGateway: + // err := s.clustersFromSnapshotMeshGateway(cfgSnap) + // if err != nil { + // return err + // } + // return nil + // TODO(proxystate): Ingress Gateways will be added in the future. + //case structs.ServiceKindIngressGateway: + // err := s.clustersFromSnapshotIngressGateway(cfgSnap) + // if err != nil { + // return err + // } + // return nil + // TODO(proxystate): API Gateways will be added in the future. + //case structs.ServiceKindAPIGateway: + // res, err := s.clustersFromSnapshotAPIGateway(cfgSnap) + // if err != nil { + // return err + // } + // return nil + default: + return fmt.Errorf("Invalid service kind: %v", cfgSnap.Kind) + } +} + +// clustersFromSnapshot returns the xDS API representation of the "clusters" +// (upstreams) in the snapshot. +func (s *Converter) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapshot) error { + // This is the list of listeners we add to. It will be empty to start. + clusters := s.proxyState.Clusters + var err error + + // Include the "app" cluster for the public listener + appCluster, err := s.makeAppCluster(cfgSnap, xdscommon.LocalAppClusterName, "", cfgSnap.Proxy.LocalServicePort) + if err != nil { + return err + } + clusters[appCluster.name] = appCluster.cluster + + if cfgSnap.Proxy.Mode == structs.ProxyModeTransparent { + passthroughs, err := s.makePassthroughClusters(cfgSnap) + if err != nil { + return fmt.Errorf("failed to make passthrough clusters for transparent proxy: %v", err) + } + for clusterName, cluster := range passthroughs { + clusters[clusterName] = cluster + } + } + + // NOTE: Any time we skip a chain below we MUST also skip that discovery chain in endpoints.go + // so that the sets of endpoints generated matches the sets of clusters. + for uid, chain := range cfgSnap.ConnectProxy.DiscoveryChain { + upstream, skip := cfgSnap.ConnectProxy.GetUpstream(uid, &cfgSnap.ProxyID.EnterpriseMeta) + if skip { + continue + } + + upstreamClusters, err := s.makeUpstreamClustersForDiscoveryChain( + uid, + upstream, + chain, + cfgSnap, + false, + ) + if err != nil { + return err + } + + for name, cluster := range upstreamClusters { + clusters[name] = cluster + } + } + + // TODO(proxystate): peering will be added in the future. + //// NOTE: Any time we skip an upstream below we MUST also skip that same + //// upstream in endpoints.go so that the sets of endpoints generated matches + //// the sets of clusters. + //for _, uid := range cfgSnap.ConnectProxy.PeeredUpstreamIDs() { + // upstream, skip := cfgSnap.ConnectProxy.GetUpstream(uid, &cfgSnap.ProxyID.EnterpriseMeta) + // if skip { + // continue + // } + // + // peerMeta, found := cfgSnap.ConnectProxy.UpstreamPeerMeta(uid) + // if !found { + // s.Logger.Warn("failed to fetch upstream peering metadata for cluster", "uid", uid) + // } + // cfg := s.getAndModifyUpstreamConfigForPeeredListener(uid, upstream, peerMeta) + // + // upstreamCluster, err := s.makeUpstreamClusterForPeerService(uid, cfg, peerMeta, cfgSnap) + // if err != nil { + // return nil, err + // } + // clusters = append(clusters, upstreamCluster) + //} + + // TODO(proxystate): L7 Intentions and JWT Auth will be added in the future. + //// add clusters for jwt-providers + //for _, prov := range cfgSnap.JWTProviders { + // //skip cluster creation for local providers + // if prov.JSONWebKeySet == nil || prov.JSONWebKeySet.Remote == nil { + // continue + // } + // + // cluster, err := makeJWTProviderCluster(prov) + // if err != nil { + // s.Logger.Warn("failed to make jwt-provider cluster", "provider name", prov.Name, "error", err) + // continue + // } + // + // clusters[cluster.GetName()] = cluster + //} + + for _, u := range cfgSnap.Proxy.Upstreams { + if u.DestinationType != structs.UpstreamDestTypePreparedQuery { + continue + } + + upstreamCluster, err := s.makeUpstreamClusterForPreparedQuery(u, cfgSnap) + if err != nil { + return err + } + clusters[upstreamCluster.name] = upstreamCluster.cluster + } + + cfgSnap.Proxy.Expose.Finalize() + paths := cfgSnap.Proxy.Expose.Paths + + // Add service health checks to the list of paths to create clusters for if needed + if cfgSnap.Proxy.Expose.Checks { + psid := structs.NewServiceID(cfgSnap.Proxy.DestinationServiceID, &cfgSnap.ProxyID.EnterpriseMeta) + for _, check := range cfgSnap.ConnectProxy.WatchedServiceChecks[psid] { + p, err := parseCheckPath(check) + if err != nil { + s.Logger.Warn("failed to create cluster for", "check", check.CheckID, "error", err) + continue + } + paths = append(paths, p) + } + } + + // Create a new cluster if we need to expose a port that is different from the service port + for _, path := range paths { + if path.LocalPathPort == cfgSnap.Proxy.LocalServicePort { + continue + } + c, err := s.makeAppCluster(cfgSnap, makeExposeClusterName(path.LocalPathPort), path.Protocol, path.LocalPathPort) + if err != nil { + s.Logger.Warn("failed to make local cluster", "path", path.Path, "error", err) + continue + } + clusters[c.name] = c.cluster + } + + return nil +} + +// TODO(proxystate): L7 Intentions and JWT Auth will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeJWTProviderCluster +// func makeJWKSDiscoveryClusterType +// func makeJWTCertValidationContext +// func parseJWTRemoteURL + func makeExposeClusterName(destinationPort int) string { return fmt.Sprintf("exposed_cluster_%d", destinationPort) } -func clusterNameForDestination(cfgSnap *proxycfg.ConfigSnapshot, name string, address string, namespace string, partition string) string { +// In transparent proxy mode there are potentially multiple passthrough clusters added. +// The first is for destinations outside of Consul's catalog. This is for a plain TCP proxy. +// All of these use Envoy's ORIGINAL_DST listener filter, which forwards to the original +// destination address (before the iptables redirection). +// The rest are for destinations inside the mesh, which require certificates for mTLS. +func (s *Converter) makePassthroughClusters(cfgSnap *proxycfg.ConfigSnapshot) (map[string]*pbproxystate.Cluster, error) { + // This size is an upper bound. + clusters := make(map[string]*pbproxystate.Cluster, 0) + if meshConf := cfgSnap.MeshConfig(); meshConf == nil || + !meshConf.TransparentProxy.MeshDestinationsOnly { + + clusters[naming.OriginalDestinationClusterName] = &pbproxystate.Cluster{ + Group: &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Passthrough{ + Passthrough: &pbproxystate.PassthroughEndpointGroup{ + Config: &pbproxystate.PassthroughEndpointGroupConfig{ + ConnectTimeout: durationpb.New(5 * time.Second), + }, + }, + }, + }, + }, + } + } + + for uid, chain := range cfgSnap.ConnectProxy.DiscoveryChain { + targetMap, ok := cfgSnap.ConnectProxy.PassthroughUpstreams[uid] + if !ok { + continue + } + + for targetID := range targetMap { + uid := proxycfg.NewUpstreamIDFromTargetID(targetID) + + sni := connect.ServiceSNI( + uid.Name, "", uid.NamespaceOrDefault(), + uid.PartitionOrDefault(), cfgSnap.Datacenter, + cfgSnap.Roots.TrustDomain) + + // Prefixed with passthrough to distinguish from non-passthrough clusters for the same upstream. + name := "passthrough~" + sni + + c := pbproxystate.Cluster{ + Group: &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Passthrough{ + Passthrough: &pbproxystate.PassthroughEndpointGroup{ + Config: &pbproxystate.PassthroughEndpointGroupConfig{ + ConnectTimeout: durationpb.New(5 * time.Second), + }, + }, + }, + }, + }, + } + + if discoTarget, ok := chain.Targets[targetID]; ok && discoTarget.ConnectTimeout > 0 { + c.GetEndpointGroup().GetPassthrough().GetConfig(). + ConnectTimeout = durationpb.New(discoTarget.ConnectTimeout) + } + + transportSocket, err := s.createOutboundMeshMTLS(cfgSnap, []string{getSpiffeID(cfgSnap, uid)}, sni) + if err != nil { + return nil, err + } + c.GetEndpointGroup().GetPassthrough().OutboundTls = transportSocket + + clusters[name] = &c + } + } + + err := cfgSnap.ConnectProxy.DestinationsUpstream.ForEachKeyE(func(uid proxycfg.UpstreamID) error { + svcConfig, ok := cfgSnap.ConnectProxy.DestinationsUpstream.Get(uid) + if !ok || svcConfig.Destination == nil { + return nil + } + + // One Cluster per Destination Address + for _, address := range svcConfig.Destination.Addresses { + name := clusterNameForDestination(cfgSnap, uid.Name, address, uid.NamespaceOrDefault(), uid.PartitionOrDefault()) + + c := &pbproxystate.Cluster{ + AltStatName: name, + Group: &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Dynamic{ + Dynamic: &pbproxystate.DynamicEndpointGroup{ + Config: &pbproxystate.DynamicEndpointGroupConfig{ + ConnectTimeout: durationpb.New(5 * time.Second), + // Endpoints are managed separately by EDS + // Having an empty config enables outlier detection with default config. + OutlierDetection: &pbproxystate.OutlierDetection{}, + }, + }, + }, + }, + }, + } + sni := connect.ServiceSNI( + uid.Name, "", uid.NamespaceOrDefault(), + uid.PartitionOrDefault(), cfgSnap.Datacenter, + cfgSnap.Roots.TrustDomain) + transportSocket, err := s.createOutboundMeshMTLS(cfgSnap, []string{getSpiffeID(cfgSnap, uid)}, sni) + if err != nil { + return err + } + c.GetEndpointGroup().GetDynamic().OutboundTls = transportSocket + clusters[name] = c + } + return nil + }) + if err != nil { + return nil, err + } + + return clusters, nil +} + +func getSpiffeID(cfgSnap *proxycfg.ConfigSnapshot, uid proxycfg.UpstreamID) string { + spiffeIDService := &connect.SpiffeIDService{ + Host: cfgSnap.Roots.TrustDomain, + Partition: uid.PartitionOrDefault(), + Namespace: uid.NamespaceOrDefault(), + Datacenter: cfgSnap.Datacenter, + Service: uid.Name, + } + return spiffeIDService.URI().String() +} +func clusterNameForDestination(cfgSnap *proxycfg.ConfigSnapshot, name string, + address string, namespace string, partition string) string { name = destinationSpecificServiceName(name, address) - sni := connect.ServiceSNI(name, "", namespace, partition, cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain) + sni := connect.ServiceSNI(name, "", namespace, partition, + cfgSnap.Datacenter, cfgSnap.Roots.TrustDomain) - // Prefixed with destination to distinguish from non-passthrough clusters for the same upstream. + // Prefixed with destination to distinguish from non-passthrough clusters + // for the same upstream. return "destination." + sni } @@ -36,6 +366,785 @@ func destinationSpecificServiceName(name string, address string) string { return fmt.Sprintf("%s.%s", address, name) } +// TODO(proxystate): Mesh Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func clustersFromSnapshotMeshGateway +// func haveVoters + +// TODO(proxystate): Peering will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makePeerServerClusters + +// TODO(proxystate): Terminating Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func clustersFromSnapshotTerminatingGateway + +// TODO(proxystate): Mesh Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeGatewayServiceClusters + +// TODO(proxystate): Cluster Peering will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeGatewayOutgoingClusterPeeringServiceClusters + +// TODO(proxystate): Terminating Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeDestinationClusters + +// TODO(proxystate): Mesh Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func injectGatewayServiceAddons + +// TODO(proxystate): Terminating Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func injectGatewayDestinationAddons + +// TODO(proxystate): Ingress Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func clustersFromSnapshotIngressGateway + +// TODO(proxystate): API Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func clustersFromSnapshotAPIGateway + +// TODO(proxystate): Ingress Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func configIngressUpstreamCluster + +func (s *Converter) makeAppCluster(cfgSnap *proxycfg.ConfigSnapshot, name, pathProtocol string, port int) (*namedCluster, error) { + var err error + namedCluster := &namedCluster{} + + cfg, err := config.ParseProxyConfig(cfgSnap.Proxy.Config) + if err != nil { + // Don't hard fail on a config typo, just warn. The parse func returns + // default config if there is an error so it's safe to continue. + s.Logger.Warn("failed to parse Connect.Proxy.Config", "error", err) + } + + //// If we have overridden local cluster config try to parse it into an Envoy cluster + //if cfg.LocalClusterJSON != "" { + // return makeClusterFromUserConfig(cfg.LocalClusterJSON) + //} + + var endpoint *pbproxystate.Endpoint + if cfgSnap.Proxy.LocalServiceSocketPath != "" { + endpoint = makeUnixSocketEndpoint(cfgSnap.Proxy.LocalServiceSocketPath) + } else { + addr := cfgSnap.Proxy.LocalServiceAddress + if addr == "" { + addr = "127.0.0.1" + } + endpoint = makeHostPortEndpoint(addr, port) + } + s.proxyState.Endpoints[name] = &pbproxystate.Endpoints{ + Endpoints: []*pbproxystate.Endpoint{endpoint}, + } + + namedCluster.name = name + namedCluster.cluster = &pbproxystate.Cluster{ + Group: &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Static{ + Static: &pbproxystate.StaticEndpointGroup{ + Config: &pbproxystate.StaticEndpointGroupConfig{ + ConnectTimeout: durationpb.New(time.Duration(cfg.LocalConnectTimeoutMs) * time.Millisecond), + }, + }, + }, + }, + }, + } + protocol := pathProtocol + if protocol == "" { + protocol = cfg.Protocol + } + namedCluster.cluster.Protocol = protocol + if cfg.MaxInboundConnections > 0 { + namedCluster.cluster.GetEndpointGroup().GetStatic().GetConfig(). + CircuitBreakers = &pbproxystate.CircuitBreakers{ + UpstreamLimits: &pbproxystate.UpstreamLimits{ + MaxConnections: response.MakeUint32Value(cfg.MaxInboundConnections), + }, + } + } + + return namedCluster, err +} + +func (s *Converter) makeUpstreamClusterForPeerService( + uid proxycfg.UpstreamID, + upstreamConfig structs.UpstreamConfig, + peerMeta structs.PeeringServiceMeta, + cfgSnap *proxycfg.ConfigSnapshot, +) (string, *pbproxystate.Cluster, *pbproxystate.Endpoints, error) { + var ( + c *pbproxystate.Cluster + e *pbproxystate.Endpoints + err error + ) + + // TODO(proxystate): escapeHatches will be implemented in the future + //if upstreamConfig.EnvoyClusterJSON != "" { + // c, err = makeClusterFromUserConfig(upstreamConfig.EnvoyClusterJSON) + // if err != nil { + // return "", c, e, err + // } + // // In the happy path don't return yet as we need to inject TLS config still. + //} + + upstreamsSnapshot, err := cfgSnap.ToConfigSnapshotUpstreams() + + if err != nil { + return "", c, e, err + } + + tbs, ok := upstreamsSnapshot.UpstreamPeerTrustBundles.Get(uid.Peer) + if !ok { + // this should never happen since we loop through upstreams with + // set trust bundles + return "", c, e, fmt.Errorf("trust bundle not ready for peer %s", uid.Peer) + } + + clusterName := generatePeeredClusterName(uid, tbs) + + outlierDetection := makeOutlierDetection(upstreamConfig.PassiveHealthCheck, nil, true) + // We can't rely on health checks for services on cluster peers because they + // don't take into account service resolvers, splitters and routers. Setting + // MaxEjectionPercent too 100% gives outlier detection the power to eject the + // entire cluster. + outlierDetection.MaxEjectionPercent = &wrapperspb.UInt32Value{Value: 100} + + s.Logger.Trace("generating cluster for", "cluster", clusterName) + if c == nil { + c = &pbproxystate.Cluster{} + + useEDS := true + if _, ok := cfgSnap.ConnectProxy.PeerUpstreamEndpointsUseHostnames[uid]; ok { + // If we're using local mesh gw, the fact that upstreams use hostnames don't matter. + // If we're not using local mesh gw, then resort to CDS. + if upstreamConfig.MeshGateway.Mode != structs.MeshGatewayModeLocal { + useEDS = false + } + } + + // If none of the service instances are addressed by a hostname we + // provide the endpoint IP addresses via EDS + if useEDS { + d := &pbproxystate.DynamicEndpointGroup{ + Config: &pbproxystate.DynamicEndpointGroupConfig{ + UseAltStatName: false, + ConnectTimeout: durationpb.New(time.Duration(upstreamConfig.ConnectTimeoutMs) * time.Millisecond), + DisablePanicThreshold: true, + CircuitBreakers: &pbproxystate.CircuitBreakers{ + UpstreamLimits: makeUpstreamLimitsIfNeeded(upstreamConfig.Limits), + }, + OutlierDetection: outlierDetection, + }, + } + c.Group = &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Dynamic{ + Dynamic: d, + }, + }, + } + transportSocket := &pbproxystate.TransportSocket{ + ConnectionTls: &pbproxystate.TransportSocket_OutboundMesh{ + OutboundMesh: &pbproxystate.OutboundMeshMTLS{ + ValidationContext: &pbproxystate.MeshOutboundValidationContext{ + SpiffeIds: peerMeta.SpiffeID, + TrustBundlePeerNameKey: uid.Peer, + }, + Sni: peerMeta.PrimarySNI(), + }, + }, + } + d.OutboundTls = transportSocket + } else { + d := &pbproxystate.DNSEndpointGroup{ + Config: &pbproxystate.DNSEndpointGroupConfig{ + UseAltStatName: false, + ConnectTimeout: durationpb.New(time.Duration(upstreamConfig.ConnectTimeoutMs) * time.Millisecond), + DisablePanicThreshold: true, + CircuitBreakers: &pbproxystate.CircuitBreakers{ + UpstreamLimits: makeUpstreamLimitsIfNeeded(upstreamConfig.Limits), + }, + OutlierDetection: outlierDetection, + }, + } + c.Group = &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Dns{ + Dns: d, + }, + }, + } + e = &pbproxystate.Endpoints{ + Endpoints: make([]*pbproxystate.Endpoint, 0), + } + + ep, _ := cfgSnap.ConnectProxy.PeerUpstreamEndpoints.Get(uid) + configureClusterWithHostnames( + s.Logger, + d, + e, + "", /*TODO:make configurable?*/ + ep, + true, /*isRemote*/ + false, /*onlyPassing*/ + ) + transportSocket := &pbproxystate.TransportSocket{ + ConnectionTls: &pbproxystate.TransportSocket_OutboundMesh{ + OutboundMesh: &pbproxystate.OutboundMeshMTLS{ + ValidationContext: &pbproxystate.MeshOutboundValidationContext{ + SpiffeIds: peerMeta.SpiffeID, + TrustBundlePeerNameKey: uid.Peer, + }, + Sni: peerMeta.PrimarySNI(), + }, + }, + } + d.OutboundTls = transportSocket + } + } + + return clusterName, c, e, nil +} + +func (s *Converter) makeUpstreamClusterForPreparedQuery(upstream structs.Upstream, cfgSnap *proxycfg.ConfigSnapshot) (*namedCluster, error) { + var c *pbproxystate.Cluster + var err error + + uid := proxycfg.NewUpstreamID(&upstream) + + dc := upstream.Datacenter + if dc == "" { + dc = cfgSnap.Datacenter + } + sni := connect.UpstreamSNI(&upstream, "", dc, cfgSnap.Roots.TrustDomain) + + cfg, _ := structs.ParseUpstreamConfig(upstream.Config) + // TODO(proxystate): add logger and enable this + //if err != nil { + // Don't hard fail on a config typo, just warn. The parse func returns + // default config if there is an error so it's safe to continue. + //s.Logger.Warn("failed to parse", "upstream", uid, "error", err) + //} + + // TODO(proxystate): escapeHatches will be implemented in the future + //if cfg.EnvoyClusterJSON != "" { + // c, err = makeClusterFromUserConfig(cfg.EnvoyClusterJSON) + // if err != nil { + // return c, err + // } + // // In the happy path don't return yet as we need to inject TLS config still. + //} + + if c == nil { + c = &pbproxystate.Cluster{ + Protocol: cfg.Protocol, + Group: &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Dynamic{ + Dynamic: &pbproxystate.DynamicEndpointGroup{ + Config: &pbproxystate.DynamicEndpointGroupConfig{ + ConnectTimeout: durationpb.New(time.Duration(cfg.ConnectTimeoutMs) * time.Millisecond), + // Endpoints are managed separately by EDS + // Having an empty config enables outlier detection with default config. + OutlierDetection: makeOutlierDetection(cfg.PassiveHealthCheck, nil, true), + DisablePanicThreshold: true, + CircuitBreakers: &pbproxystate.CircuitBreakers{ + UpstreamLimits: makeUpstreamLimitsIfNeeded(cfg.Limits), + }, + }, + }, + }, + }, + }, + } + } + + endpoints := cfgSnap.ConnectProxy.PreparedQueryEndpoints[uid] + var ( + spiffeIDs = make([]string, 0) + seen = make(map[string]struct{}) + ) + for _, e := range endpoints { + id := fmt.Sprintf("%s/%s", e.Node.Datacenter, e.Service.CompoundServiceName()) + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + + name := e.Service.Proxy.DestinationServiceName + if e.Service.Connect.Native { + name = e.Service.Service + } + + spiffeIDs = append(spiffeIDs, connect.SpiffeIDService{ + Host: cfgSnap.Roots.TrustDomain, + Namespace: e.Service.NamespaceOrDefault(), + Partition: e.Service.PartitionOrDefault(), + Datacenter: e.Node.Datacenter, + Service: name, + }.URI().String()) + } + + transportSocket, err := s.createOutboundMeshMTLS(cfgSnap, spiffeIDs, sni) + if err != nil { + return nil, err + } + c.GetEndpointGroup().GetDynamic().OutboundTls = transportSocket + + return &namedCluster{name: sni, cluster: c}, nil +} + +func finalizeUpstreamConfig(cfg structs.UpstreamConfig, chain *structs.CompiledDiscoveryChain, connectTimeout time.Duration) structs.UpstreamConfig { + if cfg.Protocol == "" { + cfg.Protocol = chain.Protocol + } + + if cfg.Protocol == "" { + cfg.Protocol = "tcp" + } + + if cfg.ConnectTimeoutMs == 0 { + cfg.ConnectTimeoutMs = int(connectTimeout / time.Millisecond) + } + return cfg +} + +func (s *Converter) createOutboundMeshMTLS(cfgSnap *proxycfg.ConfigSnapshot, spiffeIDs []string, sni string) (*pbproxystate.TransportSocket, error) { + switch cfgSnap.Kind { + case structs.ServiceKindConnectProxy: + case structs.ServiceKindMeshGateway: + default: + return nil, fmt.Errorf("cannot inject peering trust bundles for kind %q", cfgSnap.Kind) + } + + cfg, err := config.ParseProxyConfig(cfgSnap.Proxy.Config) + if err != nil { + // Don't hard fail on a config typo, just warn. The parse func returns + // default config if there is an error so it's safe to continue. + s.Logger.Warn("failed to parse Connect.Proxy.Config", "error", err) + } + + // Add all trust bundle peer names, including local. + trustBundlePeerNames := []string{"local"} + for _, tb := range cfgSnap.PeeringTrustBundles() { + trustBundlePeerNames = append(trustBundlePeerNames, tb.PeerName) + } + // Arbitrary UUID to reference the identity by. + uuid, err := uuid.GenerateUUID() + if err != nil { + return nil, err + } + // Create the transport socket + ts := &pbproxystate.TransportSocket{} + + ts.ConnectionTls = &pbproxystate.TransportSocket_OutboundMesh{ + OutboundMesh: &pbproxystate.OutboundMeshMTLS{ + IdentityKey: uuid, + ValidationContext: &pbproxystate.MeshOutboundValidationContext{ + TrustBundlePeerNameKey: trustBundlePeerNames[0], + SpiffeIds: spiffeIDs, + }, + Sni: sni, + }, + } + s.proxyState.LeafCertificates[uuid] = &pbproxystate.LeafCertificate{ + Cert: cfgSnap.Leaf().CertPEM, + Key: cfgSnap.Leaf().PrivateKeyPEM, + } + ts.TlsParameters = makeTLSParametersFromProxyTLSConfig(cfgSnap.MeshConfigTLSOutgoing()) + ts.AlpnProtocols = getAlpnProtocols(cfg.Protocol) + + return ts, nil +} +func (s *Converter) makeUpstreamClustersForDiscoveryChain( + uid proxycfg.UpstreamID, + upstream *structs.Upstream, + chain *structs.CompiledDiscoveryChain, + cfgSnap *proxycfg.ConfigSnapshot, + forMeshGateway bool, +) (map[string]*pbproxystate.Cluster, error) { + if chain == nil { + return nil, fmt.Errorf("cannot create upstream cluster without discovery chain for %s", uid) + } + + if uid.Peer != "" && forMeshGateway { + return nil, fmt.Errorf("impossible to get a peer discovery chain in a mesh gateway") + } + + upstreamConfigMap := make(map[string]interface{}) + if upstream != nil { + upstreamConfigMap = upstream.Config + } + + upstreamsSnapshot, err := cfgSnap.ToConfigSnapshotUpstreams() + + // Mesh gateways are exempt because upstreamsSnapshot is only used for + // cluster peering targets and transative failover/redirects are unsupported. + if err != nil && !forMeshGateway { + return nil, err + } + + rawUpstreamConfig, err := structs.ParseUpstreamConfigNoDefaults(upstreamConfigMap) + if err != nil { + // Don't hard fail on a config typo, just warn. The parse func returns + // default config if there is an error so it's safe to continue. + s.Logger.Warn("failed to parse", "upstream", uid, + "error", err) + } + + // TODO(proxystate): escapeHatches will be implemented in the future + //var escapeHatchCluster *pbproxystate.Cluster + //if !forMeshGateway { + // if rawUpstreamConfig.EnvoyClusterJSON != "" { + // if chain.Default { + // // If you haven't done anything to setup the discovery chain, then + // // you can use the envoy_cluster_json escape hatch. + // escapeHatchCluster = &pbproxystate.Cluster{ + // EscapeHatchClusterJson: rawUpstreamConfig.EnvoyClusterJSON, + // } + // } else { + // s.Logger.Warn("ignoring escape hatch setting, because a discovery chain is configured for", + // "discovery chain", chain.ServiceName, "upstream", uid, + // "envoy_cluster_json", chain.ServiceName) + // } + // } + //} + + out := make(map[string]*pbproxystate.Cluster) + for _, node := range chain.Nodes { + switch { + case node == nil: + return nil, fmt.Errorf("impossible to process a nil node") + case node.Type != structs.DiscoveryGraphNodeTypeResolver: + continue + case node.Resolver == nil: + return nil, fmt.Errorf("impossible to process a non-resolver node") + } + // These variables are prefixed with primary to avoid shaddowing bugs. + primaryTargetID := node.Resolver.Target + primaryTarget := chain.Targets[primaryTargetID] + primaryTargetClusterName := s.getTargetClusterName(upstreamsSnapshot, chain, primaryTargetID, forMeshGateway) + if primaryTargetClusterName == "" { + continue + } + if forMeshGateway && !cfgSnap.Locality.Matches(primaryTarget.Datacenter, primaryTarget.Partition) { + s.Logger.Warn("ignoring discovery chain target that crosses a datacenter or partition boundary in a mesh gateway", + "target", primaryTarget, + "gatewayLocality", cfgSnap.Locality, + ) + continue + } + + upstreamConfig := finalizeUpstreamConfig(rawUpstreamConfig, chain, node.Resolver.ConnectTimeout) + + mappedTargets, err := s.mapDiscoChainTargets(cfgSnap, chain, node, upstreamConfig, forMeshGateway) + if err != nil { + return nil, err + } + + targetGroups, err := mappedTargets.groupedTargets() + if err != nil { + return nil, err + } + + var failoverGroup *pbproxystate.FailoverGroup + endpointGroups := make([]*pbproxystate.EndpointGroup, 0) + if mappedTargets.failover { + // Create a failover group. The endpoint groups that are part of this failover group are created by the loop + // below. + failoverGroup = &pbproxystate.FailoverGroup{ + Config: &pbproxystate.FailoverGroupConfig{ + ConnectTimeout: durationpb.New(node.Resolver.ConnectTimeout), + }, + } + } + + // Construct the target dynamic endpoint groups. If these are not part of a failover group, they will get added + // directly to the map of pbproxystate.Cluster, if they are a part of a failover group, they will be added to + // the failover group. + for _, groupedTarget := range targetGroups { + s.Logger.Debug("generating cluster for", "cluster", groupedTarget.ClusterName) + dynamic := &pbproxystate.DynamicEndpointGroup{ + Config: &pbproxystate.DynamicEndpointGroupConfig{ + UseAltStatName: true, + ConnectTimeout: durationpb.New(node.Resolver.ConnectTimeout), + // TODO(peering): make circuit breakers or outlier detection work? + CircuitBreakers: &pbproxystate.CircuitBreakers{ + UpstreamLimits: makeUpstreamLimitsIfNeeded(upstreamConfig.Limits), + }, + OutlierDetection: makeOutlierDetection(upstreamConfig.PassiveHealthCheck, nil, true), + }, + } + ti := groupedTarget.Targets[0] + transportSocket, err := s.createOutboundMeshMTLS(cfgSnap, ti.SpiffeIDs, ti.SNI) + if err != nil { + return nil, err + } + dynamic.OutboundTls = transportSocket + + var lb *structs.LoadBalancer + if node.LoadBalancer != nil { + lb = node.LoadBalancer + } + if err := injectLBToCluster(lb, dynamic.Config); err != nil { + return nil, fmt.Errorf("failed to apply load balancer configuration to cluster %q: %v", groupedTarget.ClusterName, err) + } + + // TODO: IR: http2 options not currently supported + //if upstreamConfig.Protocol == "http2" || upstreamConfig.Protocol == "grpc" { + // if err := s.setHttp2ProtocolOptions(c); err != nil { + // return nil, err + // } + //} + + switch len(groupedTarget.Targets) { + case 0: + continue + case 1: + // We expect one target so this passes through to continue setting the cluster up. + default: + return nil, fmt.Errorf("cannot have more than one target") + } + + if targetInfo := groupedTarget.Targets[0]; targetInfo.TransportSocket != nil { + dynamic.OutboundTls = targetInfo.TransportSocket + } + + // If the endpoint group is part of a failover group, add it to the failover group. Otherwise add it + // directly to the clusters. + if failoverGroup != nil { + eg := &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Dynamic{ + Dynamic: dynamic, + }, + } + endpointGroups = append(endpointGroups, eg) + } else { + cluster := &pbproxystate.Cluster{ + AltStatName: mappedTargets.baseClusterName, + Protocol: upstreamConfig.Protocol, + Group: &pbproxystate.Cluster_EndpointGroup{ + EndpointGroup: &pbproxystate.EndpointGroup{ + Group: &pbproxystate.EndpointGroup_Dynamic{ + Dynamic: dynamic, + }, + }, + }, + } + + out[mappedTargets.baseClusterName] = cluster + } + } + + // If there's a failover group, we only add the failover group to the top level list of clusters. Its endpoint + // groups are inlined. + if failoverGroup != nil { + failoverGroup.EndpointGroups = endpointGroups + cluster := &pbproxystate.Cluster{ + AltStatName: mappedTargets.baseClusterName, + Protocol: upstreamConfig.Protocol, + Group: &pbproxystate.Cluster_FailoverGroup{ + FailoverGroup: failoverGroup, + }, + } + out[mappedTargets.baseClusterName] = cluster + } + } + + //if escapeHatchCluster != nil { + // if len(out) != 1 { + // return nil, fmt.Errorf("cannot inject escape hatch cluster when discovery chain had no nodes") + // } + // var defaultCluster *pbproxystate.Cluster + // for _, k := range out { + // defaultCluster = k + // break + // } + // + // // Overlay what the user provided. + // escapeHatchCluster.GetEndpointGroup().GetDynamic().OutboundTls.ConnectionTls = + // defaultCluster.GetEndpointGroup().GetDynamic().OutboundTls.ConnectionTls + // + // out = append(out, escapeHatchCluster) + //} + + return out, nil +} + +// TODO(proxystate): Mesh Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeExportedUpstreamClustersForMeshGateway + +// makeClusterFromUserConfig returns the listener config decoded from an +// arbitrary proto3 json format string or an error if it's invalid. +// +// For now we only support embedding in JSON strings because of the hcl parsing +// pain (see Background section in the comment for decode.HookWeakDecodeFromSlice). +// This may be fixed in decode.HookWeakDecodeFromSlice in the future. +// +// When we do that we can support just nesting the config directly into the +// JSON/hcl naturally but this is a stop-gap that gets us an escape hatch +// immediately. It's also probably not a bad thing to support long-term since +// any config generated by other systems will likely be in canonical protobuf +// from rather than our slight variant in JSON/hcl. + +// TODO(proxystate): Mesh and Terminating Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeGatewayCluster + +func configureClusterWithHostnames( + logger hclog.Logger, + dnsEndpointGroup *pbproxystate.DNSEndpointGroup, + endpointList *pbproxystate.Endpoints, + dnsDiscoveryType string, + // hostnameEndpoints is a list of endpoints with a hostname as their address + hostnameEndpoints structs.CheckServiceNodes, + // isRemote determines whether the cluster is in a remote DC or partition and we should prefer a WAN address + isRemote bool, + // onlyPassing determines whether endpoints that do not have a passing status should be considered unhealthy + onlyPassing bool, +) { + // When a service instance is addressed by a hostname we have Envoy do the DNS resolution + // by setting a DNS cluster type and passing the hostname endpoints via CDS. + if dnsEndpointGroup.Config == nil { + dnsEndpointGroup.Config = &pbproxystate.DNSEndpointGroupConfig{} + } + dnsEndpointGroup.Config.DiscoveryType = pbproxystate.DiscoveryType_DISCOVERY_TYPE_LOGICAL + if dnsDiscoveryType == "strict_dns" { + dnsEndpointGroup.Config.DiscoveryType = pbproxystate.DiscoveryType_DISCOVERY_TYPE_STRICT + } + + endpoints := make([]*envoy_endpoint_v3.LbEndpoint, 0, 1) + uniqueHostnames := make(map[string]bool) + + var ( + hostname string + idx int + fallback *pbproxystate.Endpoint + ) + for i, e := range hostnameEndpoints { + _, addr, port := e.BestAddress(isRemote) + uniqueHostnames[addr] = true + + health, weight := calculateEndpointHealthAndWeight(e, onlyPassing) + if health == pbproxystate.HealthStatus_HEALTH_STATUS_UNHEALTHY { + fallback = makeLbEndpoint(addr, port, health, weight) + continue + } + + if len(endpoints) == 0 { + endpointList.Endpoints = append(endpointList.Endpoints, makeLbEndpoint(addr, port, health, weight)) + + hostname = addr + idx = i + break + } + } + + dc := hostnameEndpoints[idx].Node.Datacenter + service := hostnameEndpoints[idx].Service.CompoundServiceName() + + // Fall back to last unhealthy endpoint if none were healthy + if len(endpoints) == 0 { + logger.Warn("upstream service does not contain any healthy instances", + "dc", dc, "service", service.String()) + + //endpoints = append(endpoints, fallback) + endpointList.Endpoints = append(endpointList.Endpoints, fallback) + } + if len(uniqueHostnames) > 1 { + logger.Warn(fmt.Sprintf("service contains instances with more than one unique hostname; only %q be resolved by Envoy", hostname), + "dc", dc, "service", service.String()) + } + +} + +// TODO(proxystate): Terminating Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeExternalIPCluster + +// TODO(proxystate): Terminating Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func makeExternalHostnameCluster + +func makeUpstreamLimitsIfNeeded(limits *structs.UpstreamLimits) *pbproxystate.UpstreamLimits { + if limits == nil { + return nil + } + + upstreamLimits := &pbproxystate.UpstreamLimits{} + + // Likewise, make sure to not set any threshold values on the zero-value in + // order to rely on Envoy defaults + if limits.MaxConnections != nil { + upstreamLimits.MaxConnections = response.MakeUint32Value(*limits.MaxConnections) + } + if limits.MaxPendingRequests != nil { + upstreamLimits.MaxPendingRequests = response.MakeUint32Value(*limits.MaxPendingRequests) + } + if limits.MaxConcurrentRequests != nil { + upstreamLimits.MaxConcurrentRequests = response.MakeUint32Value(*limits.MaxConcurrentRequests) + } + + return upstreamLimits +} + +func injectLBToCluster(ec *structs.LoadBalancer, dc *pbproxystate.DynamicEndpointGroupConfig) error { + if ec == nil { + return nil + } + + switch ec.Policy { + case "": + return nil + case structs.LBPolicyLeastRequest: + lr := &pbproxystate.DynamicEndpointGroupConfig_LeastRequest{ + LeastRequest: &pbproxystate.LBPolicyLeastRequest{}, + } + + dc.LbPolicy = lr + + if ec.LeastRequestConfig != nil { + lr.LeastRequest.ChoiceCount = &wrapperspb.UInt32Value{Value: ec.LeastRequestConfig.ChoiceCount} + } + case structs.LBPolicyRoundRobin: + dc.LbPolicy = &pbproxystate.DynamicEndpointGroupConfig_RoundRobin{ + RoundRobin: &pbproxystate.LBPolicyRoundRobin{}, + } + + case structs.LBPolicyRandom: + dc.LbPolicy = &pbproxystate.DynamicEndpointGroupConfig_Random{ + Random: &pbproxystate.LBPolicyRandom{}, + } + + case structs.LBPolicyRingHash: + rh := &pbproxystate.DynamicEndpointGroupConfig_RingHash{ + RingHash: &pbproxystate.LBPolicyRingHash{}, + } + + dc.LbPolicy = rh + + if ec.RingHashConfig != nil { + rh.RingHash.MinimumRingSize = &wrapperspb.UInt64Value{Value: ec.RingHashConfig.MinimumRingSize} + rh.RingHash.MaximumRingSize = &wrapperspb.UInt64Value{Value: ec.RingHashConfig.MaximumRingSize} + } + case structs.LBPolicyMaglev: + dc.LbPolicy = &pbproxystate.DynamicEndpointGroupConfig_Maglev{ + Maglev: &pbproxystate.LBPolicyMaglev{}, + } + + default: + return fmt.Errorf("unsupported load balancer policy %q", ec.Policy) + } + return nil +} + // generatePeeredClusterName returns an SNI-like cluster name which mimics PeeredServiceSNI // but excludes partition information which could be ambiguous (local vs remote partition). func generatePeeredClusterName(uid proxycfg.UpstreamID, tb *pbpeering.PeeringTrustBundle) string { @@ -72,3 +1181,69 @@ func (s *Converter) getTargetClusterName(upstreamsSnapshot *proxycfg.ConfigSnaps } return clusterName } + +// Return an pbproxystate.OutlierDetection populated by the values from structs.PassiveHealthCheck. +// If all values are zero a default empty OutlierDetection will be returned to +// enable outlier detection with default values. +// - If override is not nil, it will overwrite the values from p, e.g., ingress gateway defaults +// - allowZero is added to handle the legacy case where connect-proxy and mesh gateway can set 0 +// for EnforcingConsecutive5xx. Due to the definition of proto of PassiveHealthCheck, ingress +// gateway's EnforcingConsecutive5xx must be > 0. +func makeOutlierDetection(p *structs.PassiveHealthCheck, override *structs.PassiveHealthCheck, allowZero bool) *pbproxystate.OutlierDetection { + od := &pbproxystate.OutlierDetection{} + if p != nil { + + if p.Interval != 0 { + od.Interval = durationpb.New(p.Interval) + } + if p.MaxFailures != 0 { + od.Consecutive_5Xx = &wrapperspb.UInt32Value{Value: p.MaxFailures} + } + + if p.EnforcingConsecutive5xx != nil { + // NOTE: EnforcingConsecutive5xx must be greater than 0 for ingress-gateway + if *p.EnforcingConsecutive5xx != 0 { + od.EnforcingConsecutive_5Xx = &wrapperspb.UInt32Value{Value: *p.EnforcingConsecutive5xx} + } else if allowZero { + od.EnforcingConsecutive_5Xx = &wrapperspb.UInt32Value{Value: *p.EnforcingConsecutive5xx} + } + } + + if p.MaxEjectionPercent != nil { + od.MaxEjectionPercent = &wrapperspb.UInt32Value{Value: *p.MaxEjectionPercent} + } + if p.BaseEjectionTime != nil { + od.BaseEjectionTime = durationpb.New(*p.BaseEjectionTime) + } + } + + if override == nil { + return od + } + + // override the default outlier detection value + if override.Interval != 0 { + od.Interval = durationpb.New(override.Interval) + } + if override.MaxFailures != 0 { + od.Consecutive_5Xx = &wrapperspb.UInt32Value{Value: override.MaxFailures} + } + + if override.EnforcingConsecutive5xx != nil { + // NOTE: EnforcingConsecutive5xx must be great than 0 for ingress-gateway + if *override.EnforcingConsecutive5xx != 0 { + od.EnforcingConsecutive_5Xx = &wrapperspb.UInt32Value{Value: *override.EnforcingConsecutive5xx} + } + // Because only ingress gateways have overrides and they cannot have a value of 0, there is no allowZero + // override case to handle + } + + if override.MaxEjectionPercent != nil { + od.MaxEjectionPercent = &wrapperspb.UInt32Value{Value: *override.MaxEjectionPercent} + } + if override.BaseEjectionTime != nil { + od.BaseEjectionTime = durationpb.New(*override.BaseEjectionTime) + } + + return od +} diff --git a/agent/xds/proxystateconverter/converter.go b/agent/xds/proxystateconverter/converter.go index 3a8037b44b1c..061a59d6e93e 100644 --- a/agent/xds/proxystateconverter/converter.go +++ b/agent/xds/proxystateconverter/converter.go @@ -1,5 +1,5 @@ // Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 +// SPDX-License-Identifier: BUSL-1.1 package proxystateconverter @@ -57,6 +57,10 @@ func (g *Converter) resourcesFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot) erro if err != nil { return err } + //g.routesFromSnapshot(cfgSnap) + g.clustersFromSnapshot(cfgSnap) + //g.endpointsFromSnapshot(cfgSnap) + //g.secretsFromSnapshot(cfgSnap) return nil } diff --git a/agent/xds/proxystateconverter/endpoints.go b/agent/xds/proxystateconverter/endpoints.go new file mode 100644 index 000000000000..5a5dc3708098 --- /dev/null +++ b/agent/xds/proxystateconverter/endpoints.go @@ -0,0 +1,85 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package proxystateconverter + +import ( + "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/api" + "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func makeLbEndpoint(addr string, port int, health pbproxystate.HealthStatus, weight int) *pbproxystate.Endpoint { + ep := &pbproxystate.Endpoint{ + Address: &pbproxystate.Endpoint_HostPort{ + HostPort: &pbproxystate.HostPortAddress{ + Host: addr, + Port: uint32(port), + }, + }, + } + ep.HealthStatus = health + ep.LoadBalancingWeight = &wrapperspb.UInt32Value{Value: uint32(weight)} + return ep +} + +// used in clusters.go +func makeHostPortEndpoint(host string, port int) *pbproxystate.Endpoint { + return &pbproxystate.Endpoint{ + Address: &pbproxystate.Endpoint_HostPort{ + HostPort: &pbproxystate.HostPortAddress{ + Host: host, + Port: uint32(port), + }, + }, + } +} + +func makeUnixSocketEndpoint(path string) *pbproxystate.Endpoint { + return &pbproxystate.Endpoint{ + Address: &pbproxystate.Endpoint_UnixSocket{ + UnixSocket: &pbproxystate.UnixSocketAddress{ + Path: path, + // envoy's mode is particular to a pipe address and is uint32. + // it also says "The mode for the Pipe. Not applicable for abstract sockets." + // https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#config-core-v3-pipe + Mode: "0", + }, + }, + } +} + +func calculateEndpointHealthAndWeight( + ep structs.CheckServiceNode, + onlyPassing bool, +) (pbproxystate.HealthStatus, int) { + healthStatus := pbproxystate.HealthStatus_HEALTH_STATUS_HEALTHY + weight := 1 + if ep.Service.Weights != nil { + weight = ep.Service.Weights.Passing + } + + for _, chk := range ep.Checks { + if chk.Status == api.HealthCritical { + healthStatus = pbproxystate.HealthStatus_HEALTH_STATUS_UNHEALTHY + } + if onlyPassing && chk.Status != api.HealthPassing { + healthStatus = pbproxystate.HealthStatus_HEALTH_STATUS_UNHEALTHY + } + if chk.Status == api.HealthWarning && ep.Service.Weights != nil { + weight = ep.Service.Weights.Warning + } + } + // Make weights fit Envoy's limits. A zero weight means that either Warning + // (likely) or Passing (weirdly) weight has been set to 0 effectively making + // this instance unhealthy and should not be sent traffic. + if weight < 1 { + healthStatus = pbproxystate.HealthStatus_HEALTH_STATUS_UNHEALTHY + weight = 1 + } + if weight > 128 { + weight = 128 + } + return healthStatus, weight +} diff --git a/agent/xds/proxystateconverter/failover_policy.go b/agent/xds/proxystateconverter/failover_policy.go new file mode 100644 index 000000000000..6ae120eaf941 --- /dev/null +++ b/agent/xds/proxystateconverter/failover_policy.go @@ -0,0 +1,158 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package proxystateconverter + +import ( + "fmt" + + "github.com/hashicorp/consul/agent/connect" + "github.com/hashicorp/consul/agent/proxycfg" + "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/envoyextensions/xdscommon" + "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" +) + +type discoChainTargets struct { + baseClusterName string + targets []targetInfo + failover bool + failoverPolicy structs.ServiceResolverFailoverPolicy +} + +type targetInfo struct { + TargetID string + TransportSocket *pbproxystate.TransportSocket + SNI string + RootPEMs string + SpiffeIDs []string + Region *string + + PrioritizeByLocality *structs.DiscoveryPrioritizeByLocality +} + +type discoChainTargetGroup struct { + Targets []targetInfo + ClusterName string +} + +func (ft discoChainTargets) groupedTargets() ([]discoChainTargetGroup, error) { + var targetGroups []discoChainTargetGroup + + if !ft.failover { + targetGroups = append(targetGroups, discoChainTargetGroup{ + ClusterName: ft.baseClusterName, + Targets: ft.targets, + }) + return targetGroups, nil + } + + switch ft.failoverPolicy.Mode { + case "sequential", "": + return ft.sequential() + case "order-by-locality": + return ft.orderByLocality() + default: + return targetGroups, fmt.Errorf("unexpected failover policy") + } +} + +func (s *Converter) mapDiscoChainTargets(cfgSnap *proxycfg.ConfigSnapshot, chain *structs.CompiledDiscoveryChain, node *structs.DiscoveryGraphNode, upstreamConfig structs.UpstreamConfig, forMeshGateway bool) (discoChainTargets, error) { + failoverTargets := discoChainTargets{} + + if node.Resolver == nil { + return discoChainTargets{}, fmt.Errorf("impossible to process a non-resolver node") + } + + primaryTargetID := node.Resolver.Target + upstreamsSnapshot, err := cfgSnap.ToConfigSnapshotUpstreams() + if err != nil && !forMeshGateway { + return discoChainTargets{}, err + } + + failoverTargets.baseClusterName = s.getTargetClusterName(upstreamsSnapshot, chain, primaryTargetID, forMeshGateway) + + tids := []string{primaryTargetID} + failover := node.Resolver.Failover + if failover != nil && !forMeshGateway { + tids = append(tids, failover.Targets...) + failoverTargets.failover = true + if failover.Policy == nil { + failoverTargets.failoverPolicy = structs.ServiceResolverFailoverPolicy{} + } else { + failoverTargets.failoverPolicy = *failover.Policy + } + } + + for _, tid := range tids { + target := chain.Targets[tid] + targetUID := proxycfg.NewUpstreamIDFromTargetID(tid) + ti := targetInfo{TargetID: tid, PrioritizeByLocality: target.PrioritizeByLocality} + + configureTLS := true + if forMeshGateway { + // We only initiate TLS if we're doing an L7 proxy. + configureTLS = structs.IsProtocolHTTPLike(upstreamConfig.Protocol) + } + + if !configureTLS { + failoverTargets.targets = append(failoverTargets.targets, ti) + continue + } + + if targetUID.Peer != "" { + tbs, _ := upstreamsSnapshot.UpstreamPeerTrustBundles.Get(targetUID.Peer) + + peerMeta, found := upstreamsSnapshot.UpstreamPeerMeta(targetUID) + if !found { + s.Logger.Warn("failed to fetch upstream peering metadata", "target", targetUID) + continue + } + ti.SNI = peerMeta.PrimarySNI() + ti.SpiffeIDs = peerMeta.SpiffeID + region := target.Locality.GetRegion() + ti.Region = ®ion + ti.RootPEMs = tbs.ConcatenatedRootPEMs() + } else { + ti.SNI = target.SNI + ti.RootPEMs = cfgSnap.RootPEMs() + ti.SpiffeIDs = []string{connect.SpiffeIDService{ + Host: cfgSnap.Roots.TrustDomain, + Namespace: target.Namespace, + Partition: target.Partition, + Datacenter: target.Datacenter, + Service: target.Service, + }.URI().String()} + } + //commonTLSContext := makeCommonTLSContext( + // cfgSnap.Leaf(), + // rootPEMs, + // makeTLSParametersFromProxyTLSConfig(cfgSnap.MeshConfigTLSOutgoing()), + //) + // + //err := injectSANMatcher(commonTLSContext, spiffeIDs...) + //if err != nil { + // return failoverTargets, fmt.Errorf("failed to inject SAN matcher rules for cluster %q: %v", sni, err) + //} + + //tlsContext := &envoy_tls_v3.UpstreamTlsContext{ + // CommonTlsContext: commonTLSContext, + // Sni: sni, + //} + //ti.TLSContext = tlsContext + failoverTargets.targets = append(failoverTargets.targets, ti) + } + + return failoverTargets, nil +} + +func (ft discoChainTargets) sequential() ([]discoChainTargetGroup, error) { + var targetGroups []discoChainTargetGroup + for i, t := range ft.targets { + targetGroups = append(targetGroups, discoChainTargetGroup{ + ClusterName: fmt.Sprintf("%s%d~%s", xdscommon.FailoverClusterNamePrefix, i, ft.baseClusterName), + Targets: []targetInfo{t}, + }) + } + return targetGroups, nil +} diff --git a/agent/xds/proxystateconverter/failover_policy_oss.go b/agent/xds/proxystateconverter/failover_policy_oss.go new file mode 100644 index 000000000000..a9b2ec24843c --- /dev/null +++ b/agent/xds/proxystateconverter/failover_policy_oss.go @@ -0,0 +1,15 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +//go:build !consulent +// +build !consulent + +package proxystateconverter + +import ( + "fmt" +) + +func (ft discoChainTargets) orderByLocality() ([]discoChainTargetGroup, error) { + return nil, fmt.Errorf("order-by-locality is a Consul Enterprise feature") +} diff --git a/agent/xds/proxystateconverter/listeners.go b/agent/xds/proxystateconverter/listeners.go index a530fbf4fe96..7e21819260ee 100644 --- a/agent/xds/proxystateconverter/listeners.go +++ b/agent/xds/proxystateconverter/listeners.go @@ -1,5 +1,5 @@ // Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 +// SPDX-License-Identifier: BUSL-1.1 package proxystateconverter @@ -16,11 +16,6 @@ import ( envoy_http_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" "github.com/hashicorp/go-uuid" - "github.com/hashicorp/consul/agent/xds/config" - "github.com/hashicorp/consul/agent/xds/naming" - "github.com/hashicorp/consul/agent/xds/platform" - "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" - "github.com/hashicorp/go-hclog" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/wrapperspb" @@ -29,7 +24,11 @@ import ( "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/config" + "github.com/hashicorp/consul/agent/xds/naming" + "github.com/hashicorp/consul/agent/xds/platform" "github.com/hashicorp/consul/envoyextensions/xdscommon" + "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" "github.com/hashicorp/consul/sdk/iptables" "github.com/hashicorp/consul/types" ) diff --git a/agent/xds/rbac.go b/agent/xds/rbac.go index b3f9267eefb4..0c00cb92cb0c 100644 --- a/agent/xds/rbac.go +++ b/agent/xds/rbac.go @@ -19,6 +19,7 @@ import ( "github.com/hashicorp/consul/agent/connect" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/proto/private/pbpeering" ) @@ -987,7 +988,7 @@ func authenticatedPatternPrincipal(pattern string) *envoy_rbac_v3.Principal { Authenticated: &envoy_rbac_v3.Principal_Authenticated{ PrincipalName: &envoy_matcher_v3.StringMatcher{ MatchPattern: &envoy_matcher_v3.StringMatcher_SafeRegex{ - SafeRegex: makeEnvoyRegexMatch(pattern), + SafeRegex: response.MakeEnvoyRegexMatch(pattern), }, }, }, @@ -1019,7 +1020,7 @@ func xfccPrincipal(src rbacService) *envoy_rbac_v3.Principal { HeaderMatchSpecifier: &envoy_route_v3.HeaderMatcher_StringMatch{ StringMatch: &envoy_matcher_v3.StringMatcher{ MatchPattern: &envoy_matcher_v3.StringMatcher_SafeRegex{ - SafeRegex: makeEnvoyRegexMatch(pattern), + SafeRegex: response.MakeEnvoyRegexMatch(pattern), }, }, }, @@ -1224,7 +1225,7 @@ func convertPermission(perm *structs.IntentionPermission) *envoy_rbac_v3.Permiss Rule: &envoy_matcher_v3.PathMatcher_Path{ Path: &envoy_matcher_v3.StringMatcher{ MatchPattern: &envoy_matcher_v3.StringMatcher_SafeRegex{ - SafeRegex: makeEnvoyRegexMatch(perm.HTTP.PathRegex), + SafeRegex: response.MakeEnvoyRegexMatch(perm.HTTP.PathRegex), }, }, }, @@ -1245,7 +1246,7 @@ func convertPermission(perm *structs.IntentionPermission) *envoy_rbac_v3.Permiss } case hdr.Regex != "": eh.HeaderMatchSpecifier = &envoy_route_v3.HeaderMatcher_SafeRegexMatch{ - SafeRegexMatch: makeEnvoyRegexMatch(hdr.Regex), + SafeRegexMatch: response.MakeEnvoyRegexMatch(hdr.Regex), } case hdr.Prefix != "": eh.HeaderMatchSpecifier = &envoy_route_v3.HeaderMatcher_PrefixMatch{ @@ -1280,7 +1281,7 @@ func convertPermission(perm *structs.IntentionPermission) *envoy_rbac_v3.Permiss eh := &envoy_route_v3.HeaderMatcher{ Name: ":method", HeaderMatchSpecifier: &envoy_route_v3.HeaderMatcher_SafeRegexMatch{ - SafeRegexMatch: makeEnvoyRegexMatch(methodHeaderRegex), + SafeRegexMatch: response.MakeEnvoyRegexMatch(methodHeaderRegex), }, } diff --git a/agent/xds/resources_test.go b/agent/xds/resources_test.go index 22ef50bc876d..926c44fab86e 100644 --- a/agent/xds/resources_test.go +++ b/agent/xds/resources_test.go @@ -27,6 +27,7 @@ import ( "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/sdk/testutil" ) @@ -120,7 +121,7 @@ func TestAllResourcesFromSnapshot(t *testing.T) { } }) - r, err := createResponse(typeUrl, "00000001", "00000001", items) + r, err := response.CreateResponse(typeUrl, "00000001", "00000001", items) require.NoError(t, err) gotJSON := protoToJSON(t, r) diff --git a/agent/xds/response.go b/agent/xds/response/response.go similarity index 82% rename from agent/xds/response.go rename to agent/xds/response/response.go index 92e0f9d0a890..91ce6d739738 100644 --- a/agent/xds/response.go +++ b/agent/xds/response/response.go @@ -1,7 +1,7 @@ // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: BUSL-1.1 -package xds +package response import ( envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" @@ -13,7 +13,7 @@ import ( "google.golang.org/protobuf/types/known/wrapperspb" ) -func createResponse(typeURL string, version, nonce string, resources []proto.Message) (*envoy_discovery_v3.DiscoveryResponse, error) { +func CreateResponse(typeURL string, version, nonce string, resources []proto.Message) (*envoy_discovery_v3.DiscoveryResponse, error) { anys := make([]*anypb.Any, 0, len(resources)) for _, r := range resources { if r == nil { @@ -41,7 +41,7 @@ func createResponse(typeURL string, version, nonce string, resources []proto.Mes return resp, nil } -func makePipeAddress(path string, mode uint32) *envoy_core_v3.Address { +func MakePipeAddress(path string, mode uint32) *envoy_core_v3.Address { return &envoy_core_v3.Address{ Address: &envoy_core_v3.Address_Pipe{ Pipe: &envoy_core_v3.Pipe{ @@ -52,7 +52,7 @@ func makePipeAddress(path string, mode uint32) *envoy_core_v3.Address { } } -func makeAddress(ip string, port int) *envoy_core_v3.Address { +func MakeAddress(ip string, port int) *envoy_core_v3.Address { return &envoy_core_v3.Address{ Address: &envoy_core_v3.Address_SocketAddress{ SocketAddress: &envoy_core_v3.SocketAddress{ @@ -65,15 +65,15 @@ func makeAddress(ip string, port int) *envoy_core_v3.Address { } } -func makeUint32Value(n int) *wrapperspb.UInt32Value { +func MakeUint32Value(n int) *wrapperspb.UInt32Value { return &wrapperspb.UInt32Value{Value: uint32(n)} } -func makeBoolValue(n bool) *wrapperspb.BoolValue { +func MakeBoolValue(n bool) *wrapperspb.BoolValue { return &wrapperspb.BoolValue{Value: n} } -func makeEnvoyRegexMatch(patt string) *envoy_matcher_v3.RegexMatcher { +func MakeEnvoyRegexMatch(patt string) *envoy_matcher_v3.RegexMatcher { return &envoy_matcher_v3.RegexMatcher{ EngineType: &envoy_matcher_v3.RegexMatcher_GoogleRe2{ GoogleRe2: &envoy_matcher_v3.RegexMatcher_GoogleRE2{}, diff --git a/agent/xds/routes.go b/agent/xds/routes.go index 31b1c4a93241..6f0d18b05dcb 100644 --- a/agent/xds/routes.go +++ b/agent/xds/routes.go @@ -23,6 +23,7 @@ import ( "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" "github.com/hashicorp/consul/agent/xds/config" + "github.com/hashicorp/consul/agent/xds/response" ) // routesFromSnapshot returns the xDS API representation of the "routes" in the @@ -71,7 +72,7 @@ func (s *ResourceGenerator) routesForConnectProxy(cfgSnap *proxycfg.ConfigSnapsh // ValidateClusters defaults to true when defined statically and false // when done via RDS. Re-set the reasonable value of true to prevent // null-routing traffic. - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), } resources = append(resources, route) } @@ -268,7 +269,7 @@ func (s *ResourceGenerator) routesForMeshGateway(cfgSnap *proxycfg.ConfigSnapsho // ValidateClusters defaults to true when defined statically and false // when done via RDS. Re-set the reasonable value of true to prevent // null-routing traffic. - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), } resources = append(resources, route) } @@ -286,7 +287,7 @@ func makeNamedDefaultRouteWithLB(clusterName string, lb *structs.LoadBalancer, t // Configure Envoy to rewrite Host header if autoHostRewrite { action.Route.HostRewriteSpecifier = &envoy_route_v3.RouteAction_AutoHostRewrite{ - AutoHostRewrite: makeBoolValue(true), + AutoHostRewrite: response.MakeBoolValue(true), } } @@ -311,7 +312,7 @@ func makeNamedDefaultRouteWithLB(clusterName string, lb *structs.LoadBalancer, t // ValidateClusters defaults to true when defined statically and false // when done via RDS. Re-set the reasonable value of true to prevent // null-routing traffic. - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), }, nil } @@ -321,7 +322,7 @@ func makeNamedAddressesRoute(routeName string, addresses map[string]string) (*en // ValidateClusters defaults to true when defined statically and false // when done via RDS. Re-set the reasonable value of true to prevent // null-routing traffic. - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), } for clusterName, address := range addresses { action := makeRouteActionFromName(clusterName) @@ -363,7 +364,7 @@ func (s *ResourceGenerator) routesForIngressGateway(cfgSnap *proxycfg.ConfigSnap // ValidateClusters defaults to true when defined statically and false // when done via RDS. Re-set the reasonable value of true to prevent // null-routing traffic. - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), } for _, u := range upstreams { @@ -412,7 +413,7 @@ func (s *ResourceGenerator) routesForIngressGateway(cfgSnap *proxycfg.ConfigSnap } else { svcRoute := &envoy_route_v3.RouteConfiguration{ Name: svcRouteName, - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), VirtualHosts: []*envoy_route_v3.VirtualHost{virtualHost}, } result = append(result, svcRoute) @@ -448,7 +449,7 @@ func (s *ResourceGenerator) routesForAPIGateway(cfgSnap *proxycfg.ConfigSnapshot // ValidateClusters defaults to true when defined statically and false // when done via RDS. Re-set the reasonable value of true to prevent // null-routing traffic. - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), } route, ok := cfgSnap.APIGateway.HTTPRoutes.Get(routeRef) @@ -515,7 +516,7 @@ func makeHeadersValueOptions(vals map[string]string, add bool) []*envoy_core_v3. Key: k, Value: v, }, - Append: makeBoolValue(add), + Append: response.MakeBoolValue(add), } opts = append(opts, o) } @@ -752,7 +753,7 @@ func (s *ResourceGenerator) makeUpstreamRouteForDiscoveryChain( func getRetryPolicyForDestination(destination *structs.ServiceRouteDestination) *envoy_route_v3.RetryPolicy { retryPolicy := &envoy_route_v3.RetryPolicy{} if destination.NumRetries > 0 { - retryPolicy.NumRetries = makeUint32Value(int(destination.NumRetries)) + retryPolicy.NumRetries = response.MakeUint32Value(int(destination.NumRetries)) } // The RetryOn magic values come from: https://www.envoyproxy.io/docs/envoy/v1.10.0/configuration/http_filters/router_filter#config-http-filters-router-x-envoy-retry-on @@ -805,7 +806,7 @@ func makeRouteMatchForDiscoveryRoute(discoveryRoute *structs.DiscoveryRoute) *en } case match.HTTP.PathRegex != "": em.PathSpecifier = &envoy_route_v3.RouteMatch_SafeRegex{ - SafeRegex: makeEnvoyRegexMatch(match.HTTP.PathRegex), + SafeRegex: response.MakeEnvoyRegexMatch(match.HTTP.PathRegex), } default: em.PathSpecifier = &envoy_route_v3.RouteMatch_Prefix{ @@ -827,7 +828,7 @@ func makeRouteMatchForDiscoveryRoute(discoveryRoute *structs.DiscoveryRoute) *en } case hdr.Regex != "": eh.HeaderMatchSpecifier = &envoy_route_v3.HeaderMatcher_SafeRegexMatch{ - SafeRegexMatch: makeEnvoyRegexMatch(hdr.Regex), + SafeRegexMatch: response.MakeEnvoyRegexMatch(hdr.Regex), } case hdr.Prefix != "": eh.HeaderMatchSpecifier = &envoy_route_v3.HeaderMatcher_PrefixMatch{ @@ -859,7 +860,7 @@ func makeRouteMatchForDiscoveryRoute(discoveryRoute *structs.DiscoveryRoute) *en eh := &envoy_route_v3.HeaderMatcher{ Name: ":method", HeaderMatchSpecifier: &envoy_route_v3.HeaderMatcher_SafeRegexMatch{ - SafeRegexMatch: makeEnvoyRegexMatch(methodHeaderRegex), + SafeRegexMatch: response.MakeEnvoyRegexMatch(methodHeaderRegex), }, } @@ -886,7 +887,7 @@ func makeRouteMatchForDiscoveryRoute(discoveryRoute *structs.DiscoveryRoute) *en eq.QueryParameterMatchSpecifier = &envoy_route_v3.QueryParameterMatcher_StringMatch{ StringMatch: &envoy_matcher_v3.StringMatcher{ MatchPattern: &envoy_matcher_v3.StringMatcher_SafeRegex{ - SafeRegex: makeEnvoyRegexMatch(qm.Regex), + SafeRegex: response.MakeEnvoyRegexMatch(qm.Regex), }, }, } @@ -967,7 +968,7 @@ func (s *ResourceGenerator) makeRouteActionForSplitter( weight := int(split.Weight * 100) totalWeight += weight cw := &envoy_route_v3.WeightedCluster_ClusterWeight{ - Weight: makeUint32Value(weight), + Weight: response.MakeUint32Value(weight), Name: clusterName, } if err := injectHeaderManipToWeightedCluster(split.Definition, cw); err != nil { @@ -983,7 +984,7 @@ func (s *ResourceGenerator) makeRouteActionForSplitter( var envoyWeightScale *wrapperspb.UInt32Value if totalWeight == 10000 { - envoyWeightScale = makeUint32Value(10000) + envoyWeightScale = response.MakeUint32Value(10000) } return &envoy_route_v3.Route_Route{ diff --git a/agent/xds/routes_test.go b/agent/xds/routes_test.go index c691114b5e3a..8b7345d88a52 100644 --- a/agent/xds/routes_test.go +++ b/agent/xds/routes_test.go @@ -16,6 +16,7 @@ import ( "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/agent/xds/testcommon" "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/sdk/testutil" @@ -241,7 +242,7 @@ func TestRoutesFromSnapshot(t *testing.T) { sort.Slice(routes, func(i, j int) bool { return routes[i].(*envoy_route_v3.RouteConfiguration).Name < routes[j].(*envoy_route_v3.RouteConfiguration).Name }) - r, err := createResponse(xdscommon.RouteType, "00000001", "00000001", routes) + r, err := response.CreateResponse(xdscommon.RouteType, "00000001", "00000001", routes) require.NoError(t, err) t.Run("current", func(t *testing.T) { diff --git a/agent/xds/xds_protocol_helpers_test.go b/agent/xds/xds_protocol_helpers_test.go index d609268a206b..bf632217d7ee 100644 --- a/agent/xds/xds_protocol_helpers_test.go +++ b/agent/xds/xds_protocol_helpers_test.go @@ -4,6 +4,7 @@ package xds import ( + "github.com/hashicorp/consul/agent/xds/response" "sort" "sync" "testing" @@ -228,7 +229,7 @@ func xdsNewEndpoint(ip string, port int) *envoy_endpoint_v3.LbEndpoint { return &envoy_endpoint_v3.LbEndpoint{ HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: makeAddress(ip, port), + Address: response.MakeAddress(ip, port), }, }, } @@ -237,7 +238,7 @@ func xdsNewEndpoint(ip string, port int) *envoy_endpoint_v3.LbEndpoint { func xdsNewEndpointWithHealth(ip string, port int, health envoy_core_v3.HealthStatus, weight int) *envoy_endpoint_v3.LbEndpoint { ep := xdsNewEndpoint(ip, port) ep.HealthStatus = health - ep.LoadBalancingWeight = makeUint32Value(weight) + ep.LoadBalancingWeight = response.MakeUint32Value(weight) return ep } @@ -297,7 +298,7 @@ func xdsNewTransportSocket( if downstream { var requireClientCertPB *wrapperspb.BoolValue if requireClientCert { - requireClientCertPB = makeBoolValue(true) + requireClientCertPB = response.MakeBoolValue(true) } tlsContext = &envoy_tls_v3.DownstreamTlsContext{ @@ -597,7 +598,7 @@ func makeTestListener(t *testing.T, snap *proxycfg.ConfigSnapshot, fixtureName s return &envoy_listener_v3.Listener{ // Envoy can't bind to port 1 Name: "public_listener:0.0.0.0:1", - Address: makeAddress("0.0.0.0", 1), + Address: response.MakeAddress("0.0.0.0", 1), TrafficDirection: envoy_core_v3.TrafficDirection_INBOUND, FilterChains: []*envoy_listener_v3.FilterChain{ { @@ -620,7 +621,7 @@ func makeTestListener(t *testing.T, snap *proxycfg.ConfigSnapshot, fixtureName s case "tcp:public_listener": return &envoy_listener_v3.Listener{ Name: "public_listener:0.0.0.0:9999", - Address: makeAddress("0.0.0.0", 9999), + Address: response.MakeAddress("0.0.0.0", 9999), TrafficDirection: envoy_core_v3.TrafficDirection_INBOUND, FilterChains: []*envoy_listener_v3.FilterChain{ { @@ -643,7 +644,7 @@ func makeTestListener(t *testing.T, snap *proxycfg.ConfigSnapshot, fixtureName s case "tcp:db": return &envoy_listener_v3.Listener{ Name: "db:127.0.0.1:9191", - Address: makeAddress("127.0.0.1", 9191), + Address: response.MakeAddress("127.0.0.1", 9191), TrafficDirection: envoy_core_v3.TrafficDirection_OUTBOUND, FilterChains: []*envoy_listener_v3.FilterChain{ { @@ -661,7 +662,7 @@ func makeTestListener(t *testing.T, snap *proxycfg.ConfigSnapshot, fixtureName s case "http2:db": return &envoy_listener_v3.Listener{ Name: "db:127.0.0.1:9191", - Address: makeAddress("127.0.0.1", 9191), + Address: response.MakeAddress("127.0.0.1", 9191), TrafficDirection: envoy_core_v3.TrafficDirection_OUTBOUND, FilterChains: []*envoy_listener_v3.FilterChain{ { @@ -689,7 +690,7 @@ func makeTestListener(t *testing.T, snap *proxycfg.ConfigSnapshot, fixtureName s case "http2:db:rds": return &envoy_listener_v3.Listener{ Name: "db:127.0.0.1:9191", - Address: makeAddress("127.0.0.1", 9191), + Address: response.MakeAddress("127.0.0.1", 9191), TrafficDirection: envoy_core_v3.TrafficDirection_OUTBOUND, FilterChains: []*envoy_listener_v3.FilterChain{ { @@ -720,7 +721,7 @@ func makeTestListener(t *testing.T, snap *proxycfg.ConfigSnapshot, fixtureName s case "http:db:rds": return &envoy_listener_v3.Listener{ Name: "db:127.0.0.1:9191", - Address: makeAddress("127.0.0.1", 9191), + Address: response.MakeAddress("127.0.0.1", 9191), TrafficDirection: envoy_core_v3.TrafficDirection_OUTBOUND, FilterChains: []*envoy_listener_v3.FilterChain{ { @@ -751,7 +752,7 @@ func makeTestListener(t *testing.T, snap *proxycfg.ConfigSnapshot, fixtureName s case "tcp:geo-cache": return &envoy_listener_v3.Listener{ Name: "prepared_query:geo-cache:127.10.10.10:8181", - Address: makeAddress("127.10.10.10", 8181), + Address: response.MakeAddress("127.10.10.10", 8181), TrafficDirection: envoy_core_v3.TrafficDirection_OUTBOUND, FilterChains: []*envoy_listener_v3.FilterChain{ { @@ -777,7 +778,7 @@ func makeTestRoute(t *testing.T, fixtureName string) *envoy_route_v3.RouteConfig case "http2:db", "http:db": return &envoy_route_v3.RouteConfiguration{ Name: "db", - ValidateClusters: makeBoolValue(true), + ValidateClusters: response.MakeBoolValue(true), VirtualHosts: []*envoy_route_v3.VirtualHost{ { Name: "db", diff --git a/agent/xdsv2/cluster_resources.go b/agent/xdsv2/cluster_resources.go index c162d05c8d84..4c1573d5db3d 100644 --- a/agent/xdsv2/cluster_resources.go +++ b/agent/xdsv2/cluster_resources.go @@ -1,6 +1,331 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + package xdsv2 +import ( + "errors" + "fmt" + + envoy_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/config/cluster/v3" + envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + envoy_aggregate_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3" + envoy_upstreams_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" + envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" +) + +func (pr *ProxyResources) makeXDSClusters() ([]proto.Message, error) { + clusters := make([]proto.Message, 0) + + for clusterName := range pr.proxyState.Clusters { + protoCluster, err := pr.makeClusters(clusterName) + // TODO: aggregate errors for clusters and still return any properly formed clusters. + if err != nil { + return nil, err + } + clusters = append(clusters, protoCluster...) + } + + return clusters, nil +} + +func (pr *ProxyResources) makeClusters(name string) ([]proto.Message, error) { + clusters := make([]proto.Message, 0) + proxyStateCluster, ok := pr.proxyState.Clusters[name] + if !ok { + return nil, fmt.Errorf("cluster %q not found", name) + } + + switch proxyStateCluster.Group.(type) { + case *pbproxystate.Cluster_FailoverGroup: + fg := proxyStateCluster.GetFailoverGroup() + clusters, err := pr.makeEnvoyAggregateCluster(name, proxyStateCluster.Protocol, fg) + if err != nil { + return nil, err + } + for _, c := range clusters { + clusters = append(clusters, c) + } + + case *pbproxystate.Cluster_EndpointGroup: + eg := proxyStateCluster.GetEndpointGroup() + cluster, err := pr.makeEnvoyCluster(name, proxyStateCluster.Protocol, eg) + if err != nil { + return nil, err + } + clusters = append(clusters, cluster) + + default: + return nil, errors.New("cluster group type should be Endpoint Group or Failover Group") + } + return clusters, nil +} + +func (pr *ProxyResources) makeEnvoyCluster(name string, protocol string, eg *pbproxystate.EndpointGroup) (*envoy_cluster_v3.Cluster, error) { + if eg != nil { + switch t := eg.Group.(type) { + case *pbproxystate.EndpointGroup_Dynamic: + dynamic := eg.GetDynamic() + return pr.makeEnvoyDynamicCluster(name, protocol, dynamic) + case *pbproxystate.EndpointGroup_Static: + static := eg.GetStatic() + return pr.makeEnvoyStaticCluster(name, protocol, static) + case *pbproxystate.EndpointGroup_Dns: + dns := eg.GetDns() + return pr.makeEnvoyDnsCluster(name, protocol, dns) + case *pbproxystate.EndpointGroup_Passthrough: + passthrough := eg.GetPassthrough() + return pr.makeEnvoyPassthroughCluster(name, protocol, passthrough) + default: + return nil, fmt.Errorf("unsupported endpoint group type: %s", t) + } + } + return nil, fmt.Errorf("no endpoint group") +} + +func (pr *ProxyResources) makeEnvoyDynamicCluster(name string, protocol string, dynamic *pbproxystate.DynamicEndpointGroup) (*envoy_cluster_v3.Cluster, error) { + cluster := &envoy_cluster_v3.Cluster{ + Name: name, + ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_EDS}, + EdsClusterConfig: &envoy_cluster_v3.Cluster_EdsClusterConfig{ + EdsConfig: &envoy_core_v3.ConfigSource{ + ResourceApiVersion: envoy_core_v3.ApiVersion_V3, + ConfigSourceSpecifier: &envoy_core_v3.ConfigSource_Ads{ + Ads: &envoy_core_v3.AggregatedConfigSource{}, + }, + }, + }, + } + err := addHttpProtocolOptions(protocol, cluster) + if err != nil { + return nil, err + } + if dynamic.Config != nil { + if dynamic.Config.UseAltStatName { + cluster.AltStatName = name + } + cluster.ConnectTimeout = dynamic.Config.ConnectTimeout + if !dynamic.Config.DisablePanicThreshold { + cluster.CommonLbConfig = &envoy_cluster_v3.Cluster_CommonLbConfig{ + HealthyPanicThreshold: &envoy_type_v3.Percent{ + Value: 0, // disable panic threshold + }, + } + } + addEnvoyCircuitBreakers(dynamic.Config.CircuitBreakers, cluster) + addEnvoyOutlierDetection(dynamic.Config.OutlierDetection, cluster) + + err := addEnvoyLBToCluster(dynamic.Config, cluster) + if err != nil { + return nil, err + } + } + + if dynamic.OutboundTls != nil { + envoyTransportSocket, err := pr.makeEnvoyTransportSocket(dynamic.OutboundTls) + if err != nil { + return nil, err + } + cluster.TransportSocket = envoyTransportSocket + } + + return cluster, nil + +} + +func (pr *ProxyResources) makeEnvoyStaticCluster(name string, protocol string, static *pbproxystate.StaticEndpointGroup) (*envoy_cluster_v3.Cluster, error) { + endpointList, ok := pr.proxyState.Endpoints[name] + if !ok || endpointList == nil { + return nil, fmt.Errorf("static cluster %q is missing endpoints", name) + } + cluster := &envoy_cluster_v3.Cluster{ + Name: name, + ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_STATIC}, + LoadAssignment: makeEnvoyClusterLoadAssignment(name, endpointList.Endpoints), + } + err := addHttpProtocolOptions(protocol, cluster) + if err != nil { + return nil, err + } + + if static.Config != nil { + cluster.ConnectTimeout = static.Config.ConnectTimeout + addEnvoyCircuitBreakers(static.GetConfig().CircuitBreakers, cluster) + } + return cluster, nil +} +func (pr *ProxyResources) makeEnvoyDnsCluster(name string, protocol string, dns *pbproxystate.DNSEndpointGroup) (*envoy_cluster_v3.Cluster, error) { + return nil, nil +} +func (pr *ProxyResources) makeEnvoyPassthroughCluster(name string, protocol string, passthrough *pbproxystate.PassthroughEndpointGroup) (*envoy_cluster_v3.Cluster, error) { + cluster := &envoy_cluster_v3.Cluster{ + Name: name, + ConnectTimeout: passthrough.Config.ConnectTimeout, + LbPolicy: envoy_cluster_v3.Cluster_CLUSTER_PROVIDED, + ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_ORIGINAL_DST}, + } + if passthrough.OutboundTls != nil { + envoyTransportSocket, err := pr.makeEnvoyTransportSocket(passthrough.OutboundTls) + if err != nil { + return nil, err + } + cluster.TransportSocket = envoyTransportSocket + } + err := addHttpProtocolOptions(protocol, cluster) + if err != nil { + return nil, err + } + return cluster, nil +} + +func (pr *ProxyResources) makeEnvoyAggregateCluster(name string, protocol string, fg *pbproxystate.FailoverGroup) ([]*envoy_cluster_v3.Cluster, error) { + var clusters []*envoy_cluster_v3.Cluster + if fg != nil { + + var egNames []string + for _, eg := range fg.EndpointGroups { + cluster, err := pr.makeEnvoyCluster(name, protocol, eg) + if err != nil { + return nil, err + } + egNames = append(egNames, cluster.Name) + clusters = append(clusters, cluster) + } + aggregateClusterConfig, err := anypb.New(&envoy_aggregate_cluster_v3.ClusterConfig{ + Clusters: egNames, + }) + + if err != nil { + return nil, err + } + + c := &envoy_cluster_v3.Cluster{ + Name: name, + AltStatName: name, + ConnectTimeout: fg.Config.ConnectTimeout, + LbPolicy: envoy_cluster_v3.Cluster_CLUSTER_PROVIDED, + ClusterDiscoveryType: &envoy_cluster_v3.Cluster_ClusterType{ + ClusterType: &envoy_cluster_v3.Cluster_CustomClusterType{ + Name: "envoy.clusters.aggregate", + TypedConfig: aggregateClusterConfig, + }, + }, + } + err = addHttpProtocolOptions(protocol, c) + if err != nil { + return nil, err + } + clusters = append(clusters, c) + } + return clusters, nil +} + +func addHttpProtocolOptions(protocol string, c *envoy_cluster_v3.Cluster) error { + if !(protocol == "http2" || protocol == "grpc") { + // do not error. returning nil means it won't get set. + return nil + } + cfg := &envoy_upstreams_v3.HttpProtocolOptions{ + UpstreamProtocolOptions: &envoy_upstreams_v3.HttpProtocolOptions_ExplicitHttpConfig_{ + ExplicitHttpConfig: &envoy_upstreams_v3.HttpProtocolOptions_ExplicitHttpConfig{ + ProtocolConfig: &envoy_upstreams_v3.HttpProtocolOptions_ExplicitHttpConfig_Http2ProtocolOptions{ + Http2ProtocolOptions: &envoy_core_v3.Http2ProtocolOptions{}, + }, + }, + }, + } + any, err := anypb.New(cfg) + if err != nil { + return err + } + c.TypedExtensionProtocolOptions = map[string]*anypb.Any{ + "envoy.extensions.upstreams.http.v3.HttpProtocolOptions": any, + } + return nil +} + +// addEnvoyOutlierDetection will add outlier detection config to the cluster, and if nil, add empty OutlierDetection to +// enable it with default values +func addEnvoyOutlierDetection(outlierDetection *pbproxystate.OutlierDetection, c *envoy_cluster_v3.Cluster) { + if outlierDetection == nil { + return + } + od := &envoy_cluster_v3.OutlierDetection{ + BaseEjectionTime: outlierDetection.GetBaseEjectionTime(), + Consecutive_5Xx: outlierDetection.GetConsecutive_5Xx(), + EnforcingConsecutive_5Xx: outlierDetection.GetEnforcingConsecutive_5Xx(), + Interval: outlierDetection.GetInterval(), + MaxEjectionPercent: outlierDetection.GetMaxEjectionPercent(), + } + c.OutlierDetection = od + +} + +func addEnvoyCircuitBreakers(circuitBreakers *pbproxystate.CircuitBreakers, c *envoy_cluster_v3.Cluster) { + if circuitBreakers != nil { + if circuitBreakers.UpstreamLimits == nil { + c.CircuitBreakers = &envoy_cluster_v3.CircuitBreakers{} + return + } + threshold := &envoy_cluster_v3.CircuitBreakers_Thresholds{} + threshold.MaxConnections = circuitBreakers.UpstreamLimits.MaxConnections + threshold.MaxPendingRequests = circuitBreakers.UpstreamLimits.MaxPendingRequests + threshold.MaxRequests = circuitBreakers.UpstreamLimits.MaxConcurrentRequests + + c.CircuitBreakers = &envoy_cluster_v3.CircuitBreakers{ + Thresholds: []*envoy_cluster_v3.CircuitBreakers_Thresholds{threshold}, + } + } +} + +func addEnvoyLBToCluster(dynamicConfig *pbproxystate.DynamicEndpointGroupConfig, c *envoy_cluster_v3.Cluster) error { + if dynamicConfig == nil || dynamicConfig.LbPolicy == nil { + return nil + } + + switch d := dynamicConfig.LbPolicy.(type) { + case *pbproxystate.DynamicEndpointGroupConfig_LeastRequest: + c.LbPolicy = envoy_cluster_v3.Cluster_LEAST_REQUEST + + lb := dynamicConfig.LbPolicy.(*pbproxystate.DynamicEndpointGroupConfig_LeastRequest) + if lb.LeastRequest != nil { + c.LbConfig = &envoy_cluster_v3.Cluster_LeastRequestLbConfig_{ + LeastRequestLbConfig: &envoy_cluster_v3.Cluster_LeastRequestLbConfig{ + ChoiceCount: lb.LeastRequest.ChoiceCount, + }, + } + } + case *pbproxystate.DynamicEndpointGroupConfig_RoundRobin: + c.LbPolicy = envoy_cluster_v3.Cluster_ROUND_ROBIN + + case *pbproxystate.DynamicEndpointGroupConfig_Random: + c.LbPolicy = envoy_cluster_v3.Cluster_RANDOM + + case *pbproxystate.DynamicEndpointGroupConfig_RingHash: + c.LbPolicy = envoy_cluster_v3.Cluster_RING_HASH + + lb := dynamicConfig.LbPolicy.(*pbproxystate.DynamicEndpointGroupConfig_RingHash) + if lb.RingHash != nil { + c.LbConfig = &envoy_cluster_v3.Cluster_RingHashLbConfig_{ + RingHashLbConfig: &envoy_cluster_v3.Cluster_RingHashLbConfig{ + MinimumRingSize: lb.RingHash.MinimumRingSize, + MaximumRingSize: lb.RingHash.MaximumRingSize, + }, + } + } + case *pbproxystate.DynamicEndpointGroupConfig_Maglev: + c.LbPolicy = envoy_cluster_v3.Cluster_MAGLEV + + default: + return fmt.Errorf("unsupported load balancer policy %q for cluster %q", d, c.Name) + } + return nil +} + // TODO(proxystate): In a future PR this will create clusters and add it to ProxyResources.proxyState -func (pr *ProxyResources) makeCluster(name string) error { +func (pr *ProxyResources) makeEnvoyClusterFromL4Destination(name string) error { return nil } diff --git a/agent/xdsv2/endpoint_resources.go b/agent/xdsv2/endpoint_resources.go new file mode 100644 index 000000000000..f5fd1b9c5d09 --- /dev/null +++ b/agent/xdsv2/endpoint_resources.go @@ -0,0 +1,41 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package xdsv2 + +import ( + envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + "github.com/hashicorp/consul/agent/xds/response" + "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" +) + +func makeEnvoyEndpoint(endpoint *pbproxystate.Endpoint) *envoy_endpoint_v3.LbEndpoint { + var address *envoy_core_v3.Address + if endpoint.GetUnixSocket() != nil { + address = response.MakePipeAddress(endpoint.GetUnixSocket().GetPath(), 0) + } else { + address = response.MakeAddress(endpoint.GetHostPort().GetHost(), int(endpoint.GetHostPort().Port)) + } + return &envoy_endpoint_v3.LbEndpoint{ + HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ + Endpoint: &envoy_endpoint_v3.Endpoint{ + Address: address, + }, + }, + } +} + +func makeEnvoyClusterLoadAssignment(clusterName string, endpoints []*pbproxystate.Endpoint) *envoy_endpoint_v3.ClusterLoadAssignment { + localityLbEndpoints := make([]*envoy_endpoint_v3.LocalityLbEndpoints, 0, len(endpoints)) + for _, endpoint := range endpoints { + localityLbEndpoints = append(localityLbEndpoints, &envoy_endpoint_v3.LocalityLbEndpoints{ + LbEndpoints: []*envoy_endpoint_v3.LbEndpoint{makeEnvoyEndpoint(endpoint)}, + }) + } + return &envoy_endpoint_v3.ClusterLoadAssignment{ + ClusterName: clusterName, + Endpoints: localityLbEndpoints, + } + +} diff --git a/agent/xdsv2/listener_resources.go b/agent/xdsv2/listener_resources.go index 6d6ce7c8002e..2d3bf5d60da5 100644 --- a/agent/xdsv2/listener_resources.go +++ b/agent/xdsv2/listener_resources.go @@ -1,3 +1,6 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + package xdsv2 import ( @@ -281,7 +284,7 @@ func (pr *ProxyResources) makeEnvoyResourcesForSNIDestination(sni *pbproxystate. } func (pr *ProxyResources) makeEnvoyResourcesForL4Destination(l4 *pbproxystate.Router_L4) ([]*envoy_listener_v3.Filter, error) { - err := pr.makeCluster(l4.L4.Name) + err := pr.makeEnvoyClusterFromL4Destination(l4.L4.Name) if err != nil { return nil, err } @@ -632,7 +635,7 @@ func (pr *ProxyResources) makeEnvoyTransportSocket(ts *pbproxystate.TransportSoc // if tls is nil but connection tls is provided, then the proxy state is misconfigured return nil, fmt.Errorf("proxyState.Tls is required to generate router's transport socket") } - om := ts.ConnectionTls.(*pbproxystate.TransportSocket_OutboundMesh).OutboundMesh + om := ts.GetOutboundMesh() leaf, ok := pr.proxyState.LeafCertificates[om.IdentityKey] if !ok { return nil, fmt.Errorf("leaf %s not found in proxyState", om.IdentityKey) diff --git a/agent/xdsv2/resources.go b/agent/xdsv2/resources.go index 6bc1df58c664..6082f7dd56b5 100644 --- a/agent/xdsv2/resources.go +++ b/agent/xdsv2/resources.go @@ -1,5 +1,5 @@ // Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 +// SPDX-License-Identifier: BUSL-1.1 package xdsv2 @@ -47,7 +47,6 @@ func (g *ResourceGenerator) AllResourcesFromIR(proxyState *pbmesh.ProxyState) (m func (pr *ProxyResources) generateXDSResources() error { listeners := make([]proto.Message, 0) - clusters := make([]proto.Message, 0) routes := make([]proto.Message, 0) endpoints := make([]proto.Message, 0) @@ -60,6 +59,11 @@ func (pr *ProxyResources) generateXDSResources() error { listeners = append(listeners, protoListener) } + clusters, err := pr.makeXDSClusters() + if err != nil { + return err + } + pr.envoyResources[xdscommon.ListenerType] = listeners pr.envoyResources[xdscommon.ClusterType] = clusters pr.envoyResources[xdscommon.RouteType] = routes diff --git a/agent/xdsv2/route_resources.go b/agent/xdsv2/route_resources.go index 1433534e8f0a..86d82a5d306f 100644 --- a/agent/xdsv2/route_resources.go +++ b/agent/xdsv2/route_resources.go @@ -1,3 +1,6 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + package xdsv2 import ( diff --git a/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.pb.go b/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.pb.go index 9cbfe4ad6943..671e25f4ca04 100644 --- a/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.pb.go +++ b/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.pb.go @@ -86,6 +86,10 @@ type Cluster struct { Group isCluster_Group `protobuf_oneof:"group"` // escape_hatch_cluster_json configures a user configured escape hatch cluster. EscapeHatchClusterJson string `protobuf:"bytes,3,opt,name=escape_hatch_cluster_json,json=escapeHatchClusterJson,proto3" json:"escape_hatch_cluster_json,omitempty"` + // alt_stat_name is the name used for observability in place of cluster name if provided. + AltStatName string `protobuf:"bytes,4,opt,name=alt_stat_name,json=altStatName,proto3" json:"alt_stat_name,omitempty"` + // protocol is the local path protocol or the service protocol. + Protocol string `protobuf:"bytes,5,opt,name=protocol,proto3" json:"protocol,omitempty"` } func (x *Cluster) Reset() { @@ -148,6 +152,20 @@ func (x *Cluster) GetEscapeHatchClusterJson() string { return "" } +func (x *Cluster) GetAltStatName() string { + if x != nil { + return x.AltStatName + } + return "" +} + +func (x *Cluster) GetProtocol() string { + if x != nil { + return x.Protocol + } + return "" +} + type isCluster_Group interface { isCluster_Group() } @@ -1679,7 +1697,7 @@ var file_pbmesh_v1alpha1_pbproxystate_cluster_proto_rawDesc = []byte{ 0x70, 0x62, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x22, 0x97, 0x02, 0x0a, 0x07, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, + 0x6f, 0x74, 0x6f, 0x22, 0xd7, 0x02, 0x0a, 0x07, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x63, 0x0a, 0x0e, 0x66, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, @@ -1696,7 +1714,11 @@ var file_pbmesh_v1alpha1_pbproxystate_cluster_proto_rawDesc = []byte{ 0x61, 0x70, 0x65, 0x5f, 0x68, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x48, 0x61, 0x74, 0x63, 0x68, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x4a, 0x73, 0x6f, 0x6e, 0x42, 0x07, 0x0a, 0x05, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x22, 0xce, 0x01, + 0x4a, 0x73, 0x6f, 0x6e, 0x12, 0x22, 0x0a, 0x0d, 0x61, 0x6c, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x61, 0x6c, 0x74, + 0x53, 0x74, 0x61, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x63, 0x6f, 0x6c, 0x42, 0x07, 0x0a, 0x05, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x22, 0xce, 0x01, 0x0a, 0x0d, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x63, 0x0a, 0x0f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, diff --git a/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.proto b/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.proto index 9b4194d981f0..860ebf2a5cbf 100644 --- a/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.proto +++ b/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.proto @@ -18,6 +18,10 @@ message Cluster { } // escape_hatch_cluster_json configures a user configured escape hatch cluster. string escape_hatch_cluster_json = 3; + // alt_stat_name is the name used for observability in place of cluster name if provided. + string alt_stat_name = 4; + // protocol is the local path protocol or the service protocol. + string protocol = 5; } message FailoverGroup { diff --git a/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.pb.go b/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.pb.go index 9801e29c3c70..59317396be4a 100644 --- a/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.pb.go +++ b/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.pb.go @@ -231,6 +231,8 @@ type TransportSocket struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + // name of the transport socket + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // Types that are assignable to ConnectionTls: // // *TransportSocket_InboundMesh @@ -239,8 +241,8 @@ type TransportSocket struct { // *TransportSocket_OutboundNonMesh ConnectionTls isTransportSocket_ConnectionTls `protobuf_oneof:"connection_tls"` // tls_parameters can override any top level tls parameters that are configured. - TlsParameters *TLSParameters `protobuf:"bytes,5,opt,name=tls_parameters,json=tlsParameters,proto3" json:"tls_parameters,omitempty"` - AlpnProtocols []string `protobuf:"bytes,6,rep,name=alpn_protocols,json=alpnProtocols,proto3" json:"alpn_protocols,omitempty"` + TlsParameters *TLSParameters `protobuf:"bytes,6,opt,name=tls_parameters,json=tlsParameters,proto3" json:"tls_parameters,omitempty"` + AlpnProtocols []string `protobuf:"bytes,7,rep,name=alpn_protocols,json=alpnProtocols,proto3" json:"alpn_protocols,omitempty"` } func (x *TransportSocket) Reset() { @@ -275,6 +277,13 @@ func (*TransportSocket) Descriptor() ([]byte, []int) { return file_pbmesh_v1alpha1_pbproxystate_transport_socket_proto_rawDescGZIP(), []int{1} } +func (x *TransportSocket) GetName() string { + if x != nil { + return x.Name + } + return "" +} + func (m *TransportSocket) GetConnectionTls() isTransportSocket_ConnectionTls { if m != nil { return m.ConnectionTls @@ -330,22 +339,22 @@ type isTransportSocket_ConnectionTls interface { type TransportSocket_InboundMesh struct { // inbound_mesh is for incoming connections FROM the mesh. - InboundMesh *InboundMeshMTLS `protobuf:"bytes,1,opt,name=inbound_mesh,json=inboundMesh,proto3,oneof"` + InboundMesh *InboundMeshMTLS `protobuf:"bytes,2,opt,name=inbound_mesh,json=inboundMesh,proto3,oneof"` } type TransportSocket_OutboundMesh struct { // outbound_mesh is for outbound connections TO mesh destinations. - OutboundMesh *OutboundMeshMTLS `protobuf:"bytes,2,opt,name=outbound_mesh,json=outboundMesh,proto3,oneof"` + OutboundMesh *OutboundMeshMTLS `protobuf:"bytes,3,opt,name=outbound_mesh,json=outboundMesh,proto3,oneof"` } type TransportSocket_InboundNonMesh struct { // inbound_non_mesh is for incoming connections FROM non mesh. - InboundNonMesh *InboundNonMeshTLS `protobuf:"bytes,3,opt,name=inbound_non_mesh,json=inboundNonMesh,proto3,oneof"` + InboundNonMesh *InboundNonMeshTLS `protobuf:"bytes,4,opt,name=inbound_non_mesh,json=inboundNonMesh,proto3,oneof"` } type TransportSocket_OutboundNonMesh struct { // outbound_non_mesh is for outbound connections TO non mesh destinations. - OutboundNonMesh *OutboundNonMeshTLS `protobuf:"bytes,4,opt,name=outbound_non_mesh,json=outboundNonMesh,proto3,oneof"` + OutboundNonMesh *OutboundNonMeshTLS `protobuf:"bytes,5,opt,name=outbound_non_mesh,json=outboundNonMesh,proto3,oneof"` } func (*TransportSocket_InboundMesh) isTransportSocket_ConnectionTls() {} @@ -1038,213 +1047,214 @@ var file_pbmesh_v1alpha1_pbproxystate_transport_socket_proto_rawDesc = []byte{ 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x4c, 0x53, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x52, 0x15, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x54, 0x6c, 0x73, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, - 0x22, 0xd1, 0x04, 0x0a, 0x0f, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x6f, - 0x63, 0x6b, 0x65, 0x74, 0x12, 0x61, 0x0a, 0x0c, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, - 0x6d, 0x65, 0x73, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x68, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, - 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, - 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, - 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, 0x53, 0x48, 0x00, 0x52, 0x0b, 0x69, 0x6e, 0x62, 0x6f, - 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x64, 0x0a, 0x0d, 0x6f, 0x75, 0x74, 0x62, 0x6f, - 0x75, 0x6e, 0x64, 0x5f, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, + 0x22, 0xe5, 0x04, 0x0a, 0x0f, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x6f, + 0x63, 0x6b, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x61, 0x0a, 0x0c, 0x69, 0x6e, 0x62, 0x6f, + 0x75, 0x6e, 0x64, 0x5f, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, - 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4f, 0x75, 0x74, - 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, 0x53, 0x48, 0x00, 0x52, - 0x0c, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x6a, 0x0a, - 0x10, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x6e, 0x6f, 0x6e, 0x5f, 0x6d, 0x65, 0x73, - 0x68, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, - 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, - 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, - 0x4d, 0x65, 0x73, 0x68, 0x54, 0x4c, 0x53, 0x48, 0x00, 0x52, 0x0e, 0x69, 0x6e, 0x62, 0x6f, 0x75, - 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x6d, 0x0a, 0x11, 0x6f, 0x75, 0x74, - 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x6e, 0x6f, 0x6e, 0x5f, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x49, 0x6e, 0x62, + 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, 0x53, 0x48, 0x00, 0x52, 0x0b, + 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x64, 0x0a, 0x0d, 0x6f, + 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, + 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x2e, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, + 0x53, 0x48, 0x00, 0x52, 0x0c, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, + 0x68, 0x12, 0x6a, 0x0a, 0x10, 0x69, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x6e, 0x6f, 0x6e, + 0x5f, 0x6d, 0x65, 0x73, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3e, 0x2e, 0x68, 0x61, + 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, + 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, + 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, + 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x54, 0x4c, 0x53, 0x48, 0x00, 0x52, 0x0e, 0x69, + 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x6d, 0x0a, + 0x11, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x6e, 0x6f, 0x6e, 0x5f, 0x6d, 0x65, + 0x73, 0x68, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, + 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, + 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, + 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, + 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x54, 0x4c, 0x53, 0x48, 0x00, 0x52, 0x0f, 0x6f, 0x75, 0x74, + 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x61, 0x0a, 0x0e, + 0x74, 0x6c, 0x73, 0x5f, 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x06, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, - 0x74, 0x65, 0x2e, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, - 0x73, 0x68, 0x54, 0x4c, 0x53, 0x48, 0x00, 0x52, 0x0f, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, - 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x12, 0x61, 0x0a, 0x0e, 0x74, 0x6c, 0x73, 0x5f, - 0x70, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, - 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, - 0x4c, 0x53, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x52, 0x0d, 0x74, 0x6c, - 0x73, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x25, 0x0a, 0x0e, 0x61, - 0x6c, 0x70, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x73, 0x18, 0x06, 0x20, - 0x03, 0x28, 0x09, 0x52, 0x0d, 0x61, 0x6c, 0x70, 0x6e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, - 0x6c, 0x73, 0x42, 0x10, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x5f, 0x74, 0x6c, 0x73, 0x22, 0xae, 0x01, 0x0a, 0x0f, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, - 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, 0x53, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x64, 0x65, 0x6e, - 0x74, 0x69, 0x74, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x4b, 0x65, 0x79, 0x12, 0x78, 0x0a, 0x12, 0x76, - 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, - 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x49, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, - 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, - 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, - 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, - 0x78, 0x74, 0x52, 0x11, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x22, 0xc2, 0x01, 0x0a, 0x10, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, - 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, 0x53, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x64, - 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0b, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x4b, 0x65, 0x79, 0x12, 0x79, 0x0a, + 0x74, 0x65, 0x2e, 0x54, 0x4c, 0x53, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, + 0x52, 0x0d, 0x74, 0x6c, 0x73, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, + 0x25, 0x0a, 0x0e, 0x61, 0x6c, 0x70, 0x6e, 0x5f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, + 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0d, 0x61, 0x6c, 0x70, 0x6e, 0x50, 0x72, 0x6f, + 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x73, 0x42, 0x10, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, 0x6c, 0x73, 0x22, 0xae, 0x01, 0x0a, 0x0f, 0x49, 0x6e, 0x62, + 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, 0x53, 0x12, 0x21, 0x0a, 0x0c, + 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x4b, 0x65, 0x79, 0x12, + 0x78, 0x0a, 0x12, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x49, 0x2e, 0x68, 0x61, + 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, + 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, + 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x49, 0x6e, + 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, + 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x11, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x22, 0xc2, 0x01, 0x0a, 0x10, 0x4f, 0x75, + 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4d, 0x65, 0x73, 0x68, 0x4d, 0x54, 0x4c, 0x53, 0x12, 0x21, + 0x0a, 0x0c, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x4b, 0x65, + 0x79, 0x12, 0x79, 0x0a, 0x12, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4a, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, + 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x73, 0x68, + 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x11, 0x76, 0x61, 0x6c, 0x69, 0x64, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x10, 0x0a, 0x03, + 0x73, 0x6e, 0x69, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x6e, 0x69, 0x22, 0x8d, + 0x01, 0x0a, 0x11, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, + 0x68, 0x54, 0x4c, 0x53, 0x12, 0x1b, 0x0a, 0x08, 0x6c, 0x65, 0x61, 0x66, 0x5f, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x07, 0x6c, 0x65, 0x61, 0x66, 0x4b, 0x65, + 0x79, 0x12, 0x4f, 0x0a, 0x03, 0x73, 0x64, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3b, + 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, + 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, + 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x53, 0x44, 0x53, + 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x48, 0x00, 0x52, 0x03, 0x73, + 0x64, 0x73, 0x42, 0x0a, 0x0a, 0x08, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x22, 0xca, + 0x01, 0x0a, 0x12, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, + 0x73, 0x68, 0x54, 0x4c, 0x53, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x66, 0x69, + 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x65, 0x72, 0x74, 0x46, 0x69, + 0x6c, 0x65, 0x12, 0x19, 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6b, 0x65, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x7c, 0x0a, 0x12, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x74, - 0x65, 0x78, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4a, 0x2e, 0x68, 0x61, 0x73, 0x68, + 0x65, 0x78, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, - 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x4f, 0x75, 0x74, 0x62, - 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x11, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x6e, 0x69, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x73, 0x6e, 0x69, 0x22, 0x8d, 0x01, 0x0a, 0x11, 0x49, - 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x54, 0x4c, 0x53, - 0x12, 0x1b, 0x0a, 0x08, 0x6c, 0x65, 0x61, 0x66, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x48, 0x00, 0x52, 0x07, 0x6c, 0x65, 0x61, 0x66, 0x4b, 0x65, 0x79, 0x12, 0x4f, 0x0a, - 0x03, 0x73, 0x64, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, - 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, - 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x53, 0x44, 0x53, 0x43, 0x65, 0x72, 0x74, - 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x48, 0x00, 0x52, 0x03, 0x73, 0x64, 0x73, 0x42, 0x0a, - 0x0a, 0x08, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x22, 0xca, 0x01, 0x0a, 0x12, 0x4f, - 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x54, 0x4c, - 0x53, 0x12, 0x1b, 0x0a, 0x09, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x65, 0x72, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x19, - 0x0a, 0x08, 0x6b, 0x65, 0x79, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x07, 0x6b, 0x65, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x7c, 0x0a, 0x12, 0x76, 0x61, 0x6c, - 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, - 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, - 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, - 0x61, 0x74, 0x65, 0x2e, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x4f, 0x75, 0x74, 0x62, 0x6f, - 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x52, 0x11, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x22, 0x5c, 0x0a, 0x1c, 0x4d, 0x65, 0x73, 0x68, 0x49, - 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x3c, 0x0a, 0x1b, 0x74, 0x72, 0x75, 0x73, 0x74, - 0x5f, 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, - 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x17, 0x74, 0x72, - 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x50, 0x65, 0x65, 0x72, 0x4e, 0x61, 0x6d, - 0x65, 0x4b, 0x65, 0x79, 0x73, 0x22, 0x7a, 0x0a, 0x1d, 0x4d, 0x65, 0x73, 0x68, 0x4f, 0x75, 0x74, - 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, - 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x3a, 0x0a, 0x1a, 0x74, 0x72, 0x75, 0x73, 0x74, 0x5f, - 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x70, 0x65, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, - 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x74, 0x72, 0x75, 0x73, - 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x50, 0x65, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x4b, - 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x70, 0x69, 0x66, 0x66, 0x65, 0x5f, 0x69, 0x64, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x73, 0x70, 0x69, 0x66, 0x66, 0x65, 0x49, 0x64, - 0x73, 0x22, 0x3b, 0x0a, 0x20, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x4f, 0x75, 0x74, 0x62, - 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x63, 0x61, 0x5f, 0x66, 0x69, 0x6c, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x61, 0x46, 0x69, 0x6c, 0x65, 0x22, 0x58, - 0x0a, 0x0e, 0x53, 0x44, 0x53, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, - 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x4e, - 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x65, 0x72, 0x74, 0x5f, 0x72, 0x65, 0x73, 0x6f, - 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x63, 0x65, 0x72, 0x74, - 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0xa5, 0x02, 0x0a, 0x0d, 0x54, 0x4c, 0x53, - 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x58, 0x0a, 0x0b, 0x6d, 0x69, - 0x6e, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x37, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, - 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, - 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x4c, - 0x53, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x6d, 0x69, 0x6e, 0x56, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x58, 0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x37, 0x2e, 0x68, 0x61, 0x73, 0x68, + 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, 0x4f, + 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, 0x6f, + 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x11, 0x76, 0x61, 0x6c, 0x69, 0x64, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x22, 0x5c, 0x0a, 0x1c, 0x4d, + 0x65, 0x73, 0x68, 0x49, 0x6e, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x3c, 0x0a, 0x1b, 0x74, + 0x72, 0x75, 0x73, 0x74, 0x5f, 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x70, 0x65, 0x65, 0x72, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, + 0x52, 0x17, 0x74, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x50, 0x65, 0x65, + 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x4b, 0x65, 0x79, 0x73, 0x22, 0x7a, 0x0a, 0x1d, 0x4d, 0x65, 0x73, + 0x68, 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x3a, 0x0a, 0x1a, 0x74, 0x72, + 0x75, 0x73, 0x74, 0x5f, 0x62, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x5f, 0x70, 0x65, 0x65, 0x72, 0x5f, + 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, + 0x74, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x50, 0x65, 0x65, 0x72, 0x4e, + 0x61, 0x6d, 0x65, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x70, 0x69, 0x66, 0x66, 0x65, + 0x5f, 0x69, 0x64, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x09, 0x73, 0x70, 0x69, 0x66, + 0x66, 0x65, 0x49, 0x64, 0x73, 0x22, 0x3b, 0x0a, 0x20, 0x4e, 0x6f, 0x6e, 0x4d, 0x65, 0x73, 0x68, + 0x4f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x56, 0x61, 0x6c, 0x69, 0x64, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x17, 0x0a, 0x07, 0x63, 0x61, 0x5f, + 0x66, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x61, 0x46, 0x69, + 0x6c, 0x65, 0x22, 0x58, 0x0a, 0x0e, 0x53, 0x44, 0x53, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, + 0x63, 0x61, 0x74, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, + 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x63, 0x6c, 0x75, 0x73, + 0x74, 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x63, 0x65, 0x72, 0x74, 0x5f, + 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, + 0x63, 0x65, 0x72, 0x74, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x22, 0xa5, 0x02, 0x0a, + 0x0d, 0x54, 0x4c, 0x53, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x58, + 0x0a, 0x0b, 0x6d, 0x69, 0x6e, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x37, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, + 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, + 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, + 0x65, 0x2e, 0x54, 0x4c, 0x53, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x6d, 0x69, + 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x58, 0x0a, 0x0b, 0x6d, 0x61, 0x78, 0x5f, + 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x37, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, + 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x4c, 0x53, 0x56, + 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x12, 0x60, 0x0a, 0x0d, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x5f, 0x73, 0x75, 0x69, + 0x74, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, - 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x4c, 0x53, 0x56, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x52, 0x0a, 0x6d, 0x61, 0x78, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x60, - 0x0a, 0x0d, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x5f, 0x73, 0x75, 0x69, 0x74, 0x65, 0x73, 0x18, - 0x03, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, - 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, - 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, - 0x61, 0x74, 0x65, 0x2e, 0x54, 0x4c, 0x53, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, - 0x74, 0x65, 0x52, 0x0c, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x73, - 0x22, 0x37, 0x0a, 0x0f, 0x4c, 0x65, 0x61, 0x66, 0x43, 0x65, 0x72, 0x74, 0x69, 0x66, 0x69, 0x63, - 0x61, 0x74, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x65, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x63, 0x65, 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x46, 0x0a, 0x0b, 0x54, 0x72, 0x75, - 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x72, 0x75, 0x73, - 0x74, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x74, 0x72, 0x75, 0x73, 0x74, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x72, - 0x6f, 0x6f, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x72, 0x6f, 0x6f, 0x74, - 0x73, 0x2a, 0xac, 0x01, 0x0a, 0x0a, 0x54, 0x4c, 0x53, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x14, 0x0a, 0x10, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, - 0x41, 0x55, 0x54, 0x4f, 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, - 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x31, 0x5f, 0x30, 0x10, 0x01, 0x12, 0x13, 0x0a, 0x0f, 0x54, - 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x31, 0x5f, 0x31, 0x10, 0x02, - 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, - 0x31, 0x5f, 0x32, 0x10, 0x03, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, - 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x31, 0x5f, 0x33, 0x10, 0x04, 0x12, 0x17, 0x0a, 0x13, 0x54, 0x4c, - 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x49, 0x4e, 0x56, 0x41, 0x4c, 0x49, - 0x44, 0x10, 0x05, 0x12, 0x1b, 0x0a, 0x17, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, - 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, 0x45, 0x44, 0x10, 0x06, - 0x2a, 0x84, 0x05, 0x0a, 0x0e, 0x54, 0x4c, 0x53, 0x43, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, - 0x69, 0x74, 0x65, 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, - 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x45, 0x43, - 0x44, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, - 0x48, 0x41, 0x32, 0x35, 0x36, 0x10, 0x00, 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x4c, 0x53, 0x5f, 0x43, + 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x4c, 0x53, 0x43, 0x69, 0x70, 0x68, 0x65, + 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x52, 0x0c, 0x63, 0x69, 0x70, 0x68, 0x65, 0x72, 0x53, 0x75, + 0x69, 0x74, 0x65, 0x73, 0x22, 0x37, 0x0a, 0x0f, 0x4c, 0x65, 0x61, 0x66, 0x43, 0x65, 0x72, 0x74, + 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x65, 0x72, 0x74, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x65, 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, + 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x22, 0x46, 0x0a, + 0x0b, 0x54, 0x72, 0x75, 0x73, 0x74, 0x42, 0x75, 0x6e, 0x64, 0x6c, 0x65, 0x12, 0x21, 0x0a, 0x0c, + 0x74, 0x72, 0x75, 0x73, 0x74, 0x5f, 0x64, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x75, 0x73, 0x74, 0x44, 0x6f, 0x6d, 0x61, 0x69, 0x6e, 0x12, + 0x14, 0x0a, 0x05, 0x72, 0x6f, 0x6f, 0x74, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, + 0x72, 0x6f, 0x6f, 0x74, 0x73, 0x2a, 0xac, 0x01, 0x0a, 0x0a, 0x54, 0x4c, 0x53, 0x56, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x10, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, + 0x49, 0x4f, 0x4e, 0x5f, 0x41, 0x55, 0x54, 0x4f, 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x4c, + 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x31, 0x5f, 0x30, 0x10, 0x01, 0x12, + 0x13, 0x0a, 0x0f, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x31, + 0x5f, 0x31, 0x10, 0x02, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, + 0x49, 0x4f, 0x4e, 0x5f, 0x31, 0x5f, 0x32, 0x10, 0x03, 0x12, 0x13, 0x0a, 0x0f, 0x54, 0x4c, 0x53, + 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x31, 0x5f, 0x33, 0x10, 0x04, 0x12, 0x17, + 0x0a, 0x13, 0x54, 0x4c, 0x53, 0x5f, 0x56, 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x49, 0x4e, + 0x56, 0x41, 0x4c, 0x49, 0x44, 0x10, 0x05, 0x12, 0x1b, 0x0a, 0x17, 0x54, 0x4c, 0x53, 0x5f, 0x56, + 0x45, 0x52, 0x53, 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, 0x53, 0x50, 0x45, 0x43, 0x49, 0x46, 0x49, + 0x45, 0x44, 0x10, 0x06, 0x2a, 0x84, 0x05, 0x0a, 0x0e, 0x54, 0x4c, 0x53, 0x43, 0x69, 0x70, 0x68, + 0x65, 0x72, 0x53, 0x75, 0x69, 0x74, 0x65, 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, - 0x45, 0x5f, 0x45, 0x43, 0x44, 0x53, 0x41, 0x5f, 0x43, 0x48, 0x41, 0x43, 0x48, 0x41, 0x32, 0x30, - 0x5f, 0x50, 0x4f, 0x4c, 0x59, 0x31, 0x33, 0x30, 0x35, 0x10, 0x01, 0x12, 0x30, 0x0a, 0x2c, 0x54, + 0x45, 0x5f, 0x45, 0x43, 0x44, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x47, + 0x43, 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36, 0x10, 0x00, 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, - 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, - 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36, 0x10, 0x02, 0x12, 0x30, 0x0a, - 0x2c, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, - 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, 0x43, 0x48, 0x41, 0x43, - 0x48, 0x41, 0x32, 0x30, 0x5f, 0x50, 0x4f, 0x4c, 0x59, 0x31, 0x33, 0x30, 0x35, 0x10, 0x03, 0x12, - 0x2b, 0x0a, 0x27, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, - 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x53, 0x41, 0x5f, - 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x04, 0x12, 0x29, 0x0a, 0x25, - 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, - 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x31, 0x32, - 0x38, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x05, 0x12, 0x26, 0x0a, 0x22, 0x54, 0x4c, 0x53, 0x5f, 0x43, - 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x41, 0x45, 0x53, 0x31, - 0x32, 0x38, 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36, 0x10, 0x06, 0x12, - 0x1f, 0x0a, 0x1b, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, - 0x49, 0x54, 0x45, 0x5f, 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x07, - 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, - 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x53, 0x41, - 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x33, - 0x38, 0x34, 0x10, 0x08, 0x12, 0x30, 0x0a, 0x2c, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, - 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, - 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, 0x48, - 0x41, 0x33, 0x38, 0x34, 0x10, 0x09, 0x12, 0x2b, 0x0a, 0x27, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, - 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, - 0x5f, 0x45, 0x43, 0x44, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x53, 0x48, - 0x41, 0x10, 0x0a, 0x12, 0x29, 0x0a, 0x25, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, - 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, 0x53, - 0x41, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x0b, 0x12, 0x26, - 0x0a, 0x22, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, - 0x54, 0x45, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, 0x48, - 0x41, 0x33, 0x38, 0x34, 0x10, 0x0c, 0x12, 0x1f, 0x0a, 0x1b, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, - 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, - 0x36, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x0d, 0x42, 0xe0, 0x02, 0x0a, 0x2f, 0x63, 0x6f, 0x6d, 0x2e, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, - 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x42, 0x14, 0x54, 0x72, 0x61, - 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x74, - 0x6f, 0x50, 0x01, 0x5a, 0x45, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2d, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x2f, 0x70, 0x62, - 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x70, 0x62, - 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0xa2, 0x02, 0x05, 0x48, 0x43, 0x4d, - 0x56, 0x50, 0xaa, 0x02, 0x2b, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x43, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x2e, 0x56, 0x31, 0x61, 0x6c, 0x70, - 0x68, 0x61, 0x31, 0x2e, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, - 0xca, 0x02, 0x2b, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, - 0x73, 0x75, 0x6c, 0x5c, 0x4d, 0x65, 0x73, 0x68, 0x5c, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, - 0x31, 0x5c, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0xe2, 0x02, - 0x37, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, 0x73, 0x75, - 0x6c, 0x5c, 0x4d, 0x65, 0x73, 0x68, 0x5c, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x5c, - 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5c, 0x47, 0x50, 0x42, - 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x2f, 0x48, 0x61, 0x73, 0x68, 0x69, - 0x63, 0x6f, 0x72, 0x70, 0x3a, 0x3a, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x3a, 0x3a, 0x4d, 0x65, - 0x73, 0x68, 0x3a, 0x3a, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x3a, 0x3a, 0x50, 0x62, - 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x33, + 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x53, 0x41, 0x5f, 0x43, 0x48, 0x41, 0x43, + 0x48, 0x41, 0x32, 0x30, 0x5f, 0x50, 0x4f, 0x4c, 0x59, 0x31, 0x33, 0x30, 0x35, 0x10, 0x01, 0x12, + 0x30, 0x0a, 0x2c, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, + 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, 0x41, 0x45, + 0x53, 0x31, 0x32, 0x38, 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36, 0x10, + 0x02, 0x12, 0x30, 0x0a, 0x2c, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, + 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, + 0x43, 0x48, 0x41, 0x43, 0x48, 0x41, 0x32, 0x30, 0x5f, 0x50, 0x4f, 0x4c, 0x59, 0x31, 0x33, 0x30, + 0x35, 0x10, 0x03, 0x12, 0x2b, 0x0a, 0x27, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, + 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x45, 0x43, + 0x44, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x04, + 0x12, 0x29, 0x0a, 0x25, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, + 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, 0x41, + 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x05, 0x12, 0x26, 0x0a, 0x22, 0x54, + 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, + 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x47, 0x43, 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x32, 0x35, + 0x36, 0x10, 0x06, 0x12, 0x1f, 0x0a, 0x1b, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, + 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x41, 0x45, 0x53, 0x31, 0x32, 0x38, 0x5f, 0x53, + 0x48, 0x41, 0x10, 0x07, 0x12, 0x32, 0x0a, 0x2e, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, + 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, 0x45, 0x5f, 0x45, + 0x43, 0x44, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x47, 0x43, 0x4d, 0x5f, + 0x53, 0x48, 0x41, 0x33, 0x38, 0x34, 0x10, 0x08, 0x12, 0x30, 0x0a, 0x2c, 0x54, 0x4c, 0x53, 0x5f, + 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, + 0x48, 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x47, 0x43, + 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x33, 0x38, 0x34, 0x10, 0x09, 0x12, 0x2b, 0x0a, 0x27, 0x54, 0x4c, + 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, + 0x43, 0x44, 0x48, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, + 0x36, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x0a, 0x12, 0x29, 0x0a, 0x25, 0x54, 0x4c, 0x53, 0x5f, 0x43, + 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x45, 0x43, 0x44, 0x48, + 0x45, 0x5f, 0x52, 0x53, 0x41, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x53, 0x48, 0x41, + 0x10, 0x0b, 0x12, 0x26, 0x0a, 0x22, 0x54, 0x4c, 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, + 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x41, 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x47, 0x43, + 0x4d, 0x5f, 0x53, 0x48, 0x41, 0x33, 0x38, 0x34, 0x10, 0x0c, 0x12, 0x1f, 0x0a, 0x1b, 0x54, 0x4c, + 0x53, 0x5f, 0x43, 0x49, 0x50, 0x48, 0x45, 0x52, 0x5f, 0x53, 0x55, 0x49, 0x54, 0x45, 0x5f, 0x41, + 0x45, 0x53, 0x32, 0x35, 0x36, 0x5f, 0x53, 0x48, 0x41, 0x10, 0x0d, 0x42, 0xe0, 0x02, 0x0a, 0x2f, + 0x63, 0x6f, 0x6d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, + 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x42, + 0x14, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, + 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x45, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, + 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2d, 0x70, 0x75, 0x62, 0x6c, 0x69, + 0x63, 0x2f, 0x70, 0x62, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, + 0x31, 0x2f, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0xa2, 0x02, + 0x05, 0x48, 0x43, 0x4d, 0x56, 0x50, 0xaa, 0x02, 0x2b, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x2e, 0x56, + 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, + 0x74, 0x61, 0x74, 0x65, 0xca, 0x02, 0x2b, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x5c, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x4d, 0x65, 0x73, 0x68, 0x5c, 0x56, 0x31, 0x61, + 0x6c, 0x70, 0x68, 0x61, 0x31, 0x5c, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, + 0x74, 0x65, 0xe2, 0x02, 0x37, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x4d, 0x65, 0x73, 0x68, 0x5c, 0x56, 0x31, 0x61, 0x6c, 0x70, + 0x68, 0x61, 0x31, 0x5c, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x2f, 0x48, + 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x3a, 0x3a, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x3a, 0x3a, 0x4d, 0x65, 0x73, 0x68, 0x3a, 0x3a, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, + 0x3a, 0x3a, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x62, 0x06, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.proto b/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.proto index 4e53326183f9..d5173f59a5e8 100644 --- a/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.proto +++ b/proto-public/pbmesh/v1alpha1/pbproxystate/transport_socket.proto @@ -14,19 +14,21 @@ message TLS { } message TransportSocket { + // name of the transport socket + string name = 1; oneof connection_tls { // inbound_mesh is for incoming connections FROM the mesh. - InboundMeshMTLS inbound_mesh = 1; + InboundMeshMTLS inbound_mesh = 2; // outbound_mesh is for outbound connections TO mesh destinations. - OutboundMeshMTLS outbound_mesh = 2; + OutboundMeshMTLS outbound_mesh = 3; // inbound_non_mesh is for incoming connections FROM non mesh. - InboundNonMeshTLS inbound_non_mesh = 3; + InboundNonMeshTLS inbound_non_mesh = 4; // outbound_non_mesh is for outbound connections TO non mesh destinations. - OutboundNonMeshTLS outbound_non_mesh = 4; + OutboundNonMeshTLS outbound_non_mesh = 5; } // tls_parameters can override any top level tls parameters that are configured. - TLSParameters tls_parameters = 5; - repeated string alpn_protocols = 6; + TLSParameters tls_parameters = 6; + repeated string alpn_protocols = 7; } message InboundMeshMTLS { From 2f4166ece73795dbd675f264a933ec6723a3e435 Mon Sep 17 00:00:00 2001 From: John Murret Date: Wed, 16 Aug 2023 20:29:39 -0600 Subject: [PATCH 03/12] NET-4932 - xds v2 - implement base connect proxy functionality for endpoints --- agent/xds/clusters_test.go | 3 +- agent/xds/endpoints_test.go | 127 +++- agent/xds/naming/naming.go | 2 +- agent/xds/proxystateconverter/clusters.go | 6 +- agent/xds/proxystateconverter/converter.go | 17 +- agent/xds/proxystateconverter/endpoints.go | 586 ++++++++++++++++++ .../proxystateconverter/locality_policy.go | 21 + .../locality_policy_oss.go | 15 + agent/xdsv2/endpoint_resources.go | 45 +- agent/xdsv2/resources.go | 13 +- 10 files changed, 801 insertions(+), 34 deletions(-) create mode 100644 agent/xds/proxystateconverter/locality_policy.go create mode 100644 agent/xds/proxystateconverter/locality_policy_oss.go diff --git a/agent/xds/clusters_test.go b/agent/xds/clusters_test.go index 2651d77c2844..0c7cc323cc56 100644 --- a/agent/xds/clusters_test.go +++ b/agent/xds/clusters_test.go @@ -5,6 +5,7 @@ package xds import ( "bytes" + "github.com/hashicorp/consul/types" "path/filepath" "sort" "testing" @@ -26,7 +27,6 @@ import ( "github.com/hashicorp/consul/agent/xdsv2" "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/sdk/testutil" - "github.com/hashicorp/consul/types" ) type mockCfgFetcher struct { @@ -1138,6 +1138,7 @@ func TestClustersFromSnapshot(t *testing.T) { require.NoError(t, err) clusters = res[xdscommon.ClusterType] + // The order of clusters returned via CDS isn't relevant, so it's safe // to sort these for the purposes of test comparisons. sort.Slice(clusters, func(i, j int) bool { diff --git a/agent/xds/endpoints_test.go b/agent/xds/endpoints_test.go index 3156685934d5..1f0270436f13 100644 --- a/agent/xds/endpoints_test.go +++ b/agent/xds/endpoints_test.go @@ -8,22 +8,20 @@ import ( "sort" "testing" - "github.com/hashicorp/go-hclog" - envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" - - "github.com/hashicorp/consul/agent/xds/testcommon" - - "github.com/mitchellh/copystructure" - testinf "github.com/mitchellh/go-testing-interface" - "github.com/stretchr/testify/require" - "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/proxystateconverter" "github.com/hashicorp/consul/agent/xds/response" + "github.com/hashicorp/consul/agent/xds/testcommon" + "github.com/hashicorp/consul/agent/xdsv2" "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/sdk/testutil" + "github.com/hashicorp/go-hclog" + "github.com/mitchellh/copystructure" + testinf "github.com/mitchellh/go-testing-interface" + "github.com/stretchr/testify/require" ) func Test_makeLoadAssignment(t *testing.T) { @@ -244,6 +242,7 @@ type endpointTestCase struct { name string create func(t testinf.T) *proxycfg.ConfigSnapshot overrideGoldenName string + alsoRunTestForV2 bool } func makeEndpointDiscoChainTests(enterprise bool) []endpointTestCase { @@ -253,72 +252,85 @@ func makeEndpointDiscoChainTests(enterprise bool) []endpointTestCase { create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-chain-external-sni", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "external-sni", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-chain-and-overrides", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple-with-overrides", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, { name: "connect-proxy-with-chain-and-failover", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tcp-chain-failover-through-remote-gateway-triggered", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway-triggered", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tcp-chain-double-failover-through-remote-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-remote-gateway", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tcp-chain-double-failover-through-remote-gateway-triggered", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-remote-gateway-triggered", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tcp-chain-failover-through-local-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tcp-chain-failover-through-local-gateway-triggered", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway-triggered", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tcp-chain-double-failover-through-local-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-local-gateway", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tcp-chain-double-failover-through-local-gateway-triggered", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-double-local-gateway-triggered", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-default-chain-and-custom-cluster", @@ -330,12 +342,16 @@ func makeEndpointDiscoChainTests(enterprise bool) []endpointTestCase { }) }, nil) }, + // TODO(proxystate): requires custom cluster work + alsoRunTestForV2: false, }, { name: "splitter-with-resolver-redirect", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "splitter-with-resolver-redirect-multidc", enterprise, nil, nil) }, + // TODO(proxystate): requires routes work + alsoRunTestForV2: false, }, } } @@ -354,48 +370,64 @@ func TestEndpointsFromSnapshot(t *testing.T) { create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "default", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-using-federation-states", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "federation-states", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-newer-information-in-federation-states", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "newer-info-in-federation-states", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-using-federation-control-plane", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "mesh-gateway-federation", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-older-information-in-federation-states", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "older-info-in-federation-states", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-no-services", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "no-services", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-service-subsets", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "service-subsets2", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "mesh-gateway-default-service-subset", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotMeshGateway(t, "default-service-subsets2", nil, nil) }, + // TODO(proxystate): mesh gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-gateway", @@ -403,12 +435,16 @@ func TestEndpointsFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "default", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-gateway-nil-config-entry", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotIngressGateway_NilConfigEntry(t) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-gateway-no-services", @@ -416,6 +452,8 @@ func TestEndpointsFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, false, "tcp", "default", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-chain", @@ -423,6 +461,8 @@ func TestEndpointsFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "simple", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-chain-external-sni", @@ -430,6 +470,8 @@ func TestEndpointsFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "external-sni", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-chain-and-failover", @@ -437,6 +479,8 @@ func TestEndpointsFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-chain-and-failover-to-cluster-peer", @@ -444,6 +488,8 @@ func TestEndpointsFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-to-cluster-peer", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-failover-through-remote-gateway", @@ -451,6 +497,8 @@ func TestEndpointsFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-remote-gateway", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-failover-through-remote-gateway-triggered", @@ -458,6 +506,8 @@ func TestEndpointsFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-remote-gateway-triggered", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-double-failover-through-remote-gateway", @@ -465,6 +515,8 @@ func TestEndpointsFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-remote-gateway", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-double-failover-through-remote-gateway-triggered", @@ -472,6 +524,8 @@ func TestEndpointsFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-remote-gateway-triggered", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-failover-through-local-gateway", @@ -479,6 +533,8 @@ func TestEndpointsFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-local-gateway", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-failover-through-local-gateway-triggered", @@ -486,6 +542,8 @@ func TestEndpointsFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-local-gateway-triggered", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-double-failover-through-local-gateway", @@ -493,6 +551,8 @@ func TestEndpointsFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-local-gateway", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-tcp-chain-double-failover-through-local-gateway-triggered", @@ -500,6 +560,8 @@ func TestEndpointsFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "failover-through-double-local-gateway-triggered", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-splitter-with-resolver-redirect", @@ -507,30 +569,42 @@ func TestEndpointsFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "http", "splitter-with-resolver-redirect-multidc", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotTerminatingGateway(t, true, nil, nil) }, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-no-services", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotTerminatingGateway(t, false, nil, nil) }, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-service-subsets", create: proxycfg.TestConfigSnapshotTerminatingGatewayServiceSubsets, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-default-service-subset", create: proxycfg.TestConfigSnapshotTerminatingGatewayDefaultServiceSubset, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-multiple-listeners-duplicate-service", create: proxycfg.TestConfigSnapshotIngress_MultipleListenersDuplicateService, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, } @@ -564,7 +638,7 @@ func TestEndpointsFromSnapshot(t *testing.T) { r, err := response.CreateResponse(xdscommon.EndpointType, "00000001", "00000001", endpoints) require.NoError(t, err) - t.Run("current", func(t *testing.T) { + t.Run("current-xdsv1", func(t *testing.T) { gotJSON := protoToJSON(t, r) gName := tt.name @@ -574,6 +648,39 @@ func TestEndpointsFromSnapshot(t *testing.T) { require.JSONEq(t, goldenEnvoy(t, filepath.Join("endpoints", gName), envoyVersion, latestEnvoyVersion, gotJSON), gotJSON) }) + + if tt.alsoRunTestForV2 { + generator := xdsv2.NewResourceGenerator(testutil.Logger(t)) + + converter := proxystateconverter.NewConverter(testutil.Logger(t), &mockCfgFetcher{addressLan: "10.10.10.10"}) + proxyState, err := converter.ProxyStateFromSnapshot(snap) + require.NoError(t, err) + + res, err := generator.AllResourcesFromIR(proxyState) + require.NoError(t, err) + + endpoints = res[xdscommon.EndpointType] + // The order of listeners returned via LDS isn't relevant, so it's safe + // to sort these for the purposes of test comparisons. + sort.Slice(endpoints, func(i, j int) bool { + return endpoints[i].(*envoy_endpoint_v3.ClusterLoadAssignment).ClusterName < endpoints[j].(*envoy_endpoint_v3.ClusterLoadAssignment).ClusterName + }) + + r, err := response.CreateResponse(xdscommon.EndpointType, "00000001", "00000001", endpoints) + require.NoError(t, err) + + t.Run("current-xdsv2", func(t *testing.T) { + gotJSON := protoToJSON(t, r) + + gName := tt.name + if tt.overrideGoldenName != "" { + gName = tt.overrideGoldenName + } + + expectedJSON := goldenEnvoy(t, filepath.Join("endpoints", gName), envoyVersion, latestEnvoyVersion, gotJSON) + require.JSONEq(t, expectedJSON, gotJSON) + }) + } }) } }) diff --git a/agent/xds/naming/naming.go b/agent/xds/naming/naming.go index 3e19d9327003..b1de1ad12862 100644 --- a/agent/xds/naming/naming.go +++ b/agent/xds/naming/naming.go @@ -1,5 +1,5 @@ // Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: BUSL-1.1 +// SPDX-License-Identifier: MPL-2.0 package naming diff --git a/agent/xds/proxystateconverter/clusters.go b/agent/xds/proxystateconverter/clusters.go index 2a43d6eb049c..b659ba622022 100644 --- a/agent/xds/proxystateconverter/clusters.go +++ b/agent/xds/proxystateconverter/clusters.go @@ -6,7 +6,7 @@ package proxystateconverter import ( "errors" "fmt" - envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" + "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-uuid" "strings" @@ -455,6 +455,7 @@ func (s *Converter) makeAppCluster(cfgSnap *proxycfg.ConfigSnapshot, name, pathP }, }, } + protocol := pathProtocol if protocol == "" { protocol = cfg.Protocol @@ -740,6 +741,7 @@ func (s *Converter) createOutboundMeshMTLS(cfgSnap *proxycfg.ConfigSnapshot, spi if err != nil { return nil, err } + // Create the transport socket ts := &pbproxystate.TransportSocket{} @@ -1020,7 +1022,7 @@ func configureClusterWithHostnames( dnsEndpointGroup.Config.DiscoveryType = pbproxystate.DiscoveryType_DISCOVERY_TYPE_STRICT } - endpoints := make([]*envoy_endpoint_v3.LbEndpoint, 0, 1) + endpoints := make([]*pbproxystate.Endpoint, 0, 1) uniqueHostnames := make(map[string]bool) var ( diff --git a/agent/xds/proxystateconverter/converter.go b/agent/xds/proxystateconverter/converter.go index 061a59d6e93e..532f2af826cd 100644 --- a/agent/xds/proxystateconverter/converter.go +++ b/agent/xds/proxystateconverter/converter.go @@ -57,9 +57,20 @@ func (g *Converter) resourcesFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot) erro if err != nil { return err } - //g.routesFromSnapshot(cfgSnap) - g.clustersFromSnapshot(cfgSnap) - //g.endpointsFromSnapshot(cfgSnap) + + err = g.endpointsFromSnapshot(cfgSnap) + if err != nil { + return err + } + err = g.clustersFromSnapshot(cfgSnap) + if err != nil { + return err + } + //err = g.routesFromSnapshot(cfgSnap) + //if err != nil { + // return err + //} + //g.secretsFromSnapshot(cfgSnap) return nil } diff --git a/agent/xds/proxystateconverter/endpoints.go b/agent/xds/proxystateconverter/endpoints.go index 5a5dc3708098..37b484ac1992 100644 --- a/agent/xds/proxystateconverter/endpoints.go +++ b/agent/xds/proxystateconverter/endpoints.go @@ -4,9 +4,17 @@ package proxystateconverter import ( + + "errors" + "fmt" + + "github.com/hashicorp/consul/agent/connect" + "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/api" "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" + "github.com/hashicorp/go-bexpr" "google.golang.org/protobuf/types/known/wrapperspb" ) @@ -24,6 +32,273 @@ func makeLbEndpoint(addr string, port int, health pbproxystate.HealthStatus, wei return ep } +// endpointsFromSnapshot returns the mesh API representation of the "routes" in the snapshot. +func (s *Converter) endpointsFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot) error { + + if cfgSnap == nil { + return errors.New("nil config given") + } + + switch cfgSnap.Kind { + case structs.ServiceKindConnectProxy: + return s.endpointsFromSnapshotConnectProxy(cfgSnap) + //case structs.ServiceKindTerminatingGateway: + // return s.endpointsFromSnapshotTerminatingGateway(cfgSnap) + //case structs.ServiceKindMeshGateway: + // return s.endpointsFromSnapshotMeshGateway(cfgSnap) + //case structs.ServiceKindIngressGateway: + // return s.endpointsFromSnapshotIngressGateway(cfgSnap) + //case structs.ServiceKindAPIGateway: + // return s.endpointsFromSnapshotAPIGateway(cfgSnap) + default: + return fmt.Errorf("Invalid service kind: %v", cfgSnap.Kind) + } +} + +// endpointsFromSnapshotConnectProxy returns the xDS API representation of the "endpoints" +// (upstream instances) in the snapshot. +func (s *Converter) endpointsFromSnapshotConnectProxy(cfgSnap *proxycfg.ConfigSnapshot) error { + eps := make(map[string]*pbproxystate.Endpoints) + + // NOTE: Any time we skip a chain below we MUST also skip that discovery chain in clusters.go + // so that the sets of endpoints generated matches the sets of clusters. + for uid, chain := range cfgSnap.ConnectProxy.DiscoveryChain { + upstream, skip := cfgSnap.ConnectProxy.GetUpstream(uid, &cfgSnap.ProxyID.EnterpriseMeta) + if skip { + // Discovery chain is not associated with a known explicit or implicit upstream so it is skipped. + continue + } + + var upstreamConfigMap map[string]interface{} + if upstream != nil { + upstreamConfigMap = upstream.Config + } + + es, err := s.endpointsFromDiscoveryChain( + uid, + chain, + cfgSnap, + cfgSnap.Locality, + upstreamConfigMap, + cfgSnap.ConnectProxy.WatchedUpstreamEndpoints[uid], + cfgSnap.ConnectProxy.WatchedGatewayEndpoints[uid], + false, + ) + if err != nil { + return err + } + + for clusterName, endpoints := range es { + eps[clusterName] = &pbproxystate.Endpoints{ + Endpoints: endpoints, + } + + } + } + + // NOTE: Any time we skip an upstream below we MUST also skip that same + // upstream in clusters.go so that the sets of endpoints generated matches + // the sets of clusters. + for _, uid := range cfgSnap.ConnectProxy.PeeredUpstreamIDs() { + upstream, skip := cfgSnap.ConnectProxy.GetUpstream(uid, &cfgSnap.ProxyID.EnterpriseMeta) + if skip { + // Discovery chain is not associated with a known explicit or implicit upstream so it is skipped. + continue + } + + tbs, ok := cfgSnap.ConnectProxy.UpstreamPeerTrustBundles.Get(uid.Peer) + if !ok { + // this should never happen since we loop through upstreams with + // set trust bundles + return fmt.Errorf("trust bundle not ready for peer %s", uid.Peer) + } + + clusterName := generatePeeredClusterName(uid, tbs) + + mgwMode := structs.MeshGatewayModeDefault + if upstream != nil { + mgwMode = upstream.MeshGateway.Mode + } + peerServiceEndpoints, err := s.makeEndpointsForPeerService(cfgSnap, uid, mgwMode) + if err != nil { + return err + } + + if peerServiceEndpoints != nil { + pbEndpoints := &pbproxystate.Endpoints{ + Endpoints: peerServiceEndpoints, + } + + eps[clusterName] = pbEndpoints + } + } + + // Looping over explicit upstreams is only needed for prepared queries because they do not have discovery chains + for _, u := range cfgSnap.Proxy.Upstreams { + if u.DestinationType != structs.UpstreamDestTypePreparedQuery { + continue + } + uid := proxycfg.NewUpstreamID(&u) + + dc := u.Datacenter + if dc == "" { + dc = cfgSnap.Datacenter + } + clusterName := connect.UpstreamSNI(&u, "", dc, cfgSnap.Roots.TrustDomain) + + endpoints, ok := cfgSnap.ConnectProxy.PreparedQueryEndpoints[uid] + if ok { + epts := makeEndpointsForLoadAssignment( + cfgSnap, + nil, + []loadAssignmentEndpointGroup{ + {Endpoints: endpoints}, + }, + cfgSnap.Locality, + ) + pbEndpoints := &pbproxystate.Endpoints{ + Endpoints: epts, + } + + eps[clusterName] = pbEndpoints + } + } + + // Loop over potential destinations in the mesh, then grab the gateway nodes associated with each + cfgSnap.ConnectProxy.DestinationsUpstream.ForEachKey(func(uid proxycfg.UpstreamID) bool { + svcConfig, ok := cfgSnap.ConnectProxy.DestinationsUpstream.Get(uid) + if !ok || svcConfig.Destination == nil { + return true + } + + for _, address := range svcConfig.Destination.Addresses { + clusterName := clusterNameForDestination(cfgSnap, uid.Name, address, uid.NamespaceOrDefault(), uid.PartitionOrDefault()) + + endpoints, ok := cfgSnap.ConnectProxy.DestinationGateways.Get(uid) + if ok { + epts := makeEndpointsForLoadAssignment( + cfgSnap, + nil, + []loadAssignmentEndpointGroup{ + {Endpoints: endpoints}, + }, + proxycfg.GatewayKey{ /*empty so it never matches*/ }, + ) + pbEndpoints := &pbproxystate.Endpoints{ + Endpoints: epts, + } + eps[clusterName] = pbEndpoints + } + } + + return true + }) + + s.proxyState.Endpoints = eps + return nil +} + +func (s *Converter) makeEndpointsForPeerService( + cfgSnap *proxycfg.ConfigSnapshot, + uid proxycfg.UpstreamID, + upstreamGatewayMode structs.MeshGatewayMode, +) ([]*pbproxystate.Endpoint, error) { + var eps []*pbproxystate.Endpoint + + upstreamsSnapshot, err := cfgSnap.ToConfigSnapshotUpstreams() + if err != nil { + return eps, err + } + + if upstreamGatewayMode == structs.MeshGatewayModeNone { + s.Logger.Warn(fmt.Sprintf("invalid mesh gateway mode 'none', defaulting to 'remote' for %q", uid)) + } + + // If an upstream is configured with local mesh gw mode, we make a load assignment + // from the gateway endpoints instead of those of the upstreams. + if upstreamGatewayMode == structs.MeshGatewayModeLocal { + localGw, ok := cfgSnap.ConnectProxy.WatchedLocalGWEndpoints.Get(cfgSnap.Locality.String()) + if !ok { + // local GW is not ready; return early + return eps, nil + } + eps = makeEndpointsForLoadAssignment( + cfgSnap, + nil, + []loadAssignmentEndpointGroup{ + {Endpoints: localGw}, + }, + cfgSnap.Locality, + ) + return eps, nil + } + + // Also skip peer instances with a hostname as their address. EDS + // cannot resolve hostnames, so we provide them through CDS instead. + if _, ok := upstreamsSnapshot.PeerUpstreamEndpointsUseHostnames[uid]; ok { + return eps, nil + } + + endpoints, ok := upstreamsSnapshot.PeerUpstreamEndpoints.Get(uid) + if !ok { + return nil, nil + } + eps = makeEndpointsForLoadAssignment( + cfgSnap, + nil, + []loadAssignmentEndpointGroup{ + {Endpoints: endpoints}, + }, + proxycfg.GatewayKey{ /*empty so it never matches*/ }, + ) + return eps, nil +} + +func (s *Converter) filterSubsetEndpoints(subset *structs.ServiceResolverSubset, endpoints structs.CheckServiceNodes) (structs.CheckServiceNodes, error) { + // locally execute the subsets filter + if subset.Filter != "" { + filter, err := bexpr.CreateFilter(subset.Filter, nil, endpoints) + if err != nil { + return nil, err + } + + raw, err := filter.Execute(endpoints) + if err != nil { + return nil, err + } + return raw.(structs.CheckServiceNodes), nil + } + return endpoints, nil +} + +// TODO(proxystate): Terminating Gateway will be added in the future. +// Functions to add from agent/xds/endpoints.go: +// func endpointsFromSnapshotTerminatingGateway + +// TODO(proxystate): Mesh Gateway will be added in the future. +// Functions to add from agent/xds/endpoints.go: +// func endpointsFromSnapshotMeshGateway + +// TODO(proxystate): Cluster Peering will be added in the future. +// Functions to add from agent/xds/endpoints.go: +// func makeEndpointsForOutgoingPeeredServices + +// TODO(proxystate): Mesh Gateway will be added in the future. +// Functions to add from agent/xds/endpoints.go: +// func endpointsFromServicesAndResolvers + +// TODO(proxystate): Mesh Gateway will be added in the future. +// Functions to add from agent/xds/endpoints.go: +// func makePeerServerEndpointsForMeshGateway + +// TODO(proxystate): Ingress Gateway will be added in the future. +// Functions to add from agent/xds/endpoints.go: +// func endpointsFromSnapshotIngressGateway + +// TODO(proxystate): API Gateway will be added in the future. +// Functions to add from agent/xds/endpoints.go: +// func endpointsFromSnapshotAPIGateway + // used in clusters.go func makeHostPortEndpoint(host string, port int) *pbproxystate.Endpoint { return &pbproxystate.Endpoint{ @@ -50,6 +325,317 @@ func makeUnixSocketEndpoint(path string) *pbproxystate.Endpoint { } } +func (s *Converter) makeUpstreamLoadAssignmentEndpointForPeerService( + cfgSnap *proxycfg.ConfigSnapshot, + uid proxycfg.UpstreamID, + upstreamGatewayMode structs.MeshGatewayMode, +) ([]*pbproxystate.Endpoint, error) { + var eps []*pbproxystate.Endpoint + + upstreamsSnapshot, err := cfgSnap.ToConfigSnapshotUpstreams() + if err != nil { + return eps, err + } + + if upstreamGatewayMode == structs.MeshGatewayModeNone { + s.Logger.Warn(fmt.Sprintf("invalid mesh gateway mode 'none', defaulting to 'remote' for %q", uid)) + } + + // If an upstream is configured with local mesh gw mode, we make a load assignment + // from the gateway endpoints instead of those of the upstreams. + if upstreamGatewayMode == structs.MeshGatewayModeLocal { + localGw, ok := cfgSnap.ConnectProxy.WatchedLocalGWEndpoints.Get(cfgSnap.Locality.String()) + if !ok { + // local GW is not ready; return early + return eps, nil + } + eps = makeEndpointsForLoadAssignment( + cfgSnap, + nil, + []loadAssignmentEndpointGroup{ + {Endpoints: localGw}, + }, + cfgSnap.Locality, + ) + return eps, nil + } + + // Also skip peer instances with a hostname as their address. EDS + // cannot resolve hostnames, so we provide them through CDS instead. + if _, ok := upstreamsSnapshot.PeerUpstreamEndpointsUseHostnames[uid]; ok { + return eps, nil + } + + endpoints, ok := upstreamsSnapshot.PeerUpstreamEndpoints.Get(uid) + if !ok { + return nil, nil + } + eps = makeEndpointsForLoadAssignment( + cfgSnap, + nil, + []loadAssignmentEndpointGroup{ + {Endpoints: endpoints}, + }, + proxycfg.GatewayKey{ /*empty so it never matches*/ }, + ) + return eps, nil +} + +func (s *Converter) endpointsFromDiscoveryChain( + uid proxycfg.UpstreamID, + chain *structs.CompiledDiscoveryChain, + cfgSnap *proxycfg.ConfigSnapshot, + gatewayKey proxycfg.GatewayKey, + upstreamConfigMap map[string]interface{}, + upstreamEndpoints map[string]structs.CheckServiceNodes, + gatewayEndpoints map[string]structs.CheckServiceNodes, + forMeshGateway bool, +) (map[string][]*pbproxystate.Endpoint, error) { + if chain == nil { + if forMeshGateway { + return nil, fmt.Errorf("missing discovery chain for %s", uid) + } + return nil, nil + } + + if upstreamConfigMap == nil { + upstreamConfigMap = make(map[string]interface{}) // TODO:needed? + } + + clusterEndpoints := make(map[string][]*pbproxystate.Endpoint) + + // TODO(jm): escape hatches will be implemented in the future + //var escapeHatchCluster *pbproxystate.Cluster + //if !forMeshGateway { + + //cfg, err := structs.ParseUpstreamConfigNoDefaults(upstreamConfigMap) + //if err != nil { + // // Don't hard fail on a config typo, just warn. The parse func returns + // // default config if there is an error so it's safe to continue. + // s.Logger.Warn("failed to parse", "upstream", uid, + // "error", err) + //} + + //if cfg.EnvoyClusterJSON != "" { + // if chain.Default { + // // If you haven't done anything to setup the discovery chain, then + // // you can use the envoy_cluster_json escape hatch. + // escapeHatchCluster, err = makeClusterFromUserConfig(cfg.EnvoyClusterJSON) + // if err != nil { + // return ce, nil + // } + // } else { + // s.Logger.Warn("ignoring escape hatch setting, because a discovery chain is configued for", + // "discovery chain", chain.ServiceName, "upstream", uid, + // "envoy_cluster_json", chain.ServiceName) + // } + //} + //} + + mgwMode := structs.MeshGatewayModeDefault + if upstream, _ := cfgSnap.ConnectProxy.GetUpstream(uid, &cfgSnap.ProxyID.EnterpriseMeta); upstream != nil { + mgwMode = upstream.MeshGateway.Mode + } + + // Find all resolver nodes. + for _, node := range chain.Nodes { + switch { + case node == nil: + return nil, fmt.Errorf("impossible to process a nil node") + case node.Type != structs.DiscoveryGraphNodeTypeResolver: + continue + case node.Resolver == nil: + return nil, fmt.Errorf("impossible to process a non-resolver node") + } + rawUpstreamConfig, err := structs.ParseUpstreamConfigNoDefaults(upstreamConfigMap) + if err != nil { + return nil, err + } + upstreamConfig := finalizeUpstreamConfig(rawUpstreamConfig, chain, node.Resolver.ConnectTimeout) + + mappedTargets, err := s.mapDiscoChainTargets(cfgSnap, chain, node, upstreamConfig, forMeshGateway) + if err != nil { + return nil, err + } + + targetGroups, err := mappedTargets.groupedTargets() + if err != nil { + return nil, err + } + + for _, groupedTarget := range targetGroups { + clusterName := groupedTarget.ClusterName + // TODO(jm): escape hatches will be implemented in the future + //if escapeHatchCluster != nil { + // clusterName = escapeHatchCluster.Name + //} + switch len(groupedTarget.Targets) { + case 0: + continue + case 1: + // We expect one target so this passes through to continue setting the load assignment up. + default: + return nil, fmt.Errorf("cannot have more than one target") + } + ti := groupedTarget.Targets[0] + s.Logger.Debug("generating endpoints for", "cluster", clusterName, "targetID", ti.TargetID) + targetUID := proxycfg.NewUpstreamIDFromTargetID(ti.TargetID) + if targetUID.Peer != "" { + peerServiceEndpoints, err := s.makeEndpointsForPeerService(cfgSnap, targetUID, mgwMode) + if err != nil { + return nil, err + } + if peerServiceEndpoints != nil { + clusterEndpoints[clusterName] = peerServiceEndpoints + } + continue + } + + endpointGroup, valid := makeLoadAssignmentEndpointGroup( + chain.Targets, + upstreamEndpoints, + gatewayEndpoints, + ti.TargetID, + gatewayKey, + forMeshGateway, + ) + if !valid { + continue // skip the cluster if we're still populating the snapshot + } + + epts := makeEndpointsForLoadAssignment( + cfgSnap, + ti.PrioritizeByLocality, + []loadAssignmentEndpointGroup{endpointGroup}, + gatewayKey, + ) + clusterEndpoints[clusterName] = epts + } + } + + return clusterEndpoints, nil +} + +// TODO(proxystate): Mesh Gateway will be added in the future. +// Functions to add from agent/xds/endpoints.go: +// func makeExportedUpstreamEndpointsForMeshGateway + +type loadAssignmentEndpointGroup struct { + Endpoints structs.CheckServiceNodes + OnlyPassing bool + OverrideHealth pbproxystate.HealthStatus +} + +func makeEndpointsForLoadAssignment(cfgSnap *proxycfg.ConfigSnapshot, + policy *structs.DiscoveryPrioritizeByLocality, + endpointGroups []loadAssignmentEndpointGroup, + localKey proxycfg.GatewayKey) []*pbproxystate.Endpoint { + pbEndpoints := make([]*pbproxystate.Endpoint, 0, len(endpointGroups)) + + // TODO(jm): make this work in xdsv2 + //if len(endpointGroups) > 1 { + // cla.Policy = &envoy_endpoint_v3.ClusterLoadAssignment_Policy{ + // // We choose such a large value here that the failover math should + // // in effect not happen until zero instances are healthy. + // OverprovisioningFactor: response.MakeUint32Value(100000), + // } + //} + + var priority uint32 + + for _, endpointGroup := range endpointGroups { + endpointsByLocality, err := groupedEndpoints(cfgSnap.ServiceLocality, policy, endpointGroup.Endpoints) + + if err != nil { + continue + } + + for _, endpoints := range endpointsByLocality { + for _, ep := range endpoints { + // TODO (mesh-gateway) - should we respect the translate_wan_addrs configuration here or just always use the wan for cross-dc? + _, addr, port := ep.BestAddress(!localKey.Matches(ep.Node.Datacenter, ep.Node.PartitionOrDefault())) + healthStatus, weight := calculateEndpointHealthAndWeight(ep, endpointGroup.OnlyPassing) + + if endpointGroup.OverrideHealth != pbproxystate.HealthStatus_HEALTH_STATUS_UNKNOWN { + healthStatus = endpointGroup.OverrideHealth + } + + endpoint := makeHostPortEndpoint(addr, port) + endpoint.HealthStatus = healthStatus + endpoint.LoadBalancingWeight = response.MakeUint32Value(weight) + + pbEndpoints = append(pbEndpoints, endpoint) + } + + // TODO(jm): what do we do about priority downstream? + //cla.Endpoints = append(cla.Endpoints, &envoy_endpoint_v3.LocalityLbEndpoints{ + // Priority: priority, + // LbEndpoints: es, + //}) + + priority++ + } + } + + return pbEndpoints +} + +func makeLoadAssignmentEndpointGroup( + targets map[string]*structs.DiscoveryTarget, + targetHealth map[string]structs.CheckServiceNodes, + gatewayHealth map[string]structs.CheckServiceNodes, + targetID string, + localKey proxycfg.GatewayKey, + forMeshGateway bool, +) (loadAssignmentEndpointGroup, bool) { + realEndpoints, ok := targetHealth[targetID] + if !ok { + // skip the cluster if we're still populating the snapshot + return loadAssignmentEndpointGroup{}, false + } + target := targets[targetID] + + var gatewayKey proxycfg.GatewayKey + + switch target.MeshGateway.Mode { + case structs.MeshGatewayModeRemote: + gatewayKey.Datacenter = target.Datacenter + gatewayKey.Partition = target.Partition + case structs.MeshGatewayModeLocal: + gatewayKey = localKey + } + + if forMeshGateway || gatewayKey.IsEmpty() || localKey.Matches(target.Datacenter, target.Partition) { + // Gateways are not needed if the request isn't for a remote DC or partition. + return loadAssignmentEndpointGroup{ + Endpoints: realEndpoints, + OnlyPassing: target.Subset.OnlyPassing, + }, true + } + + // If using a mesh gateway we need to pull those endpoints instead. + gatewayEndpoints, ok := gatewayHealth[gatewayKey.String()] + if !ok { + // skip the cluster if we're still populating the snapshot + return loadAssignmentEndpointGroup{}, false + } + + // But we will use the health from the actual backend service. + overallHealth := pbproxystate.HealthStatus_HEALTH_STATUS_UNHEALTHY + for _, ep := range realEndpoints { + health, _ := calculateEndpointHealthAndWeight(ep, target.Subset.OnlyPassing) + if health == pbproxystate.HealthStatus_HEALTH_STATUS_HEALTHY { + overallHealth = pbproxystate.HealthStatus_HEALTH_STATUS_HEALTHY + break + } + } + + return loadAssignmentEndpointGroup{ + Endpoints: gatewayEndpoints, + OverrideHealth: overallHealth, + }, true +} + func calculateEndpointHealthAndWeight( ep structs.CheckServiceNode, onlyPassing bool, diff --git a/agent/xds/proxystateconverter/locality_policy.go b/agent/xds/proxystateconverter/locality_policy.go new file mode 100644 index 000000000000..db62e63f5f95 --- /dev/null +++ b/agent/xds/proxystateconverter/locality_policy.go @@ -0,0 +1,21 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package proxystateconverter + +import ( + "fmt" + + "github.com/hashicorp/consul/agent/structs" +) + +func groupedEndpoints(locality *structs.Locality, policy *structs.DiscoveryPrioritizeByLocality, csns structs.CheckServiceNodes) ([]structs.CheckServiceNodes, error) { + switch { + case policy == nil || policy.Mode == "" || policy.Mode == "none": + return []structs.CheckServiceNodes{csns}, nil + case policy.Mode == "failover": + return prioritizeByLocalityFailover(locality, csns), nil + default: + return nil, fmt.Errorf("unexpected priortize-by-locality mode %q", policy.Mode) + } +} diff --git a/agent/xds/proxystateconverter/locality_policy_oss.go b/agent/xds/proxystateconverter/locality_policy_oss.go new file mode 100644 index 000000000000..007501ffab7b --- /dev/null +++ b/agent/xds/proxystateconverter/locality_policy_oss.go @@ -0,0 +1,15 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +//go:build !consulent +// +build !consulent + +package proxystateconverter + +import ( + "github.com/hashicorp/consul/agent/structs" +) + +func prioritizeByLocalityFailover(locality *structs.Locality, csns structs.CheckServiceNodes) []structs.CheckServiceNodes { + return nil +} diff --git a/agent/xdsv2/endpoint_resources.go b/agent/xdsv2/endpoint_resources.go index f5fd1b9c5d09..c493c48f640d 100644 --- a/agent/xdsv2/endpoint_resources.go +++ b/agent/xdsv2/endpoint_resources.go @@ -7,35 +7,56 @@ import ( envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" "github.com/hashicorp/consul/agent/xds/response" + "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" + "google.golang.org/protobuf/proto" ) -func makeEnvoyEndpoint(endpoint *pbproxystate.Endpoint) *envoy_endpoint_v3.LbEndpoint { +func makeEnvoyLbEndpoint(endpoint *pbproxystate.Endpoint) *envoy_endpoint_v3.LbEndpoint { + hs := int32(endpoint.GetHealthStatus().Number()) + return &envoy_endpoint_v3.LbEndpoint{ + HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ + Endpoint: makeEnvoyEndpoint(endpoint), + }, + HealthStatus: envoy_core_v3.HealthStatus(hs), + LoadBalancingWeight: endpoint.GetLoadBalancingWeight(), + } +} + +func makeEnvoyEndpoint(endpoint *pbproxystate.Endpoint) *envoy_endpoint_v3.Endpoint { var address *envoy_core_v3.Address if endpoint.GetUnixSocket() != nil { address = response.MakePipeAddress(endpoint.GetUnixSocket().GetPath(), 0) } else { address = response.MakeAddress(endpoint.GetHostPort().GetHost(), int(endpoint.GetHostPort().Port)) } - return &envoy_endpoint_v3.LbEndpoint{ - HostIdentifier: &envoy_endpoint_v3.LbEndpoint_Endpoint{ - Endpoint: &envoy_endpoint_v3.Endpoint{ - Address: address, - }, - }, + + return &envoy_endpoint_v3.Endpoint{ + Address: address, } } func makeEnvoyClusterLoadAssignment(clusterName string, endpoints []*pbproxystate.Endpoint) *envoy_endpoint_v3.ClusterLoadAssignment { - localityLbEndpoints := make([]*envoy_endpoint_v3.LocalityLbEndpoints, 0, len(endpoints)) + localityLbEndpoints := &envoy_endpoint_v3.LocalityLbEndpoints{} for _, endpoint := range endpoints { - localityLbEndpoints = append(localityLbEndpoints, &envoy_endpoint_v3.LocalityLbEndpoints{ - LbEndpoints: []*envoy_endpoint_v3.LbEndpoint{makeEnvoyEndpoint(endpoint)}, - }) + localityLbEndpoints.LbEndpoints = append(localityLbEndpoints.LbEndpoints, makeEnvoyLbEndpoint(endpoint)) } return &envoy_endpoint_v3.ClusterLoadAssignment{ ClusterName: clusterName, - Endpoints: localityLbEndpoints, + Endpoints: []*envoy_endpoint_v3.LocalityLbEndpoints{localityLbEndpoints}, + } +} + +func (pr *ProxyResources) makeXDSEndpoints() ([]proto.Message, error) { + endpoints := make([]proto.Message, 0) + + for clusterName, eps := range pr.proxyState.GetEndpoints() { + // TODO(jm): this does not seem like the best way. + if clusterName != xdscommon.LocalAppClusterName { + protoEndpoint := makeEnvoyClusterLoadAssignment(clusterName, eps.Endpoints) + endpoints = append(endpoints, protoEndpoint) + } } + return endpoints, nil } diff --git a/agent/xdsv2/resources.go b/agent/xdsv2/resources.go index 6082f7dd56b5..cba6e8188c80 100644 --- a/agent/xdsv2/resources.go +++ b/agent/xdsv2/resources.go @@ -5,7 +5,6 @@ package xdsv2 import ( "fmt" - pbmesh "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1" "github.com/hashicorp/go-hclog" "google.golang.org/protobuf/proto" @@ -48,7 +47,6 @@ func (g *ResourceGenerator) AllResourcesFromIR(proxyState *pbmesh.ProxyState) (m func (pr *ProxyResources) generateXDSResources() error { listeners := make([]proto.Message, 0) routes := make([]proto.Message, 0) - endpoints := make([]proto.Message, 0) for _, l := range pr.proxyState.Listeners { protoListener, err := pr.makeListener(l) @@ -58,16 +56,21 @@ func (pr *ProxyResources) generateXDSResources() error { } listeners = append(listeners, protoListener) } + pr.envoyResources[xdscommon.ListenerType] = listeners clusters, err := pr.makeXDSClusters() if err != nil { return err } - - pr.envoyResources[xdscommon.ListenerType] = listeners pr.envoyResources[xdscommon.ClusterType] = clusters - pr.envoyResources[xdscommon.RouteType] = routes + + endpoints, err := pr.makeXDSEndpoints() + if err != nil { + return err + } pr.envoyResources[xdscommon.EndpointType] = endpoints + pr.envoyResources[xdscommon.RouteType] = routes + return nil } From a0eaa921549eec8b3a91e75e2cfcfdf8a9dd8c7f Mon Sep 17 00:00:00 2001 From: John Murret Date: Wed, 16 Aug 2023 20:36:52 -0600 Subject: [PATCH 04/12] Update endpoints_test.go --- agent/xds/endpoints_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/agent/xds/endpoints_test.go b/agent/xds/endpoints_test.go index a107e906b0b7..1f0270436f13 100644 --- a/agent/xds/endpoints_test.go +++ b/agent/xds/endpoints_test.go @@ -8,8 +8,6 @@ import ( "sort" "testing" - "github.com/hashicorp/go-hclog" - envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" "github.com/hashicorp/consul/agent/proxycfg" From e8aca7cf821a37ae0d55ede7f955ee859ee419dd Mon Sep 17 00:00:00 2001 From: John Murret Date: Wed, 16 Aug 2023 20:40:33 -0600 Subject: [PATCH 05/12] gofmt --- agent/xds/proxystateconverter/endpoints.go | 1 - 1 file changed, 1 deletion(-) diff --git a/agent/xds/proxystateconverter/endpoints.go b/agent/xds/proxystateconverter/endpoints.go index b465e4ff27a8..28156a9b034f 100644 --- a/agent/xds/proxystateconverter/endpoints.go +++ b/agent/xds/proxystateconverter/endpoints.go @@ -4,7 +4,6 @@ package proxystateconverter import ( - "errors" "fmt" From 978133d866fdd094a220874086f32fdd8b4be5d3 Mon Sep 17 00:00:00 2001 From: John Murret Date: Wed, 16 Aug 2023 20:45:06 -0600 Subject: [PATCH 06/12] NET-4858 - Make connect proxy route tests pass using xds v2 --- agent/xds/clusters_test.go | 11 +- agent/xds/endpoints_test.go | 6 +- agent/xds/listeners_test.go | 56 +- agent/xds/proxystateconverter/clusters.go | 3 +- agent/xds/proxystateconverter/converter.go | 9 +- agent/xds/proxystateconverter/routes.go | 777 ++++++++++++++++++ agent/xds/routes_test.go | 83 +- agent/xdsv2/cluster_resources.go | 29 + agent/xdsv2/listener_resources.go | 16 +- agent/xdsv2/resources.go | 17 +- agent/xdsv2/route_resources.go | 504 +++++++++++- .../v1alpha1/pbproxystate/cluster.pb.go | 754 ++++++++--------- .../v1alpha1/pbproxystate/cluster.proto | 12 +- .../pbmesh/v1alpha1/proxy_state.pb.go | 2 +- .../pbmesh/v1alpha1/proxy_state.proto | 2 +- 15 files changed, 1845 insertions(+), 436 deletions(-) create mode 100644 agent/xds/proxystateconverter/routes.go diff --git a/agent/xds/clusters_test.go b/agent/xds/clusters_test.go index 0c7cc323cc56..60635805fd9e 100644 --- a/agent/xds/clusters_test.go +++ b/agent/xds/clusters_test.go @@ -5,7 +5,6 @@ package xds import ( "bytes" - "github.com/hashicorp/consul/types" "path/filepath" "sort" "testing" @@ -27,6 +26,7 @@ import ( "github.com/hashicorp/consul/agent/xdsv2" "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/sdk/testutil" + "github.com/hashicorp/consul/types" ) type mockCfgFetcher struct { @@ -96,8 +96,7 @@ func makeClusterDiscoChainTests(enterprise bool) []clusterTestCase { create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple-with-overrides", enterprise, nil, nil) }, - // TODO(proxystate): requires routes work - alsoRunTestForV2: false, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-chain-and-failover", @@ -176,16 +175,14 @@ func makeClusterDiscoChainTests(enterprise bool) []clusterTestCase { create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "splitter-with-resolver-redirect-multidc", enterprise, nil, nil) }, - // TODO(proxystate): requires routes work - alsoRunTestForV2: false, + alsoRunTestForV2: true, }, { name: "connect-proxy-lb-in-resolver", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "lb-resolver", enterprise, nil, nil) }, - // TODO(proxystate): requires routes work - alsoRunTestForV2: false, + alsoRunTestForV2: true, }, } } diff --git a/agent/xds/endpoints_test.go b/agent/xds/endpoints_test.go index 1f0270436f13..9daf5b31f55b 100644 --- a/agent/xds/endpoints_test.go +++ b/agent/xds/endpoints_test.go @@ -266,8 +266,7 @@ func makeEndpointDiscoChainTests(enterprise bool) []endpointTestCase { create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple-with-overrides", enterprise, nil, nil) }, - // TODO(proxystate): requires routes work - alsoRunTestForV2: false, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-chain-and-failover", @@ -350,8 +349,7 @@ func makeEndpointDiscoChainTests(enterprise bool) []endpointTestCase { create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "splitter-with-resolver-redirect-multidc", enterprise, nil, nil) }, - // TODO(proxystate): requires routes work - alsoRunTestForV2: false, + alsoRunTestForV2: true, }, } } diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go index 130f2234517e..c1f98903c5de 100644 --- a/agent/xds/listeners_test.go +++ b/agent/xds/listeners_test.go @@ -37,7 +37,7 @@ type listenerTestCase struct { // test input. overrideGoldenName string generatorSetup func(*ResourceGenerator) - alsoRunTestForV2 bool + useV2 bool } func makeListenerDiscoChainTests(enterprise bool) []listenerTestCase { @@ -75,7 +75,7 @@ func makeListenerDiscoChainTests(enterprise bool) []listenerTestCase { create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", enterprise, nil, nil) }, - alsoRunTestForV2: true, + useV2: true, }, { name: "connect-proxy-with-http-chain", @@ -124,7 +124,7 @@ func makeListenerDiscoChainTests(enterprise bool) []listenerTestCase { create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "external-sni", enterprise, nil, nil) }, - alsoRunTestForV2: true, + useV2: true, }, { name: "connect-proxy-with-chain-and-overrides", @@ -137,14 +137,14 @@ func makeListenerDiscoChainTests(enterprise bool) []listenerTestCase { create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway", enterprise, nil, nil) }, - alsoRunTestForV2: true, + useV2: true, }, { name: "connect-proxy-with-tcp-chain-failover-through-local-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway", enterprise, nil, nil) }, - alsoRunTestForV2: true, + useV2: true, }, { name: "connect-proxy-with-jwt-config-entry-with-local", @@ -235,7 +235,7 @@ func TestListenersFromSnapshot(t *testing.T) { }, }) }, - alsoRunTestForV2: true, + useV2: true, }, { name: "connect-proxy-with-tls-incoming-min-version", @@ -255,7 +255,7 @@ func TestListenersFromSnapshot(t *testing.T) { }, }) }, - alsoRunTestForV2: true, + useV2: true, }, { name: "connect-proxy-with-tls-incoming-max-version", @@ -275,7 +275,7 @@ func TestListenersFromSnapshot(t *testing.T) { }, }) }, - alsoRunTestForV2: true, + useV2: true, }, { name: "connect-proxy-with-tls-incoming-cipher-suites", @@ -298,7 +298,7 @@ func TestListenersFromSnapshot(t *testing.T) { }, }) }, - alsoRunTestForV2: true, + useV2: true, }, { name: "grpc-public-listener", @@ -315,7 +315,7 @@ func TestListenersFromSnapshot(t *testing.T) { ns.Proxy.Config["bind_address"] = "127.0.0.2" }, nil) }, - alsoRunTestForV2: true, + useV2: true, }, { name: "listener-bind-port", @@ -324,7 +324,7 @@ func TestListenersFromSnapshot(t *testing.T) { ns.Proxy.Config["bind_port"] = 8888 }, nil) }, - alsoRunTestForV2: true, + useV2: true, }, { name: "listener-bind-address-port", @@ -334,7 +334,7 @@ func TestListenersFromSnapshot(t *testing.T) { ns.Proxy.Config["bind_port"] = 8888 }, nil) }, - alsoRunTestForV2: true, + useV2: true, }, { name: "listener-unix-domain-socket", @@ -346,7 +346,7 @@ func TestListenersFromSnapshot(t *testing.T) { ns.Proxy.Upstreams[0].LocalBindSocketMode = "0640" }, nil) }, - alsoRunTestForV2: true, + useV2: true, }, { name: "listener-max-inbound-connections", @@ -355,7 +355,7 @@ func TestListenersFromSnapshot(t *testing.T) { ns.Proxy.Config["max_inbound_connections"] = 222 }, nil) }, - alsoRunTestForV2: true, + useV2: true, }, { name: "http2-public-listener", @@ -372,7 +372,7 @@ func TestListenersFromSnapshot(t *testing.T) { ns.Proxy.Config["balance_inbound_connections"] = "exact_balance" }, nil) }, - alsoRunTestForV2: true, + useV2: true, }, { name: "listener-balance-outbound-connections-bind-port", @@ -381,7 +381,7 @@ func TestListenersFromSnapshot(t *testing.T) { ns.Proxy.Upstreams[0].Config["balance_outbound_connections"] = "exact_balance" }, nil) }, - alsoRunTestForV2: true, + useV2: true, }, { name: "http-public-listener", @@ -1190,19 +1190,19 @@ func TestListenersFromSnapshot(t *testing.T) { create: proxycfg.TestConfigSnapshotTransparentProxyResolverRedirectUpstream, }, { - name: "transparent-proxy-catalog-destinations-only", - create: proxycfg.TestConfigSnapshotTransparentProxyCatalogDestinationsOnly, - alsoRunTestForV2: true, + name: "transparent-proxy-catalog-destinations-only", + create: proxycfg.TestConfigSnapshotTransparentProxyCatalogDestinationsOnly, + useV2: true, }, { - name: "transparent-proxy-dial-instances-directly", - create: proxycfg.TestConfigSnapshotTransparentProxyDialDirectly, - alsoRunTestForV2: true, + name: "transparent-proxy-dial-instances-directly", + create: proxycfg.TestConfigSnapshotTransparentProxyDialDirectly, + useV2: true, }, { - name: "transparent-proxy-terminating-gateway", - create: proxycfg.TestConfigSnapshotTransparentProxyTerminatingGatewayCatalogDestinationsOnly, - alsoRunTestForV2: true, + name: "transparent-proxy-terminating-gateway", + create: proxycfg.TestConfigSnapshotTransparentProxyTerminatingGatewayCatalogDestinationsOnly, + useV2: true, }, { name: "custom-trace-listener", @@ -1265,7 +1265,7 @@ func TestListenersFromSnapshot(t *testing.T) { }, nil) }, - alsoRunTestForV2: true, + useV2: true, }, { name: "connect-proxy-without-tproxy-and-permissive-mtls", @@ -1275,7 +1275,7 @@ func TestListenersFromSnapshot(t *testing.T) { }, nil) }, - alsoRunTestForV2: true, + useV2: true, }, } @@ -1331,7 +1331,7 @@ func TestListenersFromSnapshot(t *testing.T) { require.JSONEq(t, expectedJSON, gotJSON) }) - if tt.alsoRunTestForV2 { + if tt.useV2 { generator := xdsv2.NewResourceGenerator(testutil.Logger(t)) converter := proxystateconverter.NewConverter(testutil.Logger(t), nil) proxyState, err := converter.ProxyStateFromSnapshot(snap) diff --git a/agent/xds/proxystateconverter/clusters.go b/agent/xds/proxystateconverter/clusters.go index 2e640b15121d..e12211f67078 100644 --- a/agent/xds/proxystateconverter/clusters.go +++ b/agent/xds/proxystateconverter/clusters.go @@ -11,6 +11,7 @@ import ( "strings" "time" + envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/wrapperspb" @@ -1021,7 +1022,7 @@ func configureClusterWithHostnames( dnsEndpointGroup.Config.DiscoveryType = pbproxystate.DiscoveryType_DISCOVERY_TYPE_STRICT } - endpoints := make([]*pbproxystate.Endpoint, 0, 1) + endpoints := make([]*envoy_endpoint_v3.LbEndpoint, 0, 1) uniqueHostnames := make(map[string]bool) var ( diff --git a/agent/xds/proxystateconverter/converter.go b/agent/xds/proxystateconverter/converter.go index f4e9e26f0f5e..44996ea2bc0e 100644 --- a/agent/xds/proxystateconverter/converter.go +++ b/agent/xds/proxystateconverter/converter.go @@ -66,10 +66,11 @@ func (g *Converter) resourcesFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot) erro if err != nil { return err } - //err = g.routesFromSnapshot(cfgSnap) - //if err != nil { - // return err - //} + err = g.routesFromSnapshot(cfgSnap) + if err != nil { + return err + } + //g.secretsFromSnapshot(cfgSnap) return nil } diff --git a/agent/xds/proxystateconverter/routes.go b/agent/xds/proxystateconverter/routes.go new file mode 100644 index 000000000000..f9559f7878a8 --- /dev/null +++ b/agent/xds/proxystateconverter/routes.go @@ -0,0 +1,777 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package proxystateconverter + +import ( + "errors" + "fmt" + "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" + "sort" + "strings" + "time" + + "github.com/hashicorp/consul/agent/xds/response" + + "github.com/hashicorp/consul/agent/proxycfg" + "github.com/hashicorp/consul/agent/structs" + "google.golang.org/protobuf/types/known/durationpb" +) + +// routesFromSnapshot returns the xDS API representation of the "routes" in the +// snapshot. +func (s *Converter) routesFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot) error { + if cfgSnap == nil { + return errors.New("nil config given") + } + + switch cfgSnap.Kind { + case structs.ServiceKindConnectProxy: + return s.routesForConnectProxy(cfgSnap) + // TODO(proxystate): Ingress Gateways will be added in the future. + //case structs.ServiceKindIngressGateway: + // return s.routesForIngressGateway(cfgSnap) + // TODO(proxystate): API Gateways will be added in the future. + //case structs.ServiceKindAPIGateway: + // return s.routesForAPIGateway(cfgSnap) + // TODO(proxystate): Terminating Gateways will be added in the future. + //case structs.ServiceKindTerminatingGateway: + // return s.routesForTerminatingGateway(cfgSnap) + // TODO(proxystate): Mesh Gateways will be added in the future. + //case structs.ServiceKindMeshGateway: + // return s.routesForMeshGateway(cfgSnap) + default: + return fmt.Errorf("Invalid service kind: %v", cfgSnap.Kind) + } +} + +// routesFromSnapshotConnectProxy returns the xDS API representation of the +// "routes" in the snapshot. +func (s *Converter) routesForConnectProxy(cfgSnap *proxycfg.ConfigSnapshot) error { + for uid, chain := range cfgSnap.ConnectProxy.DiscoveryChain { + if chain.Default { + continue + } + + virtualHost, err := s.makeUpstreamHostForDiscoveryChain(cfgSnap, uid, chain, []string{"*"}, false) + if err != nil { + return err + } + if virtualHost == nil { + continue + } + + route := &pbproxystate.Route{ + VirtualHosts: []*pbproxystate.VirtualHost{virtualHost}, + } + s.proxyState.Routes[uid.EnvoyID()] = route + } + addressesMap := make(map[string]map[string]string) + err := cfgSnap.ConnectProxy.DestinationsUpstream.ForEachKeyE(func(uid proxycfg.UpstreamID) error { + svcConfig, ok := cfgSnap.ConnectProxy.DestinationsUpstream.Get(uid) + if !ok || svcConfig == nil { + return nil + } + if !structs.IsProtocolHTTPLike(svcConfig.Protocol) { + // Routes can only be defined for HTTP services + return nil + } + + for _, address := range svcConfig.Destination.Addresses { + + routeName := clusterNameForDestination(cfgSnap, "~http", fmt.Sprintf("%d", svcConfig.Destination.Port), svcConfig.NamespaceOrDefault(), svcConfig.PartitionOrDefault()) + if _, ok := addressesMap[routeName]; !ok { + addressesMap[routeName] = make(map[string]string) + } + // cluster name is unique per address/port so we should not be doing any override here + clusterName := clusterNameForDestination(cfgSnap, svcConfig.Name, address, svcConfig.NamespaceOrDefault(), svcConfig.PartitionOrDefault()) + addressesMap[routeName][clusterName] = address + } + return nil + }) + + if err != nil { + return err + } + + for routeName, clusters := range addressesMap { + route, err := s.makeRouteForAddresses(clusters) + if err != nil { + return err + } + if route != nil { + s.proxyState.Routes[routeName] = route + } + } + + // TODO(rb): make sure we don't generate an empty result + return nil +} + +func (s *Converter) makeRouteForAddresses(addresses map[string]string) (*pbproxystate.Route, error) { + route, err := makeAddressesRoute(addresses) + if err != nil { + s.Logger.Error("failed to make route", "cluster", "error", err) + return nil, err + } + + return route, nil +} + +// TODO(proxystate): Terminating Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func routesForTerminatingGateway + +// TODO(proxystate): Mesh Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func routesForMeshGateway + +func makeAddressesRoute(addresses map[string]string) (*pbproxystate.Route, error) { + route := &pbproxystate.Route{} + for clusterName, address := range addresses { + destination := makeRouteDestinationFromName(clusterName) + virtualHost := &pbproxystate.VirtualHost{ + Name: clusterName, + Domains: []string{address}, + RouteRules: []*pbproxystate.RouteRule{ + { + Match: makeDefaultRouteMatch(), + Destination: destination, + }, + }, + } + route.VirtualHosts = append(route.VirtualHosts, virtualHost) + } + + // sort virtual hosts to have a stable order + sort.SliceStable(route.VirtualHosts, func(i, j int) bool { + return route.VirtualHosts[i].Name > route.VirtualHosts[j].Name + }) + return route, nil +} + +// makeRouteDestinationFromName (fka makeRouteActionFromName) +func makeRouteDestinationFromName(clusterName string) *pbproxystate.RouteDestination { + return &pbproxystate.RouteDestination{ + Destination: &pbproxystate.RouteDestination_Cluster{ + Cluster: &pbproxystate.DestinationCluster{ + Name: clusterName, + }, + }, + } +} + +// TODO(proxystate): Ingress Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func routesForIngressGateway + +// TODO(proxystate): API Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func routesForAPIGateway +// func buildHTTPRouteUpstream + +// TODO(proxystate): Ingress Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func findIngressServiceMatchingUpstream + +// TODO(proxystate): Ingress Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func generateUpstreamIngressDomains + +// TODO(proxystate): Ingress Gateways will be added in the future. +// Functions to add from agent/xds/clusters.go: +// func generateUpstreamAPIsDomains + +func (s *Converter) makeUpstreamHostForDiscoveryChain( + cfgSnap *proxycfg.ConfigSnapshot, + uid proxycfg.UpstreamID, + chain *structs.CompiledDiscoveryChain, + serviceDomains []string, + forMeshGateway bool, +) (*pbproxystate.VirtualHost, error) { + var routeRules []*pbproxystate.RouteRule + + startNode := chain.Nodes[chain.StartNode] + if startNode == nil { + return nil, fmt.Errorf("missing first node in compiled discovery chain for: %s", chain.ServiceName) + } + + upstreamsSnapshot, err := cfgSnap.ToConfigSnapshotUpstreams() + if err != nil && !forMeshGateway { + return nil, err + } + + switch startNode.Type { + case structs.DiscoveryGraphNodeTypeRouter: + routeRules = make([]*pbproxystate.RouteRule, 0, len(startNode.Routes)) + + for _, discoveryRoute := range startNode.Routes { + routeMatch := makeRouteMatchForDiscoveryRoute(discoveryRoute) + + var ( + routeDestination *pbproxystate.RouteDestination + err error + ) + + nextNode := chain.Nodes[discoveryRoute.NextNode] + + var lb *structs.LoadBalancer + if nextNode.LoadBalancer != nil { + lb = nextNode.LoadBalancer + } + + switch nextNode.Type { + case structs.DiscoveryGraphNodeTypeSplitter: + routeDestination, err = s.makeRouteDestinationForSplitter(upstreamsSnapshot, nextNode.Splits, chain, forMeshGateway) + if err != nil { + return nil, err + } + + case structs.DiscoveryGraphNodeTypeResolver: + rd, ok := s.makeRouteDestinationForChainCluster(upstreamsSnapshot, nextNode.Resolver.Target, chain, forMeshGateway) + if !ok { + continue + } + routeDestination = rd + + default: + return nil, fmt.Errorf("unexpected graph node after route %q", nextNode.Type) + } + + routeDestination.DestinationConfiguration = &pbproxystate.DestinationConfiguration{} + if err := injectLBToDestinationConfiguration(lb, routeDestination.DestinationConfiguration); err != nil { + return nil, fmt.Errorf("failed to apply load balancer configuration to route action: %v", err) + } + + // TODO(rb): Better help handle the envoy case where you need (prefix=/foo/,rewrite=/) and (exact=/foo,rewrite=/) to do a full rewrite + + destination := discoveryRoute.Definition.Destination + + routeRule := &pbproxystate.RouteRule{} + + if destination != nil { + configHandle := routeDestination.DestinationConfiguration + if destination.PrefixRewrite != "" { + configHandle.PrefixRewrite = destination.PrefixRewrite + } + + if destination.RequestTimeout > 0 || destination.IdleTimeout > 0 { + configHandle.TimeoutConfig = &pbproxystate.TimeoutConfig{} + } + if destination.RequestTimeout > 0 { + configHandle.TimeoutConfig.Timeout = durationpb.New(destination.RequestTimeout) + } + if destination.IdleTimeout > 0 { + configHandle.TimeoutConfig.IdleTimeout = durationpb.New(destination.IdleTimeout) + } + + if destination.HasRetryFeatures() { + configHandle.RetryPolicy = getRetryPolicyForDestination(destination) + } + + if err := injectHeaderManipToRoute(destination, routeRule); err != nil { + return nil, fmt.Errorf("failed to apply header manipulation configuration to route: %v", err) + } + } + + routeRule.Match = routeMatch + routeRule.Destination = routeDestination + + routeRules = append(routeRules, routeRule) + } + + case structs.DiscoveryGraphNodeTypeSplitter: + routeDestination, err := s.makeRouteDestinationForSplitter(upstreamsSnapshot, startNode.Splits, chain, forMeshGateway) + if err != nil { + return nil, err + } + var lb *structs.LoadBalancer + if startNode.LoadBalancer != nil { + lb = startNode.LoadBalancer + } + routeDestination.DestinationConfiguration = &pbproxystate.DestinationConfiguration{} + if err := injectLBToDestinationConfiguration(lb, routeDestination.DestinationConfiguration); err != nil { + return nil, fmt.Errorf("failed to apply load balancer configuration to route action: %v", err) + } + + defaultRoute := &pbproxystate.RouteRule{ + Match: makeDefaultRouteMatch(), + Destination: routeDestination, + } + + routeRules = []*pbproxystate.RouteRule{defaultRoute} + + case structs.DiscoveryGraphNodeTypeResolver: + routeDestination, ok := s.makeRouteDestinationForChainCluster(upstreamsSnapshot, startNode.Resolver.Target, chain, forMeshGateway) + if !ok { + break + } + var lb *structs.LoadBalancer + if startNode.LoadBalancer != nil { + lb = startNode.LoadBalancer + } + routeDestination.DestinationConfiguration = &pbproxystate.DestinationConfiguration{} + if err := injectLBToDestinationConfiguration(lb, routeDestination.DestinationConfiguration); err != nil { + return nil, fmt.Errorf("failed to apply load balancer configuration to route action: %v", err) + } + + if startNode.Resolver.RequestTimeout > 0 { + to := &pbproxystate.TimeoutConfig{ + Timeout: durationpb.New(startNode.Resolver.RequestTimeout), + } + routeDestination.DestinationConfiguration.TimeoutConfig = to + } + defaultRoute := &pbproxystate.RouteRule{ + Match: makeDefaultRouteMatch(), + Destination: routeDestination, + } + + routeRules = []*pbproxystate.RouteRule{defaultRoute} + + default: + return nil, fmt.Errorf("unknown first node in discovery chain of type: %s", startNode.Type) + } + + host := &pbproxystate.VirtualHost{ + Name: uid.EnvoyID(), + Domains: serviceDomains, + RouteRules: routeRules, + } + + return host, nil +} + +func getRetryPolicyForDestination(destination *structs.ServiceRouteDestination) *pbproxystate.RetryPolicy { + retryPolicy := &pbproxystate.RetryPolicy{} + if destination.NumRetries > 0 { + retryPolicy.NumRetries = response.MakeUint32Value(int(destination.NumRetries)) + } + + // The RetryOn magic values come from: https://www.envoyproxy.io/docs/envoy/v1.10.0/configuration/http_filters/router_filter#config-http-filters-router-x-envoy-retry-on + var retryStrings []string + + if len(destination.RetryOn) > 0 { + retryStrings = append(retryStrings, destination.RetryOn...) + } + + if destination.RetryOnConnectFailure { + // connect-failure can be enabled by either adding connect-failure to the RetryOn list or by using the legacy RetryOnConnectFailure option + // Check that it's not already in the RetryOn list, so we don't set it twice + connectFailureExists := false + for _, r := range retryStrings { + if r == "connect-failure" { + connectFailureExists = true + } + } + if !connectFailureExists { + retryStrings = append(retryStrings, "connect-failure") + } + } + + if len(destination.RetryOnStatusCodes) > 0 { + retryStrings = append(retryStrings, "retriable-status-codes") + retryPolicy.RetriableStatusCodes = destination.RetryOnStatusCodes + } + + retryPolicy.RetryOn = strings.Join(retryStrings, ",") + + return retryPolicy +} + +func makeRouteMatchForDiscoveryRoute(discoveryRoute *structs.DiscoveryRoute) *pbproxystate.RouteMatch { + match := discoveryRoute.Definition.Match + if match == nil || match.IsEmpty() { + return makeDefaultRouteMatch() + } + + routeMatch := &pbproxystate.RouteMatch{} + + switch { + case match.HTTP.PathExact != "": + routeMatch.PathMatch = &pbproxystate.PathMatch{ + PathMatch: &pbproxystate.PathMatch_Exact{ + Exact: match.HTTP.PathExact, + }, + } + case match.HTTP.PathPrefix != "": + routeMatch.PathMatch = &pbproxystate.PathMatch{ + PathMatch: &pbproxystate.PathMatch_Prefix{ + Prefix: match.HTTP.PathPrefix, + }, + } + case match.HTTP.PathRegex != "": + routeMatch.PathMatch = &pbproxystate.PathMatch{ + PathMatch: &pbproxystate.PathMatch_Regex{ + Regex: match.HTTP.PathRegex, + }, + } + default: + routeMatch.PathMatch = &pbproxystate.PathMatch{ + PathMatch: &pbproxystate.PathMatch_Prefix{ + Prefix: "/", + }, + } + } + + if len(match.HTTP.Header) > 0 { + routeMatch.HeaderMatches = make([]*pbproxystate.HeaderMatch, 0, len(match.HTTP.Header)) + for _, hdr := range match.HTTP.Header { + headerMatch := &pbproxystate.HeaderMatch{ + Name: hdr.Name, + } + + switch { + case hdr.Exact != "": + headerMatch.Match = &pbproxystate.HeaderMatch_Exact{ + Exact: hdr.Exact, + } + case hdr.Regex != "": + headerMatch.Match = &pbproxystate.HeaderMatch_Regex{ + Regex: hdr.Regex, + } + case hdr.Prefix != "": + headerMatch.Match = &pbproxystate.HeaderMatch_Prefix{ + Prefix: hdr.Prefix, + } + case hdr.Suffix != "": + headerMatch.Match = &pbproxystate.HeaderMatch_Suffix{ + Suffix: hdr.Suffix, + } + case hdr.Present: + headerMatch.Match = &pbproxystate.HeaderMatch_Present{ + Present: true, + } + default: + continue // skip this impossible situation + } + + if hdr.Invert { + headerMatch.InvertMatch = true + } + + routeMatch.HeaderMatches = append(routeMatch.HeaderMatches, headerMatch) + } + } + + if len(match.HTTP.Methods) > 0 { + routeMatch.MethodMatches = append(routeMatch.MethodMatches, match.HTTP.Methods...) + } + + if len(match.HTTP.QueryParam) > 0 { + routeMatch.QueryParameterMatches = make([]*pbproxystate.QueryParameterMatch, 0, len(match.HTTP.QueryParam)) + for _, qm := range match.HTTP.QueryParam { + + queryMatcher := &pbproxystate.QueryParameterMatch{ + Name: qm.Name, + } + + switch { + case qm.Exact != "": + queryMatcher.Match = &pbproxystate.QueryParameterMatch_Exact{ + Exact: qm.Exact, + } + case qm.Regex != "": + queryMatcher.Match = &pbproxystate.QueryParameterMatch_Regex{ + Regex: qm.Regex, + } + case qm.Present: + queryMatcher.Match = &pbproxystate.QueryParameterMatch_Present{ + Present: true, + } + default: + continue // skip this impossible situation + } + + routeMatch.QueryParameterMatches = append(routeMatch.QueryParameterMatches, queryMatcher) + } + } + + return routeMatch +} + +func makeDefaultRouteMatch() *pbproxystate.RouteMatch { + return &pbproxystate.RouteMatch{ + PathMatch: &pbproxystate.PathMatch{ + PathMatch: &pbproxystate.PathMatch_Prefix{ + Prefix: "/", + }, + }, + // TODO(banks) Envoy supports matching only valid GRPC + // requests which might be nice to add here for gRPC services + // but it's not supported in our current envoy SDK version + // although docs say it was supported by 1.8.0. Going to defer + // that until we've updated the deps. + } +} + +func (s *Converter) makeRouteDestinationForChainCluster( + upstreamsSnapshot *proxycfg.ConfigSnapshotUpstreams, + targetID string, + chain *structs.CompiledDiscoveryChain, + forMeshGateway bool, +) (*pbproxystate.RouteDestination, bool) { + clusterName := s.getTargetClusterName(upstreamsSnapshot, chain, targetID, forMeshGateway) + if clusterName == "" { + return nil, false + } + return makeRouteDestinationFromName(clusterName), true +} + +func (s *Converter) makeRouteDestinationForSplitter( + upstreamsSnapshot *proxycfg.ConfigSnapshotUpstreams, + splits []*structs.DiscoverySplit, + chain *structs.CompiledDiscoveryChain, + forMeshGateway bool, +) (*pbproxystate.RouteDestination, error) { + clusters := make([]*pbproxystate.L7WeightedDestinationCluster, 0, len(splits)) + for _, split := range splits { + nextNode := chain.Nodes[split.NextNode] + + if nextNode.Type != structs.DiscoveryGraphNodeTypeResolver { + return nil, fmt.Errorf("unexpected splitter destination node type: %s", nextNode.Type) + } + targetID := nextNode.Resolver.Target + + clusterName := s.getTargetClusterName(upstreamsSnapshot, chain, targetID, forMeshGateway) + if clusterName == "" { + continue + } + + // The smallest representable weight is 1/10000 or .01% but envoy + // deals with integers so scale everything up by 100x. + weight := int(split.Weight * 100) + + clusterWeight := &pbproxystate.L7WeightedDestinationCluster{ + Name: clusterName, + Weight: response.MakeUint32Value(weight), + } + if err := injectHeaderManipToWeightedCluster(split.Definition, clusterWeight); err != nil { + return nil, err + } + + clusters = append(clusters, clusterWeight) + } + + if len(clusters) <= 0 { + return nil, fmt.Errorf("number of clusters in splitter must be > 0; got %d", len(clusters)) + } + + return &pbproxystate.RouteDestination{ + Destination: &pbproxystate.RouteDestination_WeightedClusters{ + WeightedClusters: &pbproxystate.L7WeightedClusterGroup{ + Clusters: clusters, + }, + }, + }, nil +} + +func injectLBToDestinationConfiguration(lb *structs.LoadBalancer, destinationConfig *pbproxystate.DestinationConfiguration) error { + if lb == nil || !lb.IsHashBased() { + return nil + } + + result := make([]*pbproxystate.LoadBalancerHashPolicy, 0, len(lb.HashPolicies)) + for _, policy := range lb.HashPolicies { + if policy.SourceIP { + p := &pbproxystate.LoadBalancerHashPolicy{ + Policy: &pbproxystate.LoadBalancerHashPolicy_ConnectionProperties{ + ConnectionProperties: &pbproxystate.ConnectionPropertiesPolicy{ + SourceIp: true, + Terminal: policy.Terminal, + }, + }, + } + result = append(result, p) + continue + } + + switch policy.Field { + case structs.HashPolicyHeader: + p := &pbproxystate.LoadBalancerHashPolicy{ + Policy: &pbproxystate.LoadBalancerHashPolicy_Header{ + Header: &pbproxystate.HeaderPolicy{ + Name: policy.FieldValue, + Terminal: policy.Terminal, + }, + }, + } + result = append(result, p) + + case structs.HashPolicyCookie: + + cookie := &pbproxystate.CookiePolicy{ + Name: policy.FieldValue, + Terminal: policy.Terminal, + } + if policy.CookieConfig != nil { + cookie.Path = policy.CookieConfig.Path + + if policy.CookieConfig.TTL != 0*time.Second { + cookie.Ttl = durationpb.New(policy.CookieConfig.TTL) + } + + // Envoy will generate a session cookie if the ttl is present and zero. + if policy.CookieConfig.Session { + cookie.Ttl = durationpb.New(0 * time.Second) + } + } + p := &pbproxystate.LoadBalancerHashPolicy{ + Policy: &pbproxystate.LoadBalancerHashPolicy_Cookie{ + Cookie: cookie, + }, + } + result = append(result, p) + + case structs.HashPolicyQueryParam: + p := &pbproxystate.LoadBalancerHashPolicy{ + Policy: &pbproxystate.LoadBalancerHashPolicy_QueryParameter{ + QueryParameter: &pbproxystate.QueryParameterPolicy{ + Name: policy.FieldValue, + Terminal: policy.Terminal, + }, + }, + } + result = append(result, p) + + default: + return fmt.Errorf("unsupported load balancer hash policy field: %v", policy.Field) + } + } + + destinationConfig.HashPolicies = result + return nil +} + +func injectHeaderManipToRoute(dest *structs.ServiceRouteDestination, r *pbproxystate.RouteRule) error { + if !dest.RequestHeaders.IsZero() { + r.HeaderMutations = append( + r.HeaderMutations, + makeRequestHeaderAdd(dest.RequestHeaders.Add, true)..., + ) + r.HeaderMutations = append( + r.HeaderMutations, + makeRequestHeaderAdd(dest.RequestHeaders.Set, false)..., + ) + r.HeaderMutations = append( + r.HeaderMutations, + makeRequestHeaderRemove(dest.RequestHeaders.Remove), + ) + } + if !dest.ResponseHeaders.IsZero() { + r.HeaderMutations = append( + r.HeaderMutations, + makeResponseHeaderAdd(dest.ResponseHeaders.Add, true)..., + ) + r.HeaderMutations = append( + r.HeaderMutations, + makeResponseHeaderAdd(dest.ResponseHeaders.Set, false)..., + ) + r.HeaderMutations = append( + r.HeaderMutations, + makeResponseHeaderRemove(dest.ResponseHeaders.Remove), + ) + } + return nil +} + +func injectHeaderManipToWeightedCluster(split *structs.ServiceSplit, c *pbproxystate.L7WeightedDestinationCluster) error { + if !split.RequestHeaders.IsZero() { + c.HeaderMutations = append( + c.HeaderMutations, + makeRequestHeaderAdd(split.RequestHeaders.Add, true)..., + ) + c.HeaderMutations = append( + c.HeaderMutations, + makeRequestHeaderAdd(split.RequestHeaders.Set, false)..., + ) + c.HeaderMutations = append( + c.HeaderMutations, + makeRequestHeaderRemove(split.RequestHeaders.Remove), + ) + } + if !split.ResponseHeaders.IsZero() { + c.HeaderMutations = append( + c.HeaderMutations, + makeResponseHeaderAdd(split.ResponseHeaders.Add, true)..., + ) + c.HeaderMutations = append( + c.HeaderMutations, + makeResponseHeaderAdd(split.ResponseHeaders.Set, false)..., + ) + c.HeaderMutations = append( + c.HeaderMutations, + makeResponseHeaderRemove(split.ResponseHeaders.Remove), + ) + } + return nil +} + +func makeRequestHeaderAdd(vals map[string]string, add bool) []*pbproxystate.HeaderMutation { + mutations := make([]*pbproxystate.HeaderMutation, 0, len(vals)) + + appendAction := pbproxystate.AppendAction_APPEND_ACTION_OVERWRITE_IF_EXISTS_OR_ADD + if add { + appendAction = pbproxystate.AppendAction_APPEND_ACTION_APPEND_IF_EXISTS_OR_ADD + } + + for k, v := range vals { + m := &pbproxystate.HeaderMutation{ + Action: &pbproxystate.HeaderMutation_RequestHeaderAdd{ + RequestHeaderAdd: &pbproxystate.RequestHeaderAdd{ + Header: &pbproxystate.Header{ + Key: k, + Value: v, + }, + AppendAction: appendAction, + }, + }, + } + mutations = append(mutations, m) + } + return mutations +} + +func makeRequestHeaderRemove(values []string) *pbproxystate.HeaderMutation { + return &pbproxystate.HeaderMutation{ + Action: &pbproxystate.HeaderMutation_RequestHeaderRemove{ + RequestHeaderRemove: &pbproxystate.RequestHeaderRemove{ + HeaderKeys: values, + }, + }, + } +} + +func makeResponseHeaderAdd(vals map[string]string, add bool) []*pbproxystate.HeaderMutation { + mutations := make([]*pbproxystate.HeaderMutation, 0, len(vals)) + + appendAction := pbproxystate.AppendAction_APPEND_ACTION_OVERWRITE_IF_EXISTS_OR_ADD + if add { + appendAction = pbproxystate.AppendAction_APPEND_ACTION_APPEND_IF_EXISTS_OR_ADD + } + + for k, v := range vals { + m := &pbproxystate.HeaderMutation{ + Action: &pbproxystate.HeaderMutation_ResponseHeaderAdd{ + ResponseHeaderAdd: &pbproxystate.ResponseHeaderAdd{ + Header: &pbproxystate.Header{ + Key: k, + Value: v, + }, + AppendAction: appendAction, + }, + }, + } + mutations = append(mutations, m) + } + return mutations +} + +func makeResponseHeaderRemove(values []string) *pbproxystate.HeaderMutation { + return &pbproxystate.HeaderMutation{ + Action: &pbproxystate.HeaderMutation_ResponseHeaderRemove{ + ResponseHeaderRemove: &pbproxystate.ResponseHeaderRemove{ + HeaderKeys: values, + }, + }, + } +} diff --git a/agent/xds/routes_test.go b/agent/xds/routes_test.go index 8b7345d88a52..e2f7d73b5e7c 100644 --- a/agent/xds/routes_test.go +++ b/agent/xds/routes_test.go @@ -16,8 +16,10 @@ import ( "github.com/hashicorp/consul/agent/proxycfg" "github.com/hashicorp/consul/agent/structs" + "github.com/hashicorp/consul/agent/xds/proxystateconverter" "github.com/hashicorp/consul/agent/xds/response" "github.com/hashicorp/consul/agent/xds/testcommon" + "github.com/hashicorp/consul/agent/xdsv2" "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/sdk/testutil" ) @@ -26,6 +28,7 @@ type routeTestCase struct { name string create func(t testinf.T) *proxycfg.ConfigSnapshot overrideGoldenName string + alsoRunTestForV2 bool } func makeRouteDiscoChainTests(enterprise bool) []routeTestCase { @@ -35,66 +38,77 @@ func makeRouteDiscoChainTests(enterprise bool) []routeTestCase { create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-chain-external-sni", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "external-sni", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-splitter-overweight", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "splitter-overweight", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-chain-and-overrides", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple-with-overrides", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "splitter-with-resolver-redirect", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "splitter-with-resolver-redirect-multidc", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-chain-and-splitter", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "chain-and-splitter", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-grpc-router", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "grpc-router", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-chain-and-router", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "chain-and-router", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-lb-in-resolver", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "lb-resolver", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-route-to-lb-resolver", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "redirect-to-lb-node", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, { name: "connect-proxy-resolver-with-lb", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "resolver-with-lb", enterprise, nil, nil) }, + alsoRunTestForV2: true, }, } } @@ -115,6 +129,8 @@ func TestRoutesFromSnapshot(t *testing.T) { create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotIngressGateway_NilConfigEntry(t) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-defaults-no-chain", @@ -122,6 +138,8 @@ func TestRoutesFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, false, "tcp", "default", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-chain", @@ -129,6 +147,8 @@ func TestRoutesFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "simple", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-chain-external-sni", @@ -136,6 +156,8 @@ func TestRoutesFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "tcp", "external-sni", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-splitter-with-resolver-redirect", @@ -143,6 +165,8 @@ func TestRoutesFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "http", "splitter-with-resolver-redirect-multidc", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-chain-and-splitter", @@ -150,6 +174,8 @@ func TestRoutesFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "http", "chain-and-splitter", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-grpc-router", @@ -157,6 +183,8 @@ func TestRoutesFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "http", "grpc-router", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-chain-and-router", @@ -164,6 +192,8 @@ func TestRoutesFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "http", "chain-and-router", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-lb-in-resolver", @@ -171,48 +201,66 @@ func TestRoutesFromSnapshot(t *testing.T) { return proxycfg.TestConfigSnapshotIngressGateway(t, true, "http", "lb-resolver", nil, nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-http-multiple-services", create: proxycfg.TestConfigSnapshotIngress_HTTPMultipleServices, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-grpc-multiple-services", create: proxycfg.TestConfigSnapshotIngress_GRPCMultipleServices, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-chain-and-router-header-manip", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotIngressGatewayWithChain(t, "router-header-manip", nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-sds-listener-level", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotIngressGatewayWithChain(t, "sds-listener-level", nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-sds-listener-level-wildcard", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotIngressGatewayWithChain(t, "sds-listener-level-wildcard", nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-sds-service-level", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotIngressGatewayWithChain(t, "sds-service-level", nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "ingress-with-sds-service-level-mixed-tls", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotIngressGatewayWithChain(t, "sds-service-level-mixed-tls", nil, nil) }, + // TODO(proxystate): ingress gateway will come at a later time + alsoRunTestForV2: false, }, { name: "terminating-gateway-lb-config", create: proxycfg.TestConfigSnapshotTerminatingGatewayLBConfig, + // TODO(proxystate): terminating gateway will come at a later time + alsoRunTestForV2: false, }, } @@ -245,7 +293,7 @@ func TestRoutesFromSnapshot(t *testing.T) { r, err := response.CreateResponse(xdscommon.RouteType, "00000001", "00000001", routes) require.NoError(t, err) - t.Run("current", func(t *testing.T) { + t.Run("current-xdsv1", func(t *testing.T) { gotJSON := protoToJSON(t, r) gName := tt.name @@ -255,6 +303,39 @@ func TestRoutesFromSnapshot(t *testing.T) { require.JSONEq(t, goldenEnvoy(t, filepath.Join("routes", gName), envoyVersion, latestEnvoyVersion, gotJSON), gotJSON) }) + + if tt.alsoRunTestForV2 { + generator := xdsv2.NewResourceGenerator(testutil.Logger(t)) + + converter := proxystateconverter.NewConverter(testutil.Logger(t), &mockCfgFetcher{addressLan: "10.10.10.10"}) + proxyState, err := converter.ProxyStateFromSnapshot(snap) + require.NoError(t, err) + + res, err := generator.AllResourcesFromIR(proxyState) + require.NoError(t, err) + + routes = res[xdscommon.RouteType] + // The order of routes returned via RDS isn't relevant, so it's safe + // to sort these for the purposes of test comparisons. + sort.Slice(routes, func(i, j int) bool { + return routes[i].(*envoy_route_v3.Route).Name < routes[j].(*envoy_route_v3.Route).Name + }) + + r, err := response.CreateResponse(xdscommon.RouteType, "00000001", "00000001", routes) + require.NoError(t, err) + + t.Run("current-xdsv2", func(t *testing.T) { + gotJSON := protoToJSON(t, r) + + gName := tt.name + if tt.overrideGoldenName != "" { + gName = tt.overrideGoldenName + } + + expectedJSON := goldenEnvoy(t, filepath.Join("routes", gName), envoyVersion, latestEnvoyVersion, gotJSON) + require.JSONEq(t, expectedJSON, gotJSON) + }) + } }) } }) diff --git a/agent/xdsv2/cluster_resources.go b/agent/xdsv2/cluster_resources.go index 4c1573d5db3d..eb9388841ead 100644 --- a/agent/xdsv2/cluster_resources.go +++ b/agent/xdsv2/cluster_resources.go @@ -12,11 +12,22 @@ import ( envoy_aggregate_cluster_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/clusters/aggregate/v3" envoy_upstreams_v3 "github.com/envoyproxy/go-control-plane/envoy/extensions/upstreams/http/v3" envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "github.com/hashicorp/consul/envoyextensions/xdscommon" "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/anypb" ) +func (pr *ProxyResources) doesEnvoyClusterAlreadyExist(name string) bool { + // TODO(proxystate): consider using a map instead of [] for this kind of lookup + for _, envoyCluster := range pr.envoyResources[xdscommon.ClusterType] { + if envoyCluster.(*envoy_cluster_v3.Cluster).Name == name { + return true + } + } + return false +} + func (pr *ProxyResources) makeXDSClusters() ([]proto.Message, error) { clusters := make([]proto.Message, 0) @@ -39,6 +50,11 @@ func (pr *ProxyResources) makeClusters(name string) ([]proto.Message, error) { return nil, fmt.Errorf("cluster %q not found", name) } + if pr.doesEnvoyClusterAlreadyExist(name) { + // don't error + return []proto.Message{}, nil + } + switch proxyStateCluster.Group.(type) { case *pbproxystate.Cluster_FailoverGroup: fg := proxyStateCluster.GetFailoverGroup() @@ -325,6 +341,19 @@ func addEnvoyLBToCluster(dynamicConfig *pbproxystate.DynamicEndpointGroupConfig, return nil } +func makeAddress(ip string, port uint32) *envoy_core_v3.Address { + return &envoy_core_v3.Address{ + Address: &envoy_core_v3.Address_SocketAddress{ + SocketAddress: &envoy_core_v3.SocketAddress{ + Address: ip, + PortSpecifier: &envoy_core_v3.SocketAddress_PortValue{ + PortValue: port, + }, + }, + }, + } +} + // TODO(proxystate): In a future PR this will create clusters and add it to ProxyResources.proxyState func (pr *ProxyResources) makeEnvoyClusterFromL4Destination(name string) error { return nil diff --git a/agent/xdsv2/listener_resources.go b/agent/xdsv2/listener_resources.go index 2d3bf5d60da5..f3a241f0cbe3 100644 --- a/agent/xdsv2/listener_resources.go +++ b/agent/xdsv2/listener_resources.go @@ -43,6 +43,20 @@ const ( envoyHttpConnectionManagerFilterName = "envoy.filters.network.http_connection_manager" ) +func (pr *ProxyResources) makeXDSListeners() ([]proto.Message, error) { + listeners := make([]proto.Message, 0) + + for _, l := range pr.proxyState.Listeners { + protoListener, err := pr.makeListener(l) + // TODO: aggregate errors for listeners and still return any properly formed listeners. + if err != nil { + return nil, err + } + listeners = append(listeners, protoListener) + } + return listeners, nil +} + func (pr *ProxyResources) makeListener(listener *pbproxystate.Listener) (*envoy_listener_v3.Listener, error) { envoyListener := &envoy_listener_v3.Listener{} @@ -399,7 +413,7 @@ func (pr *ProxyResources) makeL7Filters(l7 *pbproxystate.L7Destination) ([]*envo }, } - routeConfig, err := pr.makeRoute(l7.Name) + routeConfig, err := pr.makeEnvoyRoute(l7.Name) if err != nil { return nil, err } diff --git a/agent/xdsv2/resources.go b/agent/xdsv2/resources.go index cba6e8188c80..4b695cbd735a 100644 --- a/agent/xdsv2/resources.go +++ b/agent/xdsv2/resources.go @@ -45,16 +45,9 @@ func (g *ResourceGenerator) AllResourcesFromIR(proxyState *pbmesh.ProxyState) (m } func (pr *ProxyResources) generateXDSResources() error { - listeners := make([]proto.Message, 0) - routes := make([]proto.Message, 0) - - for _, l := range pr.proxyState.Listeners { - protoListener, err := pr.makeListener(l) - // TODO: aggregate errors for listeners and still return any properly formed listeners. - if err != nil { - return err - } - listeners = append(listeners, protoListener) + listeners, err := pr.makeXDSListeners() + if err != nil { + return err } pr.envoyResources[xdscommon.ListenerType] = listeners @@ -70,6 +63,10 @@ func (pr *ProxyResources) generateXDSResources() error { } pr.envoyResources[xdscommon.EndpointType] = endpoints + routes, err := pr.makeXDSRoutes() + if err != nil { + return err + } pr.envoyResources[xdscommon.RouteType] = routes return nil diff --git a/agent/xdsv2/route_resources.go b/agent/xdsv2/route_resources.go index 86d82a5d306f..931d48f26078 100644 --- a/agent/xdsv2/route_resources.go +++ b/agent/xdsv2/route_resources.go @@ -5,11 +5,31 @@ package xdsv2 import ( "fmt" + envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + envoy_matcher_v3 "github.com/envoyproxy/go-control-plane/envoy/type/matcher/v3" + "github.com/hashicorp/consul/agent/xds/response" + "github.com/hashicorp/consul/envoyextensions/xdscommon" + "github.com/hashicorp/consul/proto-public/pbmesh/v1alpha1/pbproxystate" + "google.golang.org/protobuf/types/known/wrapperspb" + "strings" envoy_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3" + "google.golang.org/protobuf/proto" ) -func (pr *ProxyResources) makeRoute(name string) (*envoy_route_v3.RouteConfiguration, error) { +func (pr *ProxyResources) makeXDSRoutes() ([]proto.Message, error) { + routes := make([]proto.Message, 0) + + for name, r := range pr.proxyState.Routes { + protoRoute := pr.makeEnvoyRouteConfigFromProxystateRoute(name, r) + // TODO: aggregate errors for routes and still return any properly formed routes. + routes = append(routes, protoRoute) + } + + return routes, nil +} + +func (pr *ProxyResources) makeEnvoyRoute(name string) (*envoy_route_v3.RouteConfiguration, error) { var route *envoy_route_v3.RouteConfiguration // TODO(proxystate): This will make routes in the future. This function should distinguish between static routes // inlined into listeners and non-static routes that should be added as top level Envoy resources. @@ -21,3 +41,485 @@ func (pr *ProxyResources) makeRoute(name string) (*envoy_route_v3.RouteConfigura } return route, nil } + +// makeEnvoyRouteConfigFromProxystateRoute converts the proxystate representation of a Route into Envoy proto message +// form. We don't throw any errors here, since the proxystate has already been validated. +func (pr *ProxyResources) makeEnvoyRouteConfigFromProxystateRoute(name string, psRoute *pbproxystate.Route) *envoy_route_v3.RouteConfiguration { + + envoyRouteConfig := &envoy_route_v3.RouteConfiguration{ + Name: name, + // ValidateClusters defaults to true when defined statically and false + // when done via RDS. Re-set the reasonable value of true to prevent + // null-routing traffic. + ValidateClusters: response.MakeBoolValue(true), + } + + for _, vh := range psRoute.GetVirtualHosts() { + envoyRouteConfig.VirtualHosts = append(envoyRouteConfig.VirtualHosts, pr.makeEnvoyVHFromProxystateVH(vh)) + } + + return envoyRouteConfig +} + +func (pr *ProxyResources) makeEnvoyVHFromProxystateVH(psVirtualHost *pbproxystate.VirtualHost) *envoy_route_v3.VirtualHost { + envoyVirtualHost := &envoy_route_v3.VirtualHost{ + Name: psVirtualHost.Name, + Domains: psVirtualHost.GetDomains(), + } + + for _, rr := range psVirtualHost.GetRouteRules() { + envoyVirtualHost.Routes = append(envoyVirtualHost.Routes, pr.makeEnvoyRouteFromProxystateRouteRule(rr)) + } + + for _, hm := range psVirtualHost.GetHeaderMutations() { + injectEnvoyVirtualHostWithProxystateHeaderMutation(envoyVirtualHost, hm) + } + + return envoyVirtualHost +} + +func (pr *ProxyResources) makeEnvoyRouteFromProxystateRouteRule(psRouteRule *pbproxystate.RouteRule) *envoy_route_v3.Route { + envoyRouteRule := &envoy_route_v3.Route{ + Match: makeEnvoyRouteMatchFromProxystateRouteMatch(psRouteRule.GetMatch()), + Action: pr.makeEnvoyRouteActionFromProxystateRouteDestination(psRouteRule.GetDestination()), + } + + for _, hm := range psRouteRule.GetHeaderMutations() { + injectEnvoyRouteRuleWithProxystateHeaderMutation(envoyRouteRule, hm) + } + + return envoyRouteRule +} + +func makeEnvoyRouteMatchFromProxystateRouteMatch(psRouteMatch *pbproxystate.RouteMatch) *envoy_route_v3.RouteMatch { + envoyRouteMatch := &envoy_route_v3.RouteMatch{} + + switch psRouteMatch.PathMatch.GetPathMatch().(type) { + case *pbproxystate.PathMatch_Exact: + envoyRouteMatch.PathSpecifier = &envoy_route_v3.RouteMatch_Path{ + Path: psRouteMatch.PathMatch.GetExact(), + } + case *pbproxystate.PathMatch_Prefix: + envoyRouteMatch.PathSpecifier = &envoy_route_v3.RouteMatch_Prefix{ + Prefix: psRouteMatch.PathMatch.GetPrefix(), + } + case *pbproxystate.PathMatch_Regex: + envoyRouteMatch.PathSpecifier = &envoy_route_v3.RouteMatch_SafeRegex{ + SafeRegex: makeEnvoyRegexMatch(psRouteMatch.PathMatch.GetRegex()), + } + default: + // This shouldn't be possible considering the types of PathMatch + return nil + } + + if len(psRouteMatch.GetHeaderMatches()) > 0 { + envoyRouteMatch.Headers = make([]*envoy_route_v3.HeaderMatcher, 0, len(psRouteMatch.GetHeaderMatches())) + } + for _, psHM := range psRouteMatch.GetHeaderMatches() { + envoyRouteMatch.Headers = append(envoyRouteMatch.Headers, makeEnvoyHeaderMatcherFromProxystateHeaderMatch(psHM)) + } + + if len(psRouteMatch.MethodMatches) > 0 { + methodHeaderRegex := strings.Join(psRouteMatch.MethodMatches, "|") + + eh := &envoy_route_v3.HeaderMatcher{ + Name: ":method", + HeaderMatchSpecifier: &envoy_route_v3.HeaderMatcher_SafeRegexMatch{ + SafeRegexMatch: makeEnvoyRegexMatch(methodHeaderRegex), + }, + } + + envoyRouteMatch.Headers = append(envoyRouteMatch.Headers, eh) + } + + if len(psRouteMatch.GetQueryParameterMatches()) > 0 { + envoyRouteMatch.QueryParameters = make([]*envoy_route_v3.QueryParameterMatcher, 0, len(psRouteMatch.GetQueryParameterMatches())) + } + for _, psQM := range psRouteMatch.GetQueryParameterMatches() { + envoyRouteMatch.QueryParameters = append(envoyRouteMatch.QueryParameters, makeEnvoyQueryParamFromProxystateQueryMatch(psQM)) + } + + return envoyRouteMatch +} + +func makeEnvoyRegexMatch(pattern string) *envoy_matcher_v3.RegexMatcher { + return &envoy_matcher_v3.RegexMatcher{ + EngineType: &envoy_matcher_v3.RegexMatcher_GoogleRe2{ + GoogleRe2: &envoy_matcher_v3.RegexMatcher_GoogleRE2{}, + }, + Regex: pattern, + } +} + +func makeEnvoyHeaderMatcherFromProxystateHeaderMatch(psMatch *pbproxystate.HeaderMatch) *envoy_route_v3.HeaderMatcher { + envoyHeaderMatcher := &envoy_route_v3.HeaderMatcher{ + Name: psMatch.Name, + } + + switch psMatch.Match.(type) { + case *pbproxystate.HeaderMatch_Exact: + envoyHeaderMatcher.HeaderMatchSpecifier = &envoy_route_v3.HeaderMatcher_ExactMatch{ + ExactMatch: psMatch.GetExact(), + } + case *pbproxystate.HeaderMatch_Regex: + envoyHeaderMatcher.HeaderMatchSpecifier = &envoy_route_v3.HeaderMatcher_SafeRegexMatch{ + SafeRegexMatch: makeEnvoyRegexMatch(psMatch.GetRegex()), + } + case *pbproxystate.HeaderMatch_Prefix: + envoyHeaderMatcher.HeaderMatchSpecifier = &envoy_route_v3.HeaderMatcher_PrefixMatch{ + PrefixMatch: psMatch.GetPrefix(), + } + case *pbproxystate.HeaderMatch_Suffix: + envoyHeaderMatcher.HeaderMatchSpecifier = &envoy_route_v3.HeaderMatcher_SuffixMatch{ + SuffixMatch: psMatch.GetSuffix(), + } + case *pbproxystate.HeaderMatch_Present: + envoyHeaderMatcher.HeaderMatchSpecifier = &envoy_route_v3.HeaderMatcher_PresentMatch{ + PresentMatch: true, + } + default: + // This shouldn't be possible considering the types of HeaderMatch + return nil + } + + if psMatch.GetInvertMatch() { + envoyHeaderMatcher.InvertMatch = true + } + + return envoyHeaderMatcher +} + +func makeEnvoyQueryParamFromProxystateQueryMatch(psMatch *pbproxystate.QueryParameterMatch) *envoy_route_v3.QueryParameterMatcher { + envoyQueryParamMatcher := &envoy_route_v3.QueryParameterMatcher{ + Name: psMatch.Name, + } + + switch psMatch.Match.(type) { + case *pbproxystate.QueryParameterMatch_Exact: + envoyQueryParamMatcher.QueryParameterMatchSpecifier = &envoy_route_v3.QueryParameterMatcher_StringMatch{ + StringMatch: &envoy_matcher_v3.StringMatcher{ + MatchPattern: &envoy_matcher_v3.StringMatcher_Exact{ + Exact: psMatch.GetExact(), + }, + }, + } + + case *pbproxystate.QueryParameterMatch_Regex: + envoyQueryParamMatcher.QueryParameterMatchSpecifier = &envoy_route_v3.QueryParameterMatcher_StringMatch{ + StringMatch: &envoy_matcher_v3.StringMatcher{ + MatchPattern: &envoy_matcher_v3.StringMatcher_SafeRegex{ + SafeRegex: makeEnvoyRegexMatch(psMatch.GetRegex()), + }, + }, + } + case *pbproxystate.QueryParameterMatch_Present: + envoyQueryParamMatcher.QueryParameterMatchSpecifier = &envoy_route_v3.QueryParameterMatcher_PresentMatch{ + PresentMatch: true, + } + default: + // This shouldn't be possible considering the types of QueryMatch + return nil + } + + return envoyQueryParamMatcher +} + +// TODO (dans): Will this always be envoy_route_v3.Route_Route? +// Definitely for connect proxies this is the only option. +func (pr *ProxyResources) makeEnvoyRouteActionFromProxystateRouteDestination(psRouteDestination *pbproxystate.RouteDestination) *envoy_route_v3.Route_Route { + envoyRouteRoute := &envoy_route_v3.Route_Route{ + Route: &envoy_route_v3.RouteAction{}, + } + + switch psRouteDestination.Destination.(type) { + case *pbproxystate.RouteDestination_Cluster: + psCluster := psRouteDestination.GetCluster() + envoyRouteRoute.Route.ClusterSpecifier = &envoy_route_v3.RouteAction_Cluster{ + Cluster: psCluster.GetName(), + } + clusters, _ := pr.makeClusters(psCluster.Name) + pr.envoyResources[xdscommon.ClusterType] = append(pr.envoyResources[xdscommon.ClusterType], clusters...) + + case *pbproxystate.RouteDestination_WeightedClusters: + psWeightedClusters := psRouteDestination.GetWeightedClusters() + envoyClusters := make([]*envoy_route_v3.WeightedCluster_ClusterWeight, 0, len(psWeightedClusters.GetClusters())) + totalWeight := 0 + for _, psCluster := range psWeightedClusters.GetClusters() { + clusters, _ := pr.makeClusters(psCluster.Name) + pr.envoyResources[xdscommon.ClusterType] = append(pr.envoyResources[xdscommon.ClusterType], clusters...) + totalWeight += int(psCluster.Weight.GetValue()) + envoyClusters = append(envoyClusters, makeEnvoyClusterWeightFromProxystateWeightedCluster(psCluster)) + } + var envoyWeightScale *wrapperspb.UInt32Value + if totalWeight == 10000 { + envoyWeightScale = response.MakeUint32Value(10000) + } + + envoyRouteRoute.Route.ClusterSpecifier = &envoy_route_v3.RouteAction_WeightedClusters{ + WeightedClusters: &envoy_route_v3.WeightedCluster{ + Clusters: envoyClusters, + TotalWeight: envoyWeightScale, + }, + } + default: + // This shouldn't be possible considering the types of Destination + return nil + } + + injectEnvoyRouteActionWithProxystateDestinationConfig(envoyRouteRoute.Route, psRouteDestination.GetDestinationConfiguration()) + + if psRouteDestination.GetDestinationConfiguration() != nil { + config := psRouteDestination.GetDestinationConfiguration() + action := envoyRouteRoute.Route + + action.PrefixRewrite = config.GetPrefixRewrite() + + if config.GetTimeoutConfig().GetTimeout() != nil { + action.Timeout = config.GetTimeoutConfig().GetTimeout() + } + + if config.GetTimeoutConfig().GetTimeout() != nil { + action.Timeout = config.GetTimeoutConfig().GetTimeout() + } + + if config.GetTimeoutConfig().GetIdleTimeout() != nil { + action.IdleTimeout = config.GetTimeoutConfig().GetIdleTimeout() + } + + if config.GetRetryPolicy() != nil { + action.RetryPolicy = makeEnvoyRetryPolicyFromProxystateRetryPolicy(config.GetRetryPolicy()) + } + } + + return envoyRouteRoute +} + +func makeEnvoyClusterWeightFromProxystateWeightedCluster(cluster *pbproxystate.L7WeightedDestinationCluster) *envoy_route_v3.WeightedCluster_ClusterWeight { + envoyClusterWeight := &envoy_route_v3.WeightedCluster_ClusterWeight{ + Name: cluster.GetName(), + Weight: cluster.GetWeight(), + } + + for _, hm := range cluster.GetHeaderMutations() { + injectEnvoyClusterWeightWithProxystateHeaderMutation(envoyClusterWeight, hm) + } + + return envoyClusterWeight +} + +func injectEnvoyClusterWeightWithProxystateHeaderMutation(envoyClusterWeight *envoy_route_v3.WeightedCluster_ClusterWeight, mutation *pbproxystate.HeaderMutation) { + + mutation.GetAction() + switch mutation.GetAction().(type) { + case *pbproxystate.HeaderMutation_RequestHeaderAdd: + action := mutation.GetRequestHeaderAdd() + header := action.GetHeader() + app := action.GetAppendAction() == pbproxystate.AppendAction_APPEND_ACTION_APPEND_IF_EXISTS_OR_ADD + + hvo := &envoy_core_v3.HeaderValueOption{ + Header: &envoy_core_v3.HeaderValue{ + Key: header.GetKey(), + Value: header.GetValue(), + }, + Append: response.MakeBoolValue(app), + } + envoyClusterWeight.RequestHeadersToAdd = append(envoyClusterWeight.RequestHeadersToAdd, hvo) + + case *pbproxystate.HeaderMutation_RequestHeaderRemove: + action := mutation.GetRequestHeaderRemove() + envoyClusterWeight.RequestHeadersToRemove = append(envoyClusterWeight.RequestHeadersToRemove, action.GetHeaderKeys()...) + + case *pbproxystate.HeaderMutation_ResponseHeaderAdd: + action := mutation.GetResponseHeaderAdd() + header := action.GetHeader() + app := action.GetAppendAction() == pbproxystate.AppendAction_APPEND_ACTION_APPEND_IF_EXISTS_OR_ADD + + hvo := &envoy_core_v3.HeaderValueOption{ + Header: &envoy_core_v3.HeaderValue{ + Key: header.GetKey(), + Value: header.GetValue(), + }, + Append: response.MakeBoolValue(app), + } + envoyClusterWeight.ResponseHeadersToAdd = append(envoyClusterWeight.ResponseHeadersToAdd, hvo) + + case *pbproxystate.HeaderMutation_ResponseHeaderRemove: + action := mutation.GetResponseHeaderRemove() + envoyClusterWeight.ResponseHeadersToRemove = append(envoyClusterWeight.ResponseHeadersToRemove, action.GetHeaderKeys()...) + + default: + // This shouldn't be possible considering the types of Destination + return + } +} + +func injectEnvoyRouteActionWithProxystateDestinationConfig(envoyAction *envoy_route_v3.RouteAction, config *pbproxystate.DestinationConfiguration) { + if config == nil { + return + } + + if len(config.GetHashPolicies()) > 0 { + envoyAction.HashPolicy = make([]*envoy_route_v3.RouteAction_HashPolicy, 0, len(config.GetHashPolicies())) + } + for _, policy := range config.GetHashPolicies() { + envoyPolicy := makeEnvoyHashPolicyFromProxystateLBHashPolicy(policy) + envoyAction.HashPolicy = append(envoyAction.HashPolicy, envoyPolicy) + } + + if config.AutoHostRewrite != nil { + envoyAction.HostRewriteSpecifier = &envoy_route_v3.RouteAction_AutoHostRewrite{ + AutoHostRewrite: config.AutoHostRewrite, + } + } +} + +func makeEnvoyHashPolicyFromProxystateLBHashPolicy(psPolicy *pbproxystate.LoadBalancerHashPolicy) *envoy_route_v3.RouteAction_HashPolicy { + + switch psPolicy.GetPolicy().(type) { + case *pbproxystate.LoadBalancerHashPolicy_ConnectionProperties: + return &envoy_route_v3.RouteAction_HashPolicy{ + PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_ConnectionProperties_{ + ConnectionProperties: &envoy_route_v3.RouteAction_HashPolicy_ConnectionProperties{ + SourceIp: true, // always true + }, + }, + Terminal: psPolicy.GetConnectionProperties().GetTerminal(), + } + + case *pbproxystate.LoadBalancerHashPolicy_Header: + return &envoy_route_v3.RouteAction_HashPolicy{ + PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_Header_{ + Header: &envoy_route_v3.RouteAction_HashPolicy_Header{ + HeaderName: psPolicy.GetHeader().GetName(), + }, + }, + Terminal: psPolicy.GetHeader().GetTerminal(), + } + + case *pbproxystate.LoadBalancerHashPolicy_Cookie: + cookie := &envoy_route_v3.RouteAction_HashPolicy_Cookie{ + Name: psPolicy.GetCookie().GetName(), + Path: psPolicy.GetCookie().GetPath(), + Ttl: psPolicy.GetCookie().GetTtl(), + } + + return &envoy_route_v3.RouteAction_HashPolicy{ + PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_Cookie_{ + Cookie: cookie, + }, + Terminal: psPolicy.GetCookie().GetTerminal(), + } + + case *pbproxystate.LoadBalancerHashPolicy_QueryParameter: + return &envoy_route_v3.RouteAction_HashPolicy{ + PolicySpecifier: &envoy_route_v3.RouteAction_HashPolicy_QueryParameter_{ + QueryParameter: &envoy_route_v3.RouteAction_HashPolicy_QueryParameter{ + Name: psPolicy.GetQueryParameter().GetName(), + }, + }, + Terminal: psPolicy.GetQueryParameter().GetTerminal(), + } + } + // This shouldn't be possible considering the types of LoadBalancerPolicy + return nil +} + +func makeEnvoyRetryPolicyFromProxystateRetryPolicy(psRetryPolicy *pbproxystate.RetryPolicy) *envoy_route_v3.RetryPolicy { + return &envoy_route_v3.RetryPolicy{ + NumRetries: psRetryPolicy.GetNumRetries(), + RetriableStatusCodes: psRetryPolicy.GetRetriableStatusCodes(), + RetryOn: psRetryPolicy.GetRetryOn(), + } +} + +func injectEnvoyRouteRuleWithProxystateHeaderMutation(envoyRouteRule *envoy_route_v3.Route, mutation *pbproxystate.HeaderMutation) { + + mutation.GetAction() + switch mutation.GetAction().(type) { + case *pbproxystate.HeaderMutation_RequestHeaderAdd: + action := mutation.GetRequestHeaderAdd() + header := action.GetHeader() + app := action.GetAppendAction() == pbproxystate.AppendAction_APPEND_ACTION_APPEND_IF_EXISTS_OR_ADD + + hvo := &envoy_core_v3.HeaderValueOption{ + Header: &envoy_core_v3.HeaderValue{ + Key: header.GetKey(), + Value: header.GetValue(), + }, + Append: response.MakeBoolValue(app), + } + envoyRouteRule.RequestHeadersToAdd = append(envoyRouteRule.RequestHeadersToAdd, hvo) + + case *pbproxystate.HeaderMutation_RequestHeaderRemove: + action := mutation.GetRequestHeaderRemove() + envoyRouteRule.RequestHeadersToRemove = append(envoyRouteRule.RequestHeadersToRemove, action.GetHeaderKeys()...) + + case *pbproxystate.HeaderMutation_ResponseHeaderAdd: + action := mutation.GetResponseHeaderAdd() + header := action.GetHeader() + app := action.GetAppendAction() == pbproxystate.AppendAction_APPEND_ACTION_APPEND_IF_EXISTS_OR_ADD + + hvo := &envoy_core_v3.HeaderValueOption{ + Header: &envoy_core_v3.HeaderValue{ + Key: header.GetKey(), + Value: header.GetValue(), + }, + Append: response.MakeBoolValue(app), + } + envoyRouteRule.ResponseHeadersToAdd = append(envoyRouteRule.ResponseHeadersToAdd, hvo) + + case *pbproxystate.HeaderMutation_ResponseHeaderRemove: + action := mutation.GetResponseHeaderRemove() + envoyRouteRule.ResponseHeadersToRemove = append(envoyRouteRule.ResponseHeadersToRemove, action.GetHeaderKeys()...) + + default: + // This shouldn't be possible considering the types of Destination + return + } +} + +func injectEnvoyVirtualHostWithProxystateHeaderMutation(envoyVirtualHost *envoy_route_v3.VirtualHost, mutation *pbproxystate.HeaderMutation) { + + mutation.GetAction() + switch mutation.GetAction().(type) { + case *pbproxystate.HeaderMutation_RequestHeaderAdd: + action := mutation.GetRequestHeaderAdd() + header := action.GetHeader() + app := action.GetAppendAction() == pbproxystate.AppendAction_APPEND_ACTION_APPEND_IF_EXISTS_OR_ADD + + hvo := &envoy_core_v3.HeaderValueOption{ + Header: &envoy_core_v3.HeaderValue{ + Key: header.GetKey(), + Value: header.GetValue(), + }, + Append: response.MakeBoolValue(app), + } + envoyVirtualHost.RequestHeadersToAdd = append(envoyVirtualHost.RequestHeadersToAdd, hvo) + + case *pbproxystate.HeaderMutation_RequestHeaderRemove: + action := mutation.GetRequestHeaderRemove() + envoyVirtualHost.RequestHeadersToRemove = append(envoyVirtualHost.RequestHeadersToRemove, action.GetHeaderKeys()...) + + case *pbproxystate.HeaderMutation_ResponseHeaderAdd: + action := mutation.GetResponseHeaderAdd() + header := action.GetHeader() + app := action.GetAppendAction() == pbproxystate.AppendAction_APPEND_ACTION_APPEND_IF_EXISTS_OR_ADD + + hvo := &envoy_core_v3.HeaderValueOption{ + Header: &envoy_core_v3.HeaderValue{ + Key: header.GetKey(), + Value: header.GetValue(), + }, + Append: response.MakeBoolValue(app), + } + envoyVirtualHost.ResponseHeadersToAdd = append(envoyVirtualHost.ResponseHeadersToAdd, hvo) + + case *pbproxystate.HeaderMutation_ResponseHeaderRemove: + action := mutation.GetResponseHeaderRemove() + envoyVirtualHost.ResponseHeadersToRemove = append(envoyVirtualHost.ResponseHeadersToRemove, action.GetHeaderKeys()...) + + default: + // This shouldn't be possible considering the types of Destination + return + } +} diff --git a/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.pb.go b/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.pb.go index 671e25f4ca04..ab573377aac2 100644 --- a/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.pb.go +++ b/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.pb.go @@ -77,6 +77,8 @@ type Cluster struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields + // name is the name of the cluster. + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` // group is either a failover group or endpoint group. If this cluster needs to failover to other clusters, use the failover group. If this cluster routes directly to endpoints, use the endpoint group. // // Types that are assignable to Group: @@ -85,11 +87,11 @@ type Cluster struct { // *Cluster_EndpointGroup Group isCluster_Group `protobuf_oneof:"group"` // escape_hatch_cluster_json configures a user configured escape hatch cluster. - EscapeHatchClusterJson string `protobuf:"bytes,3,opt,name=escape_hatch_cluster_json,json=escapeHatchClusterJson,proto3" json:"escape_hatch_cluster_json,omitempty"` + EscapeHatchClusterJson string `protobuf:"bytes,4,opt,name=escape_hatch_cluster_json,json=escapeHatchClusterJson,proto3" json:"escape_hatch_cluster_json,omitempty"` // alt_stat_name is the name used for observability in place of cluster name if provided. - AltStatName string `protobuf:"bytes,4,opt,name=alt_stat_name,json=altStatName,proto3" json:"alt_stat_name,omitempty"` + AltStatName string `protobuf:"bytes,5,opt,name=alt_stat_name,json=altStatName,proto3" json:"alt_stat_name,omitempty"` // protocol is the local path protocol or the service protocol. - Protocol string `protobuf:"bytes,5,opt,name=protocol,proto3" json:"protocol,omitempty"` + Protocol string `protobuf:"bytes,6,opt,name=protocol,proto3" json:"protocol,omitempty"` } func (x *Cluster) Reset() { @@ -124,6 +126,13 @@ func (*Cluster) Descriptor() ([]byte, []int) { return file_pbmesh_v1alpha1_pbproxystate_cluster_proto_rawDescGZIP(), []int{0} } +func (x *Cluster) GetName() string { + if x != nil { + return x.Name + } + return "" +} + func (m *Cluster) GetGroup() isCluster_Group { if m != nil { return m.Group @@ -171,11 +180,11 @@ type isCluster_Group interface { } type Cluster_FailoverGroup struct { - FailoverGroup *FailoverGroup `protobuf:"bytes,1,opt,name=failover_group,json=failoverGroup,proto3,oneof"` + FailoverGroup *FailoverGroup `protobuf:"bytes,2,opt,name=failover_group,json=failoverGroup,proto3,oneof"` } type Cluster_EndpointGroup struct { - EndpointGroup *EndpointGroup `protobuf:"bytes,2,opt,name=endpoint_group,json=endpointGroup,proto3,oneof"` + EndpointGroup *EndpointGroup `protobuf:"bytes,3,opt,name=endpoint_group,json=endpointGroup,proto3,oneof"` } func (*Cluster_FailoverGroup) isCluster_Group() {} @@ -1697,395 +1706,396 @@ var file_pbmesh_v1alpha1_pbproxystate_cluster_proto_rawDesc = []byte{ 0x70, 0x62, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2f, 0x74, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x5f, 0x73, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x22, 0xd7, 0x02, 0x0a, 0x07, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, - 0x63, 0x0a, 0x0e, 0x66, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x5f, 0x67, 0x72, 0x6f, 0x75, - 0x70, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, - 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, - 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, - 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x47, 0x72, - 0x6f, 0x75, 0x70, 0x48, 0x00, 0x52, 0x0d, 0x66, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x47, - 0x72, 0x6f, 0x75, 0x70, 0x12, 0x63, 0x0a, 0x0e, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, - 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, + 0x6f, 0x74, 0x6f, 0x22, 0xeb, 0x02, 0x0a, 0x07, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, + 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x12, 0x63, 0x0a, 0x0e, 0x66, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x5f, + 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, + 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, + 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, + 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, + 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x48, 0x00, 0x52, 0x0d, 0x66, 0x61, 0x69, 0x6c, 0x6f, + 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x63, 0x0a, 0x0e, 0x65, 0x6e, 0x64, 0x70, + 0x6f, 0x69, 0x6e, 0x74, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, + 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, + 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x45, + 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x48, 0x00, 0x52, 0x0d, + 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x39, 0x0a, + 0x19, 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x5f, 0x68, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x63, 0x6c, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x16, 0x65, 0x73, 0x63, 0x61, 0x70, 0x65, 0x48, 0x61, 0x74, 0x63, 0x68, 0x43, 0x6c, 0x75, + 0x73, 0x74, 0x65, 0x72, 0x4a, 0x73, 0x6f, 0x6e, 0x12, 0x22, 0x0a, 0x0d, 0x61, 0x6c, 0x74, 0x5f, + 0x73, 0x74, 0x61, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x61, 0x6c, 0x74, 0x53, 0x74, 0x61, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x63, 0x6f, 0x6c, 0x42, 0x07, 0x0a, 0x05, 0x67, 0x72, 0x6f, 0x75, + 0x70, 0x22, 0xce, 0x01, 0x0a, 0x0d, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x47, 0x72, + 0x6f, 0x75, 0x70, 0x12, 0x63, 0x0a, 0x0f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x5f, + 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x45, 0x6e, 0x64, 0x70, 0x6f, - 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x48, 0x00, 0x52, 0x0d, 0x65, 0x6e, 0x64, 0x70, - 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x39, 0x0a, 0x19, 0x65, 0x73, 0x63, - 0x61, 0x70, 0x65, 0x5f, 0x68, 0x61, 0x74, 0x63, 0x68, 0x5f, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, - 0x72, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x16, 0x65, 0x73, - 0x63, 0x61, 0x70, 0x65, 0x48, 0x61, 0x74, 0x63, 0x68, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, - 0x4a, 0x73, 0x6f, 0x6e, 0x12, 0x22, 0x0a, 0x0d, 0x61, 0x6c, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, - 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x61, 0x6c, 0x74, - 0x53, 0x74, 0x61, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x63, 0x6f, 0x6c, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x63, 0x6f, 0x6c, 0x42, 0x07, 0x0a, 0x05, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x22, 0xce, 0x01, - 0x0a, 0x0d, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, - 0x63, 0x0a, 0x0f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x5f, 0x67, 0x72, 0x6f, 0x75, - 0x70, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, - 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, - 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, - 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, - 0x72, 0x6f, 0x75, 0x70, 0x52, 0x0e, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, - 0x6f, 0x75, 0x70, 0x73, 0x12, 0x58, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, - 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, - 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, - 0x74, 0x65, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x84, - 0x01, 0x0a, 0x13, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x29, 0x0a, 0x11, 0x75, 0x73, 0x65, 0x5f, 0x61, 0x6c, - 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x08, 0x52, 0x0e, 0x75, 0x73, 0x65, 0x41, 0x6c, 0x74, 0x53, 0x74, 0x61, 0x74, 0x4e, 0x61, 0x6d, - 0x65, 0x12, 0x42, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x5f, 0x74, 0x69, 0x6d, - 0x65, 0x6f, 0x75, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, - 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x22, 0x91, 0x03, 0x0a, 0x0d, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, - 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x5d, 0x0a, 0x07, 0x64, 0x79, 0x6e, 0x61, 0x6d, - 0x69, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, + 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x52, 0x0e, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, + 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x12, 0x58, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, - 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x44, 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x45, 0x6e, - 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x48, 0x00, 0x52, 0x07, 0x64, - 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x12, 0x5a, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x69, 0x63, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, - 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, - 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, - 0x74, 0x61, 0x74, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x63, 0x45, 0x6e, 0x64, 0x70, 0x6f, - 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x48, 0x00, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, - 0x69, 0x63, 0x12, 0x51, 0x0a, 0x03, 0x64, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x3d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, - 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, - 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x44, 0x4e, - 0x53, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x48, 0x00, - 0x52, 0x03, 0x64, 0x6e, 0x73, 0x12, 0x69, 0x0a, 0x0b, 0x70, 0x61, 0x73, 0x73, 0x74, 0x68, 0x72, - 0x6f, 0x75, 0x67, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x45, 0x2e, 0x68, 0x61, 0x73, - 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, - 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, - 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x50, 0x61, 0x73, 0x73, 0x74, 0x68, 0x72, - 0x6f, 0x75, 0x67, 0x68, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, - 0x70, 0x48, 0x00, 0x52, 0x0b, 0x70, 0x61, 0x73, 0x73, 0x74, 0x68, 0x72, 0x6f, 0x75, 0x67, 0x68, - 0x42, 0x07, 0x0a, 0x05, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x22, 0xd8, 0x01, 0x0a, 0x14, 0x44, 0x79, - 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, - 0x75, 0x70, 0x12, 0x5f, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x47, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, - 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x2e, 0x44, 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, - 0x47, 0x72, 0x6f, 0x75, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, 0x63, 0x6f, 0x6e, - 0x66, 0x69, 0x67, 0x12, 0x5f, 0x0a, 0x0c, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, - 0x74, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x68, 0x61, 0x73, 0x68, - 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, - 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, - 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, - 0x74, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x0b, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, - 0x64, 0x54, 0x6c, 0x73, 0x22, 0xe0, 0x01, 0x0a, 0x18, 0x50, 0x61, 0x73, 0x73, 0x74, 0x68, 0x72, - 0x6f, 0x75, 0x67, 0x68, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, - 0x70, 0x12, 0x63, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x4b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, - 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, - 0x50, 0x61, 0x73, 0x73, 0x74, 0x68, 0x72, 0x6f, 0x75, 0x67, 0x68, 0x45, 0x6e, 0x64, 0x70, 0x6f, - 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x5f, 0x0a, 0x0c, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, - 0x6e, 0x64, 0x5f, 0x74, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x68, - 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, - 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, - 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, - 0x70, 0x6f, 0x72, 0x74, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x0b, 0x6f, 0x75, 0x74, 0x62, - 0x6f, 0x75, 0x6e, 0x64, 0x54, 0x6c, 0x73, 0x22, 0xd0, 0x01, 0x0a, 0x10, 0x44, 0x4e, 0x53, 0x45, - 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x5b, 0x0a, 0x06, - 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x43, 0x2e, 0x68, + 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x47, + 0x72, 0x6f, 0x75, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x22, 0x84, 0x01, 0x0a, 0x13, 0x46, 0x61, 0x69, 0x6c, 0x6f, 0x76, 0x65, 0x72, 0x47, + 0x72, 0x6f, 0x75, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x29, 0x0a, 0x11, 0x75, 0x73, + 0x65, 0x5f, 0x61, 0x6c, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x75, 0x73, 0x65, 0x41, 0x6c, 0x74, 0x53, 0x74, 0x61, + 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x42, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x22, 0x91, 0x03, 0x0a, 0x0d, 0x45, 0x6e, + 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x5d, 0x0a, 0x07, 0x64, + 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, - 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x44, 0x4e, 0x53, 0x45, 0x6e, - 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x5f, 0x0a, 0x0c, 0x6f, 0x75, 0x74, - 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x74, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x3c, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, - 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, - 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x72, - 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x0b, 0x6f, - 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x54, 0x6c, 0x73, 0x22, 0x75, 0x0a, 0x13, 0x53, 0x74, - 0x61, 0x74, 0x69, 0x63, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, - 0x70, 0x12, 0x5e, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x46, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, - 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, - 0x53, 0x74, 0x61, 0x74, 0x69, 0x63, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, - 0x6f, 0x75, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, - 0x67, 0x22, 0x7f, 0x0a, 0x16, 0x4c, 0x34, 0x57, 0x65, 0x69, 0x67, 0x68, 0x74, 0x65, 0x64, 0x43, - 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x65, 0x0a, 0x08, 0x63, - 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x49, 0x2e, - 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, - 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, - 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4c, 0x34, 0x57, 0x65, - 0x69, 0x67, 0x68, 0x74, 0x65, 0x64, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x08, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, - 0x72, 0x73, 0x22, 0x7f, 0x0a, 0x16, 0x4c, 0x37, 0x57, 0x65, 0x69, 0x67, 0x68, 0x74, 0x65, 0x64, - 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x65, 0x0a, 0x08, - 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x49, - 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, - 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, - 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4c, 0x37, 0x57, - 0x65, 0x69, 0x67, 0x68, 0x74, 0x65, 0x64, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x08, 0x63, 0x6c, 0x75, 0x73, 0x74, - 0x65, 0x72, 0x73, 0x22, 0x68, 0x0a, 0x1c, 0x4c, 0x34, 0x57, 0x65, 0x69, 0x67, 0x68, 0x74, 0x65, - 0x64, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6c, 0x75, 0x73, - 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x06, 0x77, 0x65, 0x69, 0x67, 0x68, - 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, 0x33, 0x32, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, 0x22, 0xd0, 0x01, - 0x0a, 0x1c, 0x4c, 0x37, 0x57, 0x65, 0x69, 0x67, 0x68, 0x74, 0x65, 0x64, 0x44, 0x65, 0x73, 0x74, - 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x12, - 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x06, 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, - 0x52, 0x06, 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, 0x66, 0x0a, 0x10, 0x68, 0x65, 0x61, 0x64, - 0x65, 0x72, 0x5f, 0x6d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, - 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x0f, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x22, 0x88, 0x08, 0x0a, 0x1a, 0x44, 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x45, 0x6e, 0x64, 0x70, - 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, - 0x42, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, - 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, 0x65, - 0x6f, 0x75, 0x74, 0x12, 0x36, 0x0a, 0x17, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x70, - 0x61, 0x6e, 0x69, 0x63, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x08, 0x52, 0x15, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x50, 0x61, 0x6e, - 0x69, 0x63, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x68, 0x0a, 0x0d, 0x6c, - 0x65, 0x61, 0x73, 0x74, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, - 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, - 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, - 0x2e, 0x4c, 0x42, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4c, 0x65, 0x61, 0x73, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0c, 0x6c, 0x65, 0x61, 0x73, 0x74, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x62, 0x0a, 0x0b, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x72, - 0x6f, 0x62, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x68, 0x61, 0x73, + 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x44, 0x79, 0x6e, 0x61, 0x6d, + 0x69, 0x63, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x48, + 0x00, 0x52, 0x07, 0x64, 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x12, 0x5a, 0x0a, 0x06, 0x73, 0x74, + 0x61, 0x74, 0x69, 0x63, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x40, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, - 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4c, 0x42, 0x50, 0x6f, 0x6c, 0x69, 0x63, - 0x79, 0x52, 0x6f, 0x75, 0x6e, 0x64, 0x52, 0x6f, 0x62, 0x69, 0x6e, 0x48, 0x00, 0x52, 0x0a, 0x72, - 0x6f, 0x75, 0x6e, 0x64, 0x52, 0x6f, 0x62, 0x69, 0x6e, 0x12, 0x55, 0x0a, 0x06, 0x72, 0x61, 0x6e, - 0x64, 0x6f, 0x6d, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, 0x68, - 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, - 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, - 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4c, 0x42, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, - 0x52, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x48, 0x00, 0x52, 0x06, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, - 0x12, 0x5c, 0x0a, 0x09, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x06, 0x20, + 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x63, 0x45, + 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x48, 0x00, 0x52, 0x06, + 0x73, 0x74, 0x61, 0x74, 0x69, 0x63, 0x12, 0x51, 0x0a, 0x03, 0x64, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, - 0x65, 0x2e, 0x4c, 0x42, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x69, 0x6e, 0x67, 0x48, 0x61, - 0x73, 0x68, 0x48, 0x00, 0x52, 0x08, 0x72, 0x69, 0x6e, 0x67, 0x48, 0x61, 0x73, 0x68, 0x12, 0x55, - 0x0a, 0x06, 0x6d, 0x61, 0x67, 0x6c, 0x65, 0x76, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3b, + 0x65, 0x2e, 0x44, 0x4e, 0x53, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, + 0x75, 0x70, 0x48, 0x00, 0x52, 0x03, 0x64, 0x6e, 0x73, 0x12, 0x69, 0x0a, 0x0b, 0x70, 0x61, 0x73, + 0x73, 0x74, 0x68, 0x72, 0x6f, 0x75, 0x67, 0x68, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x45, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, - 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4c, 0x42, 0x50, - 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4d, 0x61, 0x67, 0x6c, 0x65, 0x76, 0x48, 0x00, 0x52, 0x06, 0x6d, - 0x61, 0x67, 0x6c, 0x65, 0x76, 0x12, 0x67, 0x0a, 0x10, 0x63, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, - 0x5f, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x3c, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, - 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, - 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x43, 0x69, - 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x0f, 0x63, - 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x73, 0x12, 0x6a, - 0x0a, 0x11, 0x6f, 0x75, 0x74, 0x6c, 0x69, 0x65, 0x72, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x68, 0x61, 0x73, 0x68, - 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, - 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, - 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4f, 0x75, 0x74, 0x6c, 0x69, 0x65, 0x72, 0x44, - 0x65, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x10, 0x6f, 0x75, 0x74, 0x6c, 0x69, 0x65, - 0x72, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x86, 0x01, 0x0a, 0x1b, 0x75, - 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x46, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, + 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x50, 0x61, 0x73, + 0x73, 0x74, 0x68, 0x72, 0x6f, 0x75, 0x67, 0x68, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x47, 0x72, 0x6f, 0x75, 0x70, 0x48, 0x00, 0x52, 0x0b, 0x70, 0x61, 0x73, 0x73, 0x74, 0x68, 0x72, + 0x6f, 0x75, 0x67, 0x68, 0x42, 0x07, 0x0a, 0x05, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x22, 0xd8, 0x01, + 0x0a, 0x14, 0x44, 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, + 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x5f, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x47, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, + 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, + 0x74, 0x61, 0x74, 0x65, 0x2e, 0x44, 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x45, 0x6e, 0x64, 0x70, + 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, + 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x5f, 0x0a, 0x0c, 0x6f, 0x75, 0x74, 0x62, 0x6f, + 0x75, 0x6e, 0x64, 0x5f, 0x74, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, + 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, 0x72, 0x61, 0x6e, + 0x73, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x0b, 0x6f, 0x75, 0x74, + 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x54, 0x6c, 0x73, 0x22, 0xe0, 0x01, 0x0a, 0x18, 0x50, 0x61, 0x73, + 0x73, 0x74, 0x68, 0x72, 0x6f, 0x75, 0x67, 0x68, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x63, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x4b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, + 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, + 0x61, 0x74, 0x65, 0x2e, 0x50, 0x61, 0x73, 0x73, 0x74, 0x68, 0x72, 0x6f, 0x75, 0x67, 0x68, 0x45, + 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x5f, 0x0a, 0x0c, 0x6f, 0x75, + 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x74, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x3c, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, - 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x55, - 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x19, 0x75, 0x70, 0x73, 0x74, 0x72, 0x65, - 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x12, 0x29, 0x0a, 0x11, 0x75, 0x73, 0x65, 0x5f, 0x61, 0x6c, 0x74, 0x5f, 0x73, - 0x74, 0x61, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, - 0x75, 0x73, 0x65, 0x41, 0x6c, 0x74, 0x53, 0x74, 0x61, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x42, 0x0b, - 0x0a, 0x09, 0x6c, 0x62, 0x5f, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x22, 0x57, 0x0a, 0x14, 0x4c, - 0x42, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4c, 0x65, 0x61, 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x3f, 0x0a, 0x0c, 0x63, 0x68, 0x6f, 0x69, 0x63, 0x65, 0x5f, 0x63, 0x6f, - 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, - 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0b, 0x63, 0x68, 0x6f, 0x69, 0x63, 0x65, 0x43, - 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x14, 0x0a, 0x12, 0x4c, 0x42, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, - 0x52, 0x6f, 0x75, 0x6e, 0x64, 0x52, 0x6f, 0x62, 0x69, 0x6e, 0x22, 0x10, 0x0a, 0x0e, 0x4c, 0x42, - 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x22, 0xa6, 0x01, 0x0a, - 0x10, 0x4c, 0x42, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x69, 0x6e, 0x67, 0x48, 0x61, 0x73, - 0x68, 0x12, 0x48, 0x0a, 0x11, 0x6d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, 0x5f, 0x72, 0x69, 0x6e, - 0x67, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, - 0x49, 0x6e, 0x74, 0x36, 0x34, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0f, 0x6d, 0x69, 0x6e, 0x69, - 0x6d, 0x75, 0x6d, 0x52, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x48, 0x0a, 0x11, 0x6d, - 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x5f, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x69, 0x7a, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, 0x36, 0x34, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0f, 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x52, 0x69, 0x6e, - 0x67, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x10, 0x0a, 0x0e, 0x4c, 0x42, 0x50, 0x6f, 0x6c, 0x69, 0x63, - 0x79, 0x4d, 0x61, 0x67, 0x6c, 0x65, 0x76, 0x22, 0x77, 0x0a, 0x0f, 0x43, 0x69, 0x72, 0x63, 0x75, - 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x73, 0x12, 0x64, 0x0a, 0x0f, 0x75, 0x70, - 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, + 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x54, + 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x52, 0x0b, + 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x54, 0x6c, 0x73, 0x22, 0xd0, 0x01, 0x0a, 0x10, + 0x44, 0x4e, 0x53, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, + 0x12, 0x5b, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x43, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, + 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, + 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x44, + 0x4e, 0x53, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x5f, 0x0a, + 0x0c, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x5f, 0x74, 0x6c, 0x73, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, - 0x65, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, - 0x52, 0x0e, 0x75, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, - 0x22, 0xfd, 0x01, 0x0a, 0x0e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4c, 0x69, 0x6d, - 0x69, 0x74, 0x73, 0x12, 0x45, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, - 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0e, 0x6d, 0x61, 0x78, 0x43, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x4e, 0x0a, 0x14, 0x6d, 0x61, - 0x78, 0x5f, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, 0x33, - 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x12, 0x6d, 0x61, 0x78, 0x50, 0x65, 0x6e, 0x64, 0x69, - 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x12, 0x54, 0x0a, 0x17, 0x6d, 0x61, - 0x78, 0x5f, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x5f, 0x72, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, + 0x65, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x70, 0x6f, 0x72, 0x74, 0x53, 0x6f, 0x63, 0x6b, 0x65, + 0x74, 0x52, 0x0b, 0x6f, 0x75, 0x74, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x54, 0x6c, 0x73, 0x22, 0x75, + 0x0a, 0x13, 0x53, 0x74, 0x61, 0x74, 0x69, 0x63, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, + 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x5e, 0x0a, 0x06, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x46, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, + 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, + 0x61, 0x74, 0x65, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x69, 0x63, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, + 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x52, 0x06, 0x63, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x22, 0x7f, 0x0a, 0x16, 0x4c, 0x34, 0x57, 0x65, 0x69, 0x67, 0x68, + 0x74, 0x65, 0x64, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x12, + 0x65, 0x0a, 0x08, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x49, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, + 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, + 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, + 0x4c, 0x34, 0x57, 0x65, 0x69, 0x67, 0x68, 0x74, 0x65, 0x64, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x08, 0x63, 0x6c, + 0x75, 0x73, 0x74, 0x65, 0x72, 0x73, 0x22, 0x7f, 0x0a, 0x16, 0x4c, 0x37, 0x57, 0x65, 0x69, 0x67, + 0x68, 0x74, 0x65, 0x64, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x47, 0x72, 0x6f, 0x75, 0x70, + 0x12, 0x65, 0x0a, 0x08, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x49, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, + 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x2e, 0x4c, 0x37, 0x57, 0x65, 0x69, 0x67, 0x68, 0x74, 0x65, 0x64, 0x44, 0x65, 0x73, 0x74, 0x69, + 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x52, 0x08, 0x63, + 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x73, 0x22, 0x68, 0x0a, 0x1c, 0x4c, 0x34, 0x57, 0x65, 0x69, + 0x67, 0x68, 0x74, 0x65, 0x64, 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, + 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x06, 0x77, + 0x65, 0x69, 0x67, 0x68, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, - 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x15, 0x6d, 0x61, 0x78, 0x43, 0x6f, - 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, - 0x22, 0x83, 0x03, 0x0a, 0x10, 0x4f, 0x75, 0x74, 0x6c, 0x69, 0x65, 0x72, 0x44, 0x65, 0x74, 0x65, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x35, 0x0a, 0x08, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, - 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x08, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x45, 0x0a, 0x0f, - 0x63, 0x6f, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x35, 0x78, 0x78, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x74, 0x69, 0x76, 0x65, - 0x35, 0x78, 0x78, 0x12, 0x58, 0x0a, 0x19, 0x65, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x69, 0x6e, 0x67, - 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x35, 0x78, 0x78, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x77, 0x65, 0x69, 0x67, 0x68, + 0x74, 0x22, 0xd0, 0x01, 0x0a, 0x1c, 0x4c, 0x37, 0x57, 0x65, 0x69, 0x67, 0x68, 0x74, 0x65, 0x64, + 0x44, 0x65, 0x73, 0x74, 0x69, 0x6e, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6c, 0x75, 0x73, 0x74, + 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x34, 0x0a, 0x06, 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x52, 0x17, 0x65, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x69, 0x6e, 0x67, 0x43, - 0x6f, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x74, 0x69, 0x76, 0x65, 0x35, 0x78, 0x78, 0x12, 0x4e, 0x0a, - 0x14, 0x6d, 0x61, 0x78, 0x5f, 0x65, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x70, 0x65, - 0x72, 0x63, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, - 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x12, 0x6d, 0x61, 0x78, 0x45, 0x6a, - 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x12, 0x47, 0x0a, - 0x12, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x65, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x74, - 0x69, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x10, 0x62, 0x61, 0x73, 0x65, 0x45, 0x6a, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x8b, 0x02, 0x0a, 0x19, 0x55, 0x70, 0x73, 0x74, 0x72, - 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x4a, 0x0a, 0x12, 0x74, 0x63, 0x70, 0x5f, 0x6b, 0x65, 0x65, 0x70, - 0x61, 0x6c, 0x69, 0x76, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x10, - 0x74, 0x63, 0x70, 0x4b, 0x65, 0x65, 0x70, 0x61, 0x6c, 0x69, 0x76, 0x65, 0x54, 0x69, 0x6d, 0x65, - 0x12, 0x52, 0x0a, 0x16, 0x74, 0x63, 0x70, 0x5f, 0x6b, 0x65, 0x65, 0x70, 0x61, 0x6c, 0x69, 0x76, - 0x65, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x14, - 0x74, 0x63, 0x70, 0x4b, 0x65, 0x65, 0x70, 0x61, 0x6c, 0x69, 0x76, 0x65, 0x49, 0x6e, 0x74, 0x65, - 0x72, 0x76, 0x61, 0x6c, 0x12, 0x4e, 0x0a, 0x14, 0x74, 0x63, 0x70, 0x5f, 0x6b, 0x65, 0x65, 0x70, - 0x61, 0x6c, 0x69, 0x76, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, - 0x52, 0x12, 0x74, 0x63, 0x70, 0x4b, 0x65, 0x65, 0x70, 0x61, 0x6c, 0x69, 0x76, 0x65, 0x50, 0x72, - 0x6f, 0x62, 0x65, 0x73, 0x22, 0x64, 0x0a, 0x1e, 0x50, 0x61, 0x73, 0x73, 0x74, 0x68, 0x72, 0x6f, - 0x75, 0x67, 0x68, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, - 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x42, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, - 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x6e, - 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x22, 0x80, 0x05, 0x0a, 0x16, 0x44, - 0x4e, 0x53, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x42, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, - 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, - 0x63, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x36, 0x0a, 0x17, 0x64, 0x69, 0x73, - 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x6e, 0x69, 0x63, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, - 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x15, 0x64, 0x69, 0x73, 0x61, - 0x62, 0x6c, 0x65, 0x50, 0x61, 0x6e, 0x69, 0x63, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, - 0x64, 0x12, 0x61, 0x0a, 0x0e, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x5f, 0x74, - 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3a, 0x2e, 0x68, 0x61, 0x73, 0x68, - 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, - 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, - 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x44, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, - 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0d, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, - 0x54, 0x79, 0x70, 0x65, 0x12, 0x67, 0x0a, 0x10, 0x63, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x5f, - 0x62, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, + 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, 0x12, 0x66, 0x0a, 0x10, + 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x5f, 0x6d, 0x75, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, + 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, + 0x74, 0x61, 0x74, 0x65, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x75, 0x74, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x0f, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x4d, 0x75, 0x74, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x88, 0x08, 0x0a, 0x1a, 0x44, 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, + 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x43, 0x6f, 0x6e, + 0x66, 0x69, 0x67, 0x12, 0x42, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x5f, 0x74, + 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, + 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, + 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x36, 0x0a, 0x17, 0x64, 0x69, 0x73, 0x61, 0x62, + 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x6e, 0x69, 0x63, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, + 0x6c, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x15, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, + 0x65, 0x50, 0x61, 0x6e, 0x69, 0x63, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, + 0x68, 0x0a, 0x0d, 0x6c, 0x65, 0x61, 0x73, 0x74, 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x41, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, + 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, + 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, + 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4c, 0x42, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4c, 0x65, 0x61, + 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0c, 0x6c, 0x65, 0x61, + 0x73, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x62, 0x0a, 0x0b, 0x72, 0x6f, 0x75, + 0x6e, 0x64, 0x5f, 0x72, 0x6f, 0x62, 0x69, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3f, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, - 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x43, 0x69, 0x72, - 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x0f, 0x63, 0x69, - 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x73, 0x12, 0x6a, 0x0a, - 0x11, 0x6f, 0x75, 0x74, 0x6c, 0x69, 0x65, 0x72, 0x5f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, - 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, - 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, - 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4f, 0x75, 0x74, 0x6c, 0x69, 0x65, 0x72, 0x44, 0x65, - 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x10, 0x6f, 0x75, 0x74, 0x6c, 0x69, 0x65, 0x72, - 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x86, 0x01, 0x0a, 0x1b, 0x75, 0x70, - 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x46, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, - 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, - 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x55, 0x70, + 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4c, 0x42, 0x50, + 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x6f, 0x75, 0x6e, 0x64, 0x52, 0x6f, 0x62, 0x69, 0x6e, 0x48, + 0x00, 0x52, 0x0a, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x52, 0x6f, 0x62, 0x69, 0x6e, 0x12, 0x55, 0x0a, + 0x06, 0x72, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3b, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, + 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4c, 0x42, 0x50, 0x6f, + 0x6c, 0x69, 0x63, 0x79, 0x52, 0x61, 0x6e, 0x64, 0x6f, 0x6d, 0x48, 0x00, 0x52, 0x06, 0x72, 0x61, + 0x6e, 0x64, 0x6f, 0x6d, 0x12, 0x5c, 0x0a, 0x09, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x68, 0x61, 0x73, + 0x68, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, + 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, + 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4c, 0x42, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x69, + 0x6e, 0x67, 0x48, 0x61, 0x73, 0x68, 0x48, 0x00, 0x52, 0x08, 0x72, 0x69, 0x6e, 0x67, 0x48, 0x61, + 0x73, 0x68, 0x12, 0x55, 0x0a, 0x06, 0x6d, 0x61, 0x67, 0x6c, 0x65, 0x76, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, + 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x2e, 0x4c, 0x42, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4d, 0x61, 0x67, 0x6c, 0x65, 0x76, 0x48, + 0x00, 0x52, 0x06, 0x6d, 0x61, 0x67, 0x6c, 0x65, 0x76, 0x12, 0x67, 0x0a, 0x10, 0x63, 0x69, 0x72, + 0x63, 0x75, 0x69, 0x74, 0x5f, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x73, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, + 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, + 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, + 0x65, 0x2e, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, + 0x73, 0x52, 0x0f, 0x63, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, + 0x72, 0x73, 0x12, 0x6a, 0x0a, 0x11, 0x6f, 0x75, 0x74, 0x6c, 0x69, 0x65, 0x72, 0x5f, 0x64, 0x65, + 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, + 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4f, 0x75, 0x74, 0x6c, + 0x69, 0x65, 0x72, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x10, 0x6f, 0x75, + 0x74, 0x6c, 0x69, 0x65, 0x72, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x86, + 0x01, 0x0a, 0x1b, 0x75, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x0a, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x46, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, + 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, + 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, + 0x74, 0x65, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x19, 0x75, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x19, 0x75, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, - 0x6d, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x12, 0x29, 0x0a, 0x11, 0x75, 0x73, 0x65, 0x5f, 0x61, 0x6c, 0x74, 0x5f, 0x73, 0x74, - 0x61, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x75, - 0x73, 0x65, 0x41, 0x6c, 0x74, 0x53, 0x74, 0x61, 0x74, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xc8, 0x01, - 0x0a, 0x19, 0x53, 0x74, 0x61, 0x74, 0x69, 0x63, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, - 0x47, 0x72, 0x6f, 0x75, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x42, 0x0a, 0x0f, 0x63, - 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, - 0x0e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, - 0x67, 0x0a, 0x10, 0x63, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x5f, 0x62, 0x72, 0x65, 0x61, 0x6b, - 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x68, 0x61, 0x73, 0x68, - 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, - 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, - 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, - 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x0f, 0x63, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, - 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x73, 0x2a, 0x46, 0x0a, 0x0d, 0x44, 0x69, 0x73, 0x63, - 0x6f, 0x76, 0x65, 0x72, 0x79, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, 0x16, 0x44, 0x49, 0x53, - 0x43, 0x4f, 0x56, 0x45, 0x52, 0x59, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4c, 0x4f, 0x47, 0x49, - 0x43, 0x41, 0x4c, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x44, 0x49, 0x53, 0x43, 0x4f, 0x56, 0x45, - 0x52, 0x59, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x54, 0x52, 0x49, 0x43, 0x54, 0x10, 0x01, - 0x42, 0xd8, 0x02, 0x0a, 0x2f, 0x63, 0x6f, 0x6d, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, - 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, - 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, - 0x74, 0x61, 0x74, 0x65, 0x42, 0x0c, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x50, 0x72, 0x6f, - 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x45, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, - 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x75, - 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2d, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x2f, 0x70, - 0x62, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2f, 0x70, - 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0xa2, 0x02, 0x05, 0x48, 0x43, - 0x4d, 0x56, 0x50, 0xaa, 0x02, 0x2b, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, - 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x2e, 0x56, 0x31, 0x61, 0x6c, - 0x70, 0x68, 0x61, 0x31, 0x2e, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, - 0x65, 0xca, 0x02, 0x2b, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, - 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x4d, 0x65, 0x73, 0x68, 0x5c, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, - 0x61, 0x31, 0x5c, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0xe2, - 0x02, 0x37, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, 0x43, 0x6f, 0x6e, 0x73, - 0x75, 0x6c, 0x5c, 0x4d, 0x65, 0x73, 0x68, 0x5c, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, - 0x5c, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x5c, 0x47, 0x50, - 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x2f, 0x48, 0x61, 0x73, 0x68, - 0x69, 0x63, 0x6f, 0x72, 0x70, 0x3a, 0x3a, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x3a, 0x3a, 0x4d, - 0x65, 0x73, 0x68, 0x3a, 0x3a, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x3a, 0x3a, 0x50, - 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, + 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x29, 0x0a, 0x11, 0x75, 0x73, 0x65, 0x5f, 0x61, + 0x6c, 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0b, 0x20, 0x01, + 0x28, 0x08, 0x52, 0x0e, 0x75, 0x73, 0x65, 0x41, 0x6c, 0x74, 0x53, 0x74, 0x61, 0x74, 0x4e, 0x61, + 0x6d, 0x65, 0x42, 0x0b, 0x0a, 0x09, 0x6c, 0x62, 0x5f, 0x70, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x22, + 0x57, 0x0a, 0x14, 0x4c, 0x42, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4c, 0x65, 0x61, 0x73, 0x74, + 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3f, 0x0a, 0x0c, 0x63, 0x68, 0x6f, 0x69, 0x63, + 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x55, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0b, 0x63, 0x68, 0x6f, + 0x69, 0x63, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x14, 0x0a, 0x12, 0x4c, 0x42, 0x50, 0x6f, + 0x6c, 0x69, 0x63, 0x79, 0x52, 0x6f, 0x75, 0x6e, 0x64, 0x52, 0x6f, 0x62, 0x69, 0x6e, 0x22, 0x10, + 0x0a, 0x0e, 0x4c, 0x42, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x61, 0x6e, 0x64, 0x6f, 0x6d, + 0x22, 0xa6, 0x01, 0x0a, 0x10, 0x4c, 0x42, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x52, 0x69, 0x6e, + 0x67, 0x48, 0x61, 0x73, 0x68, 0x12, 0x48, 0x0a, 0x11, 0x6d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, + 0x5f, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, 0x36, 0x34, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0f, + 0x6d, 0x69, 0x6e, 0x69, 0x6d, 0x75, 0x6d, 0x52, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65, 0x12, + 0x48, 0x0a, 0x11, 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, 0x6d, 0x5f, 0x72, 0x69, 0x6e, 0x67, 0x5f, + 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, + 0x74, 0x36, 0x34, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0f, 0x6d, 0x61, 0x78, 0x69, 0x6d, 0x75, + 0x6d, 0x52, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65, 0x22, 0x10, 0x0a, 0x0e, 0x4c, 0x42, 0x50, + 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x4d, 0x61, 0x67, 0x6c, 0x65, 0x76, 0x22, 0x77, 0x0a, 0x0f, 0x43, + 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x73, 0x12, 0x64, + 0x0a, 0x0f, 0x75, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, + 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3b, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, + 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, + 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4c, 0x69, + 0x6d, 0x69, 0x74, 0x73, 0x52, 0x0e, 0x75, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x4c, 0x69, + 0x6d, 0x69, 0x74, 0x73, 0x22, 0xfd, 0x01, 0x0a, 0x0e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, + 0x6d, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x73, 0x12, 0x45, 0x0a, 0x0f, 0x6d, 0x61, 0x78, 0x5f, 0x63, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, + 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0e, + 0x6d, 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x4e, + 0x0a, 0x14, 0x6d, 0x61, 0x78, 0x5f, 0x70, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x72, 0x65, + 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, + 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x12, 0x6d, 0x61, 0x78, 0x50, + 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x12, 0x54, + 0x0a, 0x17, 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, + 0x5f, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x15, 0x6d, + 0x61, 0x78, 0x43, 0x6f, 0x6e, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x73, 0x22, 0x83, 0x03, 0x0a, 0x10, 0x4f, 0x75, 0x74, 0x6c, 0x69, 0x65, 0x72, + 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x35, 0x0a, 0x08, 0x69, 0x6e, 0x74, + 0x65, 0x72, 0x76, 0x61, 0x6c, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, + 0x12, 0x45, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x74, 0x69, 0x76, 0x65, 0x5f, + 0x35, 0x78, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, + 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x73, 0x65, 0x63, 0x75, + 0x74, 0x69, 0x76, 0x65, 0x35, 0x78, 0x78, 0x12, 0x58, 0x0a, 0x19, 0x65, 0x6e, 0x66, 0x6f, 0x72, + 0x63, 0x69, 0x6e, 0x67, 0x5f, 0x63, 0x6f, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x74, 0x69, 0x76, 0x65, + 0x5f, 0x35, 0x78, 0x78, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, + 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x17, 0x65, 0x6e, 0x66, 0x6f, 0x72, 0x63, + 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x74, 0x69, 0x76, 0x65, 0x35, 0x78, + 0x78, 0x12, 0x4e, 0x0a, 0x14, 0x6d, 0x61, 0x78, 0x5f, 0x65, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x5f, 0x70, 0x65, 0x72, 0x63, 0x65, 0x6e, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, + 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x12, 0x6d, + 0x61, 0x78, 0x45, 0x6a, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x50, 0x65, 0x72, 0x63, 0x65, 0x6e, + 0x74, 0x12, 0x47, 0x0a, 0x12, 0x62, 0x61, 0x73, 0x65, 0x5f, 0x65, 0x6a, 0x65, 0x63, 0x74, 0x69, + 0x6f, 0x6e, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, + 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, + 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x10, 0x62, 0x61, 0x73, 0x65, 0x45, 0x6a, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x8b, 0x02, 0x0a, 0x19, 0x55, + 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, + 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x4a, 0x0a, 0x12, 0x74, 0x63, 0x70, 0x5f, + 0x6b, 0x65, 0x65, 0x70, 0x61, 0x6c, 0x69, 0x76, 0x65, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, + 0x75, 0x65, 0x52, 0x10, 0x74, 0x63, 0x70, 0x4b, 0x65, 0x65, 0x70, 0x61, 0x6c, 0x69, 0x76, 0x65, + 0x54, 0x69, 0x6d, 0x65, 0x12, 0x52, 0x0a, 0x16, 0x74, 0x63, 0x70, 0x5f, 0x6b, 0x65, 0x65, 0x70, + 0x61, 0x6c, 0x69, 0x76, 0x65, 0x5f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, + 0x75, 0x65, 0x52, 0x14, 0x74, 0x63, 0x70, 0x4b, 0x65, 0x65, 0x70, 0x61, 0x6c, 0x69, 0x76, 0x65, + 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c, 0x12, 0x4e, 0x0a, 0x14, 0x74, 0x63, 0x70, 0x5f, + 0x6b, 0x65, 0x65, 0x70, 0x61, 0x6c, 0x69, 0x76, 0x65, 0x5f, 0x70, 0x72, 0x6f, 0x62, 0x65, 0x73, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56, + 0x61, 0x6c, 0x75, 0x65, 0x52, 0x12, 0x74, 0x63, 0x70, 0x4b, 0x65, 0x65, 0x70, 0x61, 0x6c, 0x69, + 0x76, 0x65, 0x50, 0x72, 0x6f, 0x62, 0x65, 0x73, 0x22, 0x64, 0x0a, 0x1e, 0x50, 0x61, 0x73, 0x73, + 0x74, 0x68, 0x72, 0x6f, 0x75, 0x67, 0x68, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, + 0x72, 0x6f, 0x75, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x42, 0x0a, 0x0f, 0x63, 0x6f, + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0e, + 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x22, 0x80, + 0x05, 0x0a, 0x16, 0x44, 0x4e, 0x53, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, + 0x6f, 0x75, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x42, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, + 0x6e, 0x65, 0x63, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0e, 0x63, + 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74, 0x12, 0x36, 0x0a, + 0x17, 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x70, 0x61, 0x6e, 0x69, 0x63, 0x5f, 0x74, + 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x15, + 0x64, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65, 0x50, 0x61, 0x6e, 0x69, 0x63, 0x54, 0x68, 0x72, 0x65, + 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x61, 0x0a, 0x0e, 0x64, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, + 0x72, 0x79, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x3a, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, + 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x44, 0x69, 0x73, 0x63, + 0x6f, 0x76, 0x65, 0x72, 0x79, 0x54, 0x79, 0x70, 0x65, 0x52, 0x0d, 0x64, 0x69, 0x73, 0x63, 0x6f, + 0x76, 0x65, 0x72, 0x79, 0x54, 0x79, 0x70, 0x65, 0x12, 0x67, 0x0a, 0x10, 0x63, 0x69, 0x72, 0x63, + 0x75, 0x69, 0x74, 0x5f, 0x62, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x73, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x3c, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, + 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, + 0x2e, 0x43, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x73, + 0x52, 0x0f, 0x63, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, + 0x73, 0x12, 0x6a, 0x0a, 0x11, 0x6f, 0x75, 0x74, 0x6c, 0x69, 0x65, 0x72, 0x5f, 0x64, 0x65, 0x74, + 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3d, 0x2e, 0x68, + 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, + 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, + 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x4f, 0x75, 0x74, 0x6c, 0x69, + 0x65, 0x72, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x10, 0x6f, 0x75, 0x74, + 0x6c, 0x69, 0x65, 0x72, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x86, 0x01, + 0x0a, 0x1b, 0x75, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x5f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, + 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x06, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x46, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, + 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, + 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, + 0x65, 0x2e, 0x55, 0x70, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, + 0x74, 0x69, 0x6f, 0x6e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x19, 0x75, 0x70, 0x73, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x43, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x29, 0x0a, 0x11, 0x75, 0x73, 0x65, 0x5f, 0x61, 0x6c, + 0x74, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, + 0x08, 0x52, 0x0e, 0x75, 0x73, 0x65, 0x41, 0x6c, 0x74, 0x53, 0x74, 0x61, 0x74, 0x4e, 0x61, 0x6d, + 0x65, 0x22, 0xc8, 0x01, 0x0a, 0x19, 0x53, 0x74, 0x61, 0x74, 0x69, 0x63, 0x45, 0x6e, 0x64, 0x70, + 0x6f, 0x69, 0x6e, 0x74, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, + 0x42, 0x0a, 0x0f, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x5f, 0x74, 0x69, 0x6d, 0x65, 0x6f, + 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x0e, 0x63, 0x6f, 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x54, 0x69, 0x6d, 0x65, + 0x6f, 0x75, 0x74, 0x12, 0x67, 0x0a, 0x10, 0x63, 0x69, 0x72, 0x63, 0x75, 0x69, 0x74, 0x5f, 0x62, + 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x3c, 0x2e, + 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, + 0x2e, 0x6d, 0x65, 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, + 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x2e, 0x43, 0x69, 0x72, 0x63, + 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x73, 0x52, 0x0f, 0x63, 0x69, 0x72, + 0x63, 0x75, 0x69, 0x74, 0x42, 0x72, 0x65, 0x61, 0x6b, 0x65, 0x72, 0x73, 0x2a, 0x46, 0x0a, 0x0d, + 0x44, 0x69, 0x73, 0x63, 0x6f, 0x76, 0x65, 0x72, 0x79, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1a, 0x0a, + 0x16, 0x44, 0x49, 0x53, 0x43, 0x4f, 0x56, 0x45, 0x52, 0x59, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, + 0x4c, 0x4f, 0x47, 0x49, 0x43, 0x41, 0x4c, 0x10, 0x00, 0x12, 0x19, 0x0a, 0x15, 0x44, 0x49, 0x53, + 0x43, 0x4f, 0x56, 0x45, 0x52, 0x59, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x54, 0x52, 0x49, + 0x43, 0x54, 0x10, 0x01, 0x42, 0xd8, 0x02, 0x0a, 0x2f, 0x63, 0x6f, 0x6d, 0x2e, 0x68, 0x61, 0x73, + 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2e, 0x63, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x6d, 0x65, + 0x73, 0x68, 0x2e, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x70, 0x62, 0x70, 0x72, + 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x42, 0x0c, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, + 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x45, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, + 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x68, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x2f, 0x63, + 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2d, 0x70, 0x75, 0x62, 0x6c, + 0x69, 0x63, 0x2f, 0x70, 0x62, 0x6d, 0x65, 0x73, 0x68, 0x2f, 0x76, 0x31, 0x61, 0x6c, 0x70, 0x68, + 0x61, 0x31, 0x2f, 0x70, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0xa2, + 0x02, 0x05, 0x48, 0x43, 0x4d, 0x56, 0x50, 0xaa, 0x02, 0x2b, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, + 0x6f, 0x72, 0x70, 0x2e, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x2e, 0x4d, 0x65, 0x73, 0x68, 0x2e, + 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x2e, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, + 0x73, 0x74, 0x61, 0x74, 0x65, 0xca, 0x02, 0x2b, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, + 0x70, 0x5c, 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x4d, 0x65, 0x73, 0x68, 0x5c, 0x56, 0x31, + 0x61, 0x6c, 0x70, 0x68, 0x61, 0x31, 0x5c, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, + 0x61, 0x74, 0x65, 0xe2, 0x02, 0x37, 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x5c, + 0x43, 0x6f, 0x6e, 0x73, 0x75, 0x6c, 0x5c, 0x4d, 0x65, 0x73, 0x68, 0x5c, 0x56, 0x31, 0x61, 0x6c, + 0x70, 0x68, 0x61, 0x31, 0x5c, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, + 0x65, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x2f, + 0x48, 0x61, 0x73, 0x68, 0x69, 0x63, 0x6f, 0x72, 0x70, 0x3a, 0x3a, 0x43, 0x6f, 0x6e, 0x73, 0x75, + 0x6c, 0x3a, 0x3a, 0x4d, 0x65, 0x73, 0x68, 0x3a, 0x3a, 0x56, 0x31, 0x61, 0x6c, 0x70, 0x68, 0x61, + 0x31, 0x3a, 0x3a, 0x50, 0x62, 0x70, 0x72, 0x6f, 0x78, 0x79, 0x73, 0x74, 0x61, 0x74, 0x65, 0x62, + 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.proto b/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.proto index 860ebf2a5cbf..2073302c0575 100644 --- a/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.proto +++ b/proto-public/pbmesh/v1alpha1/pbproxystate/cluster.proto @@ -11,17 +11,19 @@ import "pbmesh/v1alpha1/pbproxystate/header_mutations.proto"; import "pbmesh/v1alpha1/pbproxystate/transport_socket.proto"; message Cluster { + // name is the name of the cluster. + string name = 1; // group is either a failover group or endpoint group. If this cluster needs to failover to other clusters, use the failover group. If this cluster routes directly to endpoints, use the endpoint group. oneof group { - FailoverGroup failover_group = 1; - EndpointGroup endpoint_group = 2; + FailoverGroup failover_group = 2; + EndpointGroup endpoint_group = 3; } // escape_hatch_cluster_json configures a user configured escape hatch cluster. - string escape_hatch_cluster_json = 3; + string escape_hatch_cluster_json = 4; // alt_stat_name is the name used for observability in place of cluster name if provided. - string alt_stat_name = 4; + string alt_stat_name = 5; // protocol is the local path protocol or the service protocol. - string protocol = 5; + string protocol = 6; } message FailoverGroup { diff --git a/proto-public/pbmesh/v1alpha1/proxy_state.pb.go b/proto-public/pbmesh/v1alpha1/proxy_state.pb.go index 193050474c28..a9c7b57a5729 100644 --- a/proto-public/pbmesh/v1alpha1/proxy_state.pb.go +++ b/proto-public/pbmesh/v1alpha1/proxy_state.pb.go @@ -105,7 +105,7 @@ type ProxyState struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - // identity is a reference to the WorkloadIdentity associated with this proxy. + // identity is a reference to the identity of the workload this proxy is for. Identity *pbresource.Reference `protobuf:"bytes,1,opt,name=identity,proto3" json:"identity,omitempty"` // listeners is a list of listeners for this proxy. Listeners []*pbproxystate.Listener `protobuf:"bytes,2,rep,name=listeners,proto3" json:"listeners,omitempty"` diff --git a/proto-public/pbmesh/v1alpha1/proxy_state.proto b/proto-public/pbmesh/v1alpha1/proxy_state.proto index 756d7dd307c8..dfbe1ca1dee2 100644 --- a/proto-public/pbmesh/v1alpha1/proxy_state.proto +++ b/proto-public/pbmesh/v1alpha1/proxy_state.proto @@ -30,7 +30,7 @@ message ProxyStateTemplate { } message ProxyState { - // identity is a reference to the WorkloadIdentity associated with this proxy. + // identity is a reference to the identity of the workload this proxy is for. hashicorp.consul.resource.Reference identity = 1; // listeners is a list of listeners for this proxy. repeated pbproxystate.Listener listeners = 2; From 376b016cbdf5bb462de4f751499f3acc14c9862a Mon Sep 17 00:00:00 2001 From: John Murret Date: Thu, 17 Aug 2023 13:14:53 -0600 Subject: [PATCH 07/12] Update endpoints_test.go --- agent/xds/endpoints_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/agent/xds/endpoints_test.go b/agent/xds/endpoints_test.go index dc80636a9539..9daf5b31f55b 100644 --- a/agent/xds/endpoints_test.go +++ b/agent/xds/endpoints_test.go @@ -8,8 +8,6 @@ import ( "sort" "testing" - "github.com/hashicorp/go-hclog" - envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" "github.com/hashicorp/consul/agent/proxycfg" From 8f0060e63e630d2c5bd3774e6fd5771c17e6ead7 Mon Sep 17 00:00:00 2001 From: John Murret Date: Thu, 17 Aug 2023 13:15:32 -0600 Subject: [PATCH 08/12] Update naming.go --- agent/xds/naming/naming.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/xds/naming/naming.go b/agent/xds/naming/naming.go index b1de1ad12862..3e19d9327003 100644 --- a/agent/xds/naming/naming.go +++ b/agent/xds/naming/naming.go @@ -1,5 +1,5 @@ // Copyright (c) HashiCorp, Inc. -// SPDX-License-Identifier: MPL-2.0 +// SPDX-License-Identifier: BUSL-1.1 package naming From 502784b71423b138ee38f62dcc374269f60c1ef5 Mon Sep 17 00:00:00 2001 From: John Murret Date: Thu, 17 Aug 2023 13:18:24 -0600 Subject: [PATCH 09/12] use alsoRunTestForV2 --- agent/xds/listeners_test.go | 54 ++++++++++++++++++------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/agent/xds/listeners_test.go b/agent/xds/listeners_test.go index c58685887db6..130f2234517e 100644 --- a/agent/xds/listeners_test.go +++ b/agent/xds/listeners_test.go @@ -37,7 +37,7 @@ type listenerTestCase struct { // test input. overrideGoldenName string generatorSetup func(*ResourceGenerator) - useV2 bool + alsoRunTestForV2 bool } func makeListenerDiscoChainTests(enterprise bool) []listenerTestCase { @@ -75,7 +75,7 @@ func makeListenerDiscoChainTests(enterprise bool) []listenerTestCase { create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "simple", enterprise, nil, nil) }, - useV2: true, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-http-chain", @@ -124,7 +124,7 @@ func makeListenerDiscoChainTests(enterprise bool) []listenerTestCase { create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "external-sni", enterprise, nil, nil) }, - useV2: true, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-chain-and-overrides", @@ -137,14 +137,14 @@ func makeListenerDiscoChainTests(enterprise bool) []listenerTestCase { create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-remote-gateway", enterprise, nil, nil) }, - useV2: true, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tcp-chain-failover-through-local-gateway", create: func(t testinf.T) *proxycfg.ConfigSnapshot { return proxycfg.TestConfigSnapshotDiscoveryChain(t, "failover-through-local-gateway", enterprise, nil, nil) }, - useV2: true, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-jwt-config-entry-with-local", @@ -235,7 +235,7 @@ func TestListenersFromSnapshot(t *testing.T) { }, }) }, - useV2: true, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tls-incoming-min-version", @@ -255,7 +255,7 @@ func TestListenersFromSnapshot(t *testing.T) { }, }) }, - useV2: true, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tls-incoming-max-version", @@ -275,7 +275,7 @@ func TestListenersFromSnapshot(t *testing.T) { }, }) }, - useV2: true, + alsoRunTestForV2: true, }, { name: "connect-proxy-with-tls-incoming-cipher-suites", @@ -298,7 +298,7 @@ func TestListenersFromSnapshot(t *testing.T) { }, }) }, - useV2: true, + alsoRunTestForV2: true, }, { name: "grpc-public-listener", @@ -315,7 +315,7 @@ func TestListenersFromSnapshot(t *testing.T) { ns.Proxy.Config["bind_address"] = "127.0.0.2" }, nil) }, - useV2: true, + alsoRunTestForV2: true, }, { name: "listener-bind-port", @@ -324,7 +324,7 @@ func TestListenersFromSnapshot(t *testing.T) { ns.Proxy.Config["bind_port"] = 8888 }, nil) }, - useV2: true, + alsoRunTestForV2: true, }, { name: "listener-bind-address-port", @@ -334,7 +334,7 @@ func TestListenersFromSnapshot(t *testing.T) { ns.Proxy.Config["bind_port"] = 8888 }, nil) }, - useV2: true, + alsoRunTestForV2: true, }, { name: "listener-unix-domain-socket", @@ -346,7 +346,7 @@ func TestListenersFromSnapshot(t *testing.T) { ns.Proxy.Upstreams[0].LocalBindSocketMode = "0640" }, nil) }, - useV2: true, + alsoRunTestForV2: true, }, { name: "listener-max-inbound-connections", @@ -355,7 +355,7 @@ func TestListenersFromSnapshot(t *testing.T) { ns.Proxy.Config["max_inbound_connections"] = 222 }, nil) }, - useV2: true, + alsoRunTestForV2: true, }, { name: "http2-public-listener", @@ -372,7 +372,7 @@ func TestListenersFromSnapshot(t *testing.T) { ns.Proxy.Config["balance_inbound_connections"] = "exact_balance" }, nil) }, - useV2: true, + alsoRunTestForV2: true, }, { name: "listener-balance-outbound-connections-bind-port", @@ -381,7 +381,7 @@ func TestListenersFromSnapshot(t *testing.T) { ns.Proxy.Upstreams[0].Config["balance_outbound_connections"] = "exact_balance" }, nil) }, - useV2: true, + alsoRunTestForV2: true, }, { name: "http-public-listener", @@ -1190,19 +1190,19 @@ func TestListenersFromSnapshot(t *testing.T) { create: proxycfg.TestConfigSnapshotTransparentProxyResolverRedirectUpstream, }, { - name: "transparent-proxy-catalog-destinations-only", - create: proxycfg.TestConfigSnapshotTransparentProxyCatalogDestinationsOnly, - useV2: true, + name: "transparent-proxy-catalog-destinations-only", + create: proxycfg.TestConfigSnapshotTransparentProxyCatalogDestinationsOnly, + alsoRunTestForV2: true, }, { - name: "transparent-proxy-dial-instances-directly", - create: proxycfg.TestConfigSnapshotTransparentProxyDialDirectly, - useV2: true, + name: "transparent-proxy-dial-instances-directly", + create: proxycfg.TestConfigSnapshotTransparentProxyDialDirectly, + alsoRunTestForV2: true, }, { - name: "transparent-proxy-terminating-gateway", - create: proxycfg.TestConfigSnapshotTransparentProxyTerminatingGatewayCatalogDestinationsOnly, - useV2: true, + name: "transparent-proxy-terminating-gateway", + create: proxycfg.TestConfigSnapshotTransparentProxyTerminatingGatewayCatalogDestinationsOnly, + alsoRunTestForV2: true, }, { name: "custom-trace-listener", @@ -1265,7 +1265,7 @@ func TestListenersFromSnapshot(t *testing.T) { }, nil) }, - useV2: true, + alsoRunTestForV2: true, }, { name: "connect-proxy-without-tproxy-and-permissive-mtls", @@ -1275,7 +1275,7 @@ func TestListenersFromSnapshot(t *testing.T) { }, nil) }, - useV2: true, + alsoRunTestForV2: true, }, } From 5b114ae525445cef7c1123fe688494bb26164348 Mon Sep 17 00:00:00 2001 From: John Murret Date: Thu, 17 Aug 2023 13:23:19 -0600 Subject: [PATCH 10/12] remove unused makeAddress --- agent/xdsv2/cluster_resources.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/agent/xdsv2/cluster_resources.go b/agent/xdsv2/cluster_resources.go index 60755e27d7e8..5eedd1f21f37 100644 --- a/agent/xdsv2/cluster_resources.go +++ b/agent/xdsv2/cluster_resources.go @@ -342,19 +342,6 @@ func addEnvoyLBToCluster(dynamicConfig *pbproxystate.DynamicEndpointGroupConfig, return nil } -func makeAddress(ip string, port uint32) *envoy_core_v3.Address { - return &envoy_core_v3.Address{ - Address: &envoy_core_v3.Address_SocketAddress{ - SocketAddress: &envoy_core_v3.SocketAddress{ - Address: ip, - PortSpecifier: &envoy_core_v3.SocketAddress_PortValue{ - PortValue: port, - }, - }, - }, - } -} - // TODO(proxystate): In a future PR this will create clusters and add it to ProxyResources.proxyState func (pr *ProxyResources) makeEnvoyClusterFromL4Destination(name string) error { return nil From f91600b043fd587a26f5a839f4675cea3fd2f7de Mon Sep 17 00:00:00 2001 From: John Murret Date: Thu, 17 Aug 2023 13:49:05 -0600 Subject: [PATCH 11/12] gofmt --- agent/xds/proxystateconverter/endpoints.go | 1 - 1 file changed, 1 deletion(-) diff --git a/agent/xds/proxystateconverter/endpoints.go b/agent/xds/proxystateconverter/endpoints.go index e3af35a21ecd..28156a9b034f 100644 --- a/agent/xds/proxystateconverter/endpoints.go +++ b/agent/xds/proxystateconverter/endpoints.go @@ -32,7 +32,6 @@ func makeLbEndpoint(addr string, port int, health pbproxystate.HealthStatus, wei return ep } - // endpointsFromSnapshot returns the mesh API representation of the "routes" in the snapshot. func (s *Converter) endpointsFromSnapshot(cfgSnap *proxycfg.ConfigSnapshot) error { From 03cf5dd63c10b8c881be0de6fb55fc3334ce3e32 Mon Sep 17 00:00:00 2001 From: John Murret Date: Thu, 17 Aug 2023 14:06:01 -0600 Subject: [PATCH 12/12] fixing clusters --- agent/xds/proxystateconverter/clusters.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/agent/xds/proxystateconverter/clusters.go b/agent/xds/proxystateconverter/clusters.go index a7d823a70abd..e12211f67078 100644 --- a/agent/xds/proxystateconverter/clusters.go +++ b/agent/xds/proxystateconverter/clusters.go @@ -6,13 +6,12 @@ package proxystateconverter import ( "errors" "fmt" - - envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" "github.com/hashicorp/go-hclog" "github.com/hashicorp/go-uuid" "strings" "time" + envoy_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3" "google.golang.org/protobuf/types/known/durationpb" "google.golang.org/protobuf/types/known/wrapperspb" @@ -1023,7 +1022,7 @@ func configureClusterWithHostnames( dnsEndpointGroup.Config.DiscoveryType = pbproxystate.DiscoveryType_DISCOVERY_TYPE_STRICT } - endpoints := make([]*pbproxystate.Endpoint, 0, 1) + endpoints := make([]*envoy_endpoint_v3.LbEndpoint, 0, 1) uniqueHostnames := make(map[string]bool) var (