Merge branch 'main' of ssh://github.com/hashicorp/consul

hashicorp · Oct 26, 2023 · 87b6971 · 87b6971
2 parents d883550 + 0295b95
commit 87b6971
Show file tree

Hide file tree

Showing 295 changed files with 19,518 additions and 3,350 deletions.
diff --git a/.changelog/19268.txt b/.changelog/19268.txt
@@ -0,0 +1,3 @@
+```release-note:bug
+Mesh Gateways: Fix a bug where replicated and peered mesh gateways with hostname-based WAN addresses fail to initialize.
+```
diff --git a/.changelog/19276.txt b/.changelog/19276.txt
diff --git a/.changelog/19285.txt b/.changelog/19285.txt
@@ -0,0 +1,7 @@
+```release-note:bug
+ca: Fix bug with Vault CA provider where token renewal goroutines could leak if CA failed to initialize.
+```
+
+```release-note:bug
+ca: Fix bug with Vault CA provider where renewing a retracted token would cause retries in a tight loop, degrading performance.
+```
diff --git a/.changelog/19306.txt b/.changelog/19306.txt
@@ -0,0 +1,3 @@
+```release-note:security
+connect: update supported envoy versions to 1.24.12, 1.25.11, 1.26.6, 1.27.2 to address [CVE-2023-44487](https://github.com/envoyproxy/envoy/security/advisories/GHSA-jhv4-f7mr-xx76)
+```
diff --git a/.changelog/19311.txt b/.changelog/19311.txt
@@ -0,0 +1,3 @@
+```release-note:bug
+raft:  Fix panic during downgrade from enterprise to oss.
+```
diff --git a/.changelog/19314.txt b/.changelog/19314.txt
@@ -0,0 +1,3 @@
+```release-note:enhancement
+raft: upgrade raft-wal library version to 0.4.1.
+```
diff --git a/.changelog/19339.txt b/.changelog/19339.txt
@@ -0,0 +1,4 @@
+```release-note:bug
+connect: Fix bug where uncleanly closed xDS connections would influence connection balancing for too long and prevent envoy instances from starting. Two new configuration fields
+`performance.grpc_keepalive_timeout` and `performance.grpc_keepalive_interval` now exist to allow for configuration on how often these dead connections will be cleaned up.
+```
diff --git a/.github/scripts/verify_envoy_version.sh b/.github/scripts/verify_envoy_version.sh
@@ -18,7 +18,7 @@ if [ -z "$current_branch" ]; then
 fi
 
 if [[ "$SKIP_VERIFY_ENVOY_VERSION" = "true" ]]; then
-  echo -e "*************** VERIFY ENVOY VERSION IS DISABLED. To enable, set the environment variable SKIP_VERIFY_ENVOY_VERSION to false in .github/workflows/verify-envoy-version.yml *****************"
+  echo -e "*************** VERIFY ENVOY VERSION IS DISABLED. To enable, update environment variable in Github settings *****************"
   exit 0
 fi
 

diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -104,7 +104,7 @@ jobs:
       - name: Setup with node and yarn
         uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
         with:
-          node-version: '14'
+          node-version: '18'
           cache: 'yarn'
           cache-dependency-path: 'ui/yarn.lock'
 
@@ -193,7 +193,7 @@ jobs:
       - name: Setup with node and yarn
         uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
         with:
-          node-version: '14'
+          node-version: '18'
           cache: 'yarn'
           cache-dependency-path: 'ui/yarn.lock'
 
@@ -244,7 +244,7 @@ jobs:
       - name: Setup with node and yarn
         uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
         with:
-          node-version: '14'
+          node-version: '18'
           cache: 'yarn'
           cache-dependency-path: 'ui/yarn.lock'
 

diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml
@@ -37,10 +37,10 @@ jobs:
 
     - uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
       with:
-        node-version: '16'
+        node-version: '18'
 
     - name: Install Yarn
-      run: npm install -g yarn
+      run: corepack enable
 
     # Install dependencies.
     - name: install yarn packages
@@ -57,10 +57,10 @@ jobs:
 
     - uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
       with:
-        node-version: '16'
+        node-version: '18'
 
     - name: Install Yarn
-      run: npm install -g yarn
+      run: corepack enable
 
     # Install dependencies.
     - name: install yarn packages
@@ -86,10 +86,10 @@ jobs:
 
     - uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
       with:
-        node-version: '16'
+        node-version: '18'
 
     - name: Install Yarn
-      run: npm install -g yarn
+      run: corepack enable
 
     - name: Install Chrome
       uses: browser-actions/setup-chrome@c485fa3bab6be59dce18dbc18ef6ab7cbc8ff5f1 # v1.2.0

diff --git a/.github/workflows/nightly-test-1.17.x.yaml b/.github/workflows/nightly-test-1.17.x.yaml
@@ -24,7 +24,7 @@ jobs:
       # Not necessary to use yarn, but enables caching
       - uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
         with:
-          node-version: 14
+          node-version: 18
           cache: 'yarn'
           cache-dependency-path: ./ui/yarn.lock
 
@@ -56,7 +56,7 @@ jobs:
       # Not necessary to use yarn, but enables caching
       - uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
         with:
-          node-version: 14
+          node-version: 18
           cache: 'yarn'
           cache-dependency-path: ./ui/yarn.lock
 
@@ -95,7 +95,7 @@ jobs:
       # Not necessary to use yarn, but enables caching
       - uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
         with:
-          node-version: 14
+          node-version: 18
           cache: 'yarn'
           cache-dependency-path: ./ui/yarn.lock
 
@@ -128,7 +128,7 @@ jobs:
       # Not necessary to use yarn, but enables caching
       - uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
         with:
-          node-version: 14
+          node-version: 18
           cache: 'yarn'
           cache-dependency-path: ./ui/yarn.lock
 
@@ -167,7 +167,7 @@ jobs:
       # Not necessary to use yarn, but enables caching
       - uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
         with:
-          node-version: 14
+          node-version: 18
           cache: 'yarn'
           cache-dependency-path: ./ui/yarn.lock
 
@@ -198,7 +198,7 @@ jobs:
       # Not necessary to use yarn, but enables caching
       - uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
         with:
-          node-version: 14
+          node-version: 18
           cache: 'yarn'
           cache-dependency-path: ./ui/yarn.lock
 

diff --git a/.github/workflows/nightly-test-integrations.yml b/.github/workflows/nightly-test-integrations.yml
@@ -65,7 +65,7 @@ jobs:
           # this is further going to multiplied in envoy-integration tests by the 
           # other dimensions in the matrix.  Currently TOTAL_RUNNERS would be
           # multiplied by 8 based on these values:
-          # envoy-version: ["1.25.11", "1.26.6", "1.27.2", "1.28.0"]
+          # envoy-version: ["1.24.12", "1.25.11", "1.26.6", "1.27.2"]
           # xds-target: ["server", "client"]
           TOTAL_RUNNERS: 4 
           JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
@@ -99,7 +99,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        envoy-version: ["1.25.11", "1.26.6", "1.27.2", "1.28.0"]
+        envoy-version: ["1.24.12", "1.25.11", "1.26.6", "1.27.2"]
         xds-target: ["server", "client"]
         test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }}
     env:

diff --git a/.github/workflows/nightly-test-main.yaml b/.github/workflows/nightly-test-main.yaml
@@ -24,7 +24,7 @@ jobs:
       # Not necessary to use yarn, but enables caching
       - uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
         with:
-          node-version: 14
+          node-version: 18
           cache: 'yarn'
           cache-dependency-path: ./ui/yarn.lock
 
@@ -56,7 +56,7 @@ jobs:
       # Not necessary to use yarn, but enables caching
       - uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
         with:
-          node-version: 14
+          node-version: 18
           cache: 'yarn'
           cache-dependency-path: ./ui/yarn.lock
 
@@ -95,7 +95,7 @@ jobs:
       # Not necessary to use yarn, but enables caching
       - uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
         with:
-          node-version: 14
+          node-version: 18
           cache: 'yarn'
           cache-dependency-path: ./ui/yarn.lock
 
@@ -128,7 +128,7 @@ jobs:
       # Not necessary to use yarn, but enables caching
       - uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
         with:
-          node-version: 14
+          node-version: 18
           cache: 'yarn'
           cache-dependency-path: ./ui/yarn.lock
 
@@ -167,7 +167,7 @@ jobs:
       # Not necessary to use yarn, but enables caching
       - uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
         with:
-          node-version: 14
+          node-version: 18
           cache: 'yarn'
           cache-dependency-path: ./ui/yarn.lock
 
@@ -198,7 +198,7 @@ jobs:
       # Not necessary to use yarn, but enables caching
       - uses: actions/setup-node@e33196f7422957bea03ed53f6fbb155025ffc7b8 # v3.7.0
         with:
-          node-version: 14
+          node-version: 18
           cache: 'yarn'
           cache-dependency-path: ./ui/yarn.lock
 

diff --git a/.github/workflows/test-integrations-windows.yml b/.github/workflows/test-integrations-windows.yml
@@ -54,7 +54,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        envoy-version: [ "1.28.0" ]
+        envoy-version: [ "1.27.2" ]
         xds-target: [ "server", "client" ]
     env:
       ENVOY_VERSION: ${{ matrix.envoy-version }}

diff --git a/.github/workflows/test-integrations.yml b/.github/workflows/test-integrations.yml
@@ -260,7 +260,7 @@ jobs:
           # this is further going to multiplied in envoy-integration tests by the
           # other dimensions in the matrix.  Currently TOTAL_RUNNERS would be
           # multiplied by 2 based on these values:
-          # envoy-version: ["1.28.0"]
+          # envoy-version: ["1.27.2"]
           # xds-target: ["server", "client"]
           TOTAL_RUNNERS: 4
           JQ_SLICER: '[ inputs ] | [_nwise(length / $runnercount | floor)]'
@@ -294,7 +294,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        envoy-version: ["1.28.0"]
+        envoy-version: ["1.27.2"]
         xds-target: ["server", "client"]
         test-cases: ${{ fromJSON(needs.generate-envoy-job-matrices.outputs.envoy-matrix) }}
     env:

diff --git a/.github/workflows/verify-envoy-version.yml b/.github/workflows/verify-envoy-version.yml
@@ -13,9 +13,9 @@ on:
     branches:
       - main
       - release/**
-
+      
 env:
-  SKIP_VERIFY_ENVOY_VERSION: "false" ## temporarily disabled; set to true to disable script
+  SKIP_VERIFY_ENVOY_VERSION: ${{ vars.SKIP_VERIFY_ENVOY_VERSION }}
 
 jobs:
   verify-envoy-version:

diff --git a/agent/agent.go b/agent/agent.go
@@ -34,6 +34,7 @@ import (
 	"golang.org/x/net/http2"
 	"golang.org/x/net/http2/h2c"
 	"google.golang.org/grpc"
+	"google.golang.org/grpc/keepalive"
 
 	"github.com/hashicorp/consul/acl"
 	"github.com/hashicorp/consul/acl/resolver"
@@ -722,6 +723,10 @@ func (a *Agent) Start(ctx context.Context) error {
 			metrics.Default(),
 			a.tlsConfigurator,
 			incomingRPCLimiter,
+			keepalive.ServerParameters{
+				Time:    a.config.GRPCKeepaliveInterval,
+				Timeout: a.config.GRPCKeepaliveTimeout,
+			},
 		)
 
 		var pt *proxytracker.ProxyTracker
@@ -757,6 +762,10 @@ func (a *Agent) Start(ctx context.Context) error {
 			metrics.Default(),
 			a.tlsConfigurator,
 			rpcRate.NullRequestLimitsHandler(),
+			keepalive.ServerParameters{
+				Time:    a.config.GRPCKeepaliveInterval,
+				Timeout: a.config.GRPCKeepaliveTimeout,
+			},
 		)
 
 		client, err := consul.NewClient(consulCfg, a.baseDeps.Deps)

diff --git a/agent/config/builder.go b/agent/config/builder.go
@@ -1019,6 +1019,8 @@ func (b *builder) build() (rt RuntimeConfig, err error) {
 		GRPCPort:                   grpcPort,
 		GRPCTLSAddrs:               grpcTlsAddrs,
 		GRPCTLSPort:                grpcTlsPort,
+		GRPCKeepaliveInterval:      b.durationValWithDefaultMin("performance.grpc_keepalive_interval", c.Performance.GRPCKeepaliveInterval, 30*time.Second, time.Second),
+		GRPCKeepaliveTimeout:       b.durationValWithDefaultMin("performance.grpc_keepalive_timeout", c.Performance.GRPCKeepaliveTimeout, 20*time.Second, time.Second),
 		HTTPMaxConnsPerClient:      intVal(c.Limits.HTTPMaxConnsPerClient),
 		HTTPSHandshakeTimeout:      b.durationVal("limits.https_handshake_timeout", c.Limits.HTTPSHandshakeTimeout),
 		KVMaxValueSize:             uint64Val(c.Limits.KVMaxValueSize),

diff --git a/agent/config/config.go b/agent/config/config.go
@@ -673,9 +673,11 @@ type HTTPConfig struct {
 }
 
 type Performance struct {
-	LeaveDrainTime *string `mapstructure:"leave_drain_time"`
-	RaftMultiplier *int    `mapstructure:"raft_multiplier"` // todo(fs): validate as uint
-	RPCHoldTimeout *string `mapstructure:"rpc_hold_timeout"`
+	LeaveDrainTime        *string `mapstructure:"leave_drain_time"`
+	RaftMultiplier        *int    `mapstructure:"raft_multiplier"` // todo(fs): validate as uint
+	RPCHoldTimeout        *string `mapstructure:"rpc_hold_timeout"`
+	GRPCKeepaliveInterval *string `mapstructure:"grpc_keepalive_interval"`
+	GRPCKeepaliveTimeout  *string `mapstructure:"grpc_keepalive_timeout"`
 }
 
 type Telemetry struct {

diff --git a/agent/config/default.go b/agent/config/default.go
@@ -118,6 +118,8 @@ func DefaultSource() Source {
 			leave_drain_time = "5s"
 			raft_multiplier = ` + strconv.Itoa(int(consul.DefaultRaftMultiplier)) + `
 			rpc_hold_timeout = "7s"
+			grpc_keepalive_interval = "30s"
+			grpc_keepalive_timeout = "20s"
 		}
 		ports = {
 			dns = 8600

diff --git a/agent/config/runtime.go b/agent/config/runtime.go
@@ -717,6 +717,19 @@ type RuntimeConfig struct {
 	// hcl: client_addr = string addresses { grpc_tls = string } ports { grpc_tls = int }
 	GRPCTLSAddrs []net.Addr
 
+	// GRPCKeepaliveInterval determines how frequently an HTTP2 keepalive will be broadcast
+	// whenever a GRPC connection is idle. This helps detect xds connections that have died.
+	//
+	// Since the xds load balancing between servers relies on knowing how many connections
+	// are active, this configuration ensures that they are routinely detected / cleaned up
+	// on an interval.
+	GRPCKeepaliveInterval time.Duration
+
+	// GRPCKeepaliveTimeout specifies how long a GRPC client has to reply to the keepalive
+	// messages spawned from GRPCKeepaliveInterval. If a client does not reply in this amount of
+	// time, the connection will be closed by the server.
+	GRPCKeepaliveTimeout time.Duration
+
 	// HTTPAddrs contains the list of TCP addresses and UNIX sockets the HTTP
 	// server will bind to. If the HTTP endpoint is disabled (ports.http <= 0)
 	// the list is empty.

diff --git a/agent/config/runtime_test.go b/agent/config/runtime_test.go
@@ -6560,6 +6560,8 @@ func TestLoad_FullConfig(t *testing.T) {
 		GRPCAddrs:             []net.Addr{tcpAddr("32.31.61.91:4881")},
 		GRPCTLSPort:           5201,
 		GRPCTLSAddrs:          []net.Addr{tcpAddr("23.14.88.19:5201")},
+		GRPCKeepaliveInterval: 33 * time.Second,
+		GRPCKeepaliveTimeout:  22 * time.Second,
 		HTTPAddrs:             []net.Addr{tcpAddr("83.39.91.39:7999")},
 		HTTPBlockEndpoints:    []string{"RBvAFcGD", "fWOWFznh"},
 		AllowWriteHTTPFrom:    []*net.IPNet{cidr("127.0.0.0/8"), cidr("22.33.44.55/32"), cidr("0.0.0.0/0")},

diff --git a/agent/config/testdata/TestRuntimeConfig_Sanitize.golden b/agent/config/testdata/TestRuntimeConfig_Sanitize.golden
@@ -210,6 +210,8 @@
     "GRPCPort": 0,
     "GRPCTLSAddrs": [],
     "GRPCTLSPort": 0,
+    "GRPCKeepaliveInterval": "0s",
+    "GRPCKeepaliveTimeout": "0s",
     "GossipLANGossipInterval": "0s",
     "GossipLANGossipNodes": 0,
     "GossipLANProbeInterval": "0s",

diff --git a/agent/config/testdata/full-config.hcl b/agent/config/testdata/full-config.hcl
@@ -335,6 +335,8 @@ performance {
     leave_drain_time = "8265s"
     raft_multiplier = 5
     rpc_hold_timeout = "15707s"
+    grpc_keepalive_interval = "33s"
+    grpc_keepalive_timeout = "22s"
 }
 pid_file = "43xN80Km"
 ports {

diff --git a/agent/config/testdata/full-config.json b/agent/config/testdata/full-config.json
@@ -383,7 +383,9 @@
   "performance": {
     "leave_drain_time": "8265s",
     "raft_multiplier": 5,
-    "rpc_hold_timeout": "15707s"
+    "rpc_hold_timeout": "15707s",
+    "grpc_keepalive_interval": "33s",
+    "grpc_keepalive_timeout": "22s"
   },
   "pid_file": "43xN80Km",
   "ports": {