From aabb8af0a9dee4338c2ece6c35eaee4f4a169a20 Mon Sep 17 00:00:00 2001 From: Derek Wang Date: Thu, 31 Aug 2023 08:19:37 -0700 Subject: [PATCH] feat: scaleUpCooldownSeconds and scaleDownCooldownSeconds to replace cooldownSeconds (#1008) Signed-off-by: Derek Wang --- api/json-schema/schema.json | 12 +- api/openapi-spec/swagger.json | 12 +- .../full/numaflow.numaproj.io_pipelines.yaml | 6 + .../full/numaflow.numaproj.io_vertices.yaml | 6 + config/install.yaml | 12 + config/namespace-install.yaml | 12 + docs/APIs.md | 29 +- docs/user-guide/reference/autoscaling.md | 7 +- pkg/apis/numaflow/v1alpha1/generated.pb.go | 289 +++++++++++------- pkg/apis/numaflow/v1alpha1/generated.proto | 13 +- .../numaflow/v1alpha1/openapi_generated.go | 16 +- pkg/apis/numaflow/v1alpha1/vertex_types.go | 32 +- .../numaflow/v1alpha1/vertex_types_test.go | 12 +- .../v1alpha1/zz_generated.deepcopy.go | 14 +- pkg/reconciler/vertex/scaling/scaling.go | 55 ++-- 15 files changed, 373 insertions(+), 154 deletions(-) diff --git a/api/json-schema/schema.json b/api/json-schema/schema.json index 1728b25eb2..ca13ed2b51 100644 --- a/api/json-schema/schema.json +++ b/api/json-schema/schema.json @@ -19063,7 +19063,7 @@ "description": "Scale defines the parameters for autoscaling.", "properties": { "cooldownSeconds": { - "description": "Cooldown seconds after a scaling operation before another one.", + "description": "Deprecated: Use scaleUpCooldownSeconds and scaleDownCooldownSeconds instead. Cooldown seconds after a scaling operation before another one.", "format": "int64", "type": "integer" }, @@ -19091,6 +19091,16 @@ "format": "int64", "type": "integer" }, + "scaleDownCooldownSeconds": { + "description": "ScaleDownCooldownSeconds defines the cooldown seconds after a scaling operation, before a follow-up scaling down. It defaults to the CooldownSeconds if not set.", + "format": "int64", + "type": "integer" + }, + "scaleUpCooldownSeconds": { + "description": "ScaleUpCooldownSeconds defines the cooldown seconds after a scaling operation, before a follow-up scaling up. It defaults to the CooldownSeconds if not set.", + "format": "int64", + "type": "integer" + }, "targetBufferAvailability": { "description": "TargetBufferAvailability is used to define the target percentage of the buffer availability. A valid and meaningful value should be less than the BufferUsageLimit defined in the Edge spec (or Pipeline spec), for example, 50. It only applies to UDF and Sink vertices because only they have buffers to read.", "format": "int64", diff --git a/api/openapi-spec/swagger.json b/api/openapi-spec/swagger.json index 1d265f819f..7f4d697f91 100644 --- a/api/openapi-spec/swagger.json +++ b/api/openapi-spec/swagger.json @@ -19050,7 +19050,7 @@ "type": "object", "properties": { "cooldownSeconds": { - "description": "Cooldown seconds after a scaling operation before another one.", + "description": "Deprecated: Use scaleUpCooldownSeconds and scaleDownCooldownSeconds instead. Cooldown seconds after a scaling operation before another one.", "type": "integer", "format": "int64" }, @@ -19078,6 +19078,16 @@ "type": "integer", "format": "int64" }, + "scaleDownCooldownSeconds": { + "description": "ScaleDownCooldownSeconds defines the cooldown seconds after a scaling operation, before a follow-up scaling down. It defaults to the CooldownSeconds if not set.", + "type": "integer", + "format": "int64" + }, + "scaleUpCooldownSeconds": { + "description": "ScaleUpCooldownSeconds defines the cooldown seconds after a scaling operation, before a follow-up scaling up. It defaults to the CooldownSeconds if not set.", + "type": "integer", + "format": "int64" + }, "targetBufferAvailability": { "description": "TargetBufferAvailability is used to define the target percentage of the buffer availability. A valid and meaningful value should be less than the BufferUsageLimit defined in the Edge spec (or Pipeline spec), for example, 50. It only applies to UDF and Sink vertices because only they have buffers to read.", "type": "integer", diff --git a/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml b/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml index 9df4a3969d..5d06f3012f 100644 --- a/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml +++ b/config/base/crds/full/numaflow.numaproj.io_pipelines.yaml @@ -5687,6 +5687,12 @@ spec: replicasPerScale: format: int32 type: integer + scaleDownCooldownSeconds: + format: int32 + type: integer + scaleUpCooldownSeconds: + format: int32 + type: integer targetBufferAvailability: format: int32 type: integer diff --git a/config/base/crds/full/numaflow.numaproj.io_vertices.yaml b/config/base/crds/full/numaflow.numaproj.io_vertices.yaml index 909383a310..71f00964a3 100644 --- a/config/base/crds/full/numaflow.numaproj.io_vertices.yaml +++ b/config/base/crds/full/numaflow.numaproj.io_vertices.yaml @@ -1530,6 +1530,12 @@ spec: replicasPerScale: format: int32 type: integer + scaleDownCooldownSeconds: + format: int32 + type: integer + scaleUpCooldownSeconds: + format: int32 + type: integer targetBufferAvailability: format: int32 type: integer diff --git a/config/install.yaml b/config/install.yaml index 31397588b8..ee46de76da 100644 --- a/config/install.yaml +++ b/config/install.yaml @@ -8216,6 +8216,12 @@ spec: replicasPerScale: format: int32 type: integer + scaleDownCooldownSeconds: + format: int32 + type: integer + scaleUpCooldownSeconds: + format: int32 + type: integer targetBufferAvailability: format: int32 type: integer @@ -12871,6 +12877,12 @@ spec: replicasPerScale: format: int32 type: integer + scaleDownCooldownSeconds: + format: int32 + type: integer + scaleUpCooldownSeconds: + format: int32 + type: integer targetBufferAvailability: format: int32 type: integer diff --git a/config/namespace-install.yaml b/config/namespace-install.yaml index c13cc30c56..01dd6ec22a 100644 --- a/config/namespace-install.yaml +++ b/config/namespace-install.yaml @@ -8216,6 +8216,12 @@ spec: replicasPerScale: format: int32 type: integer + scaleDownCooldownSeconds: + format: int32 + type: integer + scaleUpCooldownSeconds: + format: int32 + type: integer targetBufferAvailability: format: int32 type: integer @@ -12871,6 +12877,12 @@ spec: replicasPerScale: format: int32 type: integer + scaleDownCooldownSeconds: + format: int32 + type: integer + scaleUpCooldownSeconds: + format: int32 + type: integer targetBufferAvailability: format: int32 type: integer diff --git a/docs/APIs.md b/docs/APIs.md index a02f549e6a..49634a3758 100644 --- a/docs/APIs.md +++ b/docs/APIs.md @@ -4054,7 +4054,8 @@ processing rate. (Optional)

-Cooldown seconds after a scaling operation before another one. +Deprecated: Use scaleUpCooldownSeconds and scaleDownCooldownSeconds +instead. Cooldown seconds after a scaling operation before another one.

@@ -4112,6 +4113,32 @@ once. The is use to prevent too aggressive scaling operations

+ + +scaleUpCooldownSeconds
uint32 + + +(Optional) +

+ScaleUpCooldownSeconds defines the cooldown seconds after a scaling +operation, before a follow-up scaling up. It defaults to the +CooldownSeconds if not set. +

+ + + + +scaleDownCooldownSeconds
uint32 + + +(Optional) +

+ScaleDownCooldownSeconds defines the cooldown seconds after a scaling +operation, before a follow-up scaling down. It defaults to the +CooldownSeconds if not set. +

+ +

diff --git a/docs/user-guide/reference/autoscaling.md b/docs/user-guide/reference/autoscaling.md index b7f2983ff2..81c40de872 100644 --- a/docs/user-guide/reference/autoscaling.md +++ b/docs/user-guide/reference/autoscaling.md @@ -32,7 +32,8 @@ spec: min: 0 # Optional, minimum replicas, defaults to 0. max: 20 # Optional, maximum replicas, defaults to 50. lookbackSeconds: 120 # Optional, defaults to 120. - cooldownSeconds: 90 # Optional, defaults to 90. + scaleUpCooldownSeconds: 90 # Optional, defaults to 90. + scaleDownCooldownSeconds: 90 # Optional, defaults to 90. zeroReplicaSleepSeconds: 120 # Optional, defaults to 120. targetProcessingSeconds: 20 # Optional, defaults to 20. targetBufferAvailability: 50 # Optional, defaults to 50. @@ -43,7 +44,9 @@ spec: - `min` - Minimum replicas, valid value could be an integer >= 0. Defaults to `0`, which means it could be scaled down to 0. - `max` - Maximum replicas, positive integer which should not be less than `min`, defaults to `50`. if `max` and `min` are the same, that will be the fixed replica number. - `lookbackSeconds` - How many seconds to lookback for vertex average processing rate (tps) and pending messages calculation, defaults to `120`. Rate and pending messages metrics are critical for autoscaling, you might need to tune this parameter a bit to see better results. For example, your data source only have 1 minute data input in every 5 minutes, and you don't want the vertices to be scaled down to `0`. In this case, you need to increase `lookbackSeconds` to cover all the 5 minutes, so that the calculated average rate and pending messages won't be `0` during the silent period, to prevent scaling down to 0 from happening. -- `cooldownSeconds` - After a scaling operation, how many seconds to wait before doing another scaling on the same vertex. This is to give some time for a vertex to stabilize, defaults to 90 seconds. +- `cooldownSeconds` - **Deprecated**, use `scaleUpCooldownSeconds` and `scaleDownCooldownSeconds` instead. After a scaling operation, how many seconds to wait before doing another scaling on the same vertex. This is to give some time for a vertex to stabilize, defaults to 90 seconds. +- `scaleUpCooldownSeconds` - After a scaling operation, how many seconds to wait for the same vertex, if the follow-up operation is a scaling up, defaults to `90`. +- `scaleDownCooldownSeconds` - After a scaling operation, how many seconds to wait for the same vertex, if the follow-up operation is a scaling down, defaults to `90`. - `zeroReplicaSleepSeconds` - How many seconds it will wait after scaling down to `0`, defaults to `120`. Numaflow autoscaler periodically scales up a vertex pod to "peek" the incoming data, this is the period of time to wait before peeking. - `targetProcessingSeconds` - It is used to tune the aggressiveness of autoscaling for source vertices, it measures how fast you want the vertex to process all the pending messages, defaults to `20`. It is only effective for the `Source` vertices which support autoscaling, typically increasing the value leads to lower processing rate, thus less replicas. - `targetBufferAvailability` - Targeted buffer availability in percentage, defaults to `50`. It is only effective for `UDF` and `Sink` vertices, it determines how aggressive you want to do for autoscaling, increasing the value will bring more replicas. diff --git a/pkg/apis/numaflow/v1alpha1/generated.pb.go b/pkg/apis/numaflow/v1alpha1/generated.pb.go index 56855d95ca..20d34dfed4 100644 --- a/pkg/apis/numaflow/v1alpha1/generated.pb.go +++ b/pkg/apis/numaflow/v1alpha1/generated.pb.go @@ -2327,14 +2327,14 @@ func init() { } var fileDescriptor_9d0d1b17d3865563 = []byte{ - // 6599 bytes of a gzipped FileDescriptorProto + // 6648 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x7d, 0x5d, 0x6c, 0x24, 0xd9, 0x55, 0xf0, 0xf6, 0xaf, 0xbb, 0x4f, 0xdb, 0x9e, 0x99, 0x3b, 0x3f, 0xeb, 0xf1, 0xce, 0x4e, 0x4f, 0x2a, 0xdf, 0xee, 0x37, 0xf9, 0xbe, 0xc4, 0xf3, 0xad, 0xbf, 0x0d, 0xbb, 0x01, 0x92, 0x5d, 0xb7, 0x3d, 0xf6, 0xce, 0x8e, 0x3d, 0xe3, 0x9c, 0xb6, 0x67, 0x36, 0x59, 0x92, 0xa5, 0x5c, 0x7d, 0xdd, 0xae, 0xed, 0xea, 0xaa, 0x4e, 0xd5, 0x6d, 0xcf, 0x78, 0x43, 0x44, 0x42, 0x1e, 0x36, 0x11, 0xa0, - 0x20, 0x78, 0x89, 0x82, 0x02, 0x42, 0x42, 0xe2, 0x01, 0x45, 0x42, 0x82, 0xf0, 0x00, 0x42, 0xc0, - 0x0b, 0x0a, 0x3c, 0x40, 0x1e, 0x90, 0x12, 0x14, 0x64, 0x11, 0xf3, 0xc4, 0x03, 0x51, 0x44, 0x24, + 0x20, 0x78, 0x89, 0x82, 0x02, 0x42, 0x42, 0xe2, 0x01, 0x45, 0xe2, 0x81, 0xf0, 0x00, 0x42, 0xc0, + 0x0b, 0x0a, 0x3c, 0x40, 0x1e, 0x90, 0x12, 0x14, 0x64, 0x11, 0xf3, 0x84, 0x10, 0x51, 0x44, 0x24, 0x14, 0x8d, 0x90, 0x40, 0xf7, 0xaf, 0xfe, 0xba, 0x7a, 0xc6, 0xee, 0xb6, 0x27, 0x13, 0xc8, 0x5b, 0xd7, 0x39, 0xe7, 0x9e, 0x73, 0xeb, 0xd6, 0xbd, 0xe7, 0x9e, 0xbf, 0x7b, 0x1b, 0x56, 0xda, 0x36, 0xdb, 0xe9, 0x6f, 0xcd, 0x59, 0x5e, 0xf7, 0x9a, 0xdb, 0xef, 0x9a, 0x3d, 0xdf, 0x7b, 0x5b, 0xfc, @@ -2410,7 +2410,7 @@ var fileDescriptor_9d0d1b17d3865563 = []byte{ 0xde, 0xa6, 0xbe, 0xda, 0xf9, 0xe5, 0x9e, 0x4a, 0x28, 0x94, 0x7c, 0xda, 0xb2, 0x03, 0xd5, 0xf7, 0xa5, 0x91, 0x3f, 0x1d, 0x72, 0x2e, 0x6a, 0x0b, 0x17, 0xe3, 0x25, 0x00, 0x28, 0xb9, 0x93, 0x3e, 0x54, 0xdf, 0xa6, 0x2c, 0x60, 0x3e, 0x35, 0xbb, 0xea, 0xed, 0x5e, 0x1b, 0x59, 0xd4, 0xeb, 0x94, - 0x35, 0x05, 0xa7, 0xb8, 0xc5, 0x10, 0x02, 0x31, 0x92, 0x64, 0xfc, 0x65, 0x09, 0x26, 0x17, 0xbd, + 0x35, 0x05, 0xa7, 0xb8, 0xc5, 0x10, 0x02, 0x31, 0x92, 0x64, 0xfc, 0x45, 0x09, 0x26, 0x17, 0xbd, 0xee, 0x96, 0xed, 0xd2, 0xd6, 0xf5, 0x56, 0x9b, 0x92, 0xb7, 0xa0, 0x48, 0x5b, 0x6d, 0xaa, 0xde, 0x76, 0xf4, 0x7d, 0x96, 0x33, 0x8b, 0xac, 0x05, 0xfe, 0x84, 0x82, 0x31, 0x59, 0x85, 0xe9, 0x6d, 0xdf, 0xeb, 0x4a, 0xd5, 0xb5, 0xb1, 0xd7, 0x53, 0x56, 0x48, 0xe3, 0x7f, 0x69, 0x75, 0xb0, 0x9c, @@ -2434,7 +2434,7 @@ var fileDescriptor_9d0d1b17d3865563 = []byte{ 0x97, 0x05, 0x8d, 0x33, 0xda, 0x37, 0xd2, 0xd8, 0x00, 0x23, 0x6e, 0x64, 0x6b, 0xd0, 0xa5, 0x97, 0xfe, 0xe8, 0x7b, 0x87, 0x68, 0xf5, 0x11, 0xfc, 0xf9, 0x4f, 0xc2, 0xa9, 0xd0, 0xe7, 0x56, 0x6e, 0x9b, 0xf4, 0x50, 0x5f, 0xe4, 0xcd, 0x6f, 0x24, 0x51, 0x0f, 0xf6, 0xeb, 0xcf, 0x66, 0x38, 0x6e, - 0x11, 0x01, 0xa6, 0x99, 0x19, 0x7f, 0x5e, 0x80, 0x41, 0xb3, 0x3b, 0x39, 0x68, 0xb9, 0xe3, 0x1e, + 0x11, 0x01, 0xa6, 0x99, 0x19, 0x7f, 0x56, 0x80, 0x41, 0xb3, 0x3b, 0x39, 0x68, 0xb9, 0xe3, 0x1e, 0xb4, 0xf4, 0x0b, 0x49, 0xf5, 0xf9, 0xb2, 0x6a, 0x36, 0xfe, 0x4b, 0x65, 0x7d, 0x98, 0xc2, 0x71, 0x7f, 0x98, 0x27, 0x65, 0xed, 0x18, 0x5f, 0x28, 0xc2, 0xf4, 0x92, 0x49, 0xbb, 0x9e, 0xfb, 0x48, 0x27, 0x24, 0xf7, 0x44, 0x38, 0x21, 0x57, 0xa1, 0xe2, 0xd3, 0x9e, 0x63, 0x5b, 0x66, 0x20, 0x3e, @@ -2453,7 +2453,7 @@ var fileDescriptor_9d0d1b17d3865563 = []byte{ 0x77, 0x6c, 0xce, 0xdd, 0xf8, 0x8f, 0x1c, 0x54, 0x96, 0xfb, 0xae, 0x25, 0x3c, 0x9d, 0x47, 0x07, 0xb6, 0xf4, 0xf6, 0x9f, 0xcf, 0xdc, 0xfe, 0xfb, 0x50, 0xee, 0xdc, 0x0b, 0xcd, 0x83, 0xda, 0xfc, 0xda, 0xe8, 0xdf, 0x58, 0x75, 0x69, 0xee, 0xa6, 0xe0, 0x27, 0x83, 0xed, 0xd3, 0xaa, 0x43, 0xe5, - 0x9b, 0x77, 0x85, 0x50, 0x25, 0x6c, 0xf6, 0x43, 0x50, 0x8b, 0x91, 0x1d, 0x29, 0xba, 0xf7, 0xc7, + 0x9b, 0x77, 0x85, 0x50, 0x25, 0x6c, 0xf6, 0x43, 0x50, 0x8b, 0x91, 0x1d, 0x29, 0xba, 0xf7, 0x47, 0x45, 0x28, 0xaf, 0x34, 0x9b, 0x0b, 0xeb, 0x37, 0xc8, 0x07, 0xa1, 0xa6, 0xe2, 0xb0, 0xb7, 0xa2, 0x31, 0x08, 0xc3, 0xf0, 0xcd, 0x08, 0x85, 0x71, 0x3a, 0x6e, 0x5c, 0xf9, 0xd4, 0x74, 0xba, 0x6a, 0xea, 0x87, 0xc6, 0x15, 0x72, 0x20, 0x4a, 0x1c, 0x31, 0x61, 0x9a, 0xfb, 0x6b, 0x7c, 0x08, 0xa5, @@ -2474,14 +2474,14 @@ var fileDescriptor_9d0d1b17d3865563 = []byte{ 0xb6, 0x9a, 0xbb, 0x96, 0x98, 0x86, 0x72, 0x09, 0x5d, 0x51, 0x2b, 0x02, 0x6e, 0x34, 0x1b, 0x0a, 0xf3, 0x20, 0xf1, 0x84, 0xb1, 0x36, 0x91, 0xaf, 0x92, 0x7f, 0x88, 0xaf, 0xd2, 0x04, 0xe8, 0x45, 0xd6, 0x5e, 0x41, 0x50, 0xfe, 0x7f, 0x2d, 0xe6, 0x28, 0x86, 0x5e, 0x8c, 0xcd, 0x18, 0xf6, 0x97, - 0xf1, 0x27, 0x05, 0x98, 0x5d, 0xa1, 0x2c, 0x0c, 0x27, 0x28, 0x65, 0xd1, 0xec, 0x51, 0x8b, 0x8f, + 0xf1, 0xc7, 0x05, 0x98, 0x5d, 0xa1, 0x2c, 0x0c, 0x27, 0x28, 0x65, 0xd1, 0xec, 0x51, 0x8b, 0x8f, 0xca, 0xbb, 0x39, 0x28, 0x3b, 0xe6, 0x16, 0x75, 0xb8, 0x32, 0xe7, 0xdc, 0xdf, 0x1a, 0x59, 0x2f, 0x0e, 0x97, 0x32, 0xb7, 0x2a, 0x24, 0xa4, 0x34, 0xa5, 0x04, 0xa2, 0x12, 0xcf, 0x75, 0x9c, 0xe5, 0xf4, 0x03, 0x46, 0xfd, 0x75, 0xcf, 0x67, 0xca, 0x58, 0x0a, 0x75, 0xdc, 0x62, 0x84, 0xc2, 0x38, 0x1d, 0x99, 0x07, 0xb0, 0x1c, 0x9b, 0xba, 0x4c, 0xb4, 0x92, 0xd3, 0x8c, 0xe8, 0xf1, 0x5e, 0x0c, 0x31, 0x18, 0xa3, 0xe2, 0xa2, 0xba, 0x9e, 0x6b, 0x33, 0x4f, 0x8a, 0x2a, 0x26, 0x45, 0xad, 0x45, 0x28, 0x8c, 0xd3, 0x89, 0x66, 0x94, 0xf9, 0xb6, 0x15, 0x88, 0x66, 0xa5, 0x54, 0xb3, 0x08, 0x85, - 0x71, 0x3a, 0xbe, 0x05, 0xc4, 0xde, 0xff, 0x48, 0x5b, 0xc0, 0x9f, 0x56, 0xe0, 0x72, 0x62, 0x58, + 0x71, 0x3a, 0xbe, 0x05, 0xc4, 0xde, 0xff, 0x48, 0x5b, 0xc0, 0x9f, 0x54, 0xe0, 0x72, 0x62, 0x58, 0x99, 0xc9, 0xe8, 0x76, 0xdf, 0x69, 0x52, 0xa6, 0x3f, 0xe0, 0x88, 0x5b, 0xc3, 0x2f, 0x47, 0xdf, 0x5d, 0x66, 0xa3, 0xad, 0xe3, 0xf9, 0xee, 0x03, 0x1d, 0x3c, 0xd4, 0xb7, 0xbf, 0x06, 0x55, 0xd7, 0x64, 0x81, 0x58, 0x48, 0x6a, 0xcd, 0x84, 0x8e, 0xd5, 0x2d, 0x8d, 0xc0, 0x88, 0x86, 0xac, 0xc3, @@ -2500,7 +2500,7 @@ var fileDescriptor_9d0d1b17d3865563 = []byte{ 0xb4, 0xda, 0x7f, 0x73, 0x9c, 0xe5, 0x9f, 0x21, 0xe1, 0x50, 0xcb, 0xfe, 0x75, 0x20, 0xbe, 0x0a, 0xb3, 0x4b, 0xe7, 0x2e, 0xa6, 0xf9, 0xc3, 0xa2, 0x04, 0x1c, 0xa0, 0xc0, 0x8c, 0x56, 0xa4, 0x09, 0xe7, 0x03, 0xea, 0x32, 0xdb, 0xa5, 0x4e, 0x92, 0x9d, 0xdc, 0x12, 0x9e, 0x55, 0xec, 0xce, 0x37, - 0xb3, 0x88, 0x30, 0xbb, 0xed, 0x38, 0x83, 0xff, 0x8f, 0x55, 0xb1, 0xef, 0xca, 0xa1, 0x39, 0x36, + 0xb3, 0x88, 0x30, 0xbb, 0xed, 0x38, 0x83, 0xff, 0x0f, 0x55, 0xb1, 0xef, 0xca, 0xa1, 0x39, 0x36, 0xb5, 0xfd, 0x6e, 0x5a, 0x6d, 0xbf, 0x35, 0xfe, 0x77, 0x1b, 0x4d, 0x65, 0xcf, 0x03, 0x88, 0xaf, 0x10, 0xd7, 0xd9, 0xa1, 0xa6, 0xc2, 0x10, 0x83, 0x31, 0x2a, 0xbe, 0x0a, 0xf5, 0x38, 0xc7, 0xd5, 0x75, 0xb8, 0x0a, 0x9b, 0x71, 0x24, 0x26, 0x69, 0x87, 0xaa, 0xfc, 0xd2, 0xc8, 0x2a, 0xff, 0x75, @@ -2541,7 +2541,7 @@ var fileDescriptor_9d0d1b17d3865563 = []byte{ 0x52, 0xf1, 0x88, 0x92, 0x94, 0x6b, 0x96, 0x2e, 0x0d, 0x82, 0xc8, 0x65, 0x09, 0x35, 0xcb, 0x9a, 0x04, 0xa3, 0xc6, 0x13, 0x06, 0x65, 0x19, 0x06, 0x50, 0x4a, 0x76, 0xf4, 0x4c, 0x72, 0x46, 0x9d, 0x5a, 0xf4, 0x52, 0x2a, 0xa2, 0xa4, 0x64, 0x91, 0x39, 0x28, 0xb2, 0xa8, 0x6a, 0x48, 0x7b, 0x0e, - 0xc5, 0x0c, 0xf3, 0x40, 0xd0, 0x19, 0x7f, 0x57, 0x81, 0x0b, 0xd9, 0xdf, 0x90, 0xbf, 0xeb, 0x2e, + 0xc5, 0x0c, 0xf3, 0x40, 0xd0, 0x19, 0x7f, 0x5b, 0x81, 0x0b, 0xd9, 0xdf, 0x90, 0xbf, 0xeb, 0x2e, 0xf5, 0x03, 0xdb, 0x73, 0x95, 0xc9, 0x11, 0xd5, 0xb8, 0x4a, 0x30, 0x6a, 0xfc, 0x8f, 0x75, 0x96, 0xfa, 0xf7, 0x72, 0xdc, 0xb3, 0x91, 0xb1, 0xb7, 0xc7, 0x91, 0xa9, 0x7e, 0x56, 0x7a, 0x48, 0x43, 0x04, 0xe2, 0xf0, 0xbe, 0x90, 0xdf, 0xcd, 0xc1, 0x4c, 0x37, 0xe5, 0x3a, 0x9d, 0x60, 0x95, 0xb9, @@ -2556,7 +2556,7 @@ var fileDescriptor_9d0d1b17d3865563 = []byte{ 0x26, 0x72, 0xb8, 0xf1, 0x9f, 0x39, 0x38, 0x95, 0xaa, 0x48, 0xe5, 0x4d, 0xfa, 0xbe, 0xa3, 0xd4, 0x48, 0xd8, 0x64, 0x13, 0x57, 0x91, 0xc3, 0xc9, 0x5b, 0xca, 0x2a, 0xcc, 0x8f, 0x79, 0x92, 0xee, 0x96, 0xc9, 0x02, 0x6e, 0x06, 0x0e, 0x18, 0x84, 0x22, 0x7c, 0x1a, 0xf5, 0x47, 0xe9, 0xee, 0x58, - 0xf8, 0x34, 0xc2, 0x61, 0x82, 0x32, 0x15, 0x43, 0x28, 0x1e, 0x26, 0x86, 0x60, 0xfc, 0x4d, 0x01, + 0xf8, 0x34, 0xc2, 0x61, 0x82, 0x32, 0x15, 0x43, 0x28, 0x1e, 0x26, 0x86, 0x60, 0xfc, 0x75, 0x01, 0x6a, 0xaf, 0x7b, 0x5b, 0x3f, 0x26, 0x15, 0x46, 0xd9, 0x1a, 0x39, 0xff, 0x23, 0xd4, 0xc8, 0x9b, 0xf0, 0x34, 0x63, 0x4e, 0x93, 0x5a, 0x9e, 0xdb, 0x0a, 0x16, 0xb6, 0x19, 0xf5, 0x97, 0x6d, 0xd7, 0x0e, 0x76, 0x68, 0x4b, 0x05, 0x63, 0x9f, 0x39, 0xd8, 0xaf, 0x3f, 0xbd, 0xb1, 0xb1, 0x9a, 0x45, @@ -2599,7 +2599,7 @@ var fileDescriptor_9d0d1b17d3865563 = []byte{ 0xaf, 0xa5, 0x8d, 0xbe, 0x57, 0xb8, 0x3b, 0xbc, 0x10, 0x42, 0x1f, 0xec, 0xd7, 0x3f, 0x90, 0x95, 0x22, 0x4c, 0xbd, 0x7d, 0xd4, 0x00, 0x63, 0x2c, 0xc9, 0x27, 0x01, 0xe4, 0xa9, 0xb4, 0xb0, 0xf2, 0xf7, 0x11, 0x39, 0x80, 0x39, 0x7d, 0x62, 0x6a, 0xee, 0xa3, 0x7d, 0xd3, 0x65, 0x36, 0xdb, 0x93, - 0xc7, 0x26, 0xee, 0x84, 0x5c, 0x30, 0xc6, 0xd1, 0xf8, 0xab, 0x3c, 0x54, 0xb4, 0x31, 0xfa, 0x18, + 0xc7, 0x26, 0xee, 0x84, 0x5c, 0x30, 0xc6, 0xd1, 0xf8, 0xcb, 0x3c, 0x54, 0xb4, 0x31, 0xfa, 0x18, 0xb2, 0x3c, 0xed, 0x44, 0x96, 0x67, 0xf4, 0x93, 0x92, 0xba, 0xcb, 0x43, 0xf3, 0x3a, 0x5e, 0x2a, 0xaf, 0xb3, 0x32, 0xbe, 0xa8, 0x87, 0x67, 0x72, 0xbe, 0x96, 0x87, 0x69, 0x4d, 0xaa, 0x4e, 0xaf, 0xbe, 0x04, 0x53, 0x3e, 0x35, 0x5b, 0x0d, 0x93, 0x59, 0x3b, 0xe2, 0xf3, 0xe5, 0x44, 0xa5, 0xf5, @@ -2607,7 +2607,7 @@ var fileDescriptor_9d0d1b17d3865563 = []byte{ 0x66, 0xde, 0x97, 0x47, 0x48, 0xc4, 0x80, 0x15, 0x65, 0x4e, 0xb3, 0x91, 0x44, 0x61, 0x9a, 0x96, 0x4f, 0x6b, 0x09, 0xda, 0x0c, 0xcc, 0xb6, 0xec, 0x8c, 0x18, 0x85, 0x29, 0x39, 0xad, 0x1b, 0x29, 0x1c, 0x0e, 0x50, 0x13, 0x13, 0x6a, 0xbc, 0x47, 0x1b, 0x76, 0x97, 0x7a, 0x7d, 0x7d, 0xdf, 0xd0, - 0x51, 0x13, 0xb0, 0x62, 0x77, 0xc7, 0x88, 0x0d, 0xc6, 0x79, 0x1a, 0x7f, 0x9f, 0x83, 0xc9, 0x68, + 0x51, 0x13, 0xb0, 0x62, 0x77, 0xc7, 0x88, 0x0d, 0xc6, 0x79, 0x1a, 0x7f, 0x97, 0x83, 0xc9, 0x68, 0xbc, 0x4e, 0x3c, 0xd7, 0xb5, 0x9d, 0xcc, 0x75, 0x2d, 0x8c, 0x3d, 0x1d, 0x86, 0x64, 0xb7, 0x7e, 0x75, 0x22, 0x7a, 0x2d, 0x91, 0xcf, 0xda, 0x82, 0x59, 0x3b, 0x33, 0xc5, 0x13, 0xd3, 0x36, 0x61, 0x45, 0xe6, 0x8d, 0xa1, 0x94, 0xf8, 0x10, 0x2e, 0xa4, 0x0f, 0x95, 0x5d, 0xea, 0x33, 0xdb, 0xa2, @@ -2651,96 +2651,99 @@ var fileDescriptor_9d0d1b17d3865563 = []byte{ 0x27, 0x4a, 0xde, 0xc6, 0xbf, 0xe7, 0xa0, 0x1a, 0xe2, 0xc9, 0x26, 0x00, 0x57, 0x17, 0xea, 0xec, 0xef, 0x91, 0x6e, 0xd5, 0x12, 0x7e, 0xf0, 0x66, 0xd8, 0x18, 0x63, 0x8c, 0x32, 0x0e, 0x47, 0xe7, 0x8f, 0xfb, 0x70, 0xf4, 0x35, 0xa8, 0xee, 0x98, 0x6e, 0x2b, 0xd8, 0x31, 0x3b, 0x52, 0x6b, 0x56, - 0x22, 0x23, 0xed, 0x35, 0x8d, 0xc0, 0x88, 0xc6, 0xf8, 0xc3, 0x22, 0xc8, 0xeb, 0xe6, 0xf8, 0xba, + 0x22, 0x23, 0xed, 0x35, 0x8d, 0xc0, 0x88, 0xc6, 0xf8, 0x97, 0x12, 0xc8, 0xeb, 0xe6, 0xf8, 0xba, 0x6e, 0xd9, 0x81, 0xcc, 0xbb, 0xe7, 0x44, 0xcb, 0x70, 0x5d, 0x2f, 0x29, 0x38, 0x86, 0x14, 0xe4, 0x22, 0x14, 0xba, 0xb6, 0xab, 0xd2, 0x0d, 0x62, 0x5e, 0xad, 0xd9, 0x2e, 0x72, 0x98, 0x40, 0x99, 0xf7, 0x55, 0xea, 0x58, 0xa2, 0xcc, 0xfb, 0xc8, 0x61, 0xdc, 0xe9, 0x74, 0x3c, 0xaf, 0xb3, 0x65, - 0x5a, 0x1d, 0x9d, 0x12, 0x2b, 0x8a, 0xdd, 0x55, 0x38, 0x9d, 0xab, 0x49, 0x14, 0xa6, 0x69, 0x79, - 0x73, 0xcb, 0xf3, 0x9c, 0x96, 0x77, 0xcf, 0xd5, 0xcd, 0x4b, 0x51, 0xf3, 0xc5, 0x24, 0x0a, 0xd3, - 0xb4, 0x64, 0x13, 0x9e, 0x7e, 0x87, 0xfa, 0x9e, 0xd2, 0x68, 0x4d, 0x87, 0xd2, 0x9e, 0x66, 0x23, - 0x0d, 0x08, 0x91, 0xe7, 0xfe, 0x78, 0x36, 0x09, 0x0e, 0x6b, 0x2b, 0xd2, 0xe7, 0xa6, 0xdf, 0xa6, - 0x6c, 0xdd, 0xf7, 0x2c, 0x1a, 0x04, 0xb6, 0xdb, 0xd6, 0x6c, 0x27, 0x22, 0xb6, 0x1b, 0xd9, 0x24, - 0x38, 0xac, 0x2d, 0x79, 0x03, 0x66, 0x24, 0x4a, 0x1a, 0x16, 0x0b, 0xbb, 0xa6, 0xed, 0x98, 0x5b, - 0xb6, 0x63, 0xb3, 0x3d, 0x51, 0x6c, 0x32, 0x25, 0x73, 0x02, 0x1b, 0x43, 0x68, 0x70, 0x68, 0x6b, - 0x71, 0x1f, 0xac, 0xca, 0x08, 0xad, 0x53, 0x5f, 0x7c, 0x7d, 0x11, 0x5e, 0x56, 0xbe, 0x3b, 0xa6, - 0x70, 0x38, 0x40, 0x6d, 0x7c, 0x2b, 0x0f, 0xd5, 0xd0, 0x18, 0x3e, 0xc4, 0x55, 0x0f, 0x1e, 0x54, - 0xc3, 0xb2, 0x03, 0x35, 0xe9, 0x1b, 0xe3, 0x27, 0x1e, 0xa4, 0xfd, 0x16, 0x3e, 0x62, 0x24, 0x23, - 0x7e, 0x97, 0x64, 0x61, 0x8c, 0xbb, 0x24, 0x7b, 0x30, 0xc1, 0x7c, 0xbb, 0xdd, 0x56, 0x46, 0x45, - 0x6d, 0xfe, 0xc6, 0xf8, 0xee, 0xc4, 0x86, 0x64, 0x28, 0xf3, 0xf1, 0xea, 0x01, 0xb5, 0x18, 0xe3, - 0x6d, 0x38, 0x9d, 0xa6, 0x14, 0x3b, 0xae, 0xb5, 0x43, 0x5b, 0x7d, 0x47, 0x8f, 0x71, 0xb4, 0xe3, - 0x2a, 0x38, 0x86, 0x14, 0xdc, 0x74, 0x65, 0x76, 0x97, 0xbe, 0xe3, 0xb9, 0xda, 0x29, 0x10, 0xc6, - 0xcb, 0x86, 0x82, 0x61, 0x88, 0x35, 0xfe, 0xa5, 0x00, 0x17, 0x23, 0x97, 0x66, 0xcd, 0x74, 0xcd, - 0xf6, 0x21, 0x2e, 0x0b, 0xfd, 0x49, 0x15, 0xcd, 0x51, 0x6f, 0xdf, 0x29, 0x3c, 0x01, 0xb7, 0xef, - 0x7c, 0xa5, 0x00, 0xe2, 0x4a, 0x5e, 0x6e, 0x4e, 0x38, 0x9e, 0xb6, 0xb8, 0x46, 0x37, 0x27, 0x56, - 0xbd, 0xb6, 0xd4, 0xed, 0xab, 0x5e, 0x1b, 0x39, 0x47, 0xbe, 0x4f, 0x77, 0xcc, 0xed, 0x8e, 0x39, - 0xf6, 0xfa, 0x0e, 0xab, 0x7f, 0xe4, 0x3e, 0x2d, 0x1e, 0x51, 0xf2, 0xe6, 0x8a, 0x64, 0x4b, 0xdf, - 0x29, 0x39, 0xb6, 0x41, 0x10, 0xde, 0x4e, 0x29, 0x15, 0x49, 0xf8, 0x88, 0x91, 0x0c, 0x6e, 0xe2, - 0xf4, 0x5b, 0xe2, 0x6a, 0xe4, 0xe2, 0x98, 0x26, 0xce, 0xe6, 0x92, 0x78, 0x27, 0x61, 0xe2, 0xc8, - 0xdf, 0xa8, 0x58, 0x1b, 0x7f, 0x94, 0x83, 0xa9, 0xa6, 0x63, 0xb7, 0x6c, 0xb7, 0x7d, 0x72, 0x37, - 0x03, 0x91, 0xdb, 0x50, 0x0a, 0x1c, 0xbb, 0x45, 0x47, 0xbc, 0x34, 0x44, 0x7c, 0x0c, 0xde, 0x4b, - 0x8a, 0x92, 0x8f, 0xf1, 0x95, 0x32, 0xa8, 0x7b, 0xa4, 0x49, 0x1f, 0xaa, 0x6d, 0x7d, 0x83, 0x89, - 0xea, 0xf2, 0x6b, 0x63, 0x9c, 0x74, 0x4d, 0xdc, 0x85, 0x22, 0xbf, 0x4e, 0x08, 0xc4, 0x48, 0x12, - 0xa1, 0xc9, 0x39, 0xb7, 0x34, 0xe6, 0x9c, 0x93, 0xe2, 0x06, 0x67, 0x9d, 0x09, 0xc5, 0x1d, 0xc6, - 0x7a, 0x6a, 0xc2, 0x8d, 0x7e, 0x7a, 0x27, 0x3a, 0x98, 0x23, 0x33, 0x68, 0xfc, 0x19, 0x05, 0x6b, - 0x2e, 0xc2, 0x35, 0xc3, 0x2b, 0x30, 0x17, 0xc7, 0x4a, 0xd1, 0xc5, 0x45, 0xf0, 0x67, 0x14, 0xac, - 0xc9, 0xe7, 0x72, 0x30, 0xe9, 0xc7, 0x5c, 0x31, 0xe5, 0x52, 0x8c, 0x79, 0xfa, 0x21, 0xe1, 0xd7, - 0xc9, 0xea, 0xbe, 0x38, 0x1c, 0x13, 0x22, 0xb9, 0xdf, 0xc7, 0x7c, 0xd3, 0x0d, 0xb6, 0x3d, 0xbf, - 0x4b, 0x7d, 0xe5, 0x2a, 0x2f, 0x8f, 0xb1, 0xa6, 0x36, 0x22, 0x6e, 0x32, 0xe9, 0x91, 0x00, 0x61, - 0x5c, 0x1a, 0xe9, 0x40, 0xa5, 0xdf, 0x92, 0x1d, 0x55, 0xf1, 0xc8, 0x85, 0x71, 0x56, 0x73, 0x2c, - 0x1f, 0xa8, 0x9f, 0x30, 0x14, 0x60, 0x74, 0x41, 0x85, 0xea, 0x88, 0x95, 0xb8, 0xb1, 0x4c, 0x56, - 0x55, 0x5d, 0x3b, 0xdc, 0xe2, 0x0b, 0x2f, 0xdb, 0x8a, 0x5d, 0x2a, 0x91, 0x79, 0x35, 0x99, 0xf1, - 0x0f, 0x79, 0xe0, 0x9e, 0x9d, 0x3c, 0x23, 0x2d, 0xae, 0x03, 0xa4, 0xcd, 0x8e, 0xdd, 0xbb, 0x43, - 0x7d, 0x7b, 0x7b, 0x4f, 0xd9, 0xf3, 0xb1, 0x33, 0xd2, 0x69, 0x0a, 0xcc, 0x68, 0x45, 0xde, 0x84, - 0x49, 0xcb, 0x5c, 0xa4, 0x3e, 0x1b, 0xc5, 0x5b, 0x11, 0x33, 0x61, 0x71, 0x21, 0x6a, 0x8e, 0x09, - 0x66, 0xdc, 0xc7, 0xb2, 0x22, 0xd6, 0x85, 0x23, 0xfb, 0x58, 0x31, 0xc6, 0x31, 0x46, 0x04, 0xa1, - 0xda, 0xe1, 0xa4, 0x82, 0x6b, 0xf1, 0x28, 0x5c, 0x85, 0x96, 0xb9, 0xa9, 0xdb, 0x62, 0xc4, 0xc6, - 0x70, 0x61, 0x2a, 0x71, 0xf1, 0x19, 0xf9, 0x10, 0x54, 0xbc, 0x5e, 0x4c, 0xd9, 0x55, 0x45, 0x1d, - 0x51, 0xe5, 0xb6, 0x82, 0x3d, 0xd8, 0xaf, 0x4f, 0xad, 0x7a, 0x6d, 0xdb, 0xd2, 0x00, 0x0c, 0xc9, - 0x89, 0x01, 0x65, 0x51, 0xf3, 0xa5, 0xaf, 0x3d, 0x13, 0x8a, 0x5a, 0x5c, 0x89, 0x14, 0xa0, 0xc2, - 0x18, 0x9f, 0x2d, 0x42, 0x14, 0xe0, 0x26, 0x01, 0x94, 0x5b, 0xe2, 0x7a, 0x24, 0xa5, 0x57, 0x47, - 0x4f, 0x14, 0x24, 0x2f, 0x62, 0x94, 0xfe, 0x64, 0x12, 0x86, 0x4a, 0x14, 0x69, 0x43, 0xe1, 0x6d, - 0x6f, 0x6b, 0x6c, 0xb5, 0x1a, 0xab, 0xca, 0x96, 0xd1, 0xd9, 0x18, 0x00, 0xb9, 0x04, 0xf2, 0xdb, - 0x39, 0x38, 0x13, 0xa4, 0x6d, 0x50, 0x35, 0x1d, 0x70, 0x7c, 0x63, 0x3b, 0x6d, 0xd5, 0xaa, 0x82, - 0xaf, 0x61, 0x68, 0x1c, 0xec, 0x0b, 0x1f, 0x7f, 0x19, 0x79, 0x56, 0xd3, 0x69, 0x65, 0xcc, 0xab, - 0x77, 0x93, 0xe3, 0x9f, 0x84, 0xa1, 0x12, 0x65, 0xfc, 0x52, 0x1e, 0x6a, 0x31, 0x3d, 0x36, 0xf6, - 0x6d, 0x7a, 0xf7, 0x53, 0xb7, 0xe9, 0xad, 0x8f, 0x1e, 0x47, 0x8a, 0x7a, 0x75, 0xd2, 0x17, 0xea, - 0xfd, 0x75, 0x1e, 0x0a, 0x9b, 0x4b, 0xcb, 0x49, 0xef, 0x31, 0xf7, 0x18, 0xbc, 0xc7, 0x1d, 0x98, - 0xd8, 0xea, 0xdb, 0x0e, 0xb3, 0xdd, 0xb1, 0x8f, 0x48, 0xe8, 0xcb, 0x07, 0x55, 0xf9, 0xb5, 0xe4, - 0x8a, 0x9a, 0x3d, 0x69, 0xc3, 0x44, 0x5b, 0x1e, 0x91, 0x56, 0x73, 0xfe, 0xd5, 0xd1, 0xad, 0x26, - 0xc9, 0x47, 0x0a, 0x52, 0x0f, 0xa8, 0xb9, 0x1b, 0x9f, 0x01, 0x65, 0x74, 0x92, 0xe0, 0x64, 0x46, - 0x33, 0x0c, 0x33, 0x65, 0x8d, 0xa8, 0xf1, 0x69, 0x08, 0xf7, 0xc8, 0xc7, 0xfe, 0x39, 0x8d, 0x7f, - 0xcd, 0x41, 0xd2, 0x2c, 0x78, 0xfc, 0x33, 0xaa, 0x93, 0x9e, 0x51, 0x4b, 0xc7, 0xb1, 0x00, 0xb3, - 0x27, 0x95, 0xf1, 0x17, 0x79, 0x28, 0xab, 0xbf, 0x97, 0x39, 0xf9, 0x6a, 0x1b, 0x9a, 0xa8, 0xb6, - 0x59, 0x1c, 0x53, 0x39, 0x0e, 0xad, 0xb5, 0xe9, 0xa6, 0x6a, 0x6d, 0xc6, 0xbd, 0x00, 0xfd, 0x11, - 0x95, 0x36, 0x7f, 0x9b, 0x03, 0xa5, 0x9a, 0x6f, 0xb8, 0x01, 0x33, 0x5d, 0x4b, 0x78, 0x82, 0x6a, - 0x1f, 0x18, 0x37, 0xa5, 0xab, 0xca, 0x1e, 0xe4, 0xd6, 0x2f, 0x7e, 0x6b, 0xbd, 0x4f, 0xde, 0x0f, - 0x95, 0x1d, 0x2f, 0x60, 0x42, 0xd7, 0xe7, 0x93, 0xa1, 0x9e, 0xd7, 0x14, 0x1c, 0x43, 0x8a, 0x74, - 0xd6, 0xa6, 0x34, 0x3c, 0x6b, 0x63, 0xfc, 0x7e, 0x1e, 0x26, 0x13, 0xd7, 0xde, 0x8f, 0x5c, 0x38, - 0x94, 0xaa, 0xdb, 0xc9, 0x1f, 0x7f, 0xdd, 0x4e, 0x56, 0x6d, 0x52, 0x61, 0xcc, 0xda, 0xa4, 0xe2, - 0x51, 0x6a, 0x93, 0x8c, 0x6f, 0xe6, 0x00, 0xf4, 0x68, 0x9d, 0x78, 0xd9, 0x50, 0x2b, 0x59, 0x36, - 0x34, 0xf6, 0xbc, 0xca, 0x2e, 0x1a, 0xfa, 0xb3, 0x92, 0x7e, 0x25, 0x51, 0x32, 0xf4, 0x6e, 0x0e, - 0xa6, 0xcd, 0x44, 0x19, 0xce, 0xd8, 0xe6, 0x65, 0xaa, 0xaa, 0x27, 0xfc, 0x03, 0x9a, 0x24, 0x1c, - 0x53, 0x62, 0xc9, 0xcb, 0x30, 0xd9, 0x53, 0x35, 0x0a, 0xb7, 0xa2, 0x69, 0x1f, 0x9e, 0x15, 0x5c, - 0x8f, 0xe1, 0x30, 0x41, 0xf9, 0x88, 0xb2, 0xa7, 0xc2, 0xb1, 0x94, 0x3d, 0xc5, 0xcf, 0x56, 0x14, - 0x1f, 0x7a, 0xb6, 0x62, 0x17, 0xaa, 0xdb, 0xbe, 0xd7, 0x15, 0x95, 0x45, 0xea, 0xea, 0xf4, 0xeb, - 0x63, 0xec, 0x29, 0xd1, 0x9f, 0x86, 0x44, 0x5b, 0xeb, 0xb2, 0xe6, 0x8f, 0x91, 0x28, 0x11, 0xa3, - 0xf6, 0xa4, 0xd4, 0xf2, 0x71, 0x4a, 0x0d, 0x75, 0xc9, 0x86, 0xe4, 0x8e, 0x5a, 0x4c, 0xb2, 0x9a, - 0x68, 0xe2, 0xf1, 0x54, 0x13, 0x19, 0xdf, 0x0a, 0x15, 0x58, 0x33, 0x75, 0x9d, 0x40, 0x6e, 0xc8, - 0x75, 0x02, 0xea, 0x5a, 0x9f, 0x78, 0xdd, 0xcb, 0xf3, 0x50, 0xf6, 0xa9, 0x19, 0x78, 0xae, 0xba, - 0x35, 0x2c, 0x54, 0xff, 0x28, 0xa0, 0xa8, 0xb0, 0xf1, 0xfa, 0x98, 0xfc, 0x23, 0xea, 0x63, 0xde, - 0x1f, 0x9b, 0x20, 0xb2, 0x00, 0x32, 0x5c, 0xeb, 0x19, 0x93, 0x44, 0x24, 0xcf, 0xd5, 0xbf, 0x4a, - 0x96, 0xd2, 0xc9, 0x73, 0xf5, 0x8f, 0x8f, 0x21, 0x05, 0x69, 0xc1, 0xa4, 0x63, 0x06, 0x4c, 0xe4, - 0x5c, 0x5a, 0x0b, 0x6c, 0x84, 0xe2, 0x9b, 0x70, 0x19, 0xad, 0xc6, 0xf8, 0x60, 0x82, 0xab, 0xb1, - 0x5f, 0x80, 0x94, 0x1b, 0xf2, 0x93, 0xd8, 0xff, 0x7f, 0xab, 0xd8, 0xff, 0x6f, 0xe4, 0x20, 0x5a, - 0x53, 0x47, 0xcc, 0xf3, 0xbe, 0x01, 0x95, 0xae, 0x79, 0x7f, 0x89, 0x3a, 0xe6, 0xde, 0x38, 0x97, - 0x4d, 0xaf, 0x29, 0x1e, 0x18, 0x72, 0x33, 0xf6, 0x73, 0xa0, 0x2e, 0x6f, 0x22, 0x14, 0x4a, 0xdb, - 0xf6, 0x7d, 0xd5, 0x9f, 0x71, 0x6c, 0xe3, 0xd8, 0xe5, 0xfa, 0x32, 0x8c, 0x2b, 0x00, 0x28, 0xb9, - 0x93, 0x2e, 0x4c, 0x04, 0x32, 0xca, 0xae, 0x5e, 0x65, 0xf4, 0xc0, 0x63, 0x22, 0x5a, 0xaf, 0xae, - 0x62, 0x92, 0x20, 0xd4, 0x32, 0x1a, 0x9f, 0xf8, 0xc6, 0x77, 0x2f, 0x3f, 0xf5, 0xcd, 0xef, 0x5e, - 0x7e, 0xea, 0xdb, 0xdf, 0xbd, 0xfc, 0xd4, 0x67, 0x0f, 0x2e, 0xe7, 0xbe, 0x71, 0x70, 0x39, 0xf7, - 0xcd, 0x83, 0xcb, 0xb9, 0x6f, 0x1f, 0x5c, 0xce, 0xfd, 0xd3, 0xc1, 0xe5, 0xdc, 0xaf, 0xfd, 0xf3, - 0xe5, 0xa7, 0x3e, 0xfe, 0xd2, 0x88, 0xff, 0x4d, 0xfc, 0x5f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x97, - 0x37, 0x9b, 0x14, 0xd5, 0x78, 0x00, 0x00, + 0x5a, 0x1d, 0x9d, 0x12, 0x2b, 0x8a, 0xdd, 0x55, 0x38, 0x9d, 0xab, 0x49, 0x14, 0xa6, 0x69, 0xc9, + 0x0a, 0x9c, 0xb2, 0x3c, 0xcf, 0x69, 0x79, 0xf7, 0x5c, 0xdd, 0x5c, 0x1a, 0x0b, 0x22, 0x8c, 0xbf, + 0x44, 0x7b, 0x3e, 0xb5, 0xb8, 0x45, 0xb1, 0x98, 0x24, 0xc2, 0x74, 0x2b, 0xb2, 0x09, 0x4f, 0xbf, + 0x43, 0x7d, 0x4f, 0xe9, 0xb6, 0xa6, 0x43, 0x69, 0x4f, 0x33, 0x94, 0xa6, 0x84, 0xc8, 0x78, 0x7f, + 0x3c, 0x9b, 0x04, 0x87, 0xb5, 0x15, 0x89, 0x74, 0xd3, 0x6f, 0x53, 0xb6, 0xee, 0x7b, 0x16, 0x0d, + 0x02, 0xdb, 0x6d, 0x6b, 0xb6, 0x13, 0x11, 0xdb, 0x8d, 0x6c, 0x12, 0x1c, 0xd6, 0x96, 0xbc, 0x01, + 0x33, 0x12, 0x25, 0x4d, 0x8c, 0x85, 0x5d, 0xd3, 0x76, 0xcc, 0x2d, 0xdb, 0xd1, 0x77, 0xcc, 0x4e, + 0xc9, 0xec, 0xc0, 0xc6, 0x10, 0x1a, 0x1c, 0xda, 0x5a, 0xdc, 0x0c, 0xab, 0x72, 0x43, 0xeb, 0xd4, + 0x17, 0xf3, 0x40, 0x04, 0x9a, 0x95, 0x17, 0x8f, 0x29, 0x1c, 0x0e, 0x50, 0x13, 0x84, 0x0b, 0xe2, + 0x9a, 0xc2, 0xcd, 0x5e, 0x6a, 0xd0, 0x45, 0x41, 0xcc, 0x94, 0x4c, 0x02, 0x35, 0x33, 0x29, 0x70, + 0x48, 0x4b, 0xfe, 0xbe, 0x02, 0xb3, 0xe4, 0xdd, 0x73, 0xd3, 0x5c, 0x6b, 0xd1, 0xfb, 0x36, 0x87, + 0xd0, 0xe0, 0xd0, 0xd6, 0xc6, 0xb7, 0xf2, 0x50, 0x0d, 0x8d, 0xf8, 0x43, 0x5c, 0x51, 0xe1, 0x41, + 0x35, 0x2c, 0x97, 0x50, 0x8b, 0xb5, 0x31, 0x7e, 0xc2, 0x44, 0xda, 0x9d, 0xe1, 0x23, 0x46, 0x32, + 0xe2, 0x77, 0x60, 0x16, 0xc6, 0xb8, 0x03, 0xb3, 0x07, 0x13, 0xcc, 0xb7, 0xdb, 0x6d, 0x65, 0x0c, + 0xd5, 0xe6, 0x6f, 0x8c, 0xef, 0x06, 0x6d, 0x48, 0x86, 0xb2, 0x8e, 0x40, 0x3d, 0xa0, 0x16, 0x63, + 0xbc, 0x0d, 0xa7, 0xd3, 0x94, 0xc2, 0x52, 0xb0, 0x76, 0x68, 0xab, 0xef, 0xe8, 0x31, 0x8e, 0x2c, + 0x05, 0x05, 0xc7, 0x90, 0x82, 0x9b, 0xdc, 0xcc, 0xee, 0xd2, 0x77, 0x3c, 0x57, 0x3b, 0x33, 0xc2, + 0xe8, 0xda, 0x50, 0x30, 0x0c, 0xb1, 0xc6, 0x3f, 0x17, 0xe0, 0x62, 0xe4, 0x8a, 0xad, 0x99, 0xae, + 0xd9, 0x3e, 0xc4, 0x25, 0xa7, 0x3f, 0xa9, 0xfe, 0x39, 0xea, 0xad, 0x41, 0x85, 0x27, 0xe0, 0xd6, + 0xa0, 0xaf, 0x14, 0x40, 0x5c, 0x25, 0xcc, 0xcd, 0x20, 0xc7, 0xd3, 0x96, 0xe2, 0xe8, 0x66, 0xd0, + 0xaa, 0xd7, 0x96, 0x7b, 0xd2, 0xaa, 0xd7, 0x46, 0xce, 0x91, 0xdb, 0x17, 0x1d, 0x73, 0xbb, 0x63, + 0x8e, 0xbd, 0xbe, 0xc3, 0xaa, 0x25, 0x69, 0x5f, 0x88, 0x47, 0x94, 0xbc, 0xb9, 0x22, 0xd9, 0xd2, + 0x77, 0x61, 0x8e, 0x6d, 0xc8, 0x84, 0xb7, 0x6a, 0x4a, 0x45, 0x12, 0x3e, 0x62, 0x24, 0x83, 0x9b, + 0x66, 0xfd, 0x96, 0xb8, 0xd2, 0xb9, 0x38, 0xa6, 0x69, 0xb6, 0xb9, 0x24, 0xde, 0x49, 0x98, 0x66, + 0xf2, 0x37, 0x2a, 0xd6, 0xc6, 0x1f, 0xe6, 0x60, 0xaa, 0xe9, 0xd8, 0x2d, 0xdb, 0x6d, 0x9f, 0xdc, + 0x8d, 0x46, 0xe4, 0x36, 0x94, 0x02, 0xc7, 0x6e, 0xd1, 0x11, 0x2f, 0x3b, 0x11, 0x1f, 0x83, 0xf7, + 0x92, 0xa2, 0xe4, 0x63, 0x7c, 0xa5, 0x0c, 0xea, 0xfe, 0x6b, 0xd2, 0x87, 0x6a, 0x5b, 0xdf, 0xbc, + 0xa2, 0xba, 0xfc, 0xda, 0x18, 0x27, 0x74, 0x13, 0x77, 0xb8, 0xc8, 0xaf, 0x13, 0x02, 0x31, 0x92, + 0x44, 0x68, 0x72, 0xce, 0x2d, 0x8d, 0x39, 0xe7, 0xa4, 0xb8, 0xc1, 0x59, 0x67, 0x42, 0x71, 0x87, + 0xb1, 0x9e, 0x9a, 0x70, 0xa3, 0x9f, 0x3a, 0x8a, 0x0e, 0x14, 0xc9, 0xcc, 0x1f, 0x7f, 0x46, 0xc1, + 0x9a, 0x8b, 0x70, 0xcd, 0xf0, 0xea, 0xce, 0xc5, 0xb1, 0x52, 0x8b, 0x71, 0x11, 0xfc, 0x19, 0x05, + 0x6b, 0xf2, 0xb9, 0x1c, 0x4c, 0xfa, 0x31, 0x17, 0x52, 0xb9, 0x42, 0x63, 0x9e, 0xda, 0x48, 0xf8, + 0xa3, 0xb2, 0x2a, 0x31, 0x0e, 0xc7, 0x84, 0x48, 0xee, 0xaf, 0x32, 0xdf, 0x74, 0x83, 0x6d, 0xcf, + 0xef, 0x52, 0x5f, 0xb9, 0xf8, 0xcb, 0x63, 0xac, 0xa9, 0x8d, 0x88, 0x9b, 0x4c, 0xd6, 0x24, 0x40, + 0x18, 0x97, 0x46, 0x3a, 0x50, 0xe9, 0xb7, 0x64, 0x47, 0x55, 0x1c, 0x75, 0x61, 0x9c, 0xd5, 0x1c, + 0xcb, 0x63, 0xea, 0x27, 0x0c, 0x05, 0x18, 0x5d, 0x50, 0x21, 0x46, 0x62, 0x25, 0x6e, 0x5a, 0x93, + 0xd5, 0x60, 0xd7, 0x0e, 0xb7, 0xf8, 0xc2, 0x4b, 0xc2, 0x62, 0x97, 0x61, 0x64, 0x5e, 0xa9, 0x66, + 0xfc, 0x7d, 0x1e, 0xb8, 0x47, 0x2a, 0xcf, 0x76, 0x8b, 0x6b, 0x0c, 0x69, 0xb3, 0x63, 0xf7, 0xee, + 0x50, 0xdf, 0xde, 0xde, 0x53, 0x7e, 0x48, 0xec, 0x6c, 0x77, 0x9a, 0x02, 0x33, 0x5a, 0x91, 0x37, + 0x61, 0xd2, 0x32, 0x17, 0xa9, 0xcf, 0x46, 0xf1, 0xb2, 0xc4, 0x4c, 0x58, 0x5c, 0x88, 0x9a, 0x63, + 0x82, 0x19, 0xf7, 0x0d, 0xad, 0x88, 0x75, 0xe1, 0xc8, 0xbe, 0x61, 0x8c, 0x71, 0x8c, 0x11, 0x41, + 0xa8, 0x76, 0x38, 0xa9, 0xe0, 0x5a, 0x3c, 0x0a, 0x57, 0xa1, 0x65, 0x6e, 0xea, 0xb6, 0x18, 0xb1, + 0x31, 0x5c, 0x98, 0x4a, 0x5c, 0xd8, 0x46, 0x3e, 0x04, 0x15, 0xaf, 0x17, 0x53, 0x76, 0x55, 0xe1, + 0x38, 0x55, 0x6e, 0x2b, 0xd8, 0x83, 0xfd, 0xfa, 0xd4, 0xaa, 0xd7, 0xb6, 0x2d, 0x0d, 0xc0, 0x90, + 0x9c, 0x18, 0x50, 0x16, 0xb5, 0x6a, 0xfa, 0xba, 0x36, 0xa1, 0xa8, 0xc5, 0x55, 0x4e, 0x01, 0x2a, + 0x8c, 0xf1, 0xd9, 0x22, 0x44, 0x81, 0x79, 0x12, 0x40, 0xb9, 0x25, 0xae, 0x75, 0x52, 0x7a, 0x75, + 0xf4, 0x04, 0x47, 0xf2, 0x02, 0x49, 0xe9, 0x07, 0x27, 0x61, 0xa8, 0x44, 0x91, 0x36, 0x14, 0xde, + 0xf6, 0xb6, 0xc6, 0x56, 0xab, 0xb1, 0x6a, 0x72, 0x19, 0x55, 0x8e, 0x01, 0x90, 0x4b, 0x20, 0xbf, + 0x9d, 0x83, 0x33, 0x41, 0xda, 0x06, 0x55, 0xd3, 0x01, 0xc7, 0x37, 0xb6, 0xd3, 0x56, 0xad, 0x2a, + 0x54, 0x1b, 0x86, 0xc6, 0xc1, 0xbe, 0xf0, 0xf1, 0x97, 0x11, 0x73, 0x35, 0x9d, 0x56, 0xc6, 0xbc, + 0x32, 0x38, 0x39, 0xfe, 0x49, 0x18, 0x2a, 0x51, 0xc6, 0x2f, 0xe5, 0xa1, 0x16, 0xd3, 0x63, 0x63, + 0xdf, 0x02, 0x78, 0x3f, 0x75, 0x0b, 0xe0, 0xfa, 0xe8, 0xf1, 0xaf, 0xa8, 0x57, 0x27, 0x7d, 0x11, + 0xe0, 0x5f, 0xe5, 0xa1, 0xb0, 0xb9, 0xb4, 0x9c, 0xf4, 0x1e, 0x73, 0x8f, 0xc1, 0x7b, 0xdc, 0x81, + 0x89, 0xad, 0xbe, 0xed, 0x30, 0xdb, 0x1d, 0xfb, 0x68, 0x87, 0xbe, 0x34, 0x51, 0x95, 0x8d, 0x4b, + 0xae, 0xa8, 0xd9, 0x93, 0x36, 0x4c, 0xb4, 0xe5, 0xd1, 0x6e, 0x35, 0xe7, 0x5f, 0x1d, 0xdd, 0x6a, + 0x92, 0x7c, 0xa4, 0x20, 0xf5, 0x80, 0x9a, 0xbb, 0xf1, 0x19, 0x50, 0x46, 0x27, 0x09, 0x4e, 0x66, + 0x34, 0xc3, 0xf0, 0x58, 0xd6, 0x88, 0x1a, 0x9f, 0x86, 0x70, 0x8f, 0x7c, 0xec, 0x9f, 0xd3, 0xf8, + 0xd7, 0x1c, 0x24, 0xcd, 0x82, 0xc7, 0x3f, 0xa3, 0x3a, 0xe9, 0x19, 0xb5, 0x74, 0x1c, 0x0b, 0x30, + 0x7b, 0x52, 0x19, 0x7f, 0x9e, 0x87, 0xb2, 0xfa, 0x5b, 0x9c, 0x93, 0xaf, 0x12, 0xa2, 0x89, 0x2a, + 0xa1, 0xc5, 0x31, 0x95, 0xe3, 0xd0, 0x1a, 0xa1, 0x6e, 0xaa, 0x46, 0x68, 0xdc, 0x8b, 0xdb, 0x1f, + 0x51, 0x21, 0xf4, 0x37, 0x39, 0x50, 0xaa, 0xf9, 0x86, 0x1b, 0x30, 0xd3, 0xb5, 0x84, 0x27, 0xa8, + 0xf6, 0x81, 0x71, 0x53, 0xd1, 0xaa, 0x5c, 0x43, 0x6e, 0xfd, 0xe2, 0xb7, 0xd6, 0xfb, 0xe4, 0xfd, + 0x50, 0xd9, 0xf1, 0x02, 0x26, 0x74, 0x7d, 0x3e, 0x19, 0xea, 0x79, 0x4d, 0xc1, 0x31, 0xa4, 0x48, + 0x67, 0x9b, 0x4a, 0xc3, 0xb3, 0x4d, 0xc6, 0xef, 0xe7, 0x61, 0x32, 0x71, 0x5d, 0xff, 0xc8, 0x05, + 0x4f, 0xa9, 0x7a, 0xa3, 0xfc, 0xf1, 0xd7, 0x1b, 0x65, 0xd5, 0x54, 0x15, 0xc6, 0xac, 0xa9, 0x2a, + 0x1e, 0xa5, 0xa6, 0xca, 0xf8, 0x66, 0x0e, 0x40, 0x8f, 0xd6, 0x89, 0x97, 0x3b, 0xb5, 0x92, 0xe5, + 0x4e, 0x63, 0xcf, 0xab, 0xec, 0x62, 0xa7, 0x3f, 0x2d, 0xe9, 0x57, 0x12, 0xa5, 0x4e, 0xef, 0xe6, + 0x60, 0xda, 0x4c, 0x94, 0x0f, 0x8d, 0x6d, 0x5e, 0xa6, 0xaa, 0x91, 0xc2, 0x3f, 0xce, 0x49, 0xc2, + 0x31, 0x25, 0x96, 0xbc, 0x0c, 0x93, 0x3d, 0x55, 0x5b, 0x71, 0x2b, 0x9a, 0xf6, 0xe1, 0x19, 0xc7, + 0xf5, 0x18, 0x0e, 0x13, 0x94, 0x8f, 0x28, 0xd7, 0x2a, 0x1c, 0x4b, 0xb9, 0x56, 0xfc, 0x4c, 0x48, + 0xf1, 0xa1, 0x67, 0x42, 0x76, 0xa1, 0xba, 0xed, 0x7b, 0x5d, 0x51, 0x11, 0xa5, 0xae, 0x7c, 0xbf, + 0x3e, 0xc6, 0x9e, 0x12, 0xfd, 0xd9, 0x49, 0xb4, 0xb5, 0x2e, 0x6b, 0xfe, 0x18, 0x89, 0x12, 0x31, + 0x6a, 0x4f, 0x4a, 0x2d, 0x1f, 0xa7, 0xd4, 0x50, 0x97, 0x6c, 0x48, 0xee, 0xa8, 0xc5, 0x24, 0xab, + 0xa0, 0x26, 0x1e, 0x4f, 0x15, 0x94, 0xf1, 0xad, 0x50, 0x81, 0x35, 0x53, 0xd7, 0x20, 0xe4, 0x86, + 0x5c, 0x83, 0xa0, 0xae, 0x23, 0x8a, 0xd7, 0xeb, 0x3c, 0x0f, 0x65, 0x9f, 0x9a, 0x81, 0xe7, 0xaa, + 0xdb, 0xce, 0x42, 0xf5, 0x8f, 0x02, 0x8a, 0x0a, 0x1b, 0xaf, 0xeb, 0xc9, 0x3f, 0xa2, 0xae, 0xe7, + 0xfd, 0xb1, 0x09, 0x22, 0x0b, 0x37, 0xc3, 0xb5, 0x9e, 0x31, 0x49, 0x44, 0xd2, 0x5f, 0xfd, 0x1b, + 0x66, 0x29, 0x9d, 0xf4, 0x57, 0xff, 0x54, 0x19, 0x52, 0x90, 0x16, 0x4c, 0x3a, 0x66, 0xc0, 0x44, + 0x82, 0xa6, 0xb5, 0xc0, 0x46, 0x28, 0x1a, 0x0a, 0x97, 0xd1, 0x6a, 0x8c, 0x0f, 0x26, 0xb8, 0x1a, + 0xfb, 0x05, 0x48, 0xb9, 0x21, 0x3f, 0x89, 0xfd, 0xff, 0xb7, 0x8a, 0xfd, 0xff, 0x46, 0x0e, 0xa2, + 0x35, 0x75, 0xc4, 0xfc, 0xf4, 0x1b, 0x50, 0xe9, 0x9a, 0xf7, 0x97, 0xa8, 0x63, 0xee, 0x8d, 0x73, + 0x49, 0xf6, 0x9a, 0xe2, 0x81, 0x21, 0x37, 0x63, 0x3f, 0x07, 0xea, 0xd2, 0x29, 0x42, 0xa1, 0xb4, + 0x6d, 0xdf, 0x57, 0xfd, 0x19, 0xc7, 0x36, 0x8e, 0xfd, 0x29, 0x80, 0x0c, 0xe3, 0x0a, 0x00, 0x4a, + 0xee, 0xa4, 0x0b, 0x13, 0x81, 0x8c, 0xb2, 0xab, 0x57, 0x19, 0x3d, 0xf0, 0x98, 0x88, 0xd6, 0xab, + 0x2b, 0xa4, 0x24, 0x08, 0xb5, 0x8c, 0xc6, 0x27, 0xbe, 0xf1, 0xdd, 0xcb, 0x4f, 0x7d, 0xf3, 0xbb, + 0x97, 0x9f, 0xfa, 0xf6, 0x77, 0x2f, 0x3f, 0xf5, 0xd9, 0x83, 0xcb, 0xb9, 0x6f, 0x1c, 0x5c, 0xce, + 0x7d, 0xf3, 0xe0, 0x72, 0xee, 0xdb, 0x07, 0x97, 0x73, 0xff, 0x78, 0x70, 0x39, 0xf7, 0x6b, 0xff, + 0x74, 0xf9, 0xa9, 0x8f, 0xbf, 0x34, 0xe2, 0x7f, 0x2a, 0xff, 0x57, 0x00, 0x00, 0x00, 0xff, 0xff, + 0x5c, 0x95, 0xe3, 0xc7, 0x8d, 0x79, 0x00, 0x00, } func (m *AbstractPodTemplate) Marshal() (dAtA []byte, err error) { @@ -6227,8 +6230,13 @@ func (m *Scale) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.TargetBufferAvailability != nil { - i = encodeVarintGenerated(dAtA, i, uint64(*m.TargetBufferAvailability)) + if m.ScaleDownCooldownSeconds != nil { + i = encodeVarintGenerated(dAtA, i, uint64(*m.ScaleDownCooldownSeconds)) + i-- + dAtA[i] = 0x58 + } + if m.ScaleUpCooldownSeconds != nil { + i = encodeVarintGenerated(dAtA, i, uint64(*m.ScaleUpCooldownSeconds)) i-- dAtA[i] = 0x50 } @@ -6237,6 +6245,11 @@ func (m *Scale) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x48 } + if m.TargetBufferAvailability != nil { + i = encodeVarintGenerated(dAtA, i, uint64(*m.TargetBufferAvailability)) + i-- + dAtA[i] = 0x40 + } if m.TargetProcessingSeconds != nil { i = encodeVarintGenerated(dAtA, i, uint64(*m.TargetProcessingSeconds)) i-- @@ -6247,8 +6260,8 @@ func (m *Scale) MarshalToSizedBuffer(dAtA []byte) (int, error) { i-- dAtA[i] = 0x30 } - if m.CooldownSeconds != nil { - i = encodeVarintGenerated(dAtA, i, uint64(*m.CooldownSeconds)) + if m.DeprecatedCooldownSeconds != nil { + i = encodeVarintGenerated(dAtA, i, uint64(*m.DeprecatedCooldownSeconds)) i-- dAtA[i] = 0x28 } @@ -8909,8 +8922,8 @@ func (m *Scale) Size() (n int) { if m.LookbackSeconds != nil { n += 1 + sovGenerated(uint64(*m.LookbackSeconds)) } - if m.CooldownSeconds != nil { - n += 1 + sovGenerated(uint64(*m.CooldownSeconds)) + if m.DeprecatedCooldownSeconds != nil { + n += 1 + sovGenerated(uint64(*m.DeprecatedCooldownSeconds)) } if m.ZeroReplicaSleepSeconds != nil { n += 1 + sovGenerated(uint64(*m.ZeroReplicaSleepSeconds)) @@ -8918,11 +8931,17 @@ func (m *Scale) Size() (n int) { if m.TargetProcessingSeconds != nil { n += 1 + sovGenerated(uint64(*m.TargetProcessingSeconds)) } + if m.TargetBufferAvailability != nil { + n += 1 + sovGenerated(uint64(*m.TargetBufferAvailability)) + } if m.ReplicasPerScale != nil { n += 1 + sovGenerated(uint64(*m.ReplicasPerScale)) } - if m.TargetBufferAvailability != nil { - n += 1 + sovGenerated(uint64(*m.TargetBufferAvailability)) + if m.ScaleUpCooldownSeconds != nil { + n += 1 + sovGenerated(uint64(*m.ScaleUpCooldownSeconds)) + } + if m.ScaleDownCooldownSeconds != nil { + n += 1 + sovGenerated(uint64(*m.ScaleDownCooldownSeconds)) } return n } @@ -10319,11 +10338,13 @@ func (this *Scale) String() string { `Min:` + valueToStringGenerated(this.Min) + `,`, `Max:` + valueToStringGenerated(this.Max) + `,`, `LookbackSeconds:` + valueToStringGenerated(this.LookbackSeconds) + `,`, - `CooldownSeconds:` + valueToStringGenerated(this.CooldownSeconds) + `,`, + `DeprecatedCooldownSeconds:` + valueToStringGenerated(this.DeprecatedCooldownSeconds) + `,`, `ZeroReplicaSleepSeconds:` + valueToStringGenerated(this.ZeroReplicaSleepSeconds) + `,`, `TargetProcessingSeconds:` + valueToStringGenerated(this.TargetProcessingSeconds) + `,`, - `ReplicasPerScale:` + valueToStringGenerated(this.ReplicasPerScale) + `,`, `TargetBufferAvailability:` + valueToStringGenerated(this.TargetBufferAvailability) + `,`, + `ReplicasPerScale:` + valueToStringGenerated(this.ReplicasPerScale) + `,`, + `ScaleUpCooldownSeconds:` + valueToStringGenerated(this.ScaleUpCooldownSeconds) + `,`, + `ScaleDownCooldownSeconds:` + valueToStringGenerated(this.ScaleDownCooldownSeconds) + `,`, `}`, }, "") return s @@ -22073,7 +22094,7 @@ func (m *Scale) Unmarshal(dAtA []byte) error { m.LookbackSeconds = &v case 5: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field CooldownSeconds", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field DeprecatedCooldownSeconds", wireType) } var v uint32 for shift := uint(0); ; shift += 7 { @@ -22090,7 +22111,7 @@ func (m *Scale) Unmarshal(dAtA []byte) error { break } } - m.CooldownSeconds = &v + m.DeprecatedCooldownSeconds = &v case 6: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ZeroReplicaSleepSeconds", wireType) @@ -22131,6 +22152,26 @@ func (m *Scale) Unmarshal(dAtA []byte) error { } } m.TargetProcessingSeconds = &v + case 8: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field TargetBufferAvailability", wireType) + } + var v uint32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.TargetBufferAvailability = &v case 9: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ReplicasPerScale", wireType) @@ -22153,7 +22194,7 @@ func (m *Scale) Unmarshal(dAtA []byte) error { m.ReplicasPerScale = &v case 10: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field TargetBufferAvailability", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ScaleUpCooldownSeconds", wireType) } var v uint32 for shift := uint(0); ; shift += 7 { @@ -22170,7 +22211,27 @@ func (m *Scale) Unmarshal(dAtA []byte) error { break } } - m.TargetBufferAvailability = &v + m.ScaleUpCooldownSeconds = &v + case 11: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ScaleDownCooldownSeconds", wireType) + } + var v uint32 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowGenerated + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.ScaleDownCooldownSeconds = &v default: iNdEx = preIndex skippy, err := skipGenerated(dAtA[iNdEx:]) diff --git a/pkg/apis/numaflow/v1alpha1/generated.proto b/pkg/apis/numaflow/v1alpha1/generated.proto index 21935d9e3a..559e05901f 100644 --- a/pkg/apis/numaflow/v1alpha1/generated.proto +++ b/pkg/apis/numaflow/v1alpha1/generated.proto @@ -1036,6 +1036,7 @@ message Scale { // +optional optional uint32 lookbackSeconds = 4; + // Deprecated: Use scaleUpCooldownSeconds and scaleDownCooldownSeconds instead. // Cooldown seconds after a scaling operation before another one. // +optional optional uint32 cooldownSeconds = 5; @@ -1054,12 +1055,22 @@ message Scale { // A valid and meaningful value should be less than the BufferUsageLimit defined in the Edge spec (or Pipeline spec), for example, 50. // It only applies to UDF and Sink vertices because only they have buffers to read. // +optional - optional uint32 targetBufferAvailability = 10; + optional uint32 targetBufferAvailability = 8; // ReplicasPerScale defines maximum replicas can be scaled up or down at once. // The is use to prevent too aggressive scaling operations // +optional optional uint32 replicasPerScale = 9; + + // ScaleUpCooldownSeconds defines the cooldown seconds after a scaling operation, before a follow-up scaling up. + // It defaults to the CooldownSeconds if not set. + // +optional + optional uint32 scaleUpCooldownSeconds = 10; + + // ScaleDownCooldownSeconds defines the cooldown seconds after a scaling operation, before a follow-up scaling down. + // It defaults to the CooldownSeconds if not set. + // +optional + optional uint32 scaleDownCooldownSeconds = 11; } // SideInput defines information of a Side Input diff --git a/pkg/apis/numaflow/v1alpha1/openapi_generated.go b/pkg/apis/numaflow/v1alpha1/openapi_generated.go index 20c559b42b..a7646d86ba 100644 --- a/pkg/apis/numaflow/v1alpha1/openapi_generated.go +++ b/pkg/apis/numaflow/v1alpha1/openapi_generated.go @@ -3480,7 +3480,7 @@ func schema_pkg_apis_numaflow_v1alpha1_Scale(ref common.ReferenceCallback) commo }, "cooldownSeconds": { SchemaProps: spec.SchemaProps{ - Description: "Cooldown seconds after a scaling operation before another one.", + Description: "Deprecated: Use scaleUpCooldownSeconds and scaleDownCooldownSeconds instead. Cooldown seconds after a scaling operation before another one.", Type: []string{"integer"}, Format: "int64", }, @@ -3513,6 +3513,20 @@ func schema_pkg_apis_numaflow_v1alpha1_Scale(ref common.ReferenceCallback) commo Format: "int64", }, }, + "scaleUpCooldownSeconds": { + SchemaProps: spec.SchemaProps{ + Description: "ScaleUpCooldownSeconds defines the cooldown seconds after a scaling operation, before a follow-up scaling up. It defaults to the CooldownSeconds if not set.", + Type: []string{"integer"}, + Format: "int64", + }, + }, + "scaleDownCooldownSeconds": { + SchemaProps: spec.SchemaProps{ + Description: "ScaleDownCooldownSeconds defines the cooldown seconds after a scaling operation, before a follow-up scaling down. It defaults to the CooldownSeconds if not set.", + Type: []string{"integer"}, + Format: "int64", + }, + }, }, }, }, diff --git a/pkg/apis/numaflow/v1alpha1/vertex_types.go b/pkg/apis/numaflow/v1alpha1/vertex_types.go index c4a9c1d7d9..2974ea9ce5 100644 --- a/pkg/apis/numaflow/v1alpha1/vertex_types.go +++ b/pkg/apis/numaflow/v1alpha1/vertex_types.go @@ -560,9 +560,10 @@ type Scale struct { // Lookback seconds to calculate the average pending messages and processing rate. // +optional LookbackSeconds *uint32 `json:"lookbackSeconds,omitempty" protobuf:"varint,4,opt,name=lookbackSeconds"` + // Deprecated: Use scaleUpCooldownSeconds and scaleDownCooldownSeconds instead. // Cooldown seconds after a scaling operation before another one. // +optional - CooldownSeconds *uint32 `json:"cooldownSeconds,omitempty" protobuf:"varint,5,opt,name=cooldownSeconds"` + DeprecatedCooldownSeconds *uint32 `json:"cooldownSeconds,omitempty" protobuf:"varint,5,opt,name=cooldownSeconds"` // After scaling down to 0, sleep how many seconds before scaling up to peek. // +optional ZeroReplicaSleepSeconds *uint32 `json:"zeroReplicaSleepSeconds,omitempty" protobuf:"varint,6,opt,name=zeroReplicaSleepSeconds"` @@ -575,11 +576,19 @@ type Scale struct { // A valid and meaningful value should be less than the BufferUsageLimit defined in the Edge spec (or Pipeline spec), for example, 50. // It only applies to UDF and Sink vertices because only they have buffers to read. // +optional - TargetBufferAvailability *uint32 `json:"targetBufferAvailability,omitempty" protobuf:"varint,10,opt,name=targetBufferAvailability"` + TargetBufferAvailability *uint32 `json:"targetBufferAvailability,omitempty" protobuf:"varint,8,opt,name=targetBufferAvailability"` // ReplicasPerScale defines maximum replicas can be scaled up or down at once. // The is use to prevent too aggressive scaling operations // +optional ReplicasPerScale *uint32 `json:"replicasPerScale,omitempty" protobuf:"varint,9,opt,name=replicasPerScale"` + // ScaleUpCooldownSeconds defines the cooldown seconds after a scaling operation, before a follow-up scaling up. + // It defaults to the CooldownSeconds if not set. + // +optional + ScaleUpCooldownSeconds *uint32 `json:"scaleUpCooldownSeconds,omitempty" protobuf:"varint,10,opt,name=scaleUpCooldownSeconds"` + // ScaleDownCooldownSeconds defines the cooldown seconds after a scaling operation, before a follow-up scaling down. + // It defaults to the CooldownSeconds if not set. + // +optional + ScaleDownCooldownSeconds *uint32 `json:"scaleDownCooldownSeconds,omitempty" protobuf:"varint,11,opt,name=scaleDownCooldownSeconds"` } func (s Scale) GetLookbackSeconds() int { @@ -589,9 +598,22 @@ func (s Scale) GetLookbackSeconds() int { return DefaultLookbackSeconds } -func (s Scale) GetCooldownSeconds() int { - if s.CooldownSeconds != nil { - return int(*s.CooldownSeconds) +func (s Scale) GetScaleUpCooldownSeconds() int { + if s.ScaleUpCooldownSeconds != nil { + return int(*s.ScaleUpCooldownSeconds) + } + if s.DeprecatedCooldownSeconds != nil { + return int(*s.DeprecatedCooldownSeconds) + } + return DefaultCooldownSeconds +} + +func (s Scale) GetScaleDownCooldownSeconds() int { + if s.ScaleDownCooldownSeconds != nil { + return int(*s.ScaleDownCooldownSeconds) + } + if s.DeprecatedCooldownSeconds != nil { + return int(*s.DeprecatedCooldownSeconds) } return DefaultCooldownSeconds } diff --git a/pkg/apis/numaflow/v1alpha1/vertex_types_test.go b/pkg/apis/numaflow/v1alpha1/vertex_types_test.go index 12dbd1cc4c..24cfc541d9 100644 --- a/pkg/apis/numaflow/v1alpha1/vertex_types_test.go +++ b/pkg/apis/numaflow/v1alpha1/vertex_types_test.go @@ -541,13 +541,15 @@ func Test_Scale_Parameters(t *testing.T) { s := Scale{} assert.Equal(t, int32(0), s.GetMinReplicas()) assert.Equal(t, int32(DefaultMaxReplicas), s.GetMaxReplicas()) - assert.Equal(t, DefaultCooldownSeconds, s.GetCooldownSeconds()) + assert.Equal(t, DefaultCooldownSeconds, s.GetScaleUpCooldownSeconds()) + assert.Equal(t, DefaultCooldownSeconds, s.GetScaleDownCooldownSeconds()) assert.Equal(t, DefaultLookbackSeconds, s.GetLookbackSeconds()) assert.Equal(t, DefaultReplicasPerScale, s.GetReplicasPerScale()) assert.Equal(t, DefaultTargetBufferAvailability, s.GetTargetBufferAvailability()) assert.Equal(t, DefaultTargetProcessingSeconds, s.GetTargetProcessingSeconds()) assert.Equal(t, DefaultZeroReplicaSleepSeconds, s.GetZeroReplicaSleepSeconds()) - cds := uint32(100) + upcds := uint32(100) + downcds := uint32(99) lbs := uint32(101) rps := uint32(3) tps := uint32(102) @@ -556,7 +558,8 @@ func Test_Scale_Parameters(t *testing.T) { s = Scale{ Min: pointer.Int32(2), Max: pointer.Int32(4), - CooldownSeconds: &cds, + ScaleUpCooldownSeconds: &upcds, + ScaleDownCooldownSeconds: &downcds, LookbackSeconds: &lbs, ReplicasPerScale: &rps, TargetProcessingSeconds: &tps, @@ -565,7 +568,8 @@ func Test_Scale_Parameters(t *testing.T) { } assert.Equal(t, int32(2), s.GetMinReplicas()) assert.Equal(t, int32(4), s.GetMaxReplicas()) - assert.Equal(t, int(cds), s.GetCooldownSeconds()) + assert.Equal(t, int(upcds), s.GetScaleUpCooldownSeconds()) + assert.Equal(t, int(downcds), s.GetScaleDownCooldownSeconds()) assert.Equal(t, int(lbs), s.GetLookbackSeconds()) assert.Equal(t, int(rps), s.GetReplicasPerScale()) assert.Equal(t, int(tbu), s.GetTargetBufferAvailability()) diff --git a/pkg/apis/numaflow/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/numaflow/v1alpha1/zz_generated.deepcopy.go index 11a227c694..6e3ef4b8cb 100644 --- a/pkg/apis/numaflow/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/numaflow/v1alpha1/zz_generated.deepcopy.go @@ -1679,8 +1679,8 @@ func (in *Scale) DeepCopyInto(out *Scale) { *out = new(uint32) **out = **in } - if in.CooldownSeconds != nil { - in, out := &in.CooldownSeconds, &out.CooldownSeconds + if in.DeprecatedCooldownSeconds != nil { + in, out := &in.DeprecatedCooldownSeconds, &out.DeprecatedCooldownSeconds *out = new(uint32) **out = **in } @@ -1704,6 +1704,16 @@ func (in *Scale) DeepCopyInto(out *Scale) { *out = new(uint32) **out = **in } + if in.ScaleUpCooldownSeconds != nil { + in, out := &in.ScaleUpCooldownSeconds, &out.ScaleUpCooldownSeconds + *out = new(uint32) + **out = **in + } + if in.ScaleDownCooldownSeconds != nil { + in, out := &in.ScaleDownCooldownSeconds, &out.ScaleDownCooldownSeconds + *out = new(uint32) + **out = **in + } return } diff --git a/pkg/reconciler/vertex/scaling/scaling.go b/pkg/reconciler/vertex/scaling/scaling.go index a37b16ac2f..b8c6c2c400 100644 --- a/pkg/reconciler/vertex/scaling/scaling.go +++ b/pkg/reconciler/vertex/scaling/scaling.go @@ -145,7 +145,7 @@ func (s *Scaler) scale(ctx context.Context, id int, keyCh <-chan string) { // If there's back pressure in the downstream vertices (not connected), desiredReplicas remains the same. func (s *Scaler) scaleOneVertex(ctx context.Context, key string, worker int) error { log := logging.FromContext(ctx).With("worker", fmt.Sprint(worker)).With("vertexKey", key) - log.Debugf("Working on key: %s", key) + log.Debugf("Working on key: %s.", key) strs := strings.Split(key, "/") if len(strs) != 2 { return fmt.Errorf("invalid key %q", key) @@ -156,21 +156,25 @@ func (s *Scaler) scaleOneVertex(ctx context.Context, key string, worker int) err if err := s.client.Get(ctx, client.ObjectKey{Namespace: namespace, Name: vertexFullName}, vertex); err != nil { if apierrors.IsNotFound(err) { s.StopWatching(key) - log.Info("No corresponding Vertex found, stopped watching") + log.Info("No corresponding Vertex found, stopped watching.") return nil } return fmt.Errorf("failed to query vertex object of key %q, %w", key, err) } if !vertex.GetDeletionTimestamp().IsZero() { s.StopWatching(key) - log.Debug("Vertex being deleted") + log.Debug("Vertex being deleted.") return nil } if !vertex.Scalable() { // A vertex which is not scalable, such as HTTP source, or autoscaling disabled. s.StopWatching(key) // Remove it in case it's watched. return nil } - if time.Since(vertex.Status.LastScaledAt.Time).Seconds() < float64(vertex.Spec.Scale.GetCooldownSeconds()) { + secondsSinceLastScale := time.Since(vertex.Status.LastScaledAt.Time).Seconds() + scaleDownCooldown := float64(vertex.Spec.Scale.GetScaleDownCooldownSeconds()) + scaleUpCooldown := float64(vertex.Spec.Scale.GetScaleUpCooldownSeconds()) + if secondsSinceLastScale < scaleDownCooldown && secondsSinceLastScale < scaleUpCooldown { + // Skip scaling without needing further calculation log.Debug("Cooldown period, skip scaling.") return nil } @@ -182,30 +186,30 @@ func (s *Scaler) scaleOneVertex(ctx context.Context, key string, worker int) err if err := s.client.Get(ctx, client.ObjectKey{Namespace: namespace, Name: vertex.Spec.PipelineName}, pl); err != nil { if apierrors.IsNotFound(err) { s.StopWatching(key) - log.Info("No corresponding Pipeline found, stopped watching") + log.Info("No corresponding Pipeline found, stopped watching.") return nil } return fmt.Errorf("failed to query Pipeline object of key %q, %w", key, err) } if !pl.GetDeletionTimestamp().IsZero() { s.StopWatching(key) - log.Debug("Corresponding Pipeline being deleted") + log.Debug("Corresponding Pipeline being deleted.") return nil } if pl.Spec.Lifecycle.GetDesiredPhase() != dfv1.PipelinePhaseRunning { - log.Debug("Corresponding Pipeline not in Running state") + log.Debug("Corresponding Pipeline not in Running state.") return nil } if int(vertex.Status.Replicas) != vertex.GetReplicas() { - log.Debugf("Vertex %s might be under processing, replicas mismatch", vertex.Name) + log.Debugf("Vertex %s might be under processing, replicas mismatch.", vertex.Name) return nil } if vertex.Status.Replicas == 0 { // Was scaled to 0 - if seconds := time.Since(vertex.Status.LastScaledAt.Time).Seconds(); seconds >= float64(vertex.Spec.Scale.GetZeroReplicaSleepSeconds()) { - log.Debugf("Vertex %s has slept %v seconds, scaling up to peek", vertex.Name, seconds) + if secondsSinceLastScale >= float64(vertex.Spec.Scale.GetZeroReplicaSleepSeconds()) { + log.Debugf("Vertex %s has slept %v seconds, scaling up to peek.", vertex.Name, secondsSinceLastScale) return s.patchVertexReplicas(ctx, vertex, 1) } else { - log.Debugf("Vertex %q has slept %v seconds, hasn't reached zeroReplicaSleepSeconds (%v seconds)", vertex.Name, seconds, vertex.Spec.Scale.GetZeroReplicaSleepSeconds()) + log.Debugf("Vertex %q has slept %v seconds, hasn't reached zeroReplicaSleepSeconds (%v seconds).", vertex.Name, secondsSinceLastScale, vertex.Spec.Scale.GetZeroReplicaSleepSeconds()) return nil } } @@ -239,7 +243,7 @@ func (s *Scaler) scaleOneVertex(ctx context.Context, key string, worker int) err rate, existing := m.ProcessingRates["default"] // If rate is not available, we skip scaling. if !existing || rate < 0 { // Rate not available - log.Debugf("Vertex %s has no rate information, skip scaling", vertex.Name) + log.Debugf("Vertex %s has no rate information, skip scaling.", vertex.Name) return nil } partitionRates = append(partitionRates, rate) @@ -248,7 +252,7 @@ func (s *Scaler) scaleOneVertex(ctx context.Context, key string, worker int) err pending, existing := m.Pendings["default"] if !existing || pending < 0 || pending == isb.PendingNotAvailable { // Pending not available, we don't do anything - log.Debugf("Vertex %s has no pending messages information, skip scaling", vertex.Name) + log.Debugf("Vertex %s has no pending messages information, skip scaling.", vertex.Name) return nil } totalPending += pending @@ -274,7 +278,6 @@ func (s *Scaler) scaleOneVertex(ctx context.Context, key string, worker int) err partitionAvailableBufferLengths = append(partitionAvailableBufferLengths, int64(float64(bInfo.GetBufferLength())*float64(vertex.Spec.Scale.GetTargetBufferAvailability())/100)) totalBufferLength += int64(float64(*bInfo.BufferLength) * *bInfo.BufferUsageLimit) targetAvailableBufferLength += int64(float64(*bInfo.BufferLength) * float64(vertex.Spec.Scale.GetTargetBufferAvailability()) / 100) - // Add to cache for back pressure calculation } } // Add processing rate information to cache for back pressure calculation @@ -289,16 +292,16 @@ func (s *Scaler) scaleOneVertex(ctx context.Context, key string, worker int) err } else { desired = s.desiredReplicas(ctx, vertex, partitionRates, partitionPending, partitionBufferLengths, partitionAvailableBufferLengths) } - log.Debugf("Calculated desired replica number of vertex %q is: %d", vertex.Name, desired) + log.Debugf("Calculated desired replica number of vertex %q is: %d.", vertex.Name, desired) max := vertex.Spec.Scale.GetMaxReplicas() min := vertex.Spec.Scale.GetMinReplicas() if desired > max { desired = max - log.Debugf("Calculated desired replica number %d of vertex %q is greater than max, using max %d", vertex.Name, desired, max) + log.Debugf("Calculated desired replica number %d of vertex %q is greater than max, using max %d.", vertex.Name, desired, max) } if desired < min { desired = min - log.Debugf("Calculated desired replica number %d of vertex %q is smaller than min, using min %d", vertex.Name, desired, min) + log.Debugf("Calculated desired replica number %d of vertex %q is smaller than min, using min %d.", vertex.Name, desired, min) } if current > max || current < min { // Someone might have manually scaled up/down the vertex return s.patchVertexReplicas(ctx, vertex, desired) @@ -309,6 +312,10 @@ func (s *Scaler) scaleOneVertex(ctx context.Context, key string, worker int) err if diff > maxAllowed { diff = maxAllowed } + if secondsSinceLastScale < scaleDownCooldown { + log.Debugf("Cooldown period for scaling down, skip scaling.") + return nil + } return s.patchVertexReplicas(ctx, vertex, current-diff) // We scale down gradually } if desired > current { @@ -316,20 +323,24 @@ func (s *Scaler) scaleOneVertex(ctx context.Context, key string, worker int) err directPressure, downstreamPressure := s.hasBackPressure(*pl, *vertex) if directPressure { if current > 1 { - log.Debugf("Vertex %s has direct back pressure from connected vertices, decreasing one replica", key) + log.Debugf("Vertex %s has direct back pressure from connected vertices, decreasing one replica.", key) return s.patchVertexReplicas(ctx, vertex, current-1) } else { - log.Debugf("Vertex %s has direct back pressure from connected vertices, skip scaling", key) + log.Debugf("Vertex %s has direct back pressure from connected vertices, skip scaling.", key) return nil } } else if downstreamPressure { - log.Debugf("Vertex %s has back pressure in downstream vertices, skip scaling", key) + log.Debugf("Vertex %s has back pressure in downstream vertices, skip scaling.", key) return nil } diff := desired - current if diff > maxAllowed { diff = maxAllowed } + if secondsSinceLastScale < scaleUpCooldown { + log.Debugf("Cooldown period for scaling up, skip scaling.") + return nil + } return s.patchVertexReplicas(ctx, vertex, current+diff) // We scale up gradually } return nil @@ -414,7 +425,7 @@ func (s *Scaler) Start(ctx context.Context) error { for { select { case <-ctx.Done(): - log.Info("Shutting down scaling job assigner") + log.Info("Shutting down scaling job assigner.") // clear the daemon clients cache s.daemonClientsCache.Purge() return nil @@ -478,7 +489,7 @@ func (s *Scaler) patchVertexReplicas(ctx context.Context, vertex *dfv1.Vertex, d if err := s.client.Patch(ctx, vertex, client.RawPatch(types.MergePatchType, body)); err != nil && !apierrors.IsNotFound(err) { return fmt.Errorf("failed to patch vertex replicas, %w", err) } - log.Infow("Auto scaling - vertex replicas changed", zap.Int32p("from", origin), zap.Int32("to", desiredReplicas), zap.String("pipeline", vertex.Spec.PipelineName), zap.String("vertex", vertex.Spec.Name)) + log.Infow("Auto scaling - vertex replicas changed.", zap.Int32p("from", origin), zap.Int32("to", desiredReplicas), zap.String("pipeline", vertex.Spec.PipelineName), zap.String("vertex", vertex.Spec.Name)) return nil }