Skip to content

Commit

Permalink
fix(operator): Use a minimum value for replay memory ceiling (#13066)
Browse files Browse the repository at this point in the history
  • Loading branch information
@xperimental
xperimental authored Jun 3, 2024
1 parent 00d3c7a commit 4f3ed77
Show file tree
Hide file tree
Showing 2 changed files with 44 additions and 34 deletions.
68 changes: 34 additions & 34 deletions operator/internal/manifests/internal/config/build_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ ingester:
wal:
enabled: true
dir: /tmp/wal
replay_memory_ceiling: 2500
replay_memory_ceiling: 536870912
ingester_client:
grpc_client_config:
max_recv_msg_size: 67108864
Expand Down Expand Up @@ -240,7 +240,7 @@ overrides:
},
WriteAheadLog: WriteAheadLog{
Directory: "/tmp/wal",
IngesterMemoryRequest: 5000,
IngesterMemoryRequest: 0,
},
ObjectStorage: storage.Options{
SharedStore: lokiv1.ObjectStorageSecretS3,
Expand Down Expand Up @@ -327,7 +327,7 @@ ingester:
wal:
enabled: true
dir: /tmp/wal
replay_memory_ceiling: 2500
replay_memory_ceiling: 2147483648
ingester_client:
grpc_client_config:
max_recv_msg_size: 67108864
Expand Down Expand Up @@ -588,7 +588,7 @@ overrides:
},
WriteAheadLog: WriteAheadLog{
Directory: "/tmp/wal",
IngesterMemoryRequest: 5000,
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
},
ObjectStorage: storage.Options{
SharedStore: lokiv1.ObjectStorageSecretS3,
Expand Down Expand Up @@ -673,7 +673,7 @@ func TestBuild_ConfigAndRuntimeConfig_CreateLokiConfigFailed(t *testing.T) {
},
WriteAheadLog: WriteAheadLog{
Directory: "/tmp/wal",
IngesterMemoryRequest: 5000,
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
},
ObjectStorage: storage.Options{
SharedStore: lokiv1.ObjectStorageSecretS3,
Expand Down Expand Up @@ -754,7 +754,7 @@ ingester:
wal:
enabled: true
dir: /tmp/wal
replay_memory_ceiling: 2500
replay_memory_ceiling: 2147483648
ingester_client:
grpc_client_config:
max_recv_msg_size: 67108864
Expand Down Expand Up @@ -1026,7 +1026,7 @@ overrides:
},
WriteAheadLog: WriteAheadLog{
Directory: "/tmp/wal",
IngesterMemoryRequest: 5000,
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
},
ObjectStorage: storage.Options{
SharedStore: lokiv1.ObjectStorageSecretS3,
Expand Down Expand Up @@ -1113,7 +1113,7 @@ ingester:
wal:
enabled: true
dir: /tmp/wal
replay_memory_ceiling: 2500
replay_memory_ceiling: 2147483648
ingester_client:
grpc_client_config:
max_recv_msg_size: 67108864
Expand Down Expand Up @@ -1386,7 +1386,7 @@ overrides:
},
WriteAheadLog: WriteAheadLog{
Directory: "/tmp/wal",
IngesterMemoryRequest: 5000,
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
},
ObjectStorage: storage.Options{
SharedStore: lokiv1.ObjectStorageSecretS3,
Expand Down Expand Up @@ -1473,7 +1473,7 @@ ingester:
wal:
enabled: true
dir: /tmp/wal
replay_memory_ceiling: 2500
replay_memory_ceiling: 2147483648
ingester_client:
grpc_client_config:
max_recv_msg_size: 67108864
Expand Down Expand Up @@ -1776,7 +1776,7 @@ overrides:
},
WriteAheadLog: WriteAheadLog{
Directory: "/tmp/wal",
IngesterMemoryRequest: 5000,
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
},
ObjectStorage: storage.Options{
SharedStore: lokiv1.ObjectStorageSecretS3,
Expand Down Expand Up @@ -1866,7 +1866,7 @@ ingester:
wal:
enabled: true
dir: /tmp/wal
replay_memory_ceiling: 2500
replay_memory_ceiling: 2147483648
ingester_client:
grpc_client_config:
max_recv_msg_size: 67108864
Expand Down Expand Up @@ -2112,7 +2112,7 @@ overrides:
},
WriteAheadLog: WriteAheadLog{
Directory: "/tmp/wal",
IngesterMemoryRequest: 5000,
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
},
ObjectStorage: storage.Options{
SharedStore: lokiv1.ObjectStorageSecretS3,
Expand Down Expand Up @@ -2202,7 +2202,7 @@ ingester:
wal:
enabled: true
dir: /tmp/wal
replay_memory_ceiling: 2500
replay_memory_ceiling: 2147483648
ingester_client:
grpc_client_config:
max_recv_msg_size: 67108864
Expand Down Expand Up @@ -2535,7 +2535,7 @@ overrides:
},
WriteAheadLog: WriteAheadLog{
Directory: "/tmp/wal",
IngesterMemoryRequest: 5000,
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
},
ObjectStorage: storage.Options{
SharedStore: lokiv1.ObjectStorageSecretS3,
Expand Down Expand Up @@ -2635,7 +2635,7 @@ ingester:
wal:
enabled: true
dir: /tmp/wal
replay_memory_ceiling: 2500
replay_memory_ceiling: 2147483648
ingester_client:
grpc_client_config:
max_recv_msg_size: 67108864
Expand Down Expand Up @@ -2881,7 +2881,7 @@ overrides:
},
WriteAheadLog: WriteAheadLog{
Directory: "/tmp/wal",
IngesterMemoryRequest: 5000,
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
},
ObjectStorage: storage.Options{
SharedStore: lokiv1.ObjectStorageSecretS3,
Expand Down Expand Up @@ -2967,7 +2967,7 @@ ingester:
wal:
enabled: true
dir: /tmp/wal
replay_memory_ceiling: 2500
replay_memory_ceiling: 2147483648
ingester_client:
grpc_client_config:
max_recv_msg_size: 67108864
Expand Down Expand Up @@ -3377,7 +3377,7 @@ overrides:
},
WriteAheadLog: WriteAheadLog{
Directory: "/tmp/wal",
IngesterMemoryRequest: 5000,
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
},
ObjectStorage: storage.Options{
SharedStore: lokiv1.ObjectStorageSecretS3,
Expand Down Expand Up @@ -3465,7 +3465,7 @@ ingester:
wal:
enabled: true
dir: /tmp/wal
replay_memory_ceiling: 2500
replay_memory_ceiling: 2147483648
ingester_client:
grpc_client_config:
max_recv_msg_size: 67108864
Expand Down Expand Up @@ -3638,7 +3638,7 @@ overrides:
},
WriteAheadLog: WriteAheadLog{
Directory: "/tmp/wal",
IngesterMemoryRequest: 5000,
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
},
ObjectStorage: storage.Options{
SharedStore: lokiv1.ObjectStorageSecretS3,
Expand Down Expand Up @@ -3727,7 +3727,7 @@ ingester:
wal:
enabled: true
dir: /tmp/wal
replay_memory_ceiling: 2500
replay_memory_ceiling: 2147483648
ingester_client:
grpc_client_config:
max_recv_msg_size: 67108864
Expand Down Expand Up @@ -3901,7 +3901,7 @@ overrides:
},
WriteAheadLog: WriteAheadLog{
Directory: "/tmp/wal",
IngesterMemoryRequest: 5000,
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
},
ObjectStorage: storage.Options{
SharedStore: lokiv1.ObjectStorageSecretS3,
Expand Down Expand Up @@ -3990,7 +3990,7 @@ ingester:
wal:
enabled: true
dir: /tmp/wal
replay_memory_ceiling: 2500
replay_memory_ceiling: 2147483648
ingester_client:
grpc_client_config:
max_recv_msg_size: 67108864
Expand Down Expand Up @@ -4162,7 +4162,7 @@ overrides:
},
WriteAheadLog: WriteAheadLog{
Directory: "/tmp/wal",
IngesterMemoryRequest: 5000,
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
},
ObjectStorage: storage.Options{
SharedStore: lokiv1.ObjectStorageSecretS3,
Expand Down Expand Up @@ -4254,7 +4254,7 @@ ingester:
wal:
enabled: true
dir: /tmp/wal
replay_memory_ceiling: 2500
replay_memory_ceiling: 2147483648
ingester_client:
grpc_client_config:
max_recv_msg_size: 67108864
Expand Down Expand Up @@ -4460,7 +4460,7 @@ overrides:
},
WriteAheadLog: WriteAheadLog{
Directory: "/tmp/wal",
IngesterMemoryRequest: 5000,
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
},
ObjectStorage: storage.Options{
SharedStore: lokiv1.ObjectStorageSecretS3,
Expand Down Expand Up @@ -4554,7 +4554,7 @@ ingester:
wal:
enabled: true
dir: /tmp/wal
replay_memory_ceiling: 2500
replay_memory_ceiling: 2147483648
ingester_client:
grpc_client_config:
max_recv_msg_size: 67108864
Expand Down Expand Up @@ -4760,7 +4760,7 @@ overrides:
},
WriteAheadLog: WriteAheadLog{
Directory: "/tmp/wal",
IngesterMemoryRequest: 5000,
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
},
ObjectStorage: storage.Options{
SharedStore: lokiv1.ObjectStorageSecretS3,
Expand Down Expand Up @@ -4852,7 +4852,7 @@ ingester:
wal:
enabled: true
dir: /tmp/wal
replay_memory_ceiling: 2500
replay_memory_ceiling: 2147483648
ingester_client:
grpc_client_config:
max_recv_msg_size: 67108864
Expand Down Expand Up @@ -5019,7 +5019,7 @@ overrides:
},
WriteAheadLog: WriteAheadLog{
Directory: "/tmp/wal",
IngesterMemoryRequest: 5000,
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
},
ObjectStorage: storage.Options{
SharedStore: lokiv1.ObjectStorageSecretS3,
Expand Down Expand Up @@ -5109,7 +5109,7 @@ func defaultOptions() Options {
},
WriteAheadLog: WriteAheadLog{
Directory: "/tmp/wal",
IngesterMemoryRequest: 5000,
IngesterMemoryRequest: 4 * 1024 * 1024 * 1024,
},
ObjectStorage: storage.Options{
SharedStore: lokiv1.ObjectStorageSecretS3,
Expand Down Expand Up @@ -5343,7 +5343,7 @@ ingester:
wal:
enabled: true
dir: /tmp/wal
replay_memory_ceiling: 2500
replay_memory_ceiling: 2147483648
ingester_client:
grpc_client_config:
max_recv_msg_size: 67108864
Expand Down Expand Up @@ -5518,7 +5518,7 @@ ingester:
wal:
enabled: true
dir: /tmp/wal
replay_memory_ceiling: 2500
replay_memory_ceiling: 2147483648
ingester_client:
grpc_client_config:
max_recv_msg_size: 67108864
Expand Down
10 changes: 10 additions & 0 deletions operator/internal/manifests/internal/config/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -223,10 +223,20 @@ type WriteAheadLog struct {
IngesterMemoryRequest int64
}

const (
// minimumReplayCeiling contains the minimum value that will be used for the replay_memory_ceiling.
// It is set, so that even when the ingester has a low memory request, the replay will not flush each block
// on its own.
minimumReplayCeiling = 512 * 1024 * 1024
)

// ReplayMemoryCeiling calculates 50% of the ingester memory
// for the ingester to use for the write-ahead-log capbability.
func (w WriteAheadLog) ReplayMemoryCeiling() string {
value := int64(math.Ceil(float64(w.IngesterMemoryRequest) * float64(0.5)))
if value < minimumReplayCeiling {
value = minimumReplayCeiling
}
return fmt.Sprintf("%d", value)
}

Expand Down

0 comments on commit 4f3ed77

Please sign in to comment.