diff --git a/go/vt/vttablet/tabletserver/throttle/base/http.go b/go/vt/vttablet/tabletserver/throttle/base/http.go index 6f657766ad1..bbf4662d6cf 100644 --- a/go/vt/vttablet/tabletserver/throttle/base/http.go +++ b/go/vt/vttablet/tabletserver/throttle/base/http.go @@ -1,7 +1,42 @@ /* - Copyright 2017 GitHub Inc. +Copyright 2023 The Vitess Authors. - Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This codebase originates from https://github.com/github/freno, See https://github.com/github/freno/blob/master/LICENSE +/* + MIT License + + Copyright (c) 2017 GitHub + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. */ package base diff --git a/go/vt/vttablet/tabletserver/throttle/base/metric_health.go b/go/vt/vttablet/tabletserver/throttle/base/metric_health.go index e970888bf13..458e8e28264 100644 --- a/go/vt/vttablet/tabletserver/throttle/base/metric_health.go +++ b/go/vt/vttablet/tabletserver/throttle/base/metric_health.go @@ -1,7 +1,42 @@ /* - Copyright 2017 GitHub Inc. +Copyright 2023 The Vitess Authors. - Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This codebase originates from https://github.com/github/freno, See https://github.com/github/freno/blob/master/LICENSE +/* + MIT License + + Copyright (c) 2017 GitHub + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. */ package base diff --git a/go/vt/vttablet/tabletserver/throttle/base/metric_health_test.go b/go/vt/vttablet/tabletserver/throttle/base/metric_health_test.go index d11ecd7b8e5..a1a1ad4e0c0 100644 --- a/go/vt/vttablet/tabletserver/throttle/base/metric_health_test.go +++ b/go/vt/vttablet/tabletserver/throttle/base/metric_health_test.go @@ -1,9 +1,43 @@ /* - Copyright 2017 GitHub Inc. +Copyright 2023 The Vitess Authors. - Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ +// This codebase originates from https://github.com/github/freno, See https://github.com/github/freno/blob/master/LICENSE +/* + MIT License + + Copyright (c) 2017 GitHub + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ package base import ( diff --git a/go/vt/vttablet/tabletserver/throttle/base/recent_app.go b/go/vt/vttablet/tabletserver/throttle/base/recent_app.go index 2c629fbff25..64527c4cc1c 100644 --- a/go/vt/vttablet/tabletserver/throttle/base/recent_app.go +++ b/go/vt/vttablet/tabletserver/throttle/base/recent_app.go @@ -1,9 +1,43 @@ /* - Copyright 2017 GitHub Inc. +Copyright 2023 The Vitess Authors. - Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ +// This codebase originates from https://github.com/github/freno, See https://github.com/github/freno/blob/master/LICENSE +/* + MIT License + + Copyright (c) 2017 GitHub + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. +*/ package base import ( diff --git a/go/vt/vttablet/tabletserver/throttle/base/throttle_metric.go b/go/vt/vttablet/tabletserver/throttle/base/throttle_metric.go index 7070febb3bc..3d4c4f95a2e 100644 --- a/go/vt/vttablet/tabletserver/throttle/base/throttle_metric.go +++ b/go/vt/vttablet/tabletserver/throttle/base/throttle_metric.go @@ -1,7 +1,42 @@ /* - Copyright 2017 GitHub Inc. +Copyright 2023 The Vitess Authors. - Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This codebase originates from https://github.com/github/freno, See https://github.com/github/freno/blob/master/LICENSE +/* + MIT License + + Copyright (c) 2017 GitHub + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. */ package base diff --git a/go/vt/vttablet/tabletserver/throttle/base/throttle_metric_app.go b/go/vt/vttablet/tabletserver/throttle/base/throttle_metric_app.go index ce77f7068b6..482f319365f 100644 --- a/go/vt/vttablet/tabletserver/throttle/base/throttle_metric_app.go +++ b/go/vt/vttablet/tabletserver/throttle/base/throttle_metric_app.go @@ -1,7 +1,42 @@ /* - Copyright 2017 GitHub Inc. +Copyright 2023 The Vitess Authors. - Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This codebase originates from https://github.com/github/freno, See https://github.com/github/freno/blob/master/LICENSE +/* + MIT License + + Copyright (c) 2017 GitHub + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. */ package base diff --git a/go/vt/vttablet/tabletserver/throttle/check.go b/go/vt/vttablet/tabletserver/throttle/check.go index dd209a0c423..9dfbade8af6 100644 --- a/go/vt/vttablet/tabletserver/throttle/check.go +++ b/go/vt/vttablet/tabletserver/throttle/check.go @@ -1,7 +1,42 @@ /* - Copyright 2017 GitHub Inc. +Copyright 2023 The Vitess Authors. - Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This codebase originates from https://github.com/github/freno, See https://github.com/github/freno/blob/master/LICENSE +/* + MIT License + + Copyright (c) 2017 GitHub + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. */ package throttle @@ -11,7 +46,6 @@ import ( "fmt" "net/http" "strings" - "sync/atomic" "time" "vitess.io/vitess/go/stats" @@ -114,7 +148,7 @@ func (check *ThrottlerCheck) Check(ctx context.Context, appName string, storeTyp } checkResult = check.checkAppMetricResult(ctx, appName, storeType, storeName, metricResultFunc, flags) - atomic.StoreInt64(&check.throttler.lastCheckTimeNano, time.Now().UnixNano()) + check.throttler.lastCheckTimeNano.Store(time.Now().UnixNano()) go func(statusCode int) { stats.GetOrNewCounter("ThrottlerCheckAnyTotal", "total number of checks").Add(1) diff --git a/go/vt/vttablet/tabletserver/throttle/check_result.go b/go/vt/vttablet/tabletserver/throttle/check_result.go index 3bc162b623a..41a1b240934 100644 --- a/go/vt/vttablet/tabletserver/throttle/check_result.go +++ b/go/vt/vttablet/tabletserver/throttle/check_result.go @@ -1,7 +1,42 @@ /* - Copyright 2017 GitHub Inc. +Copyright 2023 The Vitess Authors. - Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This codebase originates from https://github.com/github/freno, See https://github.com/github/freno/blob/master/LICENSE +/* + MIT License + + Copyright (c) 2017 GitHub + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. */ package throttle diff --git a/go/vt/vttablet/tabletserver/throttle/config/config.go b/go/vt/vttablet/tabletserver/throttle/config/config.go index b1f3ad61f80..5241d686d10 100644 --- a/go/vt/vttablet/tabletserver/throttle/config/config.go +++ b/go/vt/vttablet/tabletserver/throttle/config/config.go @@ -1,7 +1,42 @@ /* - Copyright 2017 GitHub Inc. +Copyright 2023 The Vitess Authors. - Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This codebase originates from https://github.com/github/freno, See https://github.com/github/freno/blob/master/LICENSE +/* + MIT License + + Copyright (c) 2017 GitHub + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. */ package config diff --git a/go/vt/vttablet/tabletserver/throttle/config/mysql_config.go b/go/vt/vttablet/tabletserver/throttle/config/mysql_config.go index 3e3e82adff4..3aa0607fb28 100644 --- a/go/vt/vttablet/tabletserver/throttle/config/mysql_config.go +++ b/go/vt/vttablet/tabletserver/throttle/config/mysql_config.go @@ -1,7 +1,42 @@ /* - Copyright 2017 GitHub Inc. +Copyright 2023 The Vitess Authors. - Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This codebase originates from https://github.com/github/freno, See https://github.com/github/freno/blob/master/LICENSE +/* + MIT License + + Copyright (c) 2017 GitHub + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. */ package config diff --git a/go/vt/vttablet/tabletserver/throttle/config/store_config.go b/go/vt/vttablet/tabletserver/throttle/config/store_config.go index 9a19025df05..7e5594050d9 100644 --- a/go/vt/vttablet/tabletserver/throttle/config/store_config.go +++ b/go/vt/vttablet/tabletserver/throttle/config/store_config.go @@ -1,7 +1,42 @@ /* - Copyright 2017 GitHub Inc. +Copyright 2023 The Vitess Authors. - Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This codebase originates from https://github.com/github/freno, See https://github.com/github/freno/blob/master/LICENSE +/* + MIT License + + Copyright (c) 2017 GitHub + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. */ package config diff --git a/go/vt/vttablet/tabletserver/throttle/mysql.go b/go/vt/vttablet/tabletserver/throttle/mysql.go index 350ad465b73..81a967ddacb 100644 --- a/go/vt/vttablet/tabletserver/throttle/mysql.go +++ b/go/vt/vttablet/tabletserver/throttle/mysql.go @@ -1,7 +1,42 @@ /* - Copyright 2017 GitHub Inc. +Copyright 2023 The Vitess Authors. - Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This codebase originates from https://github.com/github/freno, See https://github.com/github/freno/blob/master/LICENSE +/* + MIT License + + Copyright (c) 2017 GitHub + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. */ package throttle @@ -16,9 +51,9 @@ import ( func aggregateMySQLProbes( ctx context.Context, - probes *mysql.Probes, + probes mysql.Probes, clusterName string, - instanceResultsMap mysql.InstanceMetricResultMap, + tabletResultsMap mysql.TabletResultMap, ignoreHostsCount int, IgnoreDialTCPErrors bool, ignoreHostsThreshold float64, @@ -26,13 +61,13 @@ func aggregateMySQLProbes( // probes is known not to change. It can be *replaced*, but not changed. // so it's safe to iterate it probeValues := []float64{} - for _, probe := range *probes { - instanceMetricResult, ok := instanceResultsMap[mysql.GetClusterInstanceKey(clusterName, &probe.Key)] + for _, probe := range probes { + tabletMetricResult, ok := tabletResultsMap[mysql.GetClusterTablet(clusterName, probe.Alias)] if !ok { return base.NoMetricResultYet } - value, err := instanceMetricResult.Get() + value, err := tabletMetricResult.Get() if err != nil { if IgnoreDialTCPErrors && base.IsDialTCPError(err) { continue @@ -42,7 +77,7 @@ func aggregateMySQLProbes( ignoreHostsCount = ignoreHostsCount - 1 continue } - return instanceMetricResult + return tabletMetricResult } // No error diff --git a/go/vt/vttablet/tabletserver/throttle/mysql/instance_key.go b/go/vt/vttablet/tabletserver/throttle/mysql/instance_key.go deleted file mode 100644 index adcd6f422fb..00000000000 --- a/go/vt/vttablet/tabletserver/throttle/mysql/instance_key.go +++ /dev/null @@ -1,98 +0,0 @@ -/* - Copyright 2015 Shlomi Noach, courtesy Booking.com - See https://github.com/github/freno/blob/master/LICENSE -*/ - -package mysql - -import ( - "fmt" - "strconv" - "strings" -) - -// InstanceKey is an instance indicator, identified by hostname and port -type InstanceKey struct { - Hostname string - Port int -} - -// SelfInstanceKey is a special indicator for "this instance", e.g. denoting the MySQL server associated with local tablet -// The values of this key are immaterial and are intentionally descriptive -var SelfInstanceKey = &InstanceKey{Hostname: "(self)", Port: 1} - -// newRawInstanceKey will parse an InstanceKey from a string representation such as 127.0.0.1:3306 -// It expects such format and returns with error if input differs in format -func newRawInstanceKey(hostPort string) (*InstanceKey, error) { - tokens := strings.SplitN(hostPort, ":", 2) - if len(tokens) != 2 { - return nil, fmt.Errorf("Cannot parse InstanceKey from %s. Expected format is host:port", hostPort) - } - instanceKey := &InstanceKey{Hostname: tokens[0]} - var err error - if instanceKey.Port, err = strconv.Atoi(tokens[1]); err != nil { - return instanceKey, fmt.Errorf("Invalid port: %s", tokens[1]) - } - - return instanceKey, nil -} - -// ParseInstanceKey will parse an InstanceKey from a string representation such as 127.0.0.1:3306 or some.hostname -// `defaultPort` is used if `hostPort` does not include a port. -func ParseInstanceKey(hostPort string, defaultPort int) (*InstanceKey, error) { - if !strings.Contains(hostPort, ":") { - return &InstanceKey{Hostname: hostPort, Port: defaultPort}, nil - } - return newRawInstanceKey(hostPort) -} - -// Equals tests equality between this key and another key -func (i *InstanceKey) Equals(other *InstanceKey) bool { - if other == nil { - return false - } - return i.Hostname == other.Hostname && i.Port == other.Port -} - -// SmallerThan returns true if this key is dictionary-smaller than another. -// This is used for consistent sorting/ordering; there's nothing magical about it. -func (i *InstanceKey) SmallerThan(other *InstanceKey) bool { - if i.Hostname < other.Hostname { - return true - } - if i.Hostname == other.Hostname && i.Port < other.Port { - return true - } - return false -} - -// IsValid uses simple heuristics to see whether this key represents an actual instance -func (i *InstanceKey) IsValid() bool { - if i.Hostname == "_" { - return false - } - return len(i.Hostname) > 0 && i.Port > 0 -} - -// IsSelf checks if this is the special "self" instance key -func (i *InstanceKey) IsSelf() bool { - if SelfInstanceKey == i { - return true - } - return SelfInstanceKey.Equals(i) -} - -// StringCode returns an official string representation of this key -func (i *InstanceKey) StringCode() string { - return fmt.Sprintf("%s:%d", i.Hostname, i.Port) -} - -// DisplayString returns a user-friendly string representation of this key -func (i *InstanceKey) DisplayString() string { - return i.StringCode() -} - -// String returns a user-friendly string representation of this key -func (i InstanceKey) String() string { - return i.StringCode() -} diff --git a/go/vt/vttablet/tabletserver/throttle/mysql/instance_key_test.go b/go/vt/vttablet/tabletserver/throttle/mysql/instance_key_test.go deleted file mode 100644 index a8d3424c36a..00000000000 --- a/go/vt/vttablet/tabletserver/throttle/mysql/instance_key_test.go +++ /dev/null @@ -1,66 +0,0 @@ -/* - Copyright 2017 GitHub Inc. - - Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE -*/ - -package mysql - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestNewRawInstanceKey(t *testing.T) { - { - key, err := newRawInstanceKey("127.0.0.1:3307") - assert.NoError(t, err) - assert.Equal(t, key.Hostname, "127.0.0.1") - assert.Equal(t, key.Port, 3307) - } - { - _, err := newRawInstanceKey("127.0.0.1:abcd") - assert.Error(t, err) - } - { - _, err := newRawInstanceKey("127.0.0.1:") - assert.Error(t, err) - } - { - _, err := newRawInstanceKey("127.0.0.1") - assert.Error(t, err) - } -} - -func TestParseInstanceKey(t *testing.T) { - { - key, err := ParseInstanceKey("127.0.0.1:3307", 3306) - assert.NoError(t, err) - assert.Equal(t, "127.0.0.1", key.Hostname) - assert.Equal(t, 3307, key.Port) - } - { - key, err := ParseInstanceKey("127.0.0.1", 3306) - assert.NoError(t, err) - assert.Equal(t, "127.0.0.1", key.Hostname) - assert.Equal(t, 3306, key.Port) - } -} - -func TestEquals(t *testing.T) { - { - expect := &InstanceKey{Hostname: "127.0.0.1", Port: 3306} - key, err := ParseInstanceKey("127.0.0.1", 3306) - assert.NoError(t, err) - assert.True(t, key.Equals(expect)) - } -} - -func TestStringCode(t *testing.T) { - { - key := &InstanceKey{Hostname: "127.0.0.1", Port: 3306} - stringCode := key.StringCode() - assert.Equal(t, "127.0.0.1:3306", stringCode) - } -} diff --git a/go/vt/vttablet/tabletserver/throttle/mysql/mysql_inventory.go b/go/vt/vttablet/tabletserver/throttle/mysql/mysql_inventory.go index ace9a2853a7..744bcc99a44 100644 --- a/go/vt/vttablet/tabletserver/throttle/mysql/mysql_inventory.go +++ b/go/vt/vttablet/tabletserver/throttle/mysql/mysql_inventory.go @@ -1,7 +1,42 @@ /* - Copyright 2017 GitHub Inc. +Copyright 2023 The Vitess Authors. - Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This codebase originates from https://github.com/github/freno, See https://github.com/github/freno/blob/master/LICENSE +/* + MIT License + + Copyright (c) 2017 GitHub + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. */ package mysql @@ -10,35 +45,35 @@ import ( "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/base" ) -// ClusterInstanceKey combines a cluster name with an instance key -type ClusterInstanceKey struct { +// ClusterTablet combines a cluster name with a tablet alias +type ClusterTablet struct { ClusterName string - Key InstanceKey + Alias string } -// GetClusterInstanceKey creates a ClusterInstanceKey object -func GetClusterInstanceKey(clusterName string, key *InstanceKey) ClusterInstanceKey { - return ClusterInstanceKey{ClusterName: clusterName, Key: *key} +// GetClusterTablet creates a GetClusterTablet object +func GetClusterTablet(clusterName string, alias string) ClusterTablet { + return ClusterTablet{ClusterName: clusterName, Alias: alias} } -// InstanceMetricResultMap maps a cluster-instance to a result -type InstanceMetricResultMap map[ClusterInstanceKey]base.MetricResult +// TabletResultMap maps a cluster-tablet to a result +type TabletResultMap map[ClusterTablet]base.MetricResult // Inventory has the operational data about probes, their metrics, and relevant configuration type Inventory struct { - ClustersProbes map[string](*Probes) + ClustersProbes map[string](Probes) IgnoreHostsCount map[string]int IgnoreHostsThreshold map[string]float64 - InstanceKeyMetrics InstanceMetricResultMap + TabletMetrics TabletResultMap } // NewInventory creates a Inventory func NewInventory() *Inventory { inventory := &Inventory{ - ClustersProbes: make(map[string](*Probes)), + ClustersProbes: make(map[string](Probes)), IgnoreHostsCount: make(map[string]int), IgnoreHostsThreshold: make(map[string]float64), - InstanceKeyMetrics: make(map[ClusterInstanceKey]base.MetricResult), + TabletMetrics: make(map[ClusterTablet]base.MetricResult), } return inventory } diff --git a/go/vt/vttablet/tabletserver/throttle/mysql/mysql_throttle_metric.go b/go/vt/vttablet/tabletserver/throttle/mysql/mysql_throttle_metric.go index a7e3650f8f3..966c7a93d7f 100644 --- a/go/vt/vttablet/tabletserver/throttle/mysql/mysql_throttle_metric.go +++ b/go/vt/vttablet/tabletserver/throttle/mysql/mysql_throttle_metric.go @@ -1,7 +1,42 @@ /* - Copyright 2017 GitHub Inc. +Copyright 2023 The Vitess Authors. - Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This codebase originates from https://github.com/github/freno, See https://github.com/github/freno/blob/master/LICENSE +/* + MIT License + + Copyright (c) 2017 GitHub + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. */ package mysql @@ -33,7 +68,7 @@ const ( var mysqlMetricCache = cache.New(cache.NoExpiration, 10*time.Second) func getMySQLMetricCacheKey(probe *Probe) string { - return fmt.Sprintf("%s:%s", probe.Key, probe.MetricQuery) + return fmt.Sprintf("%s:%s", probe.Alias, probe.MetricQuery) } func cacheMySQLThrottleMetric(probe *Probe, mySQLThrottleMetric *MySQLThrottleMetric) *MySQLThrottleMetric { @@ -71,10 +106,10 @@ func GetMetricsQueryType(query string) MetricsQueryType { return MetricsQueryTypeUnknown } -// MySQLThrottleMetric has the probed metric for a mysql instance +// MySQLThrottleMetric has the probed metric for a tablet type MySQLThrottleMetric struct { // nolint:revive ClusterName string - Key InstanceKey + Alias string Value float64 Err error } @@ -84,9 +119,9 @@ func NewMySQLThrottleMetric() *MySQLThrottleMetric { return &MySQLThrottleMetric{Value: 0} } -// GetClusterInstanceKey returns the ClusterInstanceKey part of the metric -func (metric *MySQLThrottleMetric) GetClusterInstanceKey() ClusterInstanceKey { - return GetClusterInstanceKey(metric.ClusterName, &metric.Key) +// GetClusterTablet returns the ClusterTablet part of the metric +func (metric *MySQLThrottleMetric) GetClusterTablet() ClusterTablet { + return GetClusterTablet(metric.ClusterName, metric.Alias) } // Get implements MetricResult @@ -105,7 +140,7 @@ func ReadThrottleMetric(probe *Probe, clusterName string, overrideGetMetricFunc started := time.Now() mySQLThrottleMetric = NewMySQLThrottleMetric() mySQLThrottleMetric.ClusterName = clusterName - mySQLThrottleMetric.Key = probe.Key + mySQLThrottleMetric.Alias = probe.Alias defer func(metric *MySQLThrottleMetric, started time.Time) { go func() { diff --git a/go/vt/vttablet/tabletserver/throttle/mysql/probe.go b/go/vt/vttablet/tabletserver/throttle/mysql/probe.go index 53b835497b4..8c3e069c0d1 100644 --- a/go/vt/vttablet/tabletserver/throttle/mysql/probe.go +++ b/go/vt/vttablet/tabletserver/throttle/mysql/probe.go @@ -1,7 +1,42 @@ /* - Copyright 2017 GitHub Inc. +Copyright 2023 The Vitess Authors. - Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This codebase originates from https://github.com/github/freno, See https://github.com/github/freno/blob/master/LICENSE +/* + MIT License + + Copyright (c) 2017 GitHub + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. */ package mysql @@ -14,45 +49,35 @@ import ( // Probe is the minimal configuration required to connect to a MySQL server type Probe struct { - Key InstanceKey + Alias string MetricQuery string Tablet *topodatapb.Tablet - TabletHost string - TabletPort int CacheMillis int QueryInProgress int64 } -// Probes maps instances to probe(s) -type Probes map[InstanceKey](*Probe) +// Probes maps tablet aliases to probe(s) +type Probes map[string](*Probe) // ClusterProbes has the probes for a specific cluster type ClusterProbes struct { ClusterName string IgnoreHostsCount int IgnoreHostsThreshold float64 - InstanceProbes *Probes + TabletProbes Probes } // NewProbes creates Probes -func NewProbes() *Probes { - return &Probes{} +func NewProbes() Probes { + return Probes{} } // NewProbe creates Probe func NewProbe() *Probe { - config := &Probe{ - Key: InstanceKey{}, - } - return config + return &Probe{} } // String returns a human readable string of this struct func (p *Probe) String() string { - return fmt.Sprintf("%s, tablet=%s:%d", p.Key.DisplayString(), p.TabletHost, p.TabletPort) -} - -// Equals checks if this probe has same instance key as another -func (p *Probe) Equals(other *Probe) bool { - return p.Key.Equals(&other.Key) + return fmt.Sprintf("probe alias=%s", p.Alias) } diff --git a/go/vt/vttablet/tabletserver/throttle/mysql/probe_test.go b/go/vt/vttablet/tabletserver/throttle/mysql/probe_test.go index cb63441d419..8f489f39258 100644 --- a/go/vt/vttablet/tabletserver/throttle/mysql/probe_test.go +++ b/go/vt/vttablet/tabletserver/throttle/mysql/probe_test.go @@ -1,7 +1,42 @@ /* - Copyright 2017 GitHub Inc. +Copyright 2023 The Vitess Authors. - Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This codebase originates from https://github.com/github/freno, See https://github.com/github/freno/blob/master/LICENSE +/* + MIT License + + Copyright (c) 2017 GitHub + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. */ package mysql @@ -14,6 +49,5 @@ import ( func TestNewProbe(t *testing.T) { c := NewProbe() - assert.Equal(t, "", c.Key.Hostname) - assert.Equal(t, 0, c.Key.Port) + assert.Equal(t, "", c.Alias) } diff --git a/go/vt/vttablet/tabletserver/throttle/mysql_test.go b/go/vt/vttablet/tabletserver/throttle/mysql_test.go index e90f9a69614..15d6feab03f 100644 --- a/go/vt/vttablet/tabletserver/throttle/mysql_test.go +++ b/go/vt/vttablet/tabletserver/throttle/mysql_test.go @@ -1,7 +1,42 @@ /* - Copyright 2017 GitHub Inc. +Copyright 2023 The Vitess Authors. - Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This codebase originates from https://github.com/github/freno, See https://github.com/github/freno/blob/master/LICENSE +/* + MIT License + + Copyright (c) 2017 GitHub + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. */ package throttle @@ -17,64 +52,64 @@ import ( ) var ( - key1 = mysql.InstanceKey{Hostname: "10.0.0.1", Port: 3306} - key2 = mysql.InstanceKey{Hostname: "10.0.0.2", Port: 3306} - key3 = mysql.InstanceKey{Hostname: "10.0.0.3", Port: 3306} - key4 = mysql.InstanceKey{Hostname: "10.0.0.4", Port: 3306} - key5 = mysql.InstanceKey{Hostname: "10.0.0.5", Port: 3306} + alias1 = "zone1-0001" + alias2 = "zone1-0002" + alias3 = "zone1-0003" + alias4 = "zone1-0004" + alias5 = "zone1-0005" ) func TestAggregateMySQLProbesNoErrors(t *testing.T) { ctx := context.Background() clusterName := "c0" - key1cluster := mysql.GetClusterInstanceKey(clusterName, &key1) - key2cluster := mysql.GetClusterInstanceKey(clusterName, &key2) - key3cluster := mysql.GetClusterInstanceKey(clusterName, &key3) - key4cluster := mysql.GetClusterInstanceKey(clusterName, &key4) - key5cluster := mysql.GetClusterInstanceKey(clusterName, &key5) - instanceResultsMap := mysql.InstanceMetricResultMap{ + key1cluster := mysql.GetClusterTablet(clusterName, alias1) + key2cluster := mysql.GetClusterTablet(clusterName, alias2) + key3cluster := mysql.GetClusterTablet(clusterName, alias3) + key4cluster := mysql.GetClusterTablet(clusterName, alias4) + key5cluster := mysql.GetClusterTablet(clusterName, alias5) + tabletResultsMap := mysql.TabletResultMap{ key1cluster: base.NewSimpleMetricResult(1.2), key2cluster: base.NewSimpleMetricResult(1.7), key3cluster: base.NewSimpleMetricResult(0.3), key4cluster: base.NewSimpleMetricResult(0.6), key5cluster: base.NewSimpleMetricResult(1.1), } - var probes mysql.Probes = map[mysql.InstanceKey](*mysql.Probe){} - for clusterKey := range instanceResultsMap { - probes[clusterKey.Key] = &mysql.Probe{Key: clusterKey.Key} + var probes mysql.Probes = map[string](*mysql.Probe){} + for clusterKey := range tabletResultsMap { + probes[clusterKey.Alias] = &mysql.Probe{Alias: clusterKey.Alias} } { - worstMetric := aggregateMySQLProbes(ctx, &probes, clusterName, instanceResultsMap, 0, false, 0) + worstMetric := aggregateMySQLProbes(ctx, probes, clusterName, tabletResultsMap, 0, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.7) } { - worstMetric := aggregateMySQLProbes(ctx, &probes, clusterName, instanceResultsMap, 1, false, 0) + worstMetric := aggregateMySQLProbes(ctx, probes, clusterName, tabletResultsMap, 1, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.2) } { - worstMetric := aggregateMySQLProbes(ctx, &probes, clusterName, instanceResultsMap, 2, false, 0) + worstMetric := aggregateMySQLProbes(ctx, probes, clusterName, tabletResultsMap, 2, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.1) } { - worstMetric := aggregateMySQLProbes(ctx, &probes, clusterName, instanceResultsMap, 3, false, 0) + worstMetric := aggregateMySQLProbes(ctx, probes, clusterName, tabletResultsMap, 3, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.6) } { - worstMetric := aggregateMySQLProbes(ctx, &probes, clusterName, instanceResultsMap, 4, false, 0) + worstMetric := aggregateMySQLProbes(ctx, probes, clusterName, tabletResultsMap, 4, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.3) } { - worstMetric := aggregateMySQLProbes(ctx, &probes, clusterName, instanceResultsMap, 5, false, 0) + worstMetric := aggregateMySQLProbes(ctx, probes, clusterName, tabletResultsMap, 5, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.3) @@ -84,54 +119,54 @@ func TestAggregateMySQLProbesNoErrors(t *testing.T) { func TestAggregateMySQLProbesNoErrorsIgnoreHostsThreshold(t *testing.T) { ctx := context.Background() clusterName := "c0" - key1cluster := mysql.GetClusterInstanceKey(clusterName, &key1) - key2cluster := mysql.GetClusterInstanceKey(clusterName, &key2) - key3cluster := mysql.GetClusterInstanceKey(clusterName, &key3) - key4cluster := mysql.GetClusterInstanceKey(clusterName, &key4) - key5cluster := mysql.GetClusterInstanceKey(clusterName, &key5) - instanceResultsMap := mysql.InstanceMetricResultMap{ + key1cluster := mysql.GetClusterTablet(clusterName, alias1) + key2cluster := mysql.GetClusterTablet(clusterName, alias2) + key3cluster := mysql.GetClusterTablet(clusterName, alias3) + key4cluster := mysql.GetClusterTablet(clusterName, alias4) + key5cluster := mysql.GetClusterTablet(clusterName, alias5) + tableteResultsMap := mysql.TabletResultMap{ key1cluster: base.NewSimpleMetricResult(1.2), key2cluster: base.NewSimpleMetricResult(1.7), key3cluster: base.NewSimpleMetricResult(0.3), key4cluster: base.NewSimpleMetricResult(0.6), key5cluster: base.NewSimpleMetricResult(1.1), } - var probes mysql.Probes = map[mysql.InstanceKey](*mysql.Probe){} - for clusterKey := range instanceResultsMap { - probes[clusterKey.Key] = &mysql.Probe{Key: clusterKey.Key} + var probes mysql.Probes = map[string](*mysql.Probe){} + for clusterKey := range tableteResultsMap { + probes[clusterKey.Alias] = &mysql.Probe{Alias: clusterKey.Alias} } { - worstMetric := aggregateMySQLProbes(ctx, &probes, clusterName, instanceResultsMap, 0, false, 1.0) + worstMetric := aggregateMySQLProbes(ctx, probes, clusterName, tableteResultsMap, 0, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.7) } { - worstMetric := aggregateMySQLProbes(ctx, &probes, clusterName, instanceResultsMap, 1, false, 1.0) + worstMetric := aggregateMySQLProbes(ctx, probes, clusterName, tableteResultsMap, 1, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.2) } { - worstMetric := aggregateMySQLProbes(ctx, &probes, clusterName, instanceResultsMap, 2, false, 1.0) + worstMetric := aggregateMySQLProbes(ctx, probes, clusterName, tableteResultsMap, 2, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.1) } { - worstMetric := aggregateMySQLProbes(ctx, &probes, clusterName, instanceResultsMap, 3, false, 1.0) + worstMetric := aggregateMySQLProbes(ctx, probes, clusterName, tableteResultsMap, 3, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.6) } { - worstMetric := aggregateMySQLProbes(ctx, &probes, clusterName, instanceResultsMap, 4, false, 1.0) + worstMetric := aggregateMySQLProbes(ctx, probes, clusterName, tableteResultsMap, 4, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.6) } { - worstMetric := aggregateMySQLProbes(ctx, &probes, clusterName, instanceResultsMap, 5, false, 1.0) + worstMetric := aggregateMySQLProbes(ctx, probes, clusterName, tableteResultsMap, 5, false, 1.0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 0.6) @@ -141,56 +176,56 @@ func TestAggregateMySQLProbesNoErrorsIgnoreHostsThreshold(t *testing.T) { func TestAggregateMySQLProbesWithErrors(t *testing.T) { ctx := context.Background() clusterName := "c0" - key1cluster := mysql.GetClusterInstanceKey(clusterName, &key1) - key2cluster := mysql.GetClusterInstanceKey(clusterName, &key2) - key3cluster := mysql.GetClusterInstanceKey(clusterName, &key3) - key4cluster := mysql.GetClusterInstanceKey(clusterName, &key4) - key5cluster := mysql.GetClusterInstanceKey(clusterName, &key5) - instanceResultsMap := mysql.InstanceMetricResultMap{ + key1cluster := mysql.GetClusterTablet(clusterName, alias1) + key2cluster := mysql.GetClusterTablet(clusterName, alias2) + key3cluster := mysql.GetClusterTablet(clusterName, alias3) + key4cluster := mysql.GetClusterTablet(clusterName, alias4) + key5cluster := mysql.GetClusterTablet(clusterName, alias5) + tabletResultsMap := mysql.TabletResultMap{ key1cluster: base.NewSimpleMetricResult(1.2), key2cluster: base.NewSimpleMetricResult(1.7), key3cluster: base.NewSimpleMetricResult(0.3), key4cluster: base.NoSuchMetric, key5cluster: base.NewSimpleMetricResult(1.1), } - var probes mysql.Probes = map[mysql.InstanceKey](*mysql.Probe){} - for clusterKey := range instanceResultsMap { - probes[clusterKey.Key] = &mysql.Probe{Key: clusterKey.Key} + var probes mysql.Probes = map[string](*mysql.Probe){} + for clusterKey := range tabletResultsMap { + probes[clusterKey.Alias] = &mysql.Probe{Alias: clusterKey.Alias} } { - worstMetric := aggregateMySQLProbes(ctx, &probes, clusterName, instanceResultsMap, 0, false, 0) + worstMetric := aggregateMySQLProbes(ctx, probes, clusterName, tabletResultsMap, 0, false, 0) _, err := worstMetric.Get() assert.Error(t, err) assert.Equal(t, err, base.ErrNoSuchMetric) } { - worstMetric := aggregateMySQLProbes(ctx, &probes, clusterName, instanceResultsMap, 1, false, 0) + worstMetric := aggregateMySQLProbes(ctx, probes, clusterName, tabletResultsMap, 1, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.7) } { - worstMetric := aggregateMySQLProbes(ctx, &probes, clusterName, instanceResultsMap, 2, false, 0) + worstMetric := aggregateMySQLProbes(ctx, probes, clusterName, tabletResultsMap, 2, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.2) } - instanceResultsMap[key1cluster] = base.NoSuchMetric + tabletResultsMap[key1cluster] = base.NoSuchMetric { - worstMetric := aggregateMySQLProbes(ctx, &probes, clusterName, instanceResultsMap, 0, false, 0) + worstMetric := aggregateMySQLProbes(ctx, probes, clusterName, tabletResultsMap, 0, false, 0) _, err := worstMetric.Get() assert.Error(t, err) assert.Equal(t, err, base.ErrNoSuchMetric) } { - worstMetric := aggregateMySQLProbes(ctx, &probes, clusterName, instanceResultsMap, 1, false, 0) + worstMetric := aggregateMySQLProbes(ctx, probes, clusterName, tabletResultsMap, 1, false, 0) _, err := worstMetric.Get() assert.Error(t, err) assert.Equal(t, err, base.ErrNoSuchMetric) } { - worstMetric := aggregateMySQLProbes(ctx, &probes, clusterName, instanceResultsMap, 2, false, 0) + worstMetric := aggregateMySQLProbes(ctx, probes, clusterName, tabletResultsMap, 2, false, 0) value, err := worstMetric.Get() assert.NoError(t, err) assert.Equal(t, value, 1.7) diff --git a/go/vt/vttablet/tabletserver/throttle/throttler.go b/go/vt/vttablet/tabletserver/throttle/throttler.go index 21f9b068d0e..348437239e1 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler.go @@ -1,17 +1,50 @@ /* - Copyright 2017 GitHub Inc. +Copyright 2023 The Vitess Authors. - Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// This codebase originates from https://github.com/github/freno, See https://github.com/github/freno/blob/master/LICENSE +/* + MIT License + + Copyright (c) 2017 GitHub + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE. */ package throttle import ( "context" - "encoding/json" "errors" "fmt" - "io" "math" "math/rand" "net/http" @@ -36,6 +69,7 @@ import ( "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/srvtopo" "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/topo/topoproto" "vitess.io/vitess/go/vt/vttablet/tabletserver/connpool" "vitess.io/vitess/go/vt/vttablet/tabletserver/heartbeat" "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" @@ -47,15 +81,15 @@ import ( ) const ( - leaderCheckInterval = 5 * time.Second - mysqlCollectInterval = 250 * time.Millisecond - mysqlDormantCollectInterval = 5 * time.Second - mysqlRefreshInterval = 10 * time.Second - mysqlAggregateInterval = 125 * time.Millisecond - - aggregatedMetricsExpiration = 5 * time.Second + leaderCheckInterval = 5 * time.Second + mysqlCollectInterval = 250 * time.Millisecond + mysqlDormantCollectInterval = 5 * time.Second + mysqlRefreshInterval = 10 * time.Second + mysqlAggregateInterval = 125 * time.Millisecond throttledAppsSnapshotInterval = 5 * time.Second - recentAppsExpiration = time.Hour * 24 + + aggregatedMetricsExpiration = 5 * time.Second + recentAppsExpiration = time.Hour * 24 nonDeprioritizedAppMapExpiration = time.Second @@ -130,18 +164,26 @@ type Throttler struct { isLeader atomic.Bool isOpen atomic.Bool - env tabletenv.Env - pool *connpool.Pool - tabletTypeFunc func() topodatapb.TabletType - ts throttlerTopoService - srvTopoServer srvtopo.Server - heartbeatWriter heartbeat.HeartbeatWriter + leaderCheckInterval time.Duration + mysqlCollectInterval time.Duration + mysqlDormantCollectInterval time.Duration + mysqlRefreshInterval time.Duration + mysqlAggregateInterval time.Duration + throttledAppsSnapshotInterval time.Duration + + env tabletenv.Env + pool *connpool.Pool + tabletTypeFunc func() topodatapb.TabletType + ts throttlerTopoService + srvTopoServer srvtopo.Server + heartbeatWriter heartbeat.HeartbeatWriter + overrideTmClient tmclient.TabletManagerClient // recentCheckTickerValue is an ever increasing number, incrementing once per second. - recentCheckTickerValue int64 + recentCheckTickerValue atomic.Int64 // recentCheckValue is set to match or exceed recentCheckTickerValue whenever a "check" was made (other than by the throttler itself). // when recentCheckValue < recentCheckTickerValue that means there hasn't been a recent check. - recentCheckValue int64 + recentCheckValue atomic.Int64 throttleTabletTypesMap map[topodatapb.TabletType]bool @@ -162,14 +204,15 @@ type Throttler struct { recentApps *cache.Cache metricsHealth *cache.Cache - lastCheckTimeNano int64 + lastCheckTimeNano atomic.Int64 + + initMutex sync.Mutex + enableMutex sync.Mutex + cancelOpenContext context.CancelFunc + cancelEnableContext context.CancelFunc + throttledAppsMutex sync.Mutex - initMutex sync.Mutex - enableMutex sync.Mutex - cancelOpenContext context.CancelFunc - cancelEnableContext context.CancelFunc - throttledAppsMutex sync.Mutex - watchSrvKeyspaceOnce sync.Once + readSelfThrottleMetric func(context.Context, *mysql.Probe) *mysql.MySQLThrottleMetric // overwritten by unit test nonLowPriorityAppRequestsThrottled *cache.Cache httpClient *http.Client @@ -224,7 +267,17 @@ func NewThrottler(env tabletenv.Env, srvTopoServer srvtopo.Server, ts *topo.Serv throttler.initThrottleTabletTypes() throttler.check = NewThrottlerCheck(throttler) + throttler.leaderCheckInterval = leaderCheckInterval + throttler.mysqlCollectInterval = mysqlCollectInterval + throttler.mysqlDormantCollectInterval = mysqlDormantCollectInterval + throttler.mysqlRefreshInterval = mysqlRefreshInterval + throttler.mysqlAggregateInterval = mysqlAggregateInterval + throttler.throttledAppsSnapshotInterval = throttledAppsSnapshotInterval + throttler.StoreMetricsThreshold(defaultThrottleLagThreshold.Seconds()) //default + throttler.readSelfThrottleMetric = func(ctx context.Context, p *mysql.Probe) *mysql.MySQLThrottleMetric { + return throttler.readSelfMySQLThrottleMetric(ctx, p) + } return throttler } @@ -354,9 +407,9 @@ func (throttler *Throttler) applyThrottlerConfig(ctx context.Context, throttlerC throttler.ThrottleApp(appRule.Name, protoutil.TimeFromProto(appRule.ExpiresAt).UTC(), appRule.Ratio, appRule.Exempt) } if throttlerConfig.Enabled { - go throttler.Enable(ctx) + go throttler.Enable() } else { - go throttler.Disable(ctx) + go throttler.Disable() } } @@ -384,18 +437,18 @@ func (throttler *Throttler) IsRunning() bool { // Enable activates the throttler probes; when enabled, the throttler responds to check queries based on // the collected metrics. -func (throttler *Throttler) Enable(ctx context.Context) bool { +func (throttler *Throttler) Enable() bool { throttler.enableMutex.Lock() defer throttler.enableMutex.Unlock() - isEnabled := throttler.isEnabled.Swap(true) - if isEnabled { + if wasEnabled := throttler.isEnabled.Swap(true); wasEnabled { log.Infof("Throttler: already enabled") return false } log.Infof("Throttler: enabling") - ctx, throttler.cancelEnableContext = context.WithCancel(ctx) + var ctx context.Context + ctx, throttler.cancelEnableContext = context.WithCancel(context.Background()) throttler.check.SelfChecks(ctx) throttler.Operate(ctx) @@ -407,12 +460,11 @@ func (throttler *Throttler) Enable(ctx context.Context) bool { // Disable deactivates the probes and associated operations. When disabled, the throttler responds to check // queries with "200 OK" irrespective of lag or any other metrics. -func (throttler *Throttler) Disable(ctx context.Context) bool { +func (throttler *Throttler) Disable() bool { throttler.enableMutex.Lock() defer throttler.enableMutex.Unlock() - isEnabled := throttler.isEnabled.Swap(false) - if !isEnabled { + if wasEnabled := throttler.isEnabled.Swap(false); !wasEnabled { log.Infof("Throttler: already disabled") return false } @@ -427,6 +479,54 @@ func (throttler *Throttler) Disable(ctx context.Context) bool { return true } +// retryReadAndApplyThrottlerConfig() is called by Open(), read throttler config from topo, applies it, and starts watching +// for topo changes. +// But also, we're in an Open() function, which blocks state manager's operation, and affects +// opening of all other components. We thus read the throttler config in the background. +// However, we want to handle a situation where the read errors out. +// So we kick a loop that keeps retrying reading the config, for as long as this throttler is open. +func (throttler *Throttler) retryReadAndApplyThrottlerConfig(ctx context.Context) { + var watchSrvKeyspaceOnce sync.Once + retryInterval := 10 * time.Second + retryTicker := time.NewTicker(retryInterval) + defer retryTicker.Stop() + for { + if !throttler.IsOpen() { + // Throttler is not open so no need to keep retrying. + log.Warningf("Throttler.retryReadAndApplyThrottlerConfig(): throttler no longer seems to be open, exiting") + return + } + + requestCtx, requestCancel := context.WithTimeout(ctx, 5*time.Second) + defer requestCancel() + throttlerConfig, err := throttler.readThrottlerConfig(requestCtx) + if err == nil { + log.Infof("Throttler.retryReadAndApplyThrottlerConfig(): success reading throttler config: %+v", throttlerConfig) + // It's possible that during a retry-sleep, the throttler is closed and opened again, leading + // to two (or more) instances of this goroutine. That's not a big problem; it's fine if all + // attempt to read the throttler config; but we just want to ensure they don't step on each other + // while applying the changes. + throttler.initMutex.Lock() + defer throttler.initMutex.Unlock() + throttler.applyThrottlerConfig(ctx, throttlerConfig) // may issue an Enable + go watchSrvKeyspaceOnce.Do(func() { + // We start watching SrvKeyspace only after we know it's been created. Now is that time! + // We watch using the given ctx, which is cancelled when the throttler is Close()d. + throttler.srvTopoServer.WatchSrvKeyspace(ctx, throttler.cell, throttler.keyspace, throttler.WatchSrvKeyspaceCallback) + }) + return + } + log.Errorf("Throttler.retryReadAndApplyThrottlerConfig(): error reading throttler config. Will retry in %v. Err=%+v", retryInterval, err) + select { + case <-ctx.Done(): + // Throttler is not open so no need to keep retrying. + log.Infof("Throttler.retryReadAndApplyThrottlerConfig(): throttler no longer seems to be open, exiting") + return + case <-retryTicker.C: + } + } +} + // Open opens database pool and initializes the schema func (throttler *Throttler) Open() error { log.Infof("Throttler: started execution of Open. Acquiring initMutex lock") @@ -451,52 +551,7 @@ func (throttler *Throttler) Open() error { throttler.ThrottleApp("always-throttled-app", time.Now().Add(time.Hour*24*365*10), DefaultThrottleRatio, false) - log.Infof("Throttler: throttler-config-via-topo detected") - // We want to read throttler config from topo and apply it. - // But also, we're in an Open() function, which blocks state manager's operation, and affects - // opening of all other components. We thus read the throttler config in the background. - // However, we want to handle a situation where the read errors out. - // So we kick a loop that keeps retrying reading the config, for as long as this throttler is open. - retryReadAndApplyThrottlerConfig := func(ctx context.Context) { - retryInterval := 10 * time.Second - retryTicker := time.NewTicker(retryInterval) - defer retryTicker.Stop() - for { - if !throttler.IsOpen() { - // Throttler is not open so no need to keep retrying. - log.Errorf("Throttler.retryReadAndApplyThrottlerConfig(): throttler no longer seems to be open, exiting") - return - } - - requestCtx, requestCancel := context.WithTimeout(ctx, 5*time.Second) - defer requestCancel() - throttlerConfig, err := throttler.readThrottlerConfig(requestCtx) - if err == nil { - log.Errorf("Throttler.retryReadAndApplyThrottlerConfig(): success reading throttler config: %+v", throttlerConfig) - // It's possible that during a retry-sleep, the throttler is closed and opened again, leading - // to two (or more) instances of this goroutine. That's not a big problem; it's fine if all - // attempt to read the throttler config; but we just want to ensure they don't step on each other - // while applying the changes. - throttler.initMutex.Lock() - defer throttler.initMutex.Unlock() - throttler.applyThrottlerConfig(ctx, throttlerConfig) // may issue an Enable - go throttler.watchSrvKeyspaceOnce.Do(func() { - // We start watching SrvKeyspace only after we know it's been created. Now is that time! - throttler.srvTopoServer.WatchSrvKeyspace(context.Background(), throttler.cell, throttler.keyspace, throttler.WatchSrvKeyspaceCallback) - }) - return - } - log.Errorf("Throttler.retryReadAndApplyThrottlerConfig(): error reading throttler config. Will retry in %v. Err=%+v", retryInterval, err) - select { - case <-ctx.Done(): - // Throttler is not open so no need to keep retrying. - log.Errorf("Throttler.retryReadAndApplyThrottlerConfig(): throttler no longer seems to be open, exiting") - return - case <-retryTicker.C: - } - } - } - go retryReadAndApplyThrottlerConfig(ctx) + go throttler.retryReadAndApplyThrottlerConfig(ctx) return nil } @@ -512,19 +567,24 @@ func (throttler *Throttler) Close() { log.Infof("Throttler: throttler is not open") return } - ctx := context.Background() - throttler.Disable(ctx) + throttler.Disable() throttler.isLeader.Store(false) - log.Infof("Throttler: closing pool") - throttler.pool.Close() - throttler.cancelOpenContext() + // The below " != nil " checks are relevant to unit tests, where perhaps not all + // fields are supplied. + if throttler.pool != nil { + log.Infof("Throttler: closing pool") + throttler.pool.Close() + } + if throttler.cancelOpenContext != nil { + throttler.cancelOpenContext() + } log.Infof("Throttler: finished execution of Close") } func (throttler *Throttler) generateSelfMySQLThrottleMetricFunc(ctx context.Context, probe *mysql.Probe) func() *mysql.MySQLThrottleMetric { f := func() *mysql.MySQLThrottleMetric { - return throttler.readSelfMySQLThrottleMetric(ctx, probe) + return throttler.readSelfThrottleMetric(ctx, probe) } return f } @@ -533,7 +593,7 @@ func (throttler *Throttler) generateSelfMySQLThrottleMetricFunc(ctx context.Cont func (throttler *Throttler) readSelfMySQLThrottleMetric(ctx context.Context, probe *mysql.Probe) *mysql.MySQLThrottleMetric { metric := &mysql.MySQLThrottleMetric{ ClusterName: selfStoreName, - Key: *mysql.SelfInstanceKey, + Alias: "", Value: 0, Err: nil, } @@ -588,7 +648,7 @@ func (throttler *Throttler) ThrottledApps() (result []base.AppThrottle) { // isDormant returns true when the last check was more than dormantPeriod ago func (throttler *Throttler) isDormant() bool { - lastCheckTime := time.Unix(0, atomic.LoadInt64(&throttler.lastCheckTimeNano)) + lastCheckTime := time.Unix(0, throttler.lastCheckTimeNano.Load()) return time.Since(lastCheckTime) > dormantPeriod } @@ -601,118 +661,105 @@ func (throttler *Throttler) Operate(ctx context.Context) { tickers = append(tickers, t) return t } - leaderCheckTicker := addTicker(leaderCheckInterval) - mysqlCollectTicker := addTicker(mysqlCollectInterval) - mysqlDormantCollectTicker := addTicker(mysqlDormantCollectInterval) - mysqlRefreshTicker := addTicker(mysqlRefreshInterval) - mysqlAggregateTicker := addTicker(mysqlAggregateInterval) - throttledAppsTicker := addTicker(throttledAppsSnapshotInterval) + leaderCheckTicker := addTicker(throttler.leaderCheckInterval) + mysqlCollectTicker := addTicker(throttler.mysqlCollectInterval) + mysqlDormantCollectTicker := addTicker(throttler.mysqlDormantCollectInterval) + mysqlRefreshTicker := addTicker(throttler.mysqlRefreshInterval) + mysqlAggregateTicker := addTicker(throttler.mysqlAggregateInterval) + throttledAppsTicker := addTicker(throttler.throttledAppsSnapshotInterval) recentCheckTicker := addTicker(time.Second) - tmClient := tmclient.NewTabletManagerClient() - go func() { defer log.Infof("Throttler: Operate terminated, tickers stopped") - defer tmClient.Close() for _, t := range tickers { defer t.Stop() // since we just started the tickers now, speed up the ticks by forcing an immediate tick go t.TickNow() } + tmClient := throttler.overrideTmClient + if tmClient == nil { + // This is the normal production behavior. + // throttler.overrideTmClient != nil only in unit testing + tmClient = tmclient.NewTabletManagerClient() + defer tmClient.Close() + } + for { select { case <-ctx.Done(): return case <-leaderCheckTicker.C: - { - func() { - throttler.initMutex.Lock() - defer throttler.initMutex.Unlock() - - // sparse - shouldBeLeader := false - if throttler.IsOpen() { - if throttler.tabletTypeFunc() == topodatapb.TabletType_PRIMARY { - shouldBeLeader = true - } - } - - isLeader := throttler.isLeader.Swap(shouldBeLeader) - transitionedIntoLeader := false - if shouldBeLeader && !isLeader { - log.Infof("Throttler: transition into leadership") - transitionedIntoLeader = true - } - if !shouldBeLeader && isLeader { - log.Infof("Throttler: transition out of leadership") - } - - if transitionedIntoLeader { - // transitioned into leadership, let's speed up the next 'refresh' and 'collect' ticks - go mysqlRefreshTicker.TickNow() - go throttler.heartbeatWriter.RequestHeartbeats() - } - }() - } + func() { + throttler.initMutex.Lock() + defer throttler.initMutex.Unlock() + + // sparse + shouldBeLeader := false + if throttler.IsOpen() && throttler.tabletTypeFunc() == topodatapb.TabletType_PRIMARY { + shouldBeLeader = true + } + + isLeader := throttler.isLeader.Swap(shouldBeLeader) + transitionedIntoLeader := false + if shouldBeLeader && !isLeader { + log.Infof("Throttler: transition into leadership") + transitionedIntoLeader = true + } + if !shouldBeLeader && isLeader { + log.Infof("Throttler: transition out of leadership") + } + + if transitionedIntoLeader { + // transitioned into leadership, let's speed up the next 'refresh' and 'collect' ticks + go mysqlRefreshTicker.TickNow() + go throttler.heartbeatWriter.RequestHeartbeats() + } + }() case <-mysqlCollectTicker.C: - { - if throttler.IsOpen() { - // frequent - if !throttler.isDormant() { - throttler.collectMySQLMetrics(ctx, tmClient) - } + if throttler.IsOpen() { + // frequent + if !throttler.isDormant() { + throttler.collectMySQLMetrics(ctx, tmClient) } } case <-mysqlDormantCollectTicker.C: - { - if throttler.IsOpen() { - // infrequent - if throttler.isDormant() { - throttler.collectMySQLMetrics(ctx, tmClient) - } + if throttler.IsOpen() { + // infrequent + if throttler.isDormant() { + throttler.collectMySQLMetrics(ctx, tmClient) } } case metric := <-throttler.mysqlThrottleMetricChan: - { - // incoming MySQL metric, frequent, as result of collectMySQLMetrics() - throttler.mysqlInventory.InstanceKeyMetrics[metric.GetClusterInstanceKey()] = metric - } + // incoming MySQL metric, frequent, as result of collectMySQLMetrics() + throttler.mysqlInventory.TabletMetrics[metric.GetClusterTablet()] = metric case <-mysqlRefreshTicker.C: - { - // sparse - if throttler.IsOpen() { - throttler.refreshMySQLInventory(ctx) - } + // sparse + if throttler.IsOpen() { + throttler.refreshMySQLInventory(ctx) } case probes := <-throttler.mysqlClusterProbesChan: - { - // incoming structural update, sparse, as result of refreshMySQLInventory() - throttler.updateMySQLClusterProbes(ctx, probes) - } + // incoming structural update, sparse, as result of refreshMySQLInventory() + throttler.updateMySQLClusterProbes(ctx, probes) case <-mysqlAggregateTicker.C: - { - if throttler.IsOpen() { - throttler.aggregateMySQLMetrics(ctx) - } + if throttler.IsOpen() { + throttler.aggregateMySQLMetrics(ctx) } case <-throttledAppsTicker.C: - { - if throttler.IsOpen() { - go throttler.expireThrottledApps() - } + if throttler.IsOpen() { + go throttler.expireThrottledApps() } case throttlerConfig := <-throttler.throttlerConfigChan: throttler.applyThrottlerConfig(ctx, throttlerConfig) case <-recentCheckTicker.C: // Increment recentCheckTickerValue by one. - atomic.AddInt64(&throttler.recentCheckTickerValue, 1) + throttler.recentCheckTickerValue.Add(1) } } }() } -func (throttler *Throttler) generateTabletHTTPProbeFunction(ctx context.Context, tmClient tmclient.TabletManagerClient, clusterName string, probe *mysql.Probe) (probeFunc func() *mysql.MySQLThrottleMetric) { +func (throttler *Throttler) generateTabletProbeFunction(ctx context.Context, clusterName string, tmClient tmclient.TabletManagerClient, probe *mysql.Probe) (probeFunc func() *mysql.MySQLThrottleMetric) { return func() *mysql.MySQLThrottleMetric { // Some reasonable timeout, to ensure we release connections even if they're hanging (otherwise grpc-go keeps polling those connections forever) ctx, cancel := context.WithTimeout(ctx, 4*mysqlCollectInterval) @@ -721,55 +768,24 @@ func (throttler *Throttler) generateTabletHTTPProbeFunction(ctx context.Context, // Hit a tablet's `check-self` via HTTP, and convert its CheckResult JSON output into a MySQLThrottleMetric mySQLThrottleMetric := mysql.NewMySQLThrottleMetric() mySQLThrottleMetric.ClusterName = clusterName - mySQLThrottleMetric.Key = probe.Key - - { - req := &tabletmanagerdatapb.CheckThrottlerRequest{} // We leave AppName empty; it will default to VitessName anyway, and we can save some proto space - if resp, gRPCErr := tmClient.CheckThrottler(ctx, probe.Tablet, req); gRPCErr == nil { - mySQLThrottleMetric.Value = resp.Value - if resp.StatusCode == http.StatusInternalServerError { - mySQLThrottleMetric.Err = fmt.Errorf("Status code: %d", resp.StatusCode) - } - if resp.RecentlyChecked { - // We have just probed a tablet, and it reported back that someone just recently "check"ed it. - // We therefore renew the heartbeats lease. - go throttler.heartbeatWriter.RequestHeartbeats() - } - return mySQLThrottleMetric - - // } else { - // In v18 we need to be backwards compatible. If we have a gRPC error it might be because the replica is v17 and - // does not support CheckThrottler() RPC. This is why: - // 1. We fall back to HTTP - // 2. We don't log an error (it would just spam the logs) - // In v19 we will remove all HTTP code, and will *potentially* log an error. - // log.Errorf("error in GRPC call to tablet %v: %v", probe.Tablet.GetAlias(), gRPCErr) - } - } - // Backwards compatibility to v17: if the underlying tablets do not support CheckThrottler gRPC, attempt a HTTP check: - tabletCheckSelfURL := fmt.Sprintf("http://%s:%d/throttler/check-self?app=%s", probe.TabletHost, probe.TabletPort, throttlerapp.VitessName) - resp, err := throttler.httpClient.Get(tabletCheckSelfURL) - if err != nil { - mySQLThrottleMetric.Err = err - return mySQLThrottleMetric - } - defer resp.Body.Close() - b, err := io.ReadAll(resp.Body) - if err != nil { - mySQLThrottleMetric.Err = err + mySQLThrottleMetric.Alias = probe.Alias + + if probe.Tablet == nil { + mySQLThrottleMetric.Err = fmt.Errorf("found nil tablet reference for alias %v, hostname %v", probe.Alias, probe.Tablet.Hostname) return mySQLThrottleMetric } - checkResult := &CheckResult{} - if err := json.Unmarshal(b, checkResult); err != nil { - mySQLThrottleMetric.Err = err + req := &tabletmanagerdatapb.CheckThrottlerRequest{} // We leave AppName empty; it will default to VitessName anyway, and we can save some proto space + resp, gRPCErr := tmClient.CheckThrottler(ctx, probe.Tablet, req) + if gRPCErr != nil { + resp.StatusCode = http.StatusInternalServerError + mySQLThrottleMetric.Err = fmt.Errorf("gRPC error accessing tablet %v. Err=%v", probe.Alias, gRPCErr) return mySQLThrottleMetric } - mySQLThrottleMetric.Value = checkResult.Value - - if checkResult.StatusCode == http.StatusInternalServerError { - mySQLThrottleMetric.Err = fmt.Errorf("Status code: %d", checkResult.StatusCode) + mySQLThrottleMetric.Value = resp.Value + if resp.StatusCode == http.StatusInternalServerError { + mySQLThrottleMetric.Err = fmt.Errorf("Status code: %d", resp.StatusCode) } - if checkResult.RecentlyChecked { + if resp.RecentlyChecked { // We have just probed a tablet, and it reported back that someone just recently "check"ed it. // We therefore renew the heartbeats lease. go throttler.heartbeatWriter.RequestHeartbeats() @@ -782,33 +798,33 @@ func (throttler *Throttler) collectMySQLMetrics(ctx context.Context, tmClient tm // synchronously, get lists of probes for clusterName, probes := range throttler.mysqlInventory.ClustersProbes { clusterName := clusterName - probes := probes - go func() { - // probes is known not to change. It can be *replaced*, but not changed. - // so it's safe to iterate it - for _, probe := range *probes { - probe := probe - go func() { - // Avoid querying the same server twice at the same time. If previous read is still there, - // we avoid re-reading it. - if !atomic.CompareAndSwapInt64(&probe.QueryInProgress, 0, 1) { - return - } - defer atomic.StoreInt64(&probe.QueryInProgress, 0) - - var throttleMetricFunc func() *mysql.MySQLThrottleMetric - if clusterName == selfStoreName { - // Throttler is probing its own tablet's metrics: - throttleMetricFunc = throttler.generateSelfMySQLThrottleMetricFunc(ctx, probe) - } else { - // Throttler probing other tablets: - throttleMetricFunc = throttler.generateTabletHTTPProbeFunction(ctx, tmClient, clusterName, probe) - } - throttleMetrics := mysql.ReadThrottleMetric(probe, clusterName, throttleMetricFunc) - throttler.mysqlThrottleMetricChan <- throttleMetrics - }() - } - }() + // probes is known not to change. It can be *replaced*, but not changed. + // so it's safe to iterate it + for _, probe := range probes { + go func(probe *mysql.Probe) { + // Avoid querying the same server twice at the same time. If previous read is still there, + // we avoid re-reading it. + if !atomic.CompareAndSwapInt64(&probe.QueryInProgress, 0, 1) { + return + } + defer atomic.StoreInt64(&probe.QueryInProgress, 0) + + var throttleMetricFunc func() *mysql.MySQLThrottleMetric + if clusterName == selfStoreName { + // Throttler is probing its own tablet's metrics: + throttleMetricFunc = throttler.generateSelfMySQLThrottleMetricFunc(ctx, probe) + } else { + // Throttler probing other tablets: + throttleMetricFunc = throttler.generateTabletProbeFunction(ctx, clusterName, tmClient, probe) + } + throttleMetrics := mysql.ReadThrottleMetric(probe, clusterName, throttleMetricFunc) + select { + case <-ctx.Done(): + return + case throttler.mysqlThrottleMetricChan <- throttleMetrics: + } + }(probe) + } } return nil } @@ -818,50 +834,64 @@ func (throttler *Throttler) refreshMySQLInventory(ctx context.Context) error { // distribute the query/threshold from the throttler down to the cluster settings and from there to the probes metricsQuery := throttler.GetMetricsQuery() metricsThreshold := throttler.MetricsThreshold.Load() - addInstanceKey := func(tablet *topodatapb.Tablet, tabletHost string, tabletPort int, key *mysql.InstanceKey, clusterName string, clusterSettings *config.MySQLClusterConfigurationSettings, probes *mysql.Probes) { + addProbe := func(alias string, tablet *topodatapb.Tablet, clusterName string, clusterSettings *config.MySQLClusterConfigurationSettings, probes mysql.Probes) bool { for _, ignore := range clusterSettings.IgnoreHosts { - if strings.Contains(key.StringCode(), ignore) { - log.Infof("Throttler: instance key ignored: %+v", key) - return + if strings.Contains(alias, ignore) { + log.Infof("Throttler: tablet ignored: %+v", alias) + return false } } - if !key.IsValid() && !key.IsSelf() { - log.Infof("Throttler: read invalid instance key: [%+v] for cluster %+v", key, clusterName) - return + if clusterName != selfStoreName { + if alias == "" { + log.Errorf("Throttler: got empty alias for cluster: %+v", clusterName) + return false + } + if tablet == nil { + log.Errorf("Throttler: got nil tablet for alias: %v in cluster: %+v", alias, clusterName) + return false + } } probe := &mysql.Probe{ - Key: *key, + Alias: alias, Tablet: tablet, - TabletHost: tabletHost, - TabletPort: tabletPort, MetricQuery: clusterSettings.MetricQuery, CacheMillis: clusterSettings.CacheMillis, } - (*probes)[*key] = probe + probes[alias] = probe + return true + } + + attemptWriteProbes := func(clusterProbes *mysql.ClusterProbes) error { + select { + case <-ctx.Done(): + return ctx.Err() + case throttler.mysqlClusterProbesChan <- clusterProbes: + return nil + } } for clusterName, clusterSettings := range config.Settings().Stores.MySQL.Clusters { clusterName := clusterName - clusterSettings := clusterSettings clusterSettings.MetricQuery = metricsQuery clusterSettings.ThrottleThreshold.Store(metricsThreshold) + + clusterSettingsCopy := *clusterSettings // config may dynamically change, but internal structure (config.Settings().Stores.MySQL.Clusters in our case) // is immutable and can only be _replaced_. Hence, it's safe to read in a goroutine: - go func() { - throttler.mysqlClusterThresholds.Set(clusterName, math.Float64frombits(clusterSettings.ThrottleThreshold.Load()), cache.DefaultExpiration) + collect := func() error { + throttler.mysqlClusterThresholds.Set(clusterName, math.Float64frombits(clusterSettingsCopy.ThrottleThreshold.Load()), cache.DefaultExpiration) clusterProbes := &mysql.ClusterProbes{ ClusterName: clusterName, - IgnoreHostsCount: clusterSettings.IgnoreHostsCount, - InstanceProbes: mysql.NewProbes(), + IgnoreHostsCount: clusterSettingsCopy.IgnoreHostsCount, + TabletProbes: mysql.NewProbes(), } if clusterName == selfStoreName { // special case: just looking at this tablet's MySQL server. // We will probe this "cluster" (of one server) is a special way. - addInstanceKey(nil, "", 0, mysql.SelfInstanceKey, clusterName, clusterSettings, clusterProbes.InstanceProbes) - throttler.mysqlClusterProbesChan <- clusterProbes - return + addProbe("", nil, clusterName, &clusterSettingsCopy, clusterProbes.TabletProbes) + return attemptWriteProbes(clusterProbes) } if !throttler.isLeader.Load() { // This tablet may have used to be the primary, but it isn't now. It may have a recollection @@ -870,33 +900,30 @@ func (throttler *Throttler) refreshMySQLInventory(ctx context.Context) error { // `clusterProbes` was created above as empty, and identifiable via `clusterName`. This will in turn // be used to overwrite throttler.mysqlInventory.ClustersProbes[clusterProbes.ClusterName] in // updateMySQLClusterProbes(). - throttler.mysqlClusterProbesChan <- clusterProbes + return attemptWriteProbes(clusterProbes) // not the leader (primary tablet)? Then no more work for us. - return } // The primary tablet is also in charge of collecting the shard's metrics - err := func() error { - ctx, cancel := context.WithTimeout(ctx, mysqlRefreshInterval) - defer cancel() + ctx, cancel := context.WithTimeout(ctx, mysqlRefreshInterval) + defer cancel() - tabletAliases, err := throttler.ts.FindAllTabletAliasesInShard(ctx, throttler.keyspace, throttler.shard) + tabletAliases, err := throttler.ts.FindAllTabletAliasesInShard(ctx, throttler.keyspace, throttler.shard) + if err != nil { + return err + } + for _, tabletAlias := range tabletAliases { + tablet, err := throttler.ts.GetTablet(ctx, tabletAlias) if err != nil { return err } - for _, tabletAlias := range tabletAliases { - tablet, err := throttler.ts.GetTablet(ctx, tabletAlias) - if err != nil { - return err - } - if throttler.throttleTabletTypesMap[tablet.Type] { - key := mysql.InstanceKey{Hostname: tablet.MysqlHostname, Port: int(tablet.MysqlPort)} - addInstanceKey(tablet.Tablet, tablet.Hostname, int(tablet.PortMap["vt"]), &key, clusterName, clusterSettings, clusterProbes.InstanceProbes) - } + if throttler.throttleTabletTypesMap[tablet.Type] { + addProbe(topoproto.TabletAliasString(tabletAlias), tablet.Tablet, clusterName, &clusterSettingsCopy, clusterProbes.TabletProbes) } - throttler.mysqlClusterProbesChan <- clusterProbes - return nil - }() - if err != nil { + } + return attemptWriteProbes(clusterProbes) + } + go func() { + if err := collect(); err != nil { log.Errorf("refreshMySQLInventory: %+v", err) } }() @@ -906,7 +933,7 @@ func (throttler *Throttler) refreshMySQLInventory(ctx context.Context) error { // synchronous update of inventory func (throttler *Throttler) updateMySQLClusterProbes(ctx context.Context, clusterProbes *mysql.ClusterProbes) error { - throttler.mysqlInventory.ClustersProbes[clusterProbes.ClusterName] = clusterProbes.InstanceProbes + throttler.mysqlInventory.ClustersProbes[clusterProbes.ClusterName] = clusterProbes.TabletProbes throttler.mysqlInventory.IgnoreHostsCount[clusterProbes.ClusterName] = clusterProbes.IgnoreHostsCount throttler.mysqlInventory.IgnoreHostsThreshold[clusterProbes.ClusterName] = clusterProbes.IgnoreHostsThreshold return nil @@ -918,7 +945,7 @@ func (throttler *Throttler) aggregateMySQLMetrics(ctx context.Context) error { metricName := fmt.Sprintf("mysql/%s", clusterName) ignoreHostsCount := throttler.mysqlInventory.IgnoreHostsCount[clusterName] ignoreHostsThreshold := throttler.mysqlInventory.IgnoreHostsThreshold[clusterName] - aggregatedMetric := aggregateMySQLProbes(ctx, probes, clusterName, throttler.mysqlInventory.InstanceKeyMetrics, ignoreHostsCount, config.Settings().Stores.MySQL.IgnoreDialTCPErrors, ignoreHostsThreshold) + aggregatedMetric := aggregateMySQLProbes(ctx, probes, clusterName, throttler.mysqlInventory.TabletMetrics, ignoreHostsCount, config.Settings().Stores.MySQL.IgnoreDialTCPErrors, ignoreHostsThreshold) throttler.aggregatedMetrics.Set(metricName, aggregatedMetric, cache.DefaultExpiration) } return nil @@ -1147,11 +1174,11 @@ func (throttler *Throttler) checkStore(ctx context.Context, appName string, stor // This check was made by someone other than the throttler itself, i.e. this came from online-ddl or vreplication or other. // We mark the fact that someone just made a check. If this is a REPLICA or RDONLY tables, this will be reported back // to the PRIMARY so that it knows it must renew the heartbeat lease. - atomic.StoreInt64(&throttler.recentCheckValue, 1+atomic.LoadInt64(&throttler.recentCheckTickerValue)) + throttler.recentCheckValue.Store(1 + throttler.recentCheckTickerValue.Load()) } checkResult = throttler.check.Check(ctx, appName, "mysql", storeName, remoteAddr, flags) - if atomic.LoadInt64(&throttler.recentCheckValue) >= atomic.LoadInt64(&throttler.recentCheckTickerValue) { + if throttler.recentCheckValue.Load() >= throttler.recentCheckTickerValue.Load() { // This indicates someone, who is not "vitess" ie not internal to the throttling logic, did a _recent_ `check`. // This could be online-ddl, or vreplication or whoever else. // If this tablet is a REPLICA or RDONLY, we want to advertise to the PRIMARY that someone did a recent check, diff --git a/go/vt/vttablet/tabletserver/throttle/throttler_exclude_race_test.go b/go/vt/vttablet/tabletserver/throttle/throttler_exclude_race_test.go new file mode 100644 index 00000000000..903b55b9be8 --- /dev/null +++ b/go/vt/vttablet/tabletserver/throttle/throttler_exclude_race_test.go @@ -0,0 +1,76 @@ +//go:build !race + +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package throttle + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestProbesPostDisable runs the throttler for some time, and then investigates the internal throttler maps and values. +// While the throttler is disabled, it is technically safe to iterate those structures. However, `go test -race` disagrees, +// which is why this test is in this *exclude_race* file +func TestProbesPostDisable(t *testing.T) { + throttler := newTestThrottler() + runThrottler(t, throttler, 2*time.Second, nil) + + time.Sleep(time.Second) // throttler's Operate() quits asynchronously. For sake of `go test -race` we allow a graceful wait. + probes := throttler.mysqlInventory.ClustersProbes + assert.NotEmpty(t, probes) + + selfProbes := probes[selfStoreName] + t.Run("self", func(t *testing.T) { + assert.NotEmpty(t, selfProbes) + require.Equal(t, 1, len(selfProbes)) // should always be true once refreshMySQLInventory() runs + probe, ok := selfProbes[""] + assert.True(t, ok) + assert.NotNil(t, probe) + + assert.Equal(t, "", probe.Alias) + assert.Nil(t, probe.Tablet) + assert.Equal(t, "select 1", probe.MetricQuery) + assert.Equal(t, int64(0), probe.QueryInProgress) + }) + + shardProbes := probes[shardStoreName] + t.Run("shard", func(t *testing.T) { + assert.NotEmpty(t, shardProbes) + assert.Equal(t, 2, len(shardProbes)) // see fake FindAllTabletAliasesInShard above + for _, probe := range shardProbes { + require.NotNil(t, probe) + assert.NotEmpty(t, probe.Alias) + assert.NotNil(t, probe.Tablet) + assert.Equal(t, "select 1", probe.MetricQuery) + assert.Equal(t, int64(0), probe.QueryInProgress) + } + }) + + t.Run("metrics", func(t *testing.T) { + assert.Equal(t, 3, len(throttler.mysqlInventory.TabletMetrics)) // 1 self tablet + 2 shard tablets + }) + + t.Run("aggregated", func(t *testing.T) { + assert.Equal(t, 0, throttler.aggregatedMetrics.ItemCount()) // flushed upon Disable() + aggr := throttler.aggregatedMetricsSnapshot() + assert.Equal(t, 0, len(aggr)) + }) +} diff --git a/go/vt/vttablet/tabletserver/throttle/throttler_test.go b/go/vt/vttablet/tabletserver/throttle/throttler_test.go index e1a9c5abc79..b40f3a0a9cb 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttler_test.go +++ b/go/vt/vttablet/tabletserver/throttle/throttler_test.go @@ -1,7 +1,17 @@ /* - Copyright 2017 GitHub Inc. +Copyright 2023 The Vitess Authors. - Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ package throttle @@ -9,6 +19,7 @@ package throttle import ( "context" "fmt" + "net/http" "sync/atomic" "testing" "time" @@ -18,9 +29,13 @@ import ( "github.com/stretchr/testify/require" "vitess.io/vitess/go/vt/topo" + "vitess.io/vitess/go/vt/vttablet/tabletserver/connpool" + "vitess.io/vitess/go/vt/vttablet/tabletserver/tabletenv" "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/config" "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/mysql" + "vitess.io/vitess/go/vt/vttablet/tmclient" + tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata" topodatapb "vitess.io/vitess/go/vt/proto/topodata" ) @@ -28,6 +43,23 @@ const ( waitForProbesTimeout = 30 * time.Second ) +type fakeTMClient struct { + tmclient.TabletManagerClient +} + +func (c *fakeTMClient) Close() { +} + +func (c *fakeTMClient) CheckThrottler(ctx context.Context, tablet *topodatapb.Tablet, request *tabletmanagerdatapb.CheckThrottlerRequest) (*tabletmanagerdatapb.CheckThrottlerResponse, error) { + resp := &tabletmanagerdatapb.CheckThrottlerResponse{ + StatusCode: http.StatusOK, + Value: 0, + Threshold: 1, + RecentlyChecked: true, + } + return resp, nil +} + type FakeTopoServer struct { } @@ -64,6 +96,65 @@ type FakeHeartbeatWriter struct { func (w FakeHeartbeatWriter) RequestHeartbeats() { } +func newTestThrottler() *Throttler { + metricsQuery := "select 1" + config.Settings().Stores.MySQL.Clusters = map[string]*config.MySQLClusterConfigurationSettings{ + selfStoreName: {}, + shardStoreName: {}, + } + clusters := config.Settings().Stores.MySQL.Clusters + for _, s := range clusters { + s.MetricQuery = metricsQuery + s.ThrottleThreshold = &atomic.Uint64{} + s.ThrottleThreshold.Store(1) + } + env := tabletenv.NewEnv(nil, "TabletServerTest") + throttler := &Throttler{ + mysqlClusterProbesChan: make(chan *mysql.ClusterProbes), + mysqlClusterThresholds: cache.New(cache.NoExpiration, 0), + heartbeatWriter: FakeHeartbeatWriter{}, + ts: &FakeTopoServer{}, + mysqlInventory: mysql.NewInventory(), + pool: connpool.NewPool(env, "ThrottlerPool", tabletenv.ConnPoolConfig{}), + tabletTypeFunc: func() topodatapb.TabletType { return topodatapb.TabletType_PRIMARY }, + overrideTmClient: &fakeTMClient{}, + } + throttler.mysqlThrottleMetricChan = make(chan *mysql.MySQLThrottleMetric) + throttler.mysqlInventoryChan = make(chan *mysql.Inventory, 1) + throttler.mysqlClusterProbesChan = make(chan *mysql.ClusterProbes) + throttler.throttlerConfigChan = make(chan *topodatapb.ThrottlerConfig) + throttler.mysqlInventory = mysql.NewInventory() + + throttler.throttledApps = cache.New(cache.NoExpiration, 0) + throttler.mysqlClusterThresholds = cache.New(cache.NoExpiration, 0) + throttler.aggregatedMetrics = cache.New(10*aggregatedMetricsExpiration, 0) + throttler.recentApps = cache.New(recentAppsExpiration, 0) + throttler.metricsHealth = cache.New(cache.NoExpiration, 0) + throttler.nonLowPriorityAppRequestsThrottled = cache.New(nonDeprioritizedAppMapExpiration, 0) + throttler.metricsQuery.Store(metricsQuery) + throttler.initThrottleTabletTypes() + throttler.check = NewThrottlerCheck(throttler) + + // High contention & racy itnervals: + throttler.leaderCheckInterval = 10 * time.Millisecond + throttler.mysqlCollectInterval = 10 * time.Millisecond + throttler.mysqlDormantCollectInterval = 10 * time.Millisecond + throttler.mysqlRefreshInterval = 10 * time.Millisecond + throttler.mysqlAggregateInterval = 10 * time.Millisecond + throttler.throttledAppsSnapshotInterval = 10 * time.Millisecond + + throttler.readSelfThrottleMetric = func(ctx context.Context, p *mysql.Probe) *mysql.MySQLThrottleMetric { + return &mysql.MySQLThrottleMetric{ + ClusterName: selfStoreName, + Alias: "", + Value: 1, + Err: nil, + } + } + + return throttler +} + func TestIsAppThrottled(t *testing.T) { throttler := Throttler{ throttledApps: cache.New(cache.NoExpiration, 0), @@ -153,14 +244,14 @@ func TestRefreshMySQLInventory(t *testing.T) { validateClusterProbes := func(t *testing.T, ctx context.Context) { testName := fmt.Sprintf("leader=%t", throttler.isLeader.Load()) t.Run(testName, func(t *testing.T) { - // validateProbesCount expects a number of probes according to cluster name and throttler's leadership status - validateProbesCount := func(t *testing.T, clusterName string, probes *mysql.Probes) { + // validateProbesCount expects number of probes according to cluster name and throttler's leadership status + validateProbesCount := func(t *testing.T, clusterName string, probes mysql.Probes) { if clusterName == selfStoreName { - assert.Equal(t, 1, len(*probes)) + assert.Equal(t, 1, len(probes)) } else if throttler.isLeader.Load() { - assert.NotZero(t, len(*probes)) + assert.NotZero(t, len(probes)) } else { - assert.Empty(t, *probes) + assert.Empty(t, probes) } } t.Run("waiting for probes", func(t *testing.T) { @@ -170,7 +261,7 @@ func TestRefreshMySQLInventory(t *testing.T) { for { select { case probes := <-throttler.mysqlClusterProbesChan: - // Worth noting that in this unit test, the throttler is _closed_. Its own Operate() function does + // Worth noting that in this unit test, the throttler is _closed_ and _disabled_. Its own Operate() function does // not run, and therefore there is none but us to both populate `mysqlClusterProbesChan` as well as // read from it. We do not compete here with any other goroutine. assert.NotNil(t, probes) @@ -178,7 +269,7 @@ func TestRefreshMySQLInventory(t *testing.T) { throttler.updateMySQLClusterProbes(ctx, probes) numClusterProbesResults++ - validateProbesCount(t, probes.ClusterName, probes.InstanceProbes) + validateProbesCount(t, probes.ClusterName, probes.TabletProbes) if numClusterProbesResults == len(clusters) { // Achieved our goal @@ -219,3 +310,69 @@ func TestRefreshMySQLInventory(t *testing.T) { validateClusterProbes(t, ctx) }) } + +// runThrottler opens and enables the throttler, therby making it run the Operate() function, for a given amount of time. +// Optionally, running a given function halfway while the throttler is still open and running. +func runThrottler(t *testing.T, throttler *Throttler, timeout time.Duration, f func(*testing.T)) { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + assert.False(t, throttler.IsOpen()) + assert.False(t, throttler.IsEnabled()) + + throttler.isOpen.Swap(true) + defer throttler.isOpen.Swap(false) + assert.True(t, throttler.IsOpen()) + assert.False(t, throttler.IsEnabled()) + + ok := throttler.Enable() + defer throttler.Disable() + assert.True(t, ok) + assert.True(t, throttler.IsEnabled()) + + if f != nil { + time.Sleep(timeout / 2) + f(t) + } + + <-ctx.Done() + assert.Error(t, ctx.Err()) + + throttler.Disable() + assert.False(t, throttler.IsEnabled()) +} + +// TestRace merely lets the throttler run with aggressive intervals for a few seconds, so as to detect race conditions. +// This is relevant to `go test -race` +func TestRace(t *testing.T) { + throttler := newTestThrottler() + runThrottler(t, throttler, 5*time.Second, nil) +} + +// TestProbes enables a throttler for a few seocnds, and afterwards expects to find probes and metrics. +func TestProbesWhileOperating(t *testing.T) { + throttler := newTestThrottler() + + t.Run("aggregated", func(t *testing.T) { + assert.Equal(t, 0, throttler.aggregatedMetrics.ItemCount()) + }) + runThrottler(t, throttler, 5*time.Second, func(t *testing.T) { + t.Run("aggregated", func(t *testing.T) { + assert.Equal(t, 2, throttler.aggregatedMetrics.ItemCount()) // flushed upon Disable() + aggr := throttler.aggregatedMetricsSnapshot() + assert.Equal(t, 2, len(aggr)) // "self" and "shard" clusters + for clusterName, metricResult := range aggr { + val, err := metricResult.Get() + assert.NoError(t, err) + switch clusterName { + case "mysql/self": + assert.Equal(t, float64(1), val) + case "mysql/shard": + assert.Equal(t, float64(0), val) + default: + assert.Failf(t, "unknown clusterName", "%v", clusterName) + } + } + }) + }) +} diff --git a/go/vt/vttablet/tabletserver/throttle/throttlerapp/app_test.go b/go/vt/vttablet/tabletserver/throttle/throttlerapp/app_test.go index bd14624f49b..c468009c793 100644 --- a/go/vt/vttablet/tabletserver/throttle/throttlerapp/app_test.go +++ b/go/vt/vttablet/tabletserver/throttle/throttlerapp/app_test.go @@ -1,7 +1,17 @@ /* - Copyright 2017 GitHub Inc. +Copyright 2023 The Vitess Authors. - Licensed under MIT License. See https://github.com/github/freno/blob/master/LICENSE +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ package throttlerapp