fix: the problem of high cpu and memory usage (#9015) (#9016)

apisix/core/config_util.lua

@@ -88,8 +88,14 @@ function _M.cancel_clean_handler(item, idx, fire)
     end
 
     core_tab.remove(item.clean_handlers, pos)
-    if fire then
+    if not fire then
+        return
+    end
+
+    if f then
         f(item)
+    else
+        log.error("The function used to clear the health checker is nil, please check")
     end
 end
 

apisix/upstream.lua

@@ -23,6 +23,7 @@ local error = error
 local tostring = tostring
 local ipairs = ipairs
 local pairs = pairs
+local pcall = pcall
 local ngx_var = ngx.var
 local is_http = ngx.config.subsystem == "http"
 local upstreams
@@ -103,6 +104,12 @@ local function create_checker(upstream)
         return healthcheck_parent.checker
     end
 
+    if upstream.is_creating_checker then
+        core.log.info("another request is creating new checker")
+        return nil
+    end
+    upstream.is_creating_checker = true
+
     local checker, err = healthcheck.new({
         name = get_healthchecker_name(healthcheck_parent),
         shm_name = "upstream-healthcheck",
@@ -111,12 +118,16 @@ local function create_checker(upstream)
 
     if not checker then
         core.log.error("fail to create healthcheck instance: ", err)
+        upstream.is_creating_checker = nil
         return nil
     end
 
     if healthcheck_parent.checker then
-        core.config_util.cancel_clean_handler(healthcheck_parent,
+        local ok, err = pcall(core.config_util.cancel_clean_handler, healthcheck_parent,
                                               healthcheck_parent.checker_idx, true)
+        if not ok then
+            core.log.error("cancel clean handler error: ", err)
+        end
     end
 
     core.log.info("create new checker: ", tostring(checker))
@@ -140,6 +151,8 @@ local function create_checker(upstream)
     healthcheck_parent.checker_idx =
         core.config_util.add_clean_handler(healthcheck_parent, release_checker)
 
+    upstream.is_creating_checker = nil
+
     return checker
 end
 

t/node/healthcheck3.t

@@ -0,0 +1,122 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+log_level('info');
+no_root_location();
+no_shuffle();
+worker_connections(256);
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: set route(two healthy upstream nodes)
+--- config
+    location /t {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                    "uri": "/server_port",
+                    "upstream": {
+                        "type": "roundrobin",
+                        "nodes": {
+                            "127.0.0.1:1980": 1,
+                            "127.0.0.1:1981": 1
+                        },
+                        "checks": {
+                            "active": {
+                                "http_path": "/status",
+                                "host": "foo.com",
+                                "healthy": {
+                                    "interval": 1,
+                                    "successes": 1
+                                },
+                                "unhealthy": {
+                                    "interval": 1,
+                                    "http_failures": 2
+                                }
+                            }
+                        }
+                    }
+                }]]
+                )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.say(body)
+        }
+    }
+--- request
+GET /t
+--- response_body
+passed
+--- grep_error_log eval
+qr/^.*?\[error\](?!.*process exiting).*/
+--- grep_error_log_out
+
+
+
+=== TEST 2: In case of concurrency only one request can create a checker
+--- config
+    location /t {
+        content_by_lua_block {
+            local healthcheck = require("resty.healthcheck")
+            local test = healthcheck.new
+            healthcheck.new = function(...)
+                ngx.sleep(1)
+                return test(...)
+            end
+
+            local http = require "resty.http"
+            local uri = "http://127.0.0.1:" .. ngx.var.server_port
+                        .. "/server_port"
+
+            local send_request = function()
+                local httpc = http.new()
+                local res, err = httpc:request_uri(uri, {method = "GET", keepalive = false})
+                if not res then
+                    ngx.log(ngx.ERR, err)
+                    return
+                end
+            end
+
+            local t = {}
+
+            for i = 1, 10 do
+                local th = assert(ngx.thread.spawn(send_request))
+                table.insert(t, th)
+            end
+
+            for i, th in ipairs(t) do
+                ngx.thread.wait(th)
+            end
+
+            ngx.exit(200)
+        }
+    }
+--- request
+GET /t
+--- grep_error_log eval
+qr/create new checker/
+--- grep_error_log_out
+create new checker