From 0086938b33687c727721d2c8d3f3f2ac6f1a11d1 Mon Sep 17 00:00:00 2001 From: kingluo Date: Thu, 24 Nov 2022 17:12:18 +0800 Subject: [PATCH 01/26] feat: add inspect plugin --- apisix/inspect/dbg.lua | 138 +++++++++++++++++++++++++++++++++++++ apisix/inspect/init.lua | 107 ++++++++++++++++++++++++++++ apisix/plugins/inspect.lua | 61 ++++++++++++++++ conf/config-default.yaml | 4 ++ t/lib/test_inspect.lua | 8 +++ t/plugin/inspect.t | 98 ++++++++++++++++++++++++++ 6 files changed, 416 insertions(+) create mode 100644 apisix/inspect/dbg.lua create mode 100644 apisix/inspect/init.lua create mode 100644 apisix/plugins/inspect.lua create mode 100644 t/lib/test_inspect.lua create mode 100644 t/plugin/inspect.t diff --git a/apisix/inspect/dbg.lua b/apisix/inspect/dbg.lua new file mode 100644 index 000000000000..e557d88585b4 --- /dev/null +++ b/apisix/inspect/dbg.lua @@ -0,0 +1,138 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local _M = {} + +local hooks = {} + +function _M.getname(n) + if n.what == "C" then + return n.name + end + local lc = string.format("%s:%d", n.short_src, n.currentline) + if n.what ~= "main" and n.namewhat ~= "" then + return string.format("%s (%s)", lc, n.name) + else + return lc + end +end + +local function hook(evt, arg) + local level = 2 + local finfo = debug.getinfo(level, "nSlf") + local key = finfo.source .. "#" .. arg + + local hooks2 = {} + for _, hook in ipairs(hooks) do + if key:sub(-#hook.key) == hook.key then + local filter_func = hook.filter_func + local info = {finfo = finfo, uv = {}, vals = {}} + + -- upvalues + local i = 1 + while true do + local name, value = debug.getupvalue(finfo.func, i) + if name == nil then break end + if string.sub(name, 1, 1) ~= "(" then + info.uv[name] = value + end + i = i + 1 + end + + -- local values + local i = 1 + while true do + local name, value = debug.getlocal(level, i) + if not name then break end + if string.sub(name, 1, 1) ~= "(" then + info.vals[name] = value + end + i = i + 1 + end + + local r1, r2_or_err = pcall(filter_func, info) + if not r1 then + ngx.log(ngx.ERR, r2_or_err) + end + + -- if filter_func returns false, keep the hook + if r1 and r2_or_err == false then + table.insert(hooks2, hook) + end + else + -- key not match, keep the hook + table.insert(hooks2, hook) + end + end + + -- disable debug mode if all hooks done + if #hooks2 ~= #hooks then + hooks = hooks2 + if #hooks == 0 then + debug.sethook() + end + end +end + +function _M.set_hook(file, line, func, filter_func) + if file == nil then + file = "=stdin" + end + + local key = file .. "#" .. line + table.insert(hooks, {key = key, filter_func = filter_func}) + + if jit then + jit.flush(func) + end + + debug.sethook(hook, "l") +end + +function _M.unset_hook(file, line) + if file == nil then + file = "=stdin" + end + + local hooks2 = {} + + local key = file .. "#" .. line + for i, hook in ipairs(hooks) do + if hook.key ~= key then + table.insert(hooks2, hook) + end + end + + if #hooks2 ~= #hooks then + hooks = hooks2 + if #hooks == 0 then + debug.sethook() + end + end +end + +function _M.unset_all() + if #hooks > 0 then + hooks = {} + debug.sethook() + end +end + +function _M.hooks() + return hooks +end + +return _M diff --git a/apisix/inspect/init.lua b/apisix/inspect/init.lua new file mode 100644 index 000000000000..8164d95633db --- /dev/null +++ b/apisix/inspect/init.lua @@ -0,0 +1,107 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local dbg = require 'apisix.inspect.dbg' +local lfs = require 'lfs' +local cjson = require "cjson" + +local _M = {} + +local last_modified = 0 + +local stop = false + +local running = false + +local last_report_time = 0 + +local REPORT_INTERVAL = 30 -- secs + +local function is_file_exists(file) + local f = io.open(file, "r") + if f then io.close(f) return true else return false end +end + +local function setup_hooks(file) + if is_file_exists(file) then + dbg.unset_all() + local chunk = loadfile(file) + local ok, err = pcall(chunk) + local hooks = {} + for _, hook in ipairs(dbg.hooks()) do + table.insert(hooks, hook.key) + end + ngx.log(ngx.INFO, "set hooks: err=", err, ", hooks=", cjson.encode(hooks)) + end +end + +local function reload_hooks(premature, delay, file) + if premature or stop then + stop = false + running = false + return + end + + local time, err = lfs.attributes(file, 'modification') + if err then + if last_modified ~= 0 then + ngx.log(ngx.INFO, err, ", disable all hooks") + dbg.unset_all() + last_modified = 0 + end + elseif time ~= last_modified then + setup_hooks(file) + last_modified = time + else + local ts = os.time() + if ts - last_report_time >= REPORT_INTERVAL then + local hooks = {} + for _, hook in ipairs(dbg.hooks()) do + table.insert(hooks, hook.key) + end + ngx.log(ngx.INFO, "alive hooks: ", cjson.encode(hooks)) + last_report_time = ts + end + end + + local ok, err = ngx.timer.at(delay, reload_hooks, delay, file) + if not ok then + ngx.log(ngx.ERR, "failed to create the timer: ", err) + running = false + end +end + +function _M.init(delay, file) + if not running then + file = file or "/var/run/resty_inspect_hooks.lua" + delay = delay or 3 + + setup_hooks(file) + + local ok, err = ngx.timer.at(delay, reload_hooks, delay, file) + if not ok then + ngx.log(ngx.ERR, "failed to create the timer: ", err) + return + end + running = true + end +end + +function _M.destroy() + stop = true +end + +return _M diff --git a/apisix/plugins/inspect.lua b/apisix/plugins/inspect.lua new file mode 100644 index 000000000000..112da7255943 --- /dev/null +++ b/apisix/plugins/inspect.lua @@ -0,0 +1,61 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local plugin = require("apisix.plugin") +local inspect = require("apisix.inspect") + + +local plugin_name = "inspect" + + +local schema = { + type = "object", + properties = {}, +} + + +local _M = { + version = 0.1, + priority = 200, + name = plugin_name, + schema = schema, +} + + +function _M.check_schema(conf, schema_type) + return core.schema.check(schema, conf) +end + + +function _M.init() + local attr = plugin.plugin_attr(plugin_name) + local delay + local hooks_file + if attr then + delay = attr.delay + hooks_file = attr.hooks_file + end + ngx.log(ngx.INFO, "delay=", delay, ", hooks_file=", hooks_file) + return inspect.init(delay, hooks_file) +end + + +function _M.destroy() + return inspect.destroy() +end + +return _M diff --git a/conf/config-default.yaml b/conf/config-default.yaml index 6e714b577346..3947e9bf5e1f 100755 --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -461,6 +461,7 @@ plugins: # plugin list (sorted by priority) - file-logger # priority: 399 - clickhouse-logger # priority: 398 - tencent-cloud-cls # priority: 397 + - inspect # priority: 200 #- log-rotate # priority: 100 # <- recommend to use priority (0, 100) for your custom plugins - example-plugin # priority: 0 @@ -555,6 +556,9 @@ plugin_attr: send: 60s # redirect: # https_port: 8443 # the default port for use by HTTP redirects to HTTPS + inspect: + delay: 3 + hooks_file: "/var/run/apisix_inspect_hooks.lua" deployment: role: traditional diff --git a/t/lib/test_inspect.lua b/t/lib/test_inspect.lua new file mode 100644 index 000000000000..97da16c8c64a --- /dev/null +++ b/t/lib/test_inspect.lua @@ -0,0 +1,8 @@ +local _M = {} + +function _M.run1() + local var1 = "hello" + return var1 +end + +return _M diff --git a/t/plugin/inspect.t b/t/plugin/inspect.t new file mode 100644 index 000000000000..6899ff1e88a0 --- /dev/null +++ b/t/plugin/inspect.t @@ -0,0 +1,98 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +log_level('warn'); +repeat_each(1); +no_long_string(); +no_root_location(); +no_shuffle(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!defined $block->request) { + $block->set_value("request", "GET /t"); + } +}); + +run_tests; + +__DATA__ + +=== TEST 1: simple hook in route +--- yaml_config +plugin_attr: + inspect: + delay: 1 + hooks_file: "/tmp/apisix_inspect_hooks.lua" +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code = t('/apisix/admin/routes/inspect', + ngx.HTTP_PUT, + [[{ + "methods": ["GET"], + "uri": "/inspect", + "plugins": { + "serverless-pre-function": { + "phase": "rewrite", + "functions" : ["return function() assert(require(\"lib.test_inspect\").run1() == \"hello\"); ngx.say(\"ok\"); end"] + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + } + }]]) + + if code >= 300 then + ngx.status = code + return + end + + local file = io.open("/tmp/apisix_inspect_hooks.lua", "w") + file:write([[ + local dbg = require "apisix.inspect.dbg" + + dbg.set_hook("t/lib/test_inspect.lua", 5, nil, function(info) + ngx.log(ngx.WARN, "var1=", info.vals.var1) + return true + end) + ]]) + file:close() + + ngx.sleep(2) + + local http = require "resty.http" + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/inspect" + + local httpc = http.new() + local res = httpc:request_uri(uri, {method = "GET"}) + assert(res.body == "ok\n") + + os.remove("/tmp/apisix_inspect_hooks.lua") + } + } +--- timeout: 5 +--- error_log +var1=hello +--- no_error_log +[error] From 485956f21730230747351f9ae0996db2aa50e346 Mon Sep 17 00:00:00 2001 From: kingluo Date: Fri, 25 Nov 2022 11:39:40 +0800 Subject: [PATCH 02/26] adapt APISIX coding style --- apisix/inspect/dbg.lua | 26 +++++++++++++++++--------- apisix/inspect/init.lua | 30 ++++++++++++++++++------------ 2 files changed, 35 insertions(+), 21 deletions(-) diff --git a/apisix/inspect/dbg.lua b/apisix/inspect/dbg.lua index e557d88585b4..7838dcf0602c 100644 --- a/apisix/inspect/dbg.lua +++ b/apisix/inspect/dbg.lua @@ -14,6 +14,14 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- +local core = require("apisix.core") +local string_format = string.format +local debug = debug +local ipairs = ipairs +local pcall = pcall +local table_insert = table.insert +local jit = jit + local _M = {} local hooks = {} @@ -22,9 +30,9 @@ function _M.getname(n) if n.what == "C" then return n.name end - local lc = string.format("%s:%d", n.short_src, n.currentline) + local lc = string_format("%s:%d", n.short_src, n.currentline) if n.what ~= "main" and n.namewhat ~= "" then - return string.format("%s (%s)", lc, n.name) + return string_format("%s (%s)", lc, n.name) else return lc end @@ -46,7 +54,7 @@ local function hook(evt, arg) while true do local name, value = debug.getupvalue(finfo.func, i) if name == nil then break end - if string.sub(name, 1, 1) ~= "(" then + if name:sub(1, 1) ~= "(" then info.uv[name] = value end i = i + 1 @@ -57,7 +65,7 @@ local function hook(evt, arg) while true do local name, value = debug.getlocal(level, i) if not name then break end - if string.sub(name, 1, 1) ~= "(" then + if name:sub(1, 1) ~= "(" then info.vals[name] = value end i = i + 1 @@ -65,16 +73,16 @@ local function hook(evt, arg) local r1, r2_or_err = pcall(filter_func, info) if not r1 then - ngx.log(ngx.ERR, r2_or_err) + core.log.error("inspect: pcall filter_func:", r2_or_err) end -- if filter_func returns false, keep the hook if r1 and r2_or_err == false then - table.insert(hooks2, hook) + table_insert(hooks2, hook) end else -- key not match, keep the hook - table.insert(hooks2, hook) + table_insert(hooks2, hook) end end @@ -93,7 +101,7 @@ function _M.set_hook(file, line, func, filter_func) end local key = file .. "#" .. line - table.insert(hooks, {key = key, filter_func = filter_func}) + table_insert(hooks, {key = key, filter_func = filter_func}) if jit then jit.flush(func) @@ -112,7 +120,7 @@ function _M.unset_hook(file, line) local key = file .. "#" .. line for i, hook in ipairs(hooks) do if hook.key ~= key then - table.insert(hooks2, hook) + table_insert(hooks2, hook) end end diff --git a/apisix/inspect/init.lua b/apisix/inspect/init.lua index 8164d95633db..70ff52b80add 100644 --- a/apisix/inspect/init.lua +++ b/apisix/inspect/init.lua @@ -14,9 +14,15 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- -local dbg = require 'apisix.inspect.dbg' -local lfs = require 'lfs' -local cjson = require "cjson" +local core = require("apisix.core") +local dbg = require("apisix.inspect.dbg") +local lfs = require("lfs") +local cjson = require("cjson") +local io = io +local table_insert = table.insert +local loadfile = loadfile +local pcall = pcall +local ipairs = ipairs local _M = {} @@ -39,12 +45,12 @@ local function setup_hooks(file) if is_file_exists(file) then dbg.unset_all() local chunk = loadfile(file) - local ok, err = pcall(chunk) + local _, err = pcall(chunk) local hooks = {} for _, hook in ipairs(dbg.hooks()) do - table.insert(hooks, hook.key) + table_insert(hooks, hook.key) end - ngx.log(ngx.INFO, "set hooks: err=", err, ", hooks=", cjson.encode(hooks)) + core.log.info("set hooks: err=", err, ", hooks=", cjson.encode(hooks)) end end @@ -58,7 +64,7 @@ local function reload_hooks(premature, delay, file) local time, err = lfs.attributes(file, 'modification') if err then if last_modified ~= 0 then - ngx.log(ngx.INFO, err, ", disable all hooks") + core.log.info(err, ", disable all hooks") dbg.unset_all() last_modified = 0 end @@ -70,30 +76,30 @@ local function reload_hooks(premature, delay, file) if ts - last_report_time >= REPORT_INTERVAL then local hooks = {} for _, hook in ipairs(dbg.hooks()) do - table.insert(hooks, hook.key) + table_insert(hooks, hook.key) end - ngx.log(ngx.INFO, "alive hooks: ", cjson.encode(hooks)) + core.log.info("alive hooks: ", cjson.encode(hooks)) last_report_time = ts end end local ok, err = ngx.timer.at(delay, reload_hooks, delay, file) if not ok then - ngx.log(ngx.ERR, "failed to create the timer: ", err) + core.log.error("failed to create the timer: ", err) running = false end end function _M.init(delay, file) if not running then - file = file or "/var/run/resty_inspect_hooks.lua" + file = file or "/var/run/apisix_inspect_hooks.lua" delay = delay or 3 setup_hooks(file) local ok, err = ngx.timer.at(delay, reload_hooks, delay, file) if not ok then - ngx.log(ngx.ERR, "failed to create the timer: ", err) + core.log.error("failed to create the timer: ", err) return end running = true From 25a8147f1c9c8ff15a6913956536d682a0e80a9e Mon Sep 17 00:00:00 2001 From: kingluo Date: Fri, 25 Nov 2022 11:46:39 +0800 Subject: [PATCH 03/26] fix lint --- apisix/inspect/init.lua | 2 ++ apisix/plugins/inspect.lua | 1 + 2 files changed, 3 insertions(+) diff --git a/apisix/inspect/init.lua b/apisix/inspect/init.lua index 70ff52b80add..86ee22276984 100644 --- a/apisix/inspect/init.lua +++ b/apisix/inspect/init.lua @@ -23,6 +23,8 @@ local table_insert = table.insert local loadfile = loadfile local pcall = pcall local ipairs = ipairs +local os = os +local ngx = ngx local _M = {} diff --git a/apisix/plugins/inspect.lua b/apisix/plugins/inspect.lua index 112da7255943..1602ad5fcb93 100644 --- a/apisix/plugins/inspect.lua +++ b/apisix/plugins/inspect.lua @@ -17,6 +17,7 @@ local core = require("apisix.core") local plugin = require("apisix.plugin") local inspect = require("apisix.inspect") +local ngx = ngx local plugin_name = "inspect" From e7407a79f4a349ca16847bae037873a5872d2b40 Mon Sep 17 00:00:00 2001 From: kingluo Date: Fri, 25 Nov 2022 11:53:41 +0800 Subject: [PATCH 04/26] fix lint --- apisix/inspect/init.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/apisix/inspect/init.lua b/apisix/inspect/init.lua index 86ee22276984..06fb05eaf37d 100644 --- a/apisix/inspect/init.lua +++ b/apisix/inspect/init.lua @@ -20,7 +20,6 @@ local lfs = require("lfs") local cjson = require("cjson") local io = io local table_insert = table.insert -local loadfile = loadfile local pcall = pcall local ipairs = ipairs local os = os From bcbfb266d5788f445e41a10991cd6c653ed5fd0b Mon Sep 17 00:00:00 2001 From: kingluo Date: Fri, 25 Nov 2022 12:20:35 +0800 Subject: [PATCH 05/26] fix PR --- Makefile | 3 +++ apisix/inspect/init.lua | 26 +++++++++++++++++++++++--- t/lib/test_inspect.lua | 16 ++++++++++++++++ t/plugin/inspect.t | 2 +- 4 files changed, 43 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 49468dc57e41..5da1cf3d73d5 100644 --- a/Makefile +++ b/Makefile @@ -294,6 +294,9 @@ install: runtime $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/http $(ENV_INSTALL) apisix/http/*.lua $(ENV_INST_LUADIR)/apisix/http/ + $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/inspect + $(ENV_INSTALL) apisix/inspect/*.lua $(ENV_INST_LUADIR)/apisix/inspect/ + $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/http/router $(ENV_INSTALL) apisix/http/router/*.lua $(ENV_INST_LUADIR)/apisix/http/router/ diff --git a/apisix/inspect/init.lua b/apisix/inspect/init.lua index 06fb05eaf37d..795612e17659 100644 --- a/apisix/inspect/init.lua +++ b/apisix/inspect/init.lua @@ -24,6 +24,7 @@ local pcall = pcall local ipairs = ipairs local os = os local ngx = ngx +local loadstring = loadstring local _M = {} @@ -39,14 +40,33 @@ local REPORT_INTERVAL = 30 -- secs local function is_file_exists(file) local f = io.open(file, "r") - if f then io.close(f) return true else return false end + if f then + io.close(f) + return true + else + return false + end +end + +local function run_lua_file(file) + local f, err = io.open(file, "rb") + if not f then + return false, err + end + local code = f:read("*all") + f:close() + local func, err = loadstring(code) + if not func then + return false, err + end + func() + return true end local function setup_hooks(file) if is_file_exists(file) then dbg.unset_all() - local chunk = loadfile(file) - local _, err = pcall(chunk) + local _, err = pcall(run_lua_file, file) local hooks = {} for _, hook in ipairs(dbg.hooks()) do table_insert(hooks, hook.key) diff --git a/t/lib/test_inspect.lua b/t/lib/test_inspect.lua index 97da16c8c64a..cfc6e0d456b9 100644 --- a/t/lib/test_inspect.lua +++ b/t/lib/test_inspect.lua @@ -1,3 +1,19 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- local _M = {} function _M.run1() diff --git a/t/plugin/inspect.t b/t/plugin/inspect.t index 6899ff1e88a0..378f0c0ebfd1 100644 --- a/t/plugin/inspect.t +++ b/t/plugin/inspect.t @@ -72,7 +72,7 @@ plugin_attr: file:write([[ local dbg = require "apisix.inspect.dbg" - dbg.set_hook("t/lib/test_inspect.lua", 5, nil, function(info) + dbg.set_hook("t/lib/test_inspect.lua", 21, nil, function(info) ngx.log(ngx.WARN, "var1=", info.vals.var1) return true end) From 0d8204e9a652beb021ed6dbc008162326c74fc47 Mon Sep 17 00:00:00 2001 From: kingluo Date: Fri, 25 Nov 2022 12:22:09 +0800 Subject: [PATCH 06/26] fix indent --- apisix/inspect/init.lua | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/apisix/inspect/init.lua b/apisix/inspect/init.lua index 795612e17659..79d1c075e74a 100644 --- a/apisix/inspect/init.lua +++ b/apisix/inspect/init.lua @@ -50,17 +50,17 @@ end local function run_lua_file(file) local f, err = io.open(file, "rb") - if not f then - return false, err - end + if not f then + return false, err + end local code = f:read("*all") f:close() local func, err = loadstring(code) - if not func then - return false, err - end - func() - return true + if not func then + return false, err + end + func() + return true end local function setup_hooks(file) From a44406863ffa4f961de939b9eebdba18336dc157 Mon Sep 17 00:00:00 2001 From: kingluo Date: Fri, 25 Nov 2022 12:42:26 +0800 Subject: [PATCH 07/26] fix PR --- t/admin/plugins.t | 1 + 1 file changed, 1 insertion(+) diff --git a/t/admin/plugins.t b/t/admin/plugins.t index 2ab993152fcd..5d3e88f7db24 100644 --- a/t/admin/plugins.t +++ b/t/admin/plugins.t @@ -123,6 +123,7 @@ udp-logger file-logger clickhouse-logger tencent-cloud-cls +inspect example-plugin aws-lambda azure-functions From f11309038441a56feb2769bdecbdca2c95a347dd Mon Sep 17 00:00:00 2001 From: kingluo Date: Tue, 29 Nov 2022 17:58:09 +0800 Subject: [PATCH 08/26] add new test cases --- t/lib/test_inspect.lua | 16 ++ t/plugin/inspect.t | 396 +++++++++++++++++++++++++++++++++++++---- 2 files changed, 377 insertions(+), 35 deletions(-) diff --git a/t/lib/test_inspect.lua b/t/lib/test_inspect.lua index cfc6e0d456b9..b9c999e11f5e 100644 --- a/t/lib/test_inspect.lua +++ b/t/lib/test_inspect.lua @@ -14,11 +14,27 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- + +-- +-- Don't edit existing code, because the hooks are identified by line number. +-- Instead, append new code to this file. +-- local _M = {} function _M.run1() local var1 = "hello" + local var2 = "world" return var1 end +local upvar1 = 2 +local upvar2 = "yes" +function _M.run2() + return upvar1 +end + +function _M.run3() + return upvar1 .. upvar2 +end + return _M diff --git a/t/plugin/inspect.t b/t/plugin/inspect.t index 378f0c0ebfd1..fc1dda923e2f 100644 --- a/t/plugin/inspect.t +++ b/t/plugin/inspect.t @@ -28,71 +28,397 @@ add_block_preprocessor(sub { if (!defined $block->request) { $block->set_value("request", "GET /t"); } + + my $user_yaml_config = <<_EOC_; +plugin_attr: + inspect: + delay: 1 + hooks_file: "/tmp/apisix_inspect_hooks.lua" +_EOC_ + $block->set_value("yaml_config", $user_yaml_config); + + my $extra_init_worker_by_lua = $block->extra_init_worker_by_lua // ""; + $extra_init_worker_by_lua .= <<_EOC_; +local function gen_funcs_invoke(...) + local code = "" + for _, func in ipairs({...}) do + code = code .. "test." .. func .. "();" + end + return code +end +function set_test_route(...) + func = func or 'run1' + local t = require("lib.test_admin").test + local code = [[{ + "methods": ["GET"], + "uri": "/inspect", + "plugins": { + "serverless-pre-function": { + "phase": "rewrite", + "functions" : ["return function() local test = require(\\"lib.test_inspect\\");]] + .. gen_funcs_invoke(...) + .. [[ngx.say(\\"ok\\"); end"] + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + } + }]] + return t('/apisix/admin/routes/inspect', ngx.HTTP_PUT, code) +end + +function do_request() + local http = require "resty.http" + local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/inspect" + + local httpc = http.new() + local res = httpc:request_uri(uri, {method = "GET"}) + assert(res.body == "ok\\n") +end + +function write_hooks(code, file) + local file = io.open(file or "/tmp/apisix_inspect_hooks.lua", "w") + file:write(code) + file:close() +end +_EOC_ + $block->set_value("extra_init_worker_by_lua", $extra_init_worker_by_lua); + + # note that it's different from APISIX.pm, + # here we enable no_error_log ignoreless of error_log. + if (!$block->no_error_log) { + $block->set_value("no_error_log", "[error]"); + } }); run_tests; __DATA__ -=== TEST 1: simple hook in route ---- yaml_config -plugin_attr: - inspect: - delay: 1 - hooks_file: "/tmp/apisix_inspect_hooks.lua" +=== TEST 1: simple hook --- config location /t { content_by_lua_block { - local t = require("lib.test_admin").test - local code = t('/apisix/admin/routes/inspect', - ngx.HTTP_PUT, - [[{ - "methods": ["GET"], - "uri": "/inspect", - "plugins": { - "serverless-pre-function": { - "phase": "rewrite", - "functions" : ["return function() assert(require(\"lib.test_inspect\").run1() == \"hello\"); ngx.say(\"ok\"); end"] - } - }, - "upstream": { - "type": "roundrobin", - "nodes": { - "127.0.0.1:1980": 1 - } - } - }]]) + local code = set_test_route("run1") + if code >= 300 then + ngx.status = code + return + end + write_hooks([[ + local dbg = require "apisix.inspect.dbg" + dbg.set_hook("t/lib/test_inspect.lua", 27, nil, function(info) + ngx.log(ngx.WARN, "var1=", info.vals.var1) + return true + end) + ]]) + + ngx.sleep(2) + + do_request() + + os.remove("/tmp/apisix_inspect_hooks.lua") + } + } +--- timeout: 5 +--- error_log +var1=hello + + + +=== TEST 2: filename only +--- config + location /t { + content_by_lua_block { + local code = set_test_route("run1") if code >= 300 then ngx.status = code return end - local file = io.open("/tmp/apisix_inspect_hooks.lua", "w") - file:write([[ + write_hooks([[ local dbg = require "apisix.inspect.dbg" + dbg.set_hook("test_inspect.lua", 27, nil, function(info) + ngx.log(ngx.WARN, "var1=", info.vals.var1) + return true + end) + ]]) - dbg.set_hook("t/lib/test_inspect.lua", 21, nil, function(info) + ngx.sleep(2) + + do_request() + + os.remove("/tmp/apisix_inspect_hooks.lua") + } + } +--- timeout: 5 +--- error_log +var1=hello + + + +=== TEST 3: hook lifetime +--- config + location /t { + content_by_lua_block { + local code = set_test_route("run1") + if code >= 300 then + ngx.status = code + return + end + + write_hooks([[ + local dbg = require "apisix.inspect.dbg" + local hook1_times = 2 + dbg.set_hook("test_inspect.lua", 27, nil, function(info) + ngx.log(ngx.WARN, "var1=", info.vals.var1) + hook1_times = hook1_times - 1 + return hook1_times == 0 + end) + ]]) + + ngx.sleep(2) + + -- request 3 times, but hook triggered 2 times + for _ = 1,3 do + do_request() + end + + os.remove("/tmp/apisix_inspect_hooks.lua") + } + } +--- timeout: 5 +--- error_log +var1=hello +var1=hello + + + +=== TEST 4: multiple hooks +--- config + location /t { + content_by_lua_block { + local code = set_test_route("run1") + if code >= 300 then + ngx.status = code + return + end + + write_hooks([[ + local dbg = require "apisix.inspect.dbg" + dbg.set_hook("test_inspect.lua", 26, nil, function(info) ngx.log(ngx.WARN, "var1=", info.vals.var1) return true end) + + dbg.set_hook("test_inspect.lua", 27, nil, function(info) + ngx.log(ngx.WARN, "var2=", info.vals.var2) + return true + end) ]]) - file:close() ngx.sleep(2) - local http = require "resty.http" - local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/inspect" + do_request() + + -- note that we don't remove the hook file, + -- used for next test case + } + } +--- timeout: 5 +--- error_log +var1=hello +var2=world + + + +=== TEST 5: hook file not removed, re-enabled by next startup +--- config + location /t { + content_by_lua_block { + local code = set_test_route("run1") + if code >= 300 then + ngx.status = code + return + end + + do_request() + + os.remove("/tmp/apisix_inspect_hooks.lua") + } + } +--- error_log +var1=hello + + + +=== TEST 6: soft link +--- config + location /t { + content_by_lua_block { + local code = set_test_route("run1") + if code >= 300 then + ngx.status = code + return + end + + write_hooks([[ + local dbg = require "apisix.inspect.dbg" + dbg.set_hook("t/lib/test_inspect.lua", 27, nil, function(info) + ngx.log(ngx.WARN, "var1=", info.vals.var1) + return true + end) + ]], "/tmp/test_real_tmp_file.lua") + + os.execute("ln -sf /tmp/test_real_tmp_file.lua /tmp/apisix_inspect_hooks.lua") + + ngx.sleep(2) - local httpc = http.new() - local res = httpc:request_uri(uri, {method = "GET"}) - assert(res.body == "ok\n") + do_request() os.remove("/tmp/apisix_inspect_hooks.lua") + os.remove("/tmp/test_real_tmp_file.lua") } } --- timeout: 5 --- error_log var1=hello + + + +=== TEST 7: remove soft link would disable hooks +--- config + location /t { + content_by_lua_block { + local code = set_test_route("run1") + if code >= 300 then + ngx.status = code + return + end + + write_hooks([[ + local dbg = require "apisix.inspect.dbg" + dbg.set_hook("t/lib/test_inspect.lua", 27, nil, function(info) + ngx.log(ngx.WARN, "var1=", info.vals.var1) + return true + end) + ]], "/tmp/test_real_tmp_file.lua") + + os.execute("ln -sf /tmp/test_real_tmp_file.lua /tmp/apisix_inspect_hooks.lua") + + ngx.sleep(2) + os.remove("/tmp/apisix_inspect_hooks.lua") + ngx.sleep(2) + + do_request() + + os.remove("/tmp/test_real_tmp_file.lua") + } + } +--- timeout: 8 --- no_error_log -[error] +var1=hello + + + +=== TEST 8: ensure we see all local variables till the hook line +--- config + location /t { + content_by_lua_block { + local code = set_test_route("run1") + if code >= 300 then + ngx.status = code + return + end + + write_hooks([[ + local dbg = require "apisix.inspect.dbg" + dbg.set_hook("t/lib/test_inspect.lua", 27, nil, function(info) + local count = 0 + for k,v in pairs(info.vals) do + count = count + 1 + end + ngx.log(ngx.WARN, "count=", count) + return true + end) + ]]) + + ngx.sleep(2) + + do_request() + + os.remove("/tmp/apisix_inspect_hooks.lua") + } + } +--- timeout: 5 +--- error_log +count=2 + + + +=== TEST 9: check upvalue of run2(), only upvalue used in function code are visable +--- config + location /t { + content_by_lua_block { + local code = set_test_route("run2") + if code >= 300 then + ngx.status = code + return + end + + write_hooks([[ + local dbg = require "apisix.inspect.dbg" + dbg.set_hook("t/lib/test_inspect.lua", 33, nil, function(info) + ngx.log(ngx.WARN, "upvar1=", info.uv.upvar1) + ngx.log(ngx.WARN, "upvar2=", info.uv.upvar2) + return true + end) + ]]) + + ngx.sleep(2) + + do_request() + + os.remove("/tmp/apisix_inspect_hooks.lua") + } + } +--- timeout: 5 +--- error_log +upvar1=2 +upvar2=nil + + + +=== TEST 10: check upvalue of run3(), now both upvar1 and upvar2 are visable +--- config + location /t { + content_by_lua_block { + local code = set_test_route("run3") + if code >= 300 then + ngx.status = code + return + end + + write_hooks([[ + local dbg = require "apisix.inspect.dbg" + dbg.set_hook("t/lib/test_inspect.lua", 37, nil, function(info) + ngx.log(ngx.WARN, "upvar1=", info.uv.upvar1) + ngx.log(ngx.WARN, "upvar2=", info.uv.upvar2) + return true + end) + ]]) + + ngx.sleep(2) + + do_request() + + os.remove("/tmp/apisix_inspect_hooks.lua") + } + } +--- timeout: 5 +--- error_log +upvar1=2 +upvar2=yes From 2d16a2c987a36b33c98544d9b0651a9f3b321ab7 Mon Sep 17 00:00:00 2001 From: kingluo Date: Tue, 29 Nov 2022 18:00:56 +0800 Subject: [PATCH 09/26] fix PR --- t/lib/test_inspect.lua | 2 +- t/plugin/inspect.t | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/t/lib/test_inspect.lua b/t/lib/test_inspect.lua index b9c999e11f5e..2b8afc6a54d9 100644 --- a/t/lib/test_inspect.lua +++ b/t/lib/test_inspect.lua @@ -24,7 +24,7 @@ local _M = {} function _M.run1() local var1 = "hello" local var2 = "world" - return var1 + return var1 .. var2 end local upvar1 = 2 diff --git a/t/plugin/inspect.t b/t/plugin/inspect.t index fc1dda923e2f..1d30e9e76cbf 100644 --- a/t/plugin/inspect.t +++ b/t/plugin/inspect.t @@ -359,7 +359,7 @@ count=2 -=== TEST 9: check upvalue of run2(), only upvalue used in function code are visable +=== TEST 9: check upvalue of run2(), only upvalue used in function code are visible --- config location /t { content_by_lua_block { @@ -392,7 +392,7 @@ upvar2=nil -=== TEST 10: check upvalue of run3(), now both upvar1 and upvar2 are visable +=== TEST 10: check upvalue of run3(), now both upvar1 and upvar2 are visible --- config location /t { content_by_lua_block { From 1767979a1cdfce4e6287461b2c023c87927d8766 Mon Sep 17 00:00:00 2001 From: kingluo Date: Tue, 29 Nov 2022 18:20:12 +0800 Subject: [PATCH 10/26] add default timeout --- t/plugin/inspect.t | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/t/plugin/inspect.t b/t/plugin/inspect.t index 1d30e9e76cbf..0d16654d1ab1 100644 --- a/t/plugin/inspect.t +++ b/t/plugin/inspect.t @@ -92,6 +92,10 @@ _EOC_ if (!$block->no_error_log) { $block->set_value("no_error_log", "[error]"); } + + if (!$block->timeout) { + $block->set_value("timeout", "10"); + } }); run_tests; @@ -123,7 +127,6 @@ __DATA__ os.remove("/tmp/apisix_inspect_hooks.lua") } } ---- timeout: 5 --- error_log var1=hello @@ -154,7 +157,6 @@ var1=hello os.remove("/tmp/apisix_inspect_hooks.lua") } } ---- timeout: 5 --- error_log var1=hello @@ -190,7 +192,6 @@ var1=hello os.remove("/tmp/apisix_inspect_hooks.lua") } } ---- timeout: 5 --- error_log var1=hello var1=hello @@ -228,7 +229,6 @@ var1=hello -- used for next test case } } ---- timeout: 5 --- error_log var1=hello var2=world @@ -283,7 +283,6 @@ var1=hello os.remove("/tmp/test_real_tmp_file.lua") } } ---- timeout: 5 --- error_log var1=hello @@ -318,7 +317,6 @@ var1=hello os.remove("/tmp/test_real_tmp_file.lua") } } ---- timeout: 8 --- no_error_log var1=hello @@ -353,7 +351,6 @@ var1=hello os.remove("/tmp/apisix_inspect_hooks.lua") } } ---- timeout: 5 --- error_log count=2 @@ -385,7 +382,6 @@ count=2 os.remove("/tmp/apisix_inspect_hooks.lua") } } ---- timeout: 5 --- error_log upvar1=2 upvar2=nil @@ -418,7 +414,6 @@ upvar2=nil os.remove("/tmp/apisix_inspect_hooks.lua") } } ---- timeout: 5 --- error_log upvar1=2 upvar2=yes From c9dca47e8e2a044003203ee6568536d82c1c5721 Mon Sep 17 00:00:00 2001 From: kingluo Date: Wed, 30 Nov 2022 23:09:22 +0800 Subject: [PATCH 11/26] bugfix: jit off after flush, otherwise the hooked function would be in blacklist for jit forever --- apisix/inspect/dbg.lua | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/apisix/inspect/dbg.lua b/apisix/inspect/dbg.lua index 7838dcf0602c..e732e05f8bf1 100644 --- a/apisix/inspect/dbg.lua +++ b/apisix/inspect/dbg.lua @@ -105,6 +105,7 @@ function _M.set_hook(file, line, func, filter_func) if jit then jit.flush(func) + jit.off() end debug.sethook(hook, "l") @@ -128,6 +129,9 @@ function _M.unset_hook(file, line) hooks = hooks2 if #hooks == 0 then debug.sethook() + if jit then + jit.on() + end end end end @@ -136,6 +140,9 @@ function _M.unset_all() if #hooks > 0 then hooks = {} debug.sethook() + if jit then + jit.on() + end end end From d1099c05055a124e7c4fac1a42ff98af3f338ffb Mon Sep 17 00:00:00 2001 From: kingluo Date: Wed, 30 Nov 2022 23:11:51 +0800 Subject: [PATCH 12/26] add test cases to test jit cache --- t/lib/test_inspect.lua | 22 +++++++++ t/plugin/inspect.t | 100 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 112 insertions(+), 10 deletions(-) diff --git a/t/lib/test_inspect.lua b/t/lib/test_inspect.lua index 2b8afc6a54d9..62de5993008d 100644 --- a/t/lib/test_inspect.lua +++ b/t/lib/test_inspect.lua @@ -37,4 +37,26 @@ function _M.run3() return upvar1 .. upvar2 end +local str = string.rep("a", 8192) .. "llzz" + +local sk = require("socket") + +function _M.hot1() + local t1 = sk.gettime() + for i=1,100000 do + string.find(str, "ll", 1, true) + end + local t2 = sk.gettime() + return t2 - t1 +end + +function _M.hot2() + local t1 = sk.gettime() + for i=1,100000 do + string.find(str, "ll", 1, true) + end + local t2 = sk.gettime() + return t2 - t1 +end + return _M diff --git a/t/plugin/inspect.t b/t/plugin/inspect.t index 0d16654d1ab1..e938431b4dff 100644 --- a/t/plugin/inspect.t +++ b/t/plugin/inspect.t @@ -98,6 +98,10 @@ _EOC_ } }); +add_cleanup_handler(sub { + unlink("/tmp/apisix_inspect_hooks.lua"); +}); + run_tests; __DATA__ @@ -120,7 +124,7 @@ __DATA__ end) ]]) - ngx.sleep(2) + ngx.sleep(1.5) do_request() @@ -150,7 +154,7 @@ var1=hello end) ]]) - ngx.sleep(2) + ngx.sleep(1.5) do_request() @@ -182,7 +186,7 @@ var1=hello end) ]]) - ngx.sleep(2) + ngx.sleep(1.5) -- request 3 times, but hook triggered 2 times for _ = 1,3 do @@ -221,7 +225,7 @@ var1=hello end) ]]) - ngx.sleep(2) + ngx.sleep(1.5) do_request() @@ -275,7 +279,7 @@ var1=hello os.execute("ln -sf /tmp/test_real_tmp_file.lua /tmp/apisix_inspect_hooks.lua") - ngx.sleep(2) + ngx.sleep(1.5) do_request() @@ -308,9 +312,9 @@ var1=hello os.execute("ln -sf /tmp/test_real_tmp_file.lua /tmp/apisix_inspect_hooks.lua") - ngx.sleep(2) + ngx.sleep(1.5) os.remove("/tmp/apisix_inspect_hooks.lua") - ngx.sleep(2) + ngx.sleep(1.5) do_request() @@ -344,7 +348,7 @@ var1=hello end) ]]) - ngx.sleep(2) + ngx.sleep(1.5) do_request() @@ -375,7 +379,7 @@ count=2 end) ]]) - ngx.sleep(2) + ngx.sleep(1.5) do_request() @@ -407,7 +411,7 @@ upvar2=nil end) ]]) - ngx.sleep(2) + ngx.sleep(1.5) do_request() @@ -417,3 +421,79 @@ upvar2=nil --- error_log upvar1=2 upvar2=yes + + + +=== TEST 11: flush specific JIT cache +--- config + location /t { + content_by_lua_block { + local test = require("lib.test_inspect") + + local t1 = test.hot1() + local t8 = test.hot2() + + write_hooks([[ + local test = require("lib.test_inspect") + local dbg = require "apisix.inspect.dbg" + dbg.set_hook("t/lib/test_inspect.lua", 47, test.hot1, function(info) + return false + end) + ]]) + + ngx.sleep(1.5) + + local t2 = test.hot1() + local t9 = test.hot2() + + assert(t2-t1 > t1, "hot1 consumes at least double times than before") + assert(t9-t8 < t8*0.8, "hot2 not affected") + + os.remove("/tmp/apisix_inspect_hooks.lua") + + ngx.sleep(1.5) + + local t3 = test.hot1() + local t4 = test.hot2() + assert(t3-t1 < t1*0.8, "hot1 jit recover") + assert(t4-t8 < t4*0.8, "hot2 jit recover") + } + } + + + +=== TEST 12: flush the whole JIT cache +--- config + location /t { + content_by_lua_block { + local test = require("lib.test_inspect") + + local t1 = test.hot1() + local t8 = test.hot2() + + write_hooks([[ + local test = require("lib.test_inspect") + local dbg = require "apisix.inspect.dbg" + dbg.set_hook("t/lib/test_inspect.lua", 47, nil, function(info) + return false + end) + ]]) + + ngx.sleep(1.5) + + local t2 = test.hot1() + local t9 = test.hot2() + + assert(t2-t1 > t1, "hot1 consumes at least double times than before") + assert(t9-t8 > t8, "hot2 consumes at least double times than before") + + os.remove("/tmp/apisix_inspect_hooks.lua") + + ngx.sleep(1.5) + + local t3 = test.hot1() + local t4 = test.hot2() + assert(t3-t1 < t1*0.8, "hot1 jit recover") + assert(t4-t8 < t4*0.8, "hot2 jit recover") + } + } From 72dd417e9e82ede71a6a02835d49f4358ab631b6 Mon Sep 17 00:00:00 2001 From: jinhua luo Date: Thu, 1 Dec 2022 16:14:59 +0800 Subject: [PATCH 13/26] Create inspect.md --- docs/en/latest/plugins/inspect.md | 114 ++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 docs/en/latest/plugins/inspect.md diff --git a/docs/en/latest/plugins/inspect.md b/docs/en/latest/plugins/inspect.md new file mode 100644 index 000000000000..a2f54dace5d6 --- /dev/null +++ b/docs/en/latest/plugins/inspect.md @@ -0,0 +1,114 @@ +--- +title: inspect +keywords: + - APISIX + - Plugin + - inspect + - inspect +description: This document contains information about the Apache APISIX inspect Plugin. +--- + + + +## Description + +It's useful to set arbitrary breakpoint in any lua file to inspect the context information, +e.g. print local variables if some condition satisfied. + +In this way, you don't need to modify the source code of your project, and just get diagnose information +on demand, i.e. dynamic logging. + +This plugin supports setting breakpoints within both interpretd function and jit compiled function. +The breakpoint could be at any position within the function. The function could be global/local/module/ananymous. + +## Features + +* set breakpoint at any position +* dynamic breakpoint +* Customized breakpoint handler +* you could define one-shot breakpoint +* work for jit compiled function +* if function reference specified, then performance impact is only bound to that function (JIT compiled code will not trigger debug hook, so they would run fast even if hook is enabled) +* if all breakpoints deleted, jit could recover + +## Operation Graph + +![Operation Graph](https://raw.githubusercontent.com/apache/apisix/master/docs/assets/images/plugin/inspect.png) + +## API to define hook in hooks file + +### require("resty.inspect.dbg").set_hook(file, line, func, filter_func) + +The breakpoint is specified by `file` (full qualified or short file name) and the `line` number. + +The `func` specified the scope (which function or global) of jit cache to flush: +* If the breakpoint is related to a module function or +global function, you should set it that function reference, then only the jit cache of that function would +be flushed, and it would not affect other caches to avoid slowing down other parts of the program. +* If the breakpointis related to local function or anonymous function, +then you have to set it to `nil` (because no way to get function reference), which would flush the whole jit cache of lua vm. + +You attach a `filter_func` function of the breakpoint, the function takes the `info` as argument and returns +true of false to determine whether the breakpoint would be removed. You could setup one-shot breakpoint +at ease. + +The `info` is a hash table which contains below keys: + +* `finfo`: `debug.getinfo(level, "nSlf")` +* `uv`: upvalues hash table +* `vals`: local variables hash table + +## Attributes + +| Name | Type | Required | Default | Description | +|--------------------|---------|----------|---------|------------------------------------------------------------------------------------------------| +| delay | integer | False | 3 | Time in seconds specifying how often to rotate the check the hooks file. | +| hooks_file | string | False | "/var/run/apisix_inspect_hooks.lua" | lua file to define hooks. | + +## Enabling the Plugin + +Plugin is enabled by default (`conf/config-default.yaml`): + +```yaml title="conf/config-default.yaml" +plugins: + - inspect + +plugin_attr: + inspect: + delay: 3 + hooks_file: "/var/run/apisix_inspect_hooks.lua" +``` + +## Configuration description + +TODO. + +## Example usage + +TODO. + +## Disable plugin + +To remove the `log-rotate` Plugin, you can remove it from your configuration file (`conf/config.yaml`): + +```yaml title="conf/config.yaml" +plugins: + # - log-rotate +``` From cfd6e722ae32616317e05411466ffa6c8bd91a5a Mon Sep 17 00:00:00 2001 From: jinhua luo Date: Thu, 1 Dec 2022 16:15:34 +0800 Subject: [PATCH 14/26] Add files via upload --- docs/assets/images/plugin/inspect.png | Bin 0 -> 31490 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 docs/assets/images/plugin/inspect.png diff --git a/docs/assets/images/plugin/inspect.png b/docs/assets/images/plugin/inspect.png new file mode 100644 index 0000000000000000000000000000000000000000..efe82eed82663222d44cd16d98f09ca810db4008 GIT binary patch literal 31490 zcmeFZc|6o_-#0vC4WY7B*2}$q0!XP0cI~B5L7)!R4 zEZMW~Jje7qyUz2x&-=RW^LahjAJ2Wi?mwjQ-Hz|)IF8Tqe!o8xuA`+&P02(Ffk3F$ z)o$uRAVdi8bKw*j_{+nZFD&2})J;!S5mNG*WgdJWu~k4QKpCN=P5VZ{Fu9M zpYYh7mSCy>enwO2+@pt=L*F-sPUKR{Qdqw{%_JlB!ulnA_e{D%cWBT@>Tod%?lU;t zWjb1-HJnm!xs!o<`F+!4jylfbtFoaIM>CZb_Qi92>hcCjn^I3{>4S)LAe3+@FN75H z9zmL(W_pL1@PEjk14OWFy1x$p{?j)aQV7E9*;B;d&)|J{7TtKfRfF?5ovJ_D9{W{Dl651)cGCL?^~{##$^LNues@*lmLP+wg~pA+Ftbfdy!^pP z@72;1aEbh&t>%}#tFVBrB1`dXvgsGRmo5+MqxQE-A5pVw=zJrW-+gR|+Q(-d>nkRo z>QH|i*i_1Z&SWi{%B6PgF#2}1Gf58Xs3V3a7>Aw0bOrMcpw*w$m?{t{}7&# z^e)+i9z%iCwP3(1pxZ%h`*4MAU1!2n*#U+F=9cHTwoh&&0l=aznR46NQ1T#9Ob|6co zf`7a;bEN9wy~-5`>XpH9KGUQ!a=}X#HYvj}`@j~SU?H1!h8j0OlCm{)3U!qMojffW zKuKxNVhCHT9bgKp5M0{&ao|L>W@F}TT-`b*{ebtUbgp#8GR_*!-ga~c@x5AAa4nx~ zI*a+?KJ-5AW5*lVoD_i zi&0Tp4zKL0US5)osg#1Hw-R3r;`YSlcKiyyt68T;jHYC&pR5ml?lK)WmWi`^o40Iu zJkBOIQMlZqeM9oeYd?Nc*hG;&{@s`^Y!O)`_VUhksRhd*EOslzZ*fBJhHIf1g`Hrp z#$qftYl_NuMjR)Xz^2MCfq+_J*qO~#*0Uzh>wa2jjQ3Rd!puX%;;;z#QKKsU?(^}L z$mA|Qj|oQVxy6>&gu3N~?9>bPL@wl$Tt?$jRlfHq@#d`ohkY6QBX6Q*>`wnw{(8Sd z{_376#+cr+U4ML&M*d(jAaZx*KI5~xirkX5?7J_Y`=(DPh6kt#i(QFBr8d+H$%gH} z4Z=1gx0fl)itHyRkDo@>PFK0F)OyyTRGyqS@<|&uDmS-gJ>AZbqfq$nKu$oG?r67t z)$g;UhA_MT^y<#hw~wcm@&fL+&X;lRE85=3$)#Eqd&Pnt5MhmZ>Dat$^%_PmUeUT) zyT5b=O>HqC;#V?Ld2f-$NjpHwd#S&xZ|{<12@kXQ)gABX{-l*Smc4CVGhCf6m#{hbFx9Ulnl^lWzk6^aI4Gm6Hn81c za!q`?x-^q5D?ItY;q}kqjt8ez(3FZum?Qg7Hx4bx(F?gjTj~64e0F$H#ImPS*PWXa z?!H={+tEBLjS@uuCIdU955lQTd$)Kc8}xju8rFOBI|ssF3$7%-jCv+0^df;qGN+mT zYh{4lG8LT8L2YPlqJGJGxnjcnPCDLHkMyRhE^1{4(MBniNLC0j}Wd`b7{WaG3GRGi35*%y!0yfW??- z46k#oW6CDfW!TehbpISIyf8`2_KFj|wfwoZaj3wQ)jlid9KV_)6~<27!o*Q zplPN|N{8xQ^s1pmoHp^!!=;w&s0a@;?O$yv5D7wuj+ zO(s;6&}zQOIoN2`V;s9J>r-;^^NvK=aI{9eT#cQ@n$H4mrDNj2UMt5lO|OxD+NTm* zDeQDXbz}FlWhQZ}$ai19^sgh%i=kNCgUDd)yV^M2#x7TX<^;o zwBpDqRPE0l>gCVx@1;IYTxxW1DR;Ab!^1stb{6(fh5l=%Zr8jr1$`mkbr;(#qTP|r z@ubT0=P~oIEeoHko)(zT_eI57WnFN#$xplpICD_trF5xi)-%X<%S;1 zG|p3|0zzJvzA*H&1pR6$Q|mw<1b#34QJu-NzN~5vi$Ntv7kOC5>_cBnT=hMRXVMNM zt{j;8my>ho*$)SI*F1)r6YTKaKX3X6C+G--zMc8lwx+My6;W6565??`LeS}l0!k`P zrGy#HDd5psP=wRh3{fUpm{GPD(aL9Y!_5Cw`8HbS(VRc7+t8diH?megrc-f4er4m^ zJR`QI|B5Q?EZ*_^)yKtG{#Ynu$IA-lec4H;|HoCakA9-( ziZJs_7yA2bozi3IOIar&w!JgGo2iF0Twxo*^?go-^M@jcR=t>+zK*4@ZW+}zEEU@l zDM46894C$M&NWzV){qq~g3@{5gZK^)ED<99f?c*xp9D?AGs^7yGZfo>kqzg5c+*N3 zrH9Y6>U6uZq8GEGLKjt4#1--(@>2j*$LiUM`;pGqR?m~VatxnzK>FMF_4{$$l7ho| z9eOO}uKGHWKWXMPT!u3;E1oOi6^3^e-TQd*Ts&dIna5gt+o|NzluAgfn2Z}VQdLoC zn0_)(eK##Z{^Hf3h*}0L1!#USY=mhh&p-5mMw@I5umbF z#XLt!f$F-3Nv%$`3!f4D+R?G(n-%w-lkw?u2Y=rOP;>+wsTGz){yxy8XlcGCOckvq zr)fzKHJuelHzi1Dro$(@e zW14zEl8pNs28W|*#fKiM{+qbRUsQ?*>8M_4TN=GV-)Fq=ypL?{l#i!N`DsgXSlk8k zh0RtbWe@Tz(d28Zjov1F$K!9$9QK!a`0fNw;rpykroTsbqx14r(dD$4rv*rw+|3wG z2l&|(&v5LhQhnH=y_P;iUz)e%{oGr3bzboo^U?Nh65~8-NK4>rOcvYmqUDKi@ACUr z$HuK<=fWG04#p;(+I+CR_z z#CtVX2k=cgpUrY`s(1a?$?TUEF!MeSjpAps)3%&^xZU~K;b_}B;5ODdAcqF!i|wU< z%D<(@+Fu!+vuk&&Ies|@@M<)g|pPBc>0V=2c4~KUqjoQX1K_Nn6(nLvw0~3o6N^{l<{R2Jv#eeU4uql- z9KXR@xNs^6`-)+1k`1FCqZ)*rl43+>^0Q^`ft@@{CY#Qtf@VvOo{SFS4$K_LI15f9 zCY(fs-aHBDn-bE0ORj>x)~}Dy#*b5C)ZbzlpAy52?^v~R-SI84`XGp+`CX_m|8G&K z+`_0^Je~BJV?Dg^^09wZK;6Xkh4zE#1nq&vou=JZ;g?xSxZbb+8;HDBP&>Be@3-zW zodhtU3GJ88XB{osuG#8Ow+?^$ycHtsuaJ76 zeX^--IJEPDoL$Dl<+blz*5Ndd%HA*9lV7wdiWc9(zcYzc-(s^-Ak# ze{1pMffYILFZPo)_E#r|plS`fkj(!J6xqr?N8iKPO)U62@pktK_BL62s>8-(D<(Ahr)qqsk}m=@;li;Rz}tsbNGg5s7DaeS^3| zu_!GHT&}Us)3>JJ*u=3-5iv}>&A~j02<^_-Awh!)FO_JI%qAk7nwWSFQ^@;aP5KOHP)Of+oREW__I~N%2^&z-iUCr5S2%?+X>IEAn zJev=VB!by7(tlqvgy5{JnqFi&L^s{nr$k*OPYfNB{?C9TK2KL=%rbfN`@WULI^Wiza${jP``6xa!+xRcaFy{JL{wC(n zcB!m(qa*{`s*Wv>V#qL@aO=i82lu($ZI2sR3yo;X@OR{5ou7{d|CFG+jmix!Y9kfS zo!%4=O9nYVbxo@!oqq~jxz(UGDjQW}8b54@w>JJ-!G)Q};)fnUQ01RGJ1L_S=$4VRP)K)#^&KHK?nyoZGb;G4#?)-b~^Bl!t z`-tp9S;{HT;me4dbvf=P>%6nk$rfQ!iiy(vf$ctUyi)na8i1InjoBlZaQa1uNiZ7J ztDCc$9JaqMsi>yQk=mctJ57;r`n38pDN}r6^!s!H54~cU8}~h~Q&%VFn6Tv`3gWOg z)VD@LIb2mPJFX6{+E{cH^y?zlRzE!_CM!M?sit!VFpG3VKG@-7$b%m6#J~eMnz!vRb$tOVKYz_y>`Tn4&mL`tm2NI7tt-(E(o@u zMkXR%#Y($y?z%B_f;${KT)tpu=$hGNLlNDlMJ}QJiHl5dx#0~m!9 zqrd-PVfmtn?EBaijJ?^FB%CSw`dgs_9I5-GXMzo9UE<|&AH254-ohv;%pl)mT#rVk zou|7TXZ>;0?C!{|C)nvNO4qaAW3uQ@{`4QL(h&5@nT~h_1d)8f`0m9ZEM;p+As-w$ z_^~VGn^PF~RGg&bN6xI<4psT|Tt4*1r{vC3y&U}bYl$TlZOGhbCX_CBv|>{1B|y?! zVs}+wZ1ntU@;Plvtj}_SoW-d?GkjEB72TPH#n_=!66VdC8f=;*&yyGJw<#T za3tcR(H8+sp<)>xJIzlX%DVTnQAu#E^YNZkLQAC)jhS!p`@Vbb(h)WR_Iai3Hv&Xv zlYE^UT`#>oQ(ygn$0u(wuVhI7G+OOj{G=PL_ArC?`Yc-zHtn48TPUW3iJzaJ1gD=m zP$^D=dWGmaLr~3QrJbOJJj>UX!|n_Q`}Gl16A~_H-BshR>qqMwzjQp>lwtIVSk;2 z5z`Tv#Y}~$dF0Bu+8g`LYwuBJ-uGHUFsM3N!u_IzBB)m(w28DRUpTDU6ABYQoTBwk zeB@BNON;4vBQ`-iPlRSWQ~bRSc;Qt@R0wvs)N*W9=&HbL_}5LdqQNVZI|c%py$s0% zUsxfCPbNmDQ$g6#+|t)~;mG@XIhX0^+ZBzl)zbE3Y~|dFQ8vaa={a96wZEj0>?ZLp3m3J?wYFP1z|`$bZUUxlFhdAT>hsAtJUG{JJj(f9B{(J-or zho1a6Yt$%7fNTAIg(oQlSoS)p*S$BAnDECGp?MILX!jfPeo~n1B_>b15)cpJNH;(E zt4(?2x~=%kXqPBxK^Vgk$Zn%Z|QSl&Ca@RJ6L0(nY1{*wk%-D@klT`uh| zmZ)pbrzVSfT23V#9?fgp4A`+o7w$|&IGXuh)U-=o5ip78Km^{%30yqmXOwLlb3G&u ztD5An5(rG)K>jyV2c_aFvN7!1am$SN7It{(Ux)B&#Fz=|E!@Ok_u#%A#8Vb^ShcU-p&E3%jWw3tv1L`FB7M94;%B;_@wfr(S8px;vR>|U zzc`OiU{1+fT)LJbn8bV9SScUZ@l~< z^QcVs>wcbw1A1VUC)KWQIsUK^a&7P;C(NTG<|Jl!2_@nw^scM$1cJU%b5}(uh+E9x zms?Iyj%@A&J!Se#GwOm<4X_C~W z(*iv6NJpXv2C+A0)67*iSx?sDUzBoxJnZ1RwUA^&{v>9P^3LVz zJvKvgD5lW9waB^D#db$q{z%_^z@jP8ykQj8=S{kf;iJ-+$J>57A3<~q_f$(q9}2=< zYxfukcbT^B)heXrf4MWJp1yGyAwQ%(MGmZB{l^NqxWofn<@Mu%Q-LWbJhxBxla{jJ zydlP58(Y_Wcq#MeJP6wLm@Gp9vUzHW2n}O{CxeXJ*y$SBtxt5bEl9QU=R%wBuBb(P zH9T8PLwZftC~U!|iUK7%*5p^-riPcjdY73umiM9E_xaxB<{)e=%i)L(ky@boX{`K-8EK_S~wH+MU8f=PV!E_MO zIVXaQ?f+I1|F0{FAaPv9X8SyO#oZFGjQP8{g<4xZ514V*SZC#OG8ls{)!20luom5F z!3;`TEWV=0Sf_Ha@w8pz>4H`7 z3eB8V0+rKlu)K|8DY+PygjLyq>0g%Cesxi z(TwV-h4@EbQW@`sk#&4dgG0V}yIk-&=U8(P$6e!`6|5<((VHE(eO_~)4ig@JmMjm7 z5^aA?js$^^m&MZoON3QONO_FqX#30IL;R|Z0-@zA0k23KUs%%@^95Y1Z?4|O(v&~k29cIGig5MpP(PujH z<)&9bXX=)w9uR>lm-SEbgH^x}&e!A^(O4_+W{yfT<(*qhtW{P)ms59b=^Ab4Wk5h> z{K+Ch#{*RN|B;M~o0P+$ZkX_&-j{cS>pyoW{(6qP$*GDSSAg?Np5r|479(Z!VGd;; z|CG}!@pa5-zoLMsJS4^J)QX;Me56&e5>u7f^Y)8dXCq-EqRp}T5Jc#Ot()i|EKf^w zNe}WpC+bRB_rX3tdAV>^eLc2tkQwQ{IT~d8yvB>Dwss+jD0+-b_)IDH4SU#$LWUV5}`y}3V8ZSV6vClJRLKMg0RxEDT|5m%3t{{We`=5+VB|% zMinF9JKnp2MA^VPSh)v+77*P~Snlr9}l~1Qo&7J1FXv z(W45d7a$XBK3x~*#E90reaj2L12Y~BgCGXz)O@`TL2NC=3{fKi9P;9>7aY$g0Pv3W z>P~y$;a{0+n+9Qh-BpW1K~g9>P)%hw*N_EXIHk?NkQnt!Z`pU02sW4_Z6Xc=U_$mM z+1TsYxxW_*f4Am;jd3`?lL9WQe#n3iqn;A){{w>F^?lNP0X)+RbxR%Op0~~oO}z=? ze%^z)Mf`t89t~!A%u*AOaFe2-97)Q)TKXc0+amB1^DKz4FPL*h8r*Z|-Lei?0%rIv zeHP}%CXe5J>OK=u1m?2j^Up3ST&}jpP5U6Qw#i z&;zUI)1MUQZ$OxPvEBYg*2!s<)kO2}KcC~)oyU-DG7 z+X zLLXhXaA4@HnOfTF>9A%VV=^V3KbZYp;N^|=c$qf36Mi`m6kY~wIj45<<}M3n45}q) ze}7$Sl{J&@I2@4up>cxqA(lHo$p~qaYsZRvzl@AcZlJLncPO0KYO~XIW{DWu^#>ua zo|m5vjx_GPQj!F${JKKCs$cU*yH0gze8EKa z$W<*P1*G3ig^IHWlyJ0rdk0(_wya}&1*5o9XWU;g*Uy`%2I>r9_|5Lp-<1Ywh5GuL zYtz1cj!(E?m847EMlUC*m5DObw-fti@7MiCphg*D#pbJF;Z(QmU3YvOuUNu$V-BDC zN9WHC(VxC~iAIpy4KP#uL|BCde;=PI%8@*C0 zCNNrpx=03N?h-d(6|wq6o(^7&)s)+r6eT($J8c0FW^5-{X|aM3E-@oBCnE?$gmd5A z479PVgp6;rBONd#=}}l1gG1Vi@0X@;YWN(yoBiEdx7(|U!)K>|-j>envR;G2%HCuT z0Dv{G^ZB=TAO?e|-?cSBi*UWXd$+H`OkehEsn;#WV9WGO4wD{Hs`3P7eDmIQq1%x{ z)Gb!O;)rNO^0-w}_opT_7X6kQC=u(AWz4BQB-}pmFm-K6;b#o(BK>*$RA7w37nLU9 zB#zKmUbcc&E^vJ(w_-nAos*Njm)$ddn5>;=h~~CifUg!KU6U)kvGct;+mUvxx<60; zF^zVnV5`+qj`Ga#gGMMqwN%P`JBipeu4YV~@e`G#A=E+=m#!3hKQD+o`Q0VH5^(!! zsT}!IK$$!meReyC=tX9c?qLL)3}(OyLzgLmn7}Ic6-kPjXG?m6Fa#fuj-Y?Sv3FMF=tCk@?ELja}+0Es36AYpn^;SFNI)V#(;WAi866a z176P_L>_MQ_+S-ZWK{aIxazCaqvm%P?0i=nc7ECdabIf|f!DEbTJ@ik7B_l2+Xbis z{dXU?+JJa&6)Jj74ZMefjclqtCBpVm#PoD8dVJ$(xN%#2e@6Jxp0&!Cj5=Wd6&l$r zcFepF_1SC%U?=oFPtRO+q3y=&{hL4!m{@XjxM0*|AN`QJgdSLA@1H|u0YC{%iii4O z6UE|)?yj4@f8z=408fCt7t5#(mcC2?A4id?c5mJ<;DeO{#O@KHyElu~hJvtH?C*|! z0=v;(>pX5N6FbWr3e2#*Look8&@2P8xR!d7T%$6ux+6$SvnAn3>X$j{Zoo#Vi{^dZ zP~g;?x52m$fV2PikfyX^8r%R&sfPh2sy~l&@!|v z$^dm@3C$ zQB$JyuuAf2;rmOKLHXAub9BBj%V7!m-C)hZ1{rLkl;Pb|A`(>jYq|$S3$(LD3CXZE z|MfOkN4=Z)Xw{RwlnZFN!(YvseH!cOa)loXH{D!(g+@FBzPPh$k#e@)&c)c(e~FRl z8_h3S3?Et(F#rp%@bg5L1ZF<4_zf351ZF%;P9oqDU_#}MCW*UnaUE4-io2~@c69kM zcExYC1@CZ>6^wJg8?rm1_>kgZG-^^RDh+uCtisYi5M>Ynwk$uzg`KITGl}SXxn`vr zCy~3S_x|~da*gxB(w0Z7^p5?7Rk7U4gb5T#FS$YiM>1R;9wWmPS{(J&-UW;DwAhkO zt?PttmW*@m5R)U(+GJrqI7lY)Ysnlz!M&088k2A^mVR1`1*iU z57l|Jana|I9&uCG36cXxT6=q(TyFnoj!hP?U$S}vbLvhn!*ye7bgWbP!Vd@>k|E+? zAOyZ43LsD*T2oVy0*j;mIllboG=2(7Nn!G`vpVoG6AnjPQ1qrlr{EP#cp680SS1LZ zL)$1d8eFbI%coGV^qhgM{~mr`iwp>%``+NMw7C=jFyYN8*9a$${weH`-MMIW(x+5o zQL|)hCXq~dC2-PbwhMvdX`|vLfWTH^s?x&kiyfY(18G8hn=A|DX#txA^0`ZQ(koOf zqQAm^fD$oT3aE;}V$F$jIO|t#?RAU<_uF1l)#s?jQKdwuAU3v~Y`Af@ats-v&o2Vp zsN7MO{Gs~eKEIP`9Pjui$p>>Vck*df=E~Bi>7KSKMf5eCuu4_>g4JPqEve~9#fY7+ zeO6O?ek}ba%Zqrmipnr7>Q=_*rysWrm+G=-DpXejEgQ~VGLZx;tFtc&sSl^)k%*=8 z1LIf{tj+k=&ymk=tr9~rt{Gr*wLClP%kHQeL316uv?Zd425 zof&VW!trt%yf&3OaJx z$Wfvl#ci$tJHs+>+k^eZO%>U6&sXT6^YM}z($iXvugnxz%?m3=QdiMOmuU8et2ZtK zl1?v%7?wIcbgu?5lXz;s-v9*Zrz{~7TC(X(pj9J5*yk4h=u?aW;$^8))#bWuzR=vS zu!!@arAka*VpJcD#1@v$r#kbZ!{3E&Sf8ySt(Cx#fF;cRk2P~QlhTC$419JbIb{g; z8z23?r})d^dClM6QXy0Lqm}t_^tVNdIdFaQu^xcdKkb;I@xmoq;F#FT?j= z1v75h_I=OL>WLfrdFyRy$c+#lctYRM*CZLtyin?!v-HovoQUW1BoZTf`g(_M^-kt_ zT<`&O;=+Em(4YOAi*xs#TJo%WmBfDZQ&m!!H5NGHs`Vj$^}gwP z!p?-)UaqTo7tgA+>AHI5L91|J_F$Llmik7`z$3P0)8TdXrmn@_-pyi%fDBW);pm3? z??Yj&o8t}#6Urf@U`Fjy z5*^`iI#Y))++-*#hp2ZgfG9@XBC!t!@rt{eW%$&;InWoG&70{*6Q++s2obEg0QpR#YKANc( zZ!97;W<28GK4XmbTaa!8*q6XdP30@a7pX1GJeK8p_8$BX09XKPJ`hq}m^=PP&Vit% z44Xg~5}2Vz;=e3i^aT4Zm{2_w-GP@#XNSCu6b#8>q&|FheX4Gm&~_1KQ&g=;dSUr2 zX`jjxKm30GWn21)2B+mZ&fYaOt>EILI7mVGepRHzi%F4r?(9NAh*RUaWhdTqzoskl z6Q>Me%dyhG_(q?llTK?&2_dF}Z>obn6dHtngO#P_z}UvlH@DLW2THP+$Fd^JfB`R% zF#qKHfJA3In95zx@de2$D57^htbrLz zFkk53%(wU$%j)=&wD=t#ti!J0M-~UFvp*cYpaIoX#LonzBna&}KyM@jT@g{96P)4w zEP^o~5Fz)cdFVxhP|Z;_+CixFZu_tkpss0mD-F#qkI;MQ&qO9pnZ%$hD9IspVaf)N zJA1w83KY?^W@3~_fVvL6E85SKaOJOT%Tw^A7oSKz!FC;#90?(kuQak0M#wOIPO*B* zD?R0*+*+P(cLjT)bzs7(ANwxo;~$`lE_T>;`sbG)-+=JMqvDbl2XMsVA8mj)d@y9f zcOKQL)!V1_9tlO&R+O@o&HJ%MP0L_{iu>GeS=3%SOpYMUp6{ENOf{OE4?kPhMWMc1 z=diXPawZ6CYV*U7Hx@*{f%KK2xZk_3A7D>HBwmFaDi%Yox$1I~?<`jnHSe3JXVx_e zhxg@aHECN^Gq{GInt+F zx#X__JLX$7c}`UzXjQO-f>fNwZWt6Ag=`axCS@ZSs z1{0XB!Argf1QFW)+}Ce5z+HZOLi-bB2b)8t&KV6Eroni>)dp2UqpHwx?p$h&{PCAN z;!B!Ly)uNxGU=%%c!4aX7Rmd7j5HVQ?uxv;!kAduyvaB%tJhv}GFXVzsRY>^x_b1)bnHI@fb z<0jQ{Q4*Mo+s#LE|BQsf{(A6#C(raBQR}~26r?cY2SV663xa5F0LX#W>vA6u6QRGD zG+!4X#O(^M&^T^Ky(?#&Nw34$JqqQcP&+qY5@=Mh2S)IOyP+a#=W#9XpT;9WQBLpu zyMFRc))Y=QjDiuy}$i1T)u|iO`W% zZ`O%`!dJt>=Lygw#uKBRZs{+QNsl^JP1!0pdRCLd2JIbtqJW<8`uq2@f6!ID@&~Ki zuIU?D?l7gOSxq7|t!29h&|!7+-kJ-XKBXp)X}+YqO_UyV`%M4s$iN#~&wZ^(tTTG< zpYMC{B|G^+(_oIvn@&ysWfVhX;c$En*D`lA&(-|&^tU-$i`OQqlr!#f&q+)5g1YXT z^6ZjMkp%Wy!lr?2$tO8-{ExWu+tTvsx|Tyrjkr6WQazp&PTKx57b7CZC_r5o5|%HX zp@@%`I~*1+8a-(O-4HaDZ^ApkQqc4)%>;VGt)CY6i-NHH_N$^C7ZP-$ZCd2{&KFWS znFcG#^q)O#=F0PY_dps0biBNy&jNswSBfU;Ci9xD#`pHT&o_e-AGnF<>((KxB()C1 zu>&ib@tzX&J^Rzq@pk#<=|4JC;5>mT)Afi1#sg2$1EKte=Ef@Duh(ziaA-2rN_pXK*4WeHB zbfvJ0RDEJZ{PV5-*m0@!_$JNl1KY6pvq;1z^j`FgvfGmhy*>pa_YWi_+IY z0`~o0k^!KQ_3JXEa9?noInQj@sWn!3!AtJxqe^ZO zh3#K-emli>ouYakIL;xx;%WNAfokoOol`jXjo314&~wkFVt5;%fr@1g_LMkx};UAi>H|#wMVCF!09oTP&264YVD&p{iBUxV=jd5eK9BSpK||0v@zceM5bJ+ug=WQW2kXGws=-ZrCUfAkxSu zO1wo^+%NCitFU^(8%=gugs*soZXA3yckfmFMLtPgx!oGP`IuceQk${Suj?r1neD2E zR2bGf11bCiDHY^damHW1ld-L_@zWy7(9I*uM;qpFJZoDdbs7Zy;9=P)6HtRKn5#Mj zVg2;aYge_FlEp5X(l|IcAyk)J5W2iPN4U3R;PfJqv{ zR@rk>d~AnL3XEu$HsbGi#GG2YE@3J8>k_PkhwQw?>PLRYm5X-gI#~A-hy2Th?3z^g zEZPiCplmfz=V4y2>9O^HAhEcpbgEb$0ST|A&Bo~nr1@Zy#|=O?@-^RiA3 z=V~Z^M18Z_E#T0AC(tmYF#x2zY4R`rU_cjr*l4Ww;@OM7utwf0nNisO&Jj7<*Te89}+})w#G~)R9{ayophWfFjlqv zB(QnIL`xYsvRU{zsO_*x*jwW~oUSP$btMvHsSK6!Yy_MNsH{sZo8C>D>)*cpxPHFI zut>RawdnSXOn!mFGN5datSoug0!37+TN*b5RxwvaiJKQdssaLeU=wRT)YS$(RdL{b}#6U@~CkgKf>W7V6YFZx9% zj4ObE>+qq<(a`1X62)FNM@kMv%+=xXBLaj^`-f)`%eGTekk7m_CirP!Rh#!Vm2NUy zjYaN2IM#7NQ_;k^s$$|MY_SeTNw+={J{dk)>JNm+%zMsT3Gp$IvS?}0Sp&YC042E` zCzj5Go)$<48cH=p<%TjkVmX#k9^a8g?o?>WY`hlJjFC8h?1cMxD zkSmsOW1?&~*6Pn6CHYeA$P~7CG?0dlAcpZh0|XeYq=&-v$A6I|BY`9tvLmxT@X;Bn zp2LS|9mO66|8mdvw#s1%0d1^C`E${hlAt=WVc|!-1~_xm;ve$&8Y8;pXFptkjJ=TI zi`Cug3-=dxo;25T4Cg7l(c?g*wdd>R2OMj0*F3D3&iVOC@mI*Or12>lRR0&!vk`#D z<^D8_x0AhrjiNAUwQurk;H%AZuoy&TK=)6#Q|gYSA(P3suXCpL}o0cjac{O&(7@4r5%9|hZ^|GYoo%HnEC z27B&#vDK@B^3#ArcmQjP1({d`jda4?W(Te9K+0s-m$%Po{GJCKW!n!2T!Qs-2quNy4-ieOw1j9>VK|lB|-f4WiF_kWZq z=y&wdCa_=Ndt?HtiX@{(S<%J2TbJN;^4it5B>zlh2T2mTWma+4^7Tp;=KC;&st0II z2^fu8TVjNHlEIoP+U*ip1wj`K*U1|(!A~^Q@fi}2A#c{EuB~&9y6$iwTqos7Q#}bd z+>-E%O^{&Gah$#dva?uYS}Y~XSLepkkS3|-6K-K*glcpKpl%;7bBcJvyy?EtmLs`Q`aNAn4xBSp28r9D_jrqT;`i))C_<9Q^I zVka-=lq_jTLv?@}sewLwK@XHzDeHrkRH#>44i(mqiDuz+eQS!pNPRLALcX2P4p)nY z)jSC@bmdeNnq~7S3GVHy<)lzw;WcKj%nP|~9S9MsIWwO3ne$?+Nf^w@mN8Cjf#ZUm zm38RlLV)jxEQG_L2<>NK=4tss>bRR$sXJRhv2E$(#~?o` zOO=MjB@4vE{m#dCRT~!G+ja2l4hcw~!-VGvzD^*5mAzBda1G)X6uqK`qzB4=drxJ} zxOHMR0bp>UcN%Z zd|c1#jT*htPAAd%(mSQ7x6KKU=YX*2RXfRZG!$lRJsYC;M*#q!9Bs|5(Z|yUq)tYf zs~NP@)7VB=qmr{Lh>12>PeV2OfW;FW?D6Tzj9Ou(d6+R1whH z-hxYv*8}F}eBF8%m^&>k@vk<~f1^47|KE8uFz5fGGk;zUKPR^%x!-BjxIWNZ)pX<- zm3o=p2+XBfMqkQ#jCxk`vjGVBNPSI`@BiuU%)_Z}+jpO-fs%R1Oo&3VBxIhV7O|Af z4P==!%NT`F86(R)t!2nOhb5V!jEiJ07Fwp2IrF~1dY)%L@4Mf9{GQ)F_Hpdv*#Gob z$#=T%>pri~d0x8AFkixTb3hFIke%4lc4@Xn0)TPn<)0mjWNo1aub@_uJP>P+mtII#G zh%>#EY)6iaMwD>*uJ!Uw-s2(z;w|f4Iz2qlc|#+ZEv@uU!0buqz_LE0;^oBgd!GsE z1Dx1VF?<+R$ss1Kjg5KKC+H}X1?AW|YvO>!_dTgk`*J{O}pl0|D1MfOWUvqMNQ)CM!Z&S z+3}4psGXBu!4{DOw;VacrTMUjWZ9YI@e(4A*@fS|0cA%M_4qjC7>(o<1Glwq_nbuH zKR~Co`~<&esf^!OVIyHJ!0Pnget)f-Uo7laD4N6h*yJW#2~j3Xvn^Yb{f&FLxb&W) zs-}D`1$>lqYrYAa9QuS~Xi5D@!gE>5DvysGDhT89+8e_hRmq>Rr2_GNUqLyngf@+W zxs8%+^{W5id2i2a^X3BGIhkPeSYM`gTc%~l8^Am-J)g0SD*I^s!C++lT-4O7(bNg@ zp0&E*Q9eMg%0An@1EpP9YUH--AA?VRXec_HVO z6ARKfk;A`lEfzbz-?qa?Y8BmvuUMPz*#qT|jx-8Cr4kNA+BeeB>4!`4_?C&G1h?^QtC>2a= zXTU@XQh}#=!7k@)s8^bhvC&BuEcZF-vGN05oG#c}-|SLp;9pGl8@Rikf4SJBdZR75 z%KQ4Zai)H^G9sg`gPHHic7-K&8Aah_F zs)I%JR0x?(TX@b!>cEAv=RSPs`evEz?|vlxk{S+5G1#DQ_j6vTC#d0V@HZ};H2(;} z(`JMB9{AQi5ttWI50C-XJV8Ik<1Of4{osSx=}Ev&x2?|3tCo-SxNf~BJ2p0OhIbix zu=VNOlmT^4+h)ulqaCqyo(aY;$zP=Fxdfrpk0VC+hS4G-7|-eVrkC$!hJYU{cY(v* zE(5AWtIbZpBsIF!^}`6#JNEzBB3?EyMh3?c!zGoP@-x1j-nfkEhDxM=3n_6s1>(-? zcb01B_B6*FK>J|yQ(hkkg2;CDT?9KDj#GpUxY64cahx}#PoqkUW3jumJ!@(2nU=sl z($gOk{cl^85Ul=Se+N1Wdd(M9b&Yq4G!p7J$1|aD8)kpM-6^j8J?v^S@nR7RK`tL~ zRE88=#M8)0BJCv$07`~NvGQZ0m2pF<^zB#5AbU3O+2u-` zDuP}UDwE|9^gZ@(*S@Jpze*Qr1_AZLOm)FC;L!H`l|U!7^A~!kNq~U6TPH`r??nV0 z<3G!F`RAa=|4Yx)PS4sXgq-!D(_n{bG8d|uFV8}L@G@C1XOIijORIY(1Kd6y_3%qX zPHR#C7a>MLc!7mK=+FL*`#7oG-kajb-S!+DNZ6N&6&*H&3PQ}mE|v5vm4o@G2T;7| zUhM!I?&`RzLzbV%Mhu7BFgQE8=kQnAt|QssBI6f0BVZK3Dufp(X@XrQvyW7E0*hCR zq=fxx#Te^XJQGIXNe>7&?6>m&$`LA++~Om-Nxwye2!K7JLUYerh15RPR&Q|Sh)6+` zc!jKlQSJ@*IggP?yy#gUqjvxNREF@g&ujVAY3*`;o_9@mE3U+E|TnOv$}3^EMfP~9-V`c$Dw3zy~Gn>D|W z+y+XKgc>9WW*x=!b+-^|EJoJv;oPn;Mlw8z3Pdm8S6ESnyg7nKZEgDCiqJA1{^L*3 zI~5<|0Kj5l%0*M$ziut^&6-a2zQxDPX$PeT;w>8?1okZ+@hp6fCC7<;MM7btr zTp7i{0UltIxG!Jyw|V@e=_0klz5%cMN7Y z0KVwkLkY$w|2$P3CSTBL6^Ljt-!0-l;?~ZbsUif+wnZJ9&3^juRNv{~=-du*(=nuZ z2Qt(?Qv!`koIo3aA%Va=#0EJmVUl6RBK>bF*IgDMBL{>&o^I@3AJ(;q+;nVpn#v{ zUz)0aSXK_TE-U<=<|#<)da20#9=lW>-uOR2>5wu}e9!m~<1+)zdudFTkUA4aopMCQSI@IPj3H zUR7y@az*lhJ17yjgDzB4bdsPhTwGQ^@&DI)`F}NQ<Z*5qXA{Jvlv3Hn0ENac70SW6SjSx?IAX8s_%5r*C5Ovy zqj&K88!r(J-wD>Pb6U7O-SaSCh~eLBrJJ4=1x0=;495#HpKxHHTui zBq)W(y*0u^uP-`Xv#lu7;p_dviOLXilK}#laS1Uv4#!!u`+7qSk_UDAx*^Iu`Q26AXfylLbZ{o=ebuI zrS=*#1Y*2Ea>m&z&QFXzYi}IB=BI?4UH2(NnH1IjVQ% z-6c6efvrkoz+}q-fj!zdVj8G$H>4H@Zh~EgiN*gN%~h62?^2pwIvC)^UujTD_MJt~ z)p1r3gV8L}Ubu5+ELh;8?Yuy4-kDQoZh=il3Jjm=ic!Ne{TP2j2TG*Ge~AOMJ?Xak znk*sS*vtrr$DTwo1jdV0>QAKmN-- zJaE6~&xgGGvp$Se#t}lxMi%pK16_-2x-RXa{OA8^`oE<8pG^OD?ScbKU=BghZ85F^ z!OmH|gU98LfTG?LEmBaxRTZwgIc%B8ZmewkZ+HONAgZ;DTG9e*ozoILr6m z43zkmnc<)Yo9AROY07=EG4dm*Qpcy=2U2OFd%3&c$-yOG58+760U8);N(TDFElO)u zJt4R$-G~LeGrU@;8WBhvk97OJ-eZFwf~Na(2QZ8PsdY~X4xs5L0Xpd8NH0Cbh4BmE zmFZk&gW$Muec^Nh1L#MqF~GdR+cmV$vl9lM+N}$oFTtvA!!Xf<7ViU!7)2jQB%z^7 zREG5SZK%fnUjs@01?aj1?RNs%P&57?r8f@ELBABhBA*DDYOTB^Lr%`0y7T}XY5>>9 zpABr0ns>sRNPz_ww0VmbJ(wA7z&@R2s#i%u0=Fx;m&lS9S|B-02Vxm73(@AB0N)ep zo3jbuKm0}Sdr{+DHar~n7(ouIZHFXqu;feltb*jVb#;S>AUrbr>?91jc0RjSkO~Q* zm1&y!Nf(|%X;Fb$)!^lwoO`if2T*Nu#0VcL%Fg4>afmheCpnXUHGbgf1mNhu`bSK_ z#KGQ#TXxWu-+od0^aXIi;3rxK{h=Y6_CSar6cYf#Md!6nIsH-LKv_zP2nnCKTcp3i zU^jU8-bPh}Jq^qrOjp2*xYb0yJX|k}dzsn(HGAMMwp=a)WDn8~2#?I-bQErpA8bpG zD_AcH0Aeu4sEy%8*>tVNC$t1)(3d;>Qqaoix{N`v(FZZ%+$-%v6$is0Dy@fcS zbusA*%L)?q{U}Xsu8fjgh_g*84FaafDt|gp1g_)eks|I%8lGA$M|z7_l+F~ zDbkOsqq^1gD#I+OVY1XXwy0I3_lObch1yfrcz0yR+}KTgxR>`nNgmh+4@EB@vG?|U zJRYUj(tZGH?0SR@M?{=ZXXu$j(W|#G#}E>|uC+UTCquq#(z7F_(1Si>0@d%gp>Hlt z)8yBtrUuNu$~v!bZPwNkG;e$>as)Z8!w!Vh^XZO6PqZC;9`cA@I**Ne(wnjraG!py zofY%QS-u60ZLm$(t0P`P%ddx$h$u0W$Y?9j3}I3MLOmbbHEj4Ps#-tw23)oz^~g0x zog2JnU}i>g)cp=wg}(|)h~riGlj9`=1etJo+Q4OsR7mn#y$D+jxZc|JA%VTOBT)OK zJZ6z80DZxnBLFPOmdqZhN--$xXJv#yk?y(H$x<%yJ^?iOG;9WWiC@7a*~foe;y=UV zU&!D?3sX^y<}|TM4#NjIN=YER_*I&wc4I~E`Pq4VY=T5a(}M9yIQ???s=H&Z_k-9u zYoylNoh?%IN=jCEy33Ld@*IiH#qJk2B&g#c*W2x`k3alyjXU;x>4>F*amd|opyysa z2eZV?_TR1gw)fyk<&mfN-dCwe6&cl1E6=@fqIlU4QhW?-5om+Fp{L zs<`F;66j5hKerYWUvYg0$974jX-+9Wl2nc}9L1i}z&{WAPL&|R6gWEqfFV;%$V1Rl zkZc1%=TJ)&`hSF@icYb(M{69k*pki6XVn?^#4aS(6@L2#B(I>j6rVK(N7Q-0=_g)3 z6TayIK-Ut#ZK}7m!Rzc>#2y*`*&Dy4(Oy}eH2Ip$U`HWMbC8gp0P?3gmpT5Otfp$@ ziasSP3H-qbm!xCAY3~2iwBTBZx@M-z4KXNl{KBo?AUj(~St=u8ynGs& zchl}V>fK-DHR;Dg^19yzimu^V^n&WwOmB_?QqClWE^t!YQG!D zkqd1>JJx^LVaLhfOz{)Xfu;XtF{>KXLj{>*`{0@H%1y@cK)%lHRT5wbVxSd)&c`J+ zf?$h3{Ez=DWZFUR%%8{8|0?D3p95^-a+||`9=hr7;N}kVTe8vgG=Di$8A<|Z>`al8 za6b4-1cl$$C-cgsSt|2ifnLdb3j0$EMh1y%^NAC?=*li#HSX@!7c1X8r6%Q zGjtzo=&r>kL*uDl<{7lr%!4rS+hz^Rt!yvzP3Ja2gK9BXeszrKR|O)wR|6L$R<5RJ zo<=cE1W~*H=b%BT67k*F@r#$|* zyz%K6@)N(ZCIq!nTGwLD4#31IUoZ@7-grXK319IH;rmg|{{ZN+3i);WI{=E-PZhs6 zn-84rMScU%&O|=Qc=F5ykxl^boTSeMr2wwoZ^z~=&{U}XHH^aRNPcB!y|f9UZ4lZ@ zZ%{RT&6m&@!R?5OaUX(IO;$Sp|1KCKR z-fI|)aI$zBKJW#c`foe38IggxQZE+To(S;SpO*wlfVnG1tn57&*#+556ky4FJ>1wq zNsSb~Zs;-vf-Kex3qb&dn~&*t=9VjI~XQECiyRUnP21lsnZHJXufi9_3mLL~9gEU6v`xMURCAu3h~C z6+fpfj-b9O#RxP}1&c58f;)x$>9LL>6;nyC@Ew1$bg6)$ z$F)cgKH{L!`&Or*>v`bNj~svSVSZs=D)2bO-_K@MToo%&LAjco2?B+FY;g};W7!%3 z+oRZH<*tepS<3JhagxH#^}4cxsN2gVK-q(bh&b0u7>Q!~-Hc&LMfA)%OO-mo5AWxk zXy)*VH*u00pL;2E!-H{iS;J$_s!3)gOFWJFaXkM)4jvSOx;{JzC!Fl=g8!!YWNo}5 z_flWI?$n6|hd zRLS38M2psS=Qtzv+FY-ORi2s(FX_A4-@}BPpDTmhdb({3AT9k>flIaAI^kQ%buo?E zr}*j=au`=fWy|JMmwHh{)>K<%Bkif)4^#VmpK5(f@=|Ixg)pYMZN>Bv${kq7x3A)O zO!5iWJ|&?{6q44tv0E^4%$))SLt**kpAbLWX0e&8D2U zZ}b^7SuWMc$e-<)moVt2B6V+_+vX|qZ5sqQsF#Nrt&#nK+n~aU8%zzG>Yr zuaK-g*w#-4eKze70aD@0ky9d1Y29!H>7gOnlX?|)zX*>oV&*|Q9m&-Zz_spxeUWR z^w=UK0?FssVCdQJ~s95q)0)*yAmHdF|`pjVNfv=40ArccX5S+VSSf8Ok1>d{>jk-5(+~5 zJ&;ZH(<5(~(Jx$n*yHGCfr3n6MjfHjwC`B6=}5Nq{Pmu*=-V>Mq1C#Pd)L^FCj*xr zZZ}-r6%G+_i&|+u2&2zapgSU|dl1()rzrAO25UT~Dt~IUsf6AfFUnyotu4LC*#XoSZ1p|%&M8^{$J7jM&*rlqd090Su%S{GB`8tDbL#3_qK=*> z;(7VniETaQ^cC*Sl2!>Rde`SAoXi(rks2*^PrqWJxmFl{0%j@`4`vM(rZBsgvxM8W zy5<&|8oSlIM?XM}_lyo%Mb}ad9pF1@(sH?B$>Yf!a9jRkcus9g!A@m_vw`qtC~@ty z;F#TLUGAmzTBn&TjN*?YCWQSOoSpMK)fWzn@sUYPQAs~m2f@Ft|45(ea-=2Z&g`N& zin^mVMm|{0%_G{72E8;BLuGk5c1fySmMp^Zs9xz#VM}{gc4NvL3yn9p_pG{}-s&P4 zX8xO5=&#MmzLejra4rU4%uJr3%lxuMImL^!Qs>e&ZCJ^sohPG+5d+(5C(4&AcI&#P z&yHYVUXR{cd`1at%2fJBPOYK^O3N_$>{L; zmnh!u?4^4=qfRF_H%5lj3GMPOUx<9DrA7%{s{7;{$8f)DK>;;ULB98Gd&FgV`pp5S znP4f7*P3wQ{B9EkvFEg{0d==cuOy6(SXt206_0wlf(!&2R4I;gCpXx&aLNet`Q#`0 zlkg$w*Gb{QYnT1)qq*%$sz-eZy+~edz3+2&QCriJ@L9qAMH?P<|H)ULw@z2I?De>! zng{{vldm50EWNe-1O~B%`=#2VjuAsz1`SW^py3jk^vfr-l}lK<;;6XV5aJ@|m-@m; z`(F!HR=+%!POv*u*pgn%Z-dwEwQ(nks@8DU*bB{E^Ov zx@6_4h+C!)RZG`|$A<}ZhgM&gd&dTcjTgo-9F;h;Nue0ZvyWR>dCWYrN9+hhFRVFr zk*>~Zpz!4;@dDSK*gxtjiD$?riM5z|c=d@=!;hIxQT3{&#wO&MK+vT@Ih-Oq%s&F-P{BpHd+2oY^Tq6}Yd=0$ zF3^x@=;8c2E?NvsNXR_Kj@SRTOml~$=Pc1;Cq;v==As7s=}baw-Co9a2F}IMDMCzj z?$_t-*vt|Wm7G(kSLom(oo)_G-|e|*40@KF&(cO_(W|!Q7czNbd6RitgbeAG*A2XG z%V9@rj+?yQ>evQXCL}M1$?r}wuj=Zi8mjDb)7-+8;N$g^V8gIW{kD+QD;8^ZeId-3|;Ev`uU|#} zd-HNtWXx4tU#hU#le=KJY%RxZg1n0qtbIw4>v;QB&znZ6p*uFlSNmP;c|3)DXXYL& z6>iFy6m>HNPW+IVvmP*;LA`n5@E|Lf24*?smj$UUYd0~rDi70P5e5h{qCBlV9G~Lv&3{jqioHe*8U>%T-IV;J9)3< z(GfK_yWGKZXCq3lYq$H@1PHj%y2J!cINz)H3_Isrt{*VDXh4nkSY8#1c#t~j(pyZg zpe8;Gtw+P^y>z2wp9ZADa*Ki0{m(i`hCV6%j zUwOe7uoI_%1>Ykp4{%#~x0C$6fl-PRjF6kZgY+u&^qldAa#PnHfvG)ScVzrzNm0KZ z>v^byzeEk%kn_ipeFvLr;k7qAIa#bfTti68!ClTnvxSTx^Fcr;V>}2I`Ay(MBW@kQ zf&aff;+7P~I^<0PS0BD@(+3VF(-oJlq_t}qUl*YSM-kzQr>_~o?h;oz(@Fs)?3pHz zq2~0)N8L%_;85f(291f)g2u$;)!jY{y4}{cxoRXJVIteTxHOOq)uH!gwEBsFOMU)% z7e$0=N8*TRnh=-~M(a15?D6x#^P?-08{p8G!$$;jj&+MB4kno!p8MTaR`6c!lzNTz z2Y62E95Xa6>{e3Mgddyy>u>a!)de=ky7YC&?sjH=m%P(8KN`^f8)d02sk%|j=B}c9 z$@zA0g7r`fq#? Date: Thu, 1 Dec 2022 16:18:21 +0800 Subject: [PATCH 15/26] fix doc --- docs/en/latest/plugins/inspect.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/en/latest/plugins/inspect.md b/docs/en/latest/plugins/inspect.md index a2f54dace5d6..e69ce4a6d0c0 100644 --- a/docs/en/latest/plugins/inspect.md +++ b/docs/en/latest/plugins/inspect.md @@ -59,9 +59,11 @@ The breakpoint could be at any position within the function. The function could The breakpoint is specified by `file` (full qualified or short file name) and the `line` number. The `func` specified the scope (which function or global) of jit cache to flush: + * If the breakpoint is related to a module function or global function, you should set it that function reference, then only the jit cache of that function would be flushed, and it would not affect other caches to avoid slowing down other parts of the program. + * If the breakpointis related to local function or anonymous function, then you have to set it to `nil` (because no way to get function reference), which would flush the whole jit cache of lua vm. From 4d216300fa731ffe1406c5e9663a99d4572e08ab Mon Sep 17 00:00:00 2001 From: kingluo Date: Thu, 1 Dec 2022 16:21:32 +0800 Subject: [PATCH 16/26] fix doc --- docs/en/latest/config.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/en/latest/config.json b/docs/en/latest/config.json index ae6dd95d4296..4b43afc37b98 100644 --- a/docs/en/latest/config.json +++ b/docs/en/latest/config.json @@ -57,7 +57,8 @@ "plugins/server-info", "plugins/ext-plugin-pre-req", "plugins/ext-plugin-post-req", - "plugins/ext-plugin-post-resp" + "plugins/ext-plugin-post-resp", + "plugins/inspect" ] }, { From 0f3dcb14686a11c8348e8256a981e7d4326fa86c Mon Sep 17 00:00:00 2001 From: jinhua luo Date: Mon, 5 Dec 2022 18:31:59 +0800 Subject: [PATCH 17/26] Update inspect.md --- docs/en/latest/plugins/inspect.md | 61 ++++++++++++++++++++++++++++++- 1 file changed, 60 insertions(+), 1 deletion(-) diff --git a/docs/en/latest/plugins/inspect.md b/docs/en/latest/plugins/inspect.md index e69ce4a6d0c0..7f90c334ee18 100644 --- a/docs/en/latest/plugins/inspect.md +++ b/docs/en/latest/plugins/inspect.md @@ -104,7 +104,66 @@ TODO. ## Example usage -TODO. +```bash +# create test route +curl http://127.0.0.1:9180/apisix/admin/routes/test_limit_req -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d ' +{ + "methods": ["GET"], + "uri": "/get", + "plugins": { + "limit-req": { + "rate": 100, + "burst": 0, + "rejected_code": 503, + "key_type": "var", + "key": "remote_addr" + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "httpbin.org": 1 + } + } +}' + +# create a hooks file to set a test breakpoint +# Note that the breakpint is associated with the line number, +# so if the lua code changes, you need to adjust the line number in the hooks file +cat </tmp/hooks.lua +local dbg = require "resty.inspect.dbg" + +dbg.set_hook("limit-req.lua", 88, require("apisix.plugins.limit-req").access, function(info) + ngx.log(ngx.INFO, debug.traceback("foo traceback", 3)) + ngx.log(ngx.INFO, dbg.getname(info.finfo)) + ngx.log(ngx.INFO, "conf_key=", info.vals.conf_key) + return true +end) + +--- more breakpoints could be defined via dbg.set_hook() +--- ... +EOF + +# enable the hooks file +ln -sf /tmp/hooks.lua /var/run/resty_inspect_hooks.lua + +# check errors.log to confirm the test breakpoint is enabled +2022/09/01 00:55:38 [info] 2754534#2754534: *3700 [lua] init.lua:29: setup_hooks(): set hooks: err=nil, hooks=["limit-req.lua#88"], context: ngx.timer + +# access the test route +curl -i http://127.0.0.1:9080/get + +# check errors.log to confirm the test breakpoint is triggered +2022/09/01 00:55:52 [info] 2754534#2754534: *4070 [lua] resty_inspect_hooks.lua:4: foo traceback +stack traceback: + /opt/lua-resty-inspect/lib/resty/inspect/dbg.lua:50: in function + /opt/apisix.fork/apisix/plugins/limit-req.lua:88: in function 'phase_func' + /opt/apisix.fork/apisix/plugin.lua:900: in function 'run_plugin' + /opt/apisix.fork/apisix/init.lua:456: in function 'http_access_phase' + access_by_lua(nginx.conf:303):2: in main chunk, client: 127.0.0.1, server: _, request: "GET /get HTTP/1.1", host: "127.0.0.1:9080" +2022/09/01 00:55:52 [info] 2754534#2754534: *4070 [lua] resty_inspect_hooks.lua:5: /opt/apisix.fork/apisix/plugins/limit-req.lua:88 (phase_func), client: 127.0.0.1, server: _, request: "GET /get HTTP/1.1", host: "127.0.0.1:9080" +2022/09/01 00:55:52 [info] 2754534#2754534: *4070 [lua] resty_inspect_hooks.lua:6: conf_key=remote_addr, client: 127.0.0.1, server: _, request: "GET /get HTTP/1.1", host: "127.0.0.1:9080" +``` ## Disable plugin From a19b580b384a9485b95e0a5a570dd8060c343b68 Mon Sep 17 00:00:00 2001 From: jinhua luo Date: Wed, 7 Dec 2022 16:57:35 +0800 Subject: [PATCH 18/26] Update inspect.md --- docs/en/latest/plugins/inspect.md | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/docs/en/latest/plugins/inspect.md b/docs/en/latest/plugins/inspect.md index 7f90c334ee18..fd667fe1a7b1 100644 --- a/docs/en/latest/plugins/inspect.md +++ b/docs/en/latest/plugins/inspect.md @@ -81,8 +81,8 @@ The `info` is a hash table which contains below keys: | Name | Type | Required | Default | Description | |--------------------|---------|----------|---------|------------------------------------------------------------------------------------------------| -| delay | integer | False | 3 | Time in seconds specifying how often to rotate the check the hooks file. | -| hooks_file | string | False | "/var/run/apisix_inspect_hooks.lua" | lua file to define hooks. | +| delay | integer | False | 3 | Time in seconds specifying how often to check the hooks file. | +| hooks_file | string | False | "/var/run/apisix_inspect_hooks.lua" | lua file to define hooks, which could be a link file. | ## Enabling the Plugin @@ -98,10 +98,6 @@ plugin_attr: hooks_file: "/var/run/apisix_inspect_hooks.lua" ``` -## Configuration description - -TODO. - ## Example usage ```bash @@ -128,7 +124,7 @@ curl http://127.0.0.1:9180/apisix/admin/routes/test_limit_req -H 'X-API-KEY: edd }' # create a hooks file to set a test breakpoint -# Note that the breakpint is associated with the line number, +# Note that the breakpoint is associated with the line number, # so if the lua code changes, you need to adjust the line number in the hooks file cat </tmp/hooks.lua local dbg = require "resty.inspect.dbg" @@ -167,9 +163,9 @@ stack traceback: ## Disable plugin -To remove the `log-rotate` Plugin, you can remove it from your configuration file (`conf/config.yaml`): +To remove the `inspect` Plugin, you can remove it from your configuration file (`conf/config.yaml`): ```yaml title="conf/config.yaml" plugins: - # - log-rotate + # - inspect ``` From 2a4aac6c593da380b1cd71a77ff6f6e5305e547d Mon Sep 17 00:00:00 2001 From: kingluo Date: Thu, 8 Dec 2022 17:53:52 +0800 Subject: [PATCH 19/26] fix PR --- Makefile | 6 +++--- apisix/inspect/init.lua | 21 +++++++-------------- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 5da1cf3d73d5..8aade9bdd4c6 100644 --- a/Makefile +++ b/Makefile @@ -294,9 +294,6 @@ install: runtime $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/http $(ENV_INSTALL) apisix/http/*.lua $(ENV_INST_LUADIR)/apisix/http/ - $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/inspect - $(ENV_INSTALL) apisix/inspect/*.lua $(ENV_INST_LUADIR)/apisix/inspect/ - $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/http/router $(ENV_INSTALL) apisix/http/router/*.lua $(ENV_INST_LUADIR)/apisix/http/router/ @@ -318,6 +315,9 @@ install: runtime $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/ip-restriction $(ENV_INSTALL) apisix/plugins/ip-restriction/*.lua $(ENV_INST_LUADIR)/apisix/plugins/ip-restriction/ + $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/inspect + $(ENV_INSTALL) apisix/inspect/*.lua $(ENV_INST_LUADIR)/apisix/inspect/ + $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/limit-conn $(ENV_INSTALL) apisix/plugins/limit-conn/*.lua $(ENV_INST_LUADIR)/apisix/plugins/limit-conn/ diff --git a/apisix/inspect/init.lua b/apisix/inspect/init.lua index 79d1c075e74a..ccc6d24177f4 100644 --- a/apisix/inspect/init.lua +++ b/apisix/inspect/init.lua @@ -17,7 +17,7 @@ local core = require("apisix.core") local dbg = require("apisix.inspect.dbg") local lfs = require("lfs") -local cjson = require("cjson") +local pl_path = require("pl.path") local io = io local table_insert = table.insert local pcall = pcall @@ -38,16 +38,6 @@ local last_report_time = 0 local REPORT_INTERVAL = 30 -- secs -local function is_file_exists(file) - local f = io.open(file, "r") - if f then - io.close(f) - return true - else - return false - end -end - local function run_lua_file(file) local f, err = io.open(file, "rb") if not f then @@ -55,6 +45,9 @@ local function run_lua_file(file) end local code = f:read("*all") f:close() + if code == nil then + return false, "cannot read hooks file" + end local func, err = loadstring(code) if not func then return false, err @@ -64,14 +57,14 @@ local function run_lua_file(file) end local function setup_hooks(file) - if is_file_exists(file) then + if pl_path.exists(file) then dbg.unset_all() local _, err = pcall(run_lua_file, file) local hooks = {} for _, hook in ipairs(dbg.hooks()) do table_insert(hooks, hook.key) end - core.log.info("set hooks: err=", err, ", hooks=", cjson.encode(hooks)) + core.log.info("set hooks: err=", err, ", hooks=", core.json.encode(hooks)) end end @@ -99,7 +92,7 @@ local function reload_hooks(premature, delay, file) for _, hook in ipairs(dbg.hooks()) do table_insert(hooks, hook.key) end - core.log.info("alive hooks: ", cjson.encode(hooks)) + core.log.info("alive hooks: ", core.json.encode(hooks)) last_report_time = ts end end From cf0038cf86c6090ba0d4859f29c796adedcf36c5 Mon Sep 17 00:00:00 2001 From: kingluo Date: Fri, 9 Dec 2022 11:23:51 +0800 Subject: [PATCH 20/26] fix PR --- apisix/plugins/inspect.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apisix/plugins/inspect.lua b/apisix/plugins/inspect.lua index 1602ad5fcb93..85410a128fbf 100644 --- a/apisix/plugins/inspect.lua +++ b/apisix/plugins/inspect.lua @@ -50,7 +50,7 @@ function _M.init() delay = attr.delay hooks_file = attr.hooks_file end - ngx.log(ngx.INFO, "delay=", delay, ", hooks_file=", hooks_file) + core.log.info("delay=", delay, ", hooks_file=", hooks_file) return inspect.init(delay, hooks_file) end From 9c4c58ceb502a23a20a2bc9dcd2a78efd047c5b7 Mon Sep 17 00:00:00 2001 From: kingluo Date: Fri, 9 Dec 2022 12:00:06 +0800 Subject: [PATCH 21/26] fix PR --- apisix/plugins/inspect.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/apisix/plugins/inspect.lua b/apisix/plugins/inspect.lua index 85410a128fbf..19f50c79e55b 100644 --- a/apisix/plugins/inspect.lua +++ b/apisix/plugins/inspect.lua @@ -17,7 +17,6 @@ local core = require("apisix.core") local plugin = require("apisix.plugin") local inspect = require("apisix.inspect") -local ngx = ngx local plugin_name = "inspect" From 9792e4c2e0a1b140036e241d01769b91f8357737 Mon Sep 17 00:00:00 2001 From: kingluo Date: Fri, 9 Dec 2022 15:39:25 +0800 Subject: [PATCH 22/26] fix PR --- apisix/inspect/dbg.lua | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/apisix/inspect/dbg.lua b/apisix/inspect/dbg.lua index e732e05f8bf1..9661a53de16a 100644 --- a/apisix/inspect/dbg.lua +++ b/apisix/inspect/dbg.lua @@ -74,10 +74,8 @@ local function hook(evt, arg) local r1, r2_or_err = pcall(filter_func, info) if not r1 then core.log.error("inspect: pcall filter_func:", r2_or_err) - end - - -- if filter_func returns false, keep the hook - if r1 and r2_or_err == false then + elseif r2_or_err == false then + -- if filter_func returns false, keep the hook table_insert(hooks2, hook) end else From f4fe75f0f3855883b46a61d078018a4396639f13 Mon Sep 17 00:00:00 2001 From: kingluo Date: Mon, 12 Dec 2022 15:11:20 +0800 Subject: [PATCH 23/26] fix PR --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 8aade9bdd4c6..0eb4864aa49a 100644 --- a/Makefile +++ b/Makefile @@ -300,6 +300,9 @@ install: runtime $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/include/apisix/model $(ENV_INSTALL) apisix/include/apisix/model/*.proto $(ENV_INST_LUADIR)/apisix/include/apisix/model/ + $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/inspect + $(ENV_INSTALL) apisix/inspect/*.lua $(ENV_INST_LUADIR)/apisix/inspect/ + $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins $(ENV_INSTALL) apisix/plugins/*.lua $(ENV_INST_LUADIR)/apisix/plugins/ @@ -315,9 +318,6 @@ install: runtime $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/ip-restriction $(ENV_INSTALL) apisix/plugins/ip-restriction/*.lua $(ENV_INST_LUADIR)/apisix/plugins/ip-restriction/ - $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/inspect - $(ENV_INSTALL) apisix/inspect/*.lua $(ENV_INST_LUADIR)/apisix/inspect/ - $(ENV_INSTALL) -d $(ENV_INST_LUADIR)/apisix/plugins/limit-conn $(ENV_INSTALL) apisix/plugins/limit-conn/*.lua $(ENV_INST_LUADIR)/apisix/plugins/limit-conn/ From 483ff23a59f3d9f3d9658e397914bfdfc017887b Mon Sep 17 00:00:00 2001 From: kingluo Date: Tue, 13 Dec 2022 16:02:37 +0800 Subject: [PATCH 24/26] fix PR --- apisix/inspect/dbg.lua | 2 +- apisix/inspect/init.lua | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/apisix/inspect/dbg.lua b/apisix/inspect/dbg.lua index 9661a53de16a..a8a619a261ac 100644 --- a/apisix/inspect/dbg.lua +++ b/apisix/inspect/dbg.lua @@ -38,7 +38,7 @@ function _M.getname(n) end end -local function hook(evt, arg) +local function hook(_, arg) local level = 2 local finfo = debug.getinfo(level, "nSlf") local key = finfo.source .. "#" .. arg diff --git a/apisix/inspect/init.lua b/apisix/inspect/init.lua index ccc6d24177f4..a33c30ec3c5b 100644 --- a/apisix/inspect/init.lua +++ b/apisix/inspect/init.lua @@ -25,6 +25,7 @@ local ipairs = ipairs local os = os local ngx = ngx local loadstring = loadstring +local format = string.format local _M = {} @@ -43,10 +44,10 @@ local function run_lua_file(file) if not f then return false, err end - local code = f:read("*all") + local code, err = f:read("*all") f:close() if code == nil then - return false, "cannot read hooks file" + return false, format("cannot read hooks file: %s", err) end local func, err = loadstring(code) if not func then @@ -64,7 +65,7 @@ local function setup_hooks(file) for _, hook in ipairs(dbg.hooks()) do table_insert(hooks, hook.key) end - core.log.info("set hooks: err=", err, ", hooks=", core.json.encode(hooks)) + core.log.info("set hooks: err: ", err, ", hooks: ", core.json.delay_encode(hooks)) end end From 1916a2e2528304bf1804ccaec6ff9538a9e5e2eb Mon Sep 17 00:00:00 2001 From: kingluo Date: Thu, 15 Dec 2022 15:40:06 +0800 Subject: [PATCH 25/26] fix PR --- conf/config-default.yaml | 2 +- docs/en/latest/plugins/inspect.md | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/conf/config-default.yaml b/conf/config-default.yaml index 3947e9bf5e1f..cc9d0fe60b24 100755 --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -557,7 +557,7 @@ plugin_attr: # redirect: # https_port: 8443 # the default port for use by HTTP redirects to HTTPS inspect: - delay: 3 + delay: 3 # in seconds hooks_file: "/var/run/apisix_inspect_hooks.lua" deployment: diff --git a/docs/en/latest/plugins/inspect.md b/docs/en/latest/plugins/inspect.md index fd667fe1a7b1..43fa0fd346a4 100644 --- a/docs/en/latest/plugins/inspect.md +++ b/docs/en/latest/plugins/inspect.md @@ -3,8 +3,8 @@ title: inspect keywords: - APISIX - Plugin - - inspect - - inspect + - Inspect + - Dynamic Lua Debugging description: This document contains information about the Apache APISIX inspect Plugin. --- @@ -29,7 +29,7 @@ description: This document contains information about the Apache APISIX inspect ## Description -It's useful to set arbitrary breakpoint in any lua file to inspect the context information, +It's useful to set arbitrary breakpoint in any Lua file to inspect the context information, e.g. print local variables if some condition satisfied. In this way, you don't need to modify the source code of your project, and just get diagnose information @@ -40,13 +40,13 @@ The breakpoint could be at any position within the function. The function could ## Features -* set breakpoint at any position -* dynamic breakpoint -* Customized breakpoint handler -* you could define one-shot breakpoint -* work for jit compiled function -* if function reference specified, then performance impact is only bound to that function (JIT compiled code will not trigger debug hook, so they would run fast even if hook is enabled) -* if all breakpoints deleted, jit could recover +* Set breakpoint at any position +* Dynamic breakpoint +* customized breakpoint handler +* You could define one-shot breakpoint +* Work for jit compiled function +* If function reference specified, then performance impact is only bound to that function (JIT compiled code will not trigger debug hook, so they would run fast even if hook is enabled) +* If all breakpoints deleted, jit could recover ## Operation Graph @@ -65,7 +65,7 @@ global function, you should set it that function reference, then only the jit ca be flushed, and it would not affect other caches to avoid slowing down other parts of the program. * If the breakpointis related to local function or anonymous function, -then you have to set it to `nil` (because no way to get function reference), which would flush the whole jit cache of lua vm. +then you have to set it to `nil` (because no way to get function reference), which would flush the whole jit cache of Lua vm. You attach a `filter_func` function of the breakpoint, the function takes the `info` as argument and returns true of false to determine whether the breakpoint would be removed. You could setup one-shot breakpoint @@ -82,7 +82,7 @@ The `info` is a hash table which contains below keys: | Name | Type | Required | Default | Description | |--------------------|---------|----------|---------|------------------------------------------------------------------------------------------------| | delay | integer | False | 3 | Time in seconds specifying how often to check the hooks file. | -| hooks_file | string | False | "/var/run/apisix_inspect_hooks.lua" | lua file to define hooks, which could be a link file. | +| hooks_file | string | False | "/var/run/apisix_inspect_hooks.lua" | Lua file to define hooks, which could be a link file. | ## Enabling the Plugin @@ -125,7 +125,7 @@ curl http://127.0.0.1:9180/apisix/admin/routes/test_limit_req -H 'X-API-KEY: edd # create a hooks file to set a test breakpoint # Note that the breakpoint is associated with the line number, -# so if the lua code changes, you need to adjust the line number in the hooks file +# so if the Lua code changes, you need to adjust the line number in the hooks file cat </tmp/hooks.lua local dbg = require "resty.inspect.dbg" From 62453365868db7af8815fd6a24a9fc9945e1d343 Mon Sep 17 00:00:00 2001 From: kingluo Date: Thu, 15 Dec 2022 18:24:11 +0800 Subject: [PATCH 26/26] fix PR --- conf/config-default.yaml | 2 +- docs/en/latest/plugins/inspect.md | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/conf/config-default.yaml b/conf/config-default.yaml index cc9d0fe60b24..5cbf737353cd 100755 --- a/conf/config-default.yaml +++ b/conf/config-default.yaml @@ -558,7 +558,7 @@ plugin_attr: # https_port: 8443 # the default port for use by HTTP redirects to HTTPS inspect: delay: 3 # in seconds - hooks_file: "/var/run/apisix_inspect_hooks.lua" + hooks_file: "/usr/local/apisix/plugin_inspect_hooks.lua" deployment: role: traditional diff --git a/docs/en/latest/plugins/inspect.md b/docs/en/latest/plugins/inspect.md index 43fa0fd346a4..fad20a1bb1ba 100644 --- a/docs/en/latest/plugins/inspect.md +++ b/docs/en/latest/plugins/inspect.md @@ -82,7 +82,7 @@ The `info` is a hash table which contains below keys: | Name | Type | Required | Default | Description | |--------------------|---------|----------|---------|------------------------------------------------------------------------------------------------| | delay | integer | False | 3 | Time in seconds specifying how often to check the hooks file. | -| hooks_file | string | False | "/var/run/apisix_inspect_hooks.lua" | Lua file to define hooks, which could be a link file. | +| hooks_file | string | False | "/usr/local/apisix/plugin_inspect_hooks.lua" | Lua file to define hooks, which could be a link file. Ensure only administrator could write this file, otherwise it may be a security risk. | ## Enabling the Plugin @@ -95,7 +95,7 @@ plugins: plugin_attr: inspect: delay: 3 - hooks_file: "/var/run/apisix_inspect_hooks.lua" + hooks_file: "/usr/local/apisix/plugin_inspect_hooks.lua" ``` ## Example usage @@ -126,7 +126,7 @@ curl http://127.0.0.1:9180/apisix/admin/routes/test_limit_req -H 'X-API-KEY: edd # create a hooks file to set a test breakpoint # Note that the breakpoint is associated with the line number, # so if the Lua code changes, you need to adjust the line number in the hooks file -cat </tmp/hooks.lua +cat </usr/local/apisix/example_hooks.lua local dbg = require "resty.inspect.dbg" dbg.set_hook("limit-req.lua", 88, require("apisix.plugins.limit-req").access, function(info) @@ -141,7 +141,7 @@ end) EOF # enable the hooks file -ln -sf /tmp/hooks.lua /var/run/resty_inspect_hooks.lua +ln -sf /usr/local/apisix/example_hooks.lua /usr/local/apisix/plugin_inspect_hooks.lua # check errors.log to confirm the test breakpoint is enabled 2022/09/01 00:55:38 [info] 2754534#2754534: *3700 [lua] init.lua:29: setup_hooks(): set hooks: err=nil, hooks=["limit-req.lua#88"], context: ngx.timer