Merge branch 'master' into remove-conf-server

.github/workflows/doc-lint.yml

@@ -22,7 +22,7 @@ jobs:
     steps:
       - uses: actions/[email protected]
       - name: 🚀 Use Node.js
-        uses: actions/[email protected].0
+        uses: actions/[email protected].1
         with:
           node-version: "12.x"
       - run: npm install -g [email protected]

.github/workflows/license-checker.yml

@@ -32,6 +32,6 @@ jobs:
     steps:
       - uses: actions/[email protected]
       - name: Check License Header
-        uses: apache/skywalking-eyes@v0.4.0
+        uses: apache/skywalking-eyes@v0.5.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/lint.yml

@@ -33,7 +33,7 @@ jobs:
         uses: actions/[email protected]
 
       - name: Setup Nodejs env
-        uses: actions/[email protected].0
+        uses: actions/[email protected].1
         with:
           node-version: '12'
 

.licenserc.yaml

@@ -23,6 +23,8 @@ header:
 
   paths-ignore:
     - '.gitignore'
+    - '.gitattributes'
+    - '.gitmodules'
     - 'LICENSE'
     - 'NOTICE'
     - '**/*.json'
@@ -46,7 +48,11 @@ header:
     # Exclude plugin-specific configuration files
     - 't/plugin/authz-casbin'
     - 't/coredns'
+    - 't/fuzzing/requirements.txt'
+    - 't/perf/requirements.txt'
     - 'autodocs/'
     - 'docs/**/*.md'
+    - '.ignore_words'
+    - '.luacheckrc'
 
   comment: on-failure

Makefile

@@ -26,6 +26,7 @@ VERSION                ?= master
 project_name           ?= apache-apisix
 project_release_name   ?= $(project_name)-$(VERSION)-src
 
+OTEL_CONFIG ?= ./ci/pod/otelcol-contrib/data-otlp.json
 
 # Hyperconverged Infrastructure
 ENV_OS_NAME            ?= $(shell uname -s | tr '[:upper:]' '[:lower:]')
@@ -158,7 +159,7 @@ check-rust:
 
 ### deps : Installing dependencies
 .PHONY: deps
-deps: check-rust runtime
+deps: runtime
 	$(eval ENV_LUAROCKS_VER := $(shell $(ENV_LUAROCKS) --version | grep -E -o "luarocks [0-9]+."))
 	@if [ '$(ENV_LUAROCKS_VER)' = 'luarocks 3.' ]; then \
 		mkdir -p ~/.luarocks; \
@@ -450,6 +451,8 @@ compress-tar:
 .PHONY: ci-env-up
 ci-env-up:
 	@$(call func_echo_status, "$@ -> [ Start ]")
+	touch $(OTEL_CONFIG)
+	chmod 777 $(OTEL_CONFIG)
 	$(ENV_DOCKER_COMPOSE) up -d
 	@$(call func_echo_success_status, "$@ -> [ Done ]")
 
@@ -474,5 +477,6 @@ ci-env-rebuild:
 .PHONY: ci-env-down
 ci-env-down:
 	@$(call func_echo_status, "$@ -> [ Start ]")
+	rm $(OTEL_CONFIG)
 	$(ENV_DOCKER_COMPOSE) down
 	@$(call func_echo_success_status, "$@ -> [ Done ]")

apisix/admin/init.lua

@@ -376,6 +376,41 @@ local function reload_plugins(data, event, source, pid)
 end
 
 
+local function schema_validate()
+    local uri_segs = core.utils.split_uri(ngx.var.uri)
+    core.log.info("uri: ", core.json.delay_encode(uri_segs))
+
+    local seg_res = uri_segs[6]
+    local resource = resources[seg_res]
+    if not resource then
+        core.response.exit(404, {error_msg = "Unsupported resource type: ".. seg_res})
+    end
+
+    local req_body, err = core.request.get_body(MAX_REQ_BODY)
+    if err then
+        core.log.error("failed to read request body: ", err)
+        core.response.exit(400, {error_msg = "invalid request body: " .. err})
+    end
+
+    if req_body then
+        local data, err = core.json.decode(req_body)
+        if err then
+            core.log.error("invalid request body: ", req_body, " err: ", err)
+            core.response.exit(400, {error_msg = "invalid request body: " .. err,
+                                     req_body = req_body})
+        end
+
+        req_body = data
+    end
+
+    local ok, err = core.schema.check(resource.schema, req_body)
+    if ok then
+        core.response.exit(200)
+    end
+    core.response.exit(400, {error_msg = err})
+end
+
+
 local uri_route = {
     {
         paths = [[/apisix/admin]],
@@ -392,6 +427,11 @@ local uri_route = {
         methods = {"GET"},
         handler = get_plugins_list,
     },
+    {
+        paths = [[/apisix/admin/schema/validate/*]],
+        methods = {"POST"},
+        handler = schema_validate,
+    },
     {
         paths = reload_event,
         methods = {"PUT"},

apisix/cli/etcd.lua

@@ -280,103 +280,7 @@ local function prepare_dirs_via_http(yaml_conf, args, index, host, host_count)
 end
 
 
-local function grpc_request(url, yaml_conf, key)
-    local cmd
-
-    local auth = ""
-    if yaml_conf.etcd.user then
-        local user = yaml_conf.etcd.user
-        local password = yaml_conf.etcd.password
-        auth = str_format("--user=%s:%s", user, password)
-    end
-
-    if str_sub(url, 1, 8) == "https://" then
-        local host = url:sub(9)
-
-        local verify = true
-        local certificate, pkey, cafile
-        if yaml_conf.etcd.tls then
-            local cfg = yaml_conf.etcd.tls
-
-            if cfg.verify == false then
-                verify = false
-            end
-
-            certificate = cfg.cert
-            pkey = cfg.key
-
-            local apisix_ssl = yaml_conf.apisix.ssl
-            if apisix_ssl and apisix_ssl.ssl_trusted_certificate then
-                cafile = apisix_ssl.ssl_trusted_certificate
-            end
-        end
-
-        cmd = str_format(
-            "etcdctl --insecure-transport=false %s %s %s %s " ..
-                "%s --endpoints=%s put %s init_dir",
-            verify and "" or "--insecure-skip-tls-verify",
-            certificate and "--cert " .. certificate or "",
-            pkey and "--key " .. pkey or "",
-            cafile and "--cacert " .. cafile or "",
-            auth, host, key)
-    else
-        local host = url:sub(#("http://") + 1)
-
-        cmd = str_format(
-            "etcdctl %s --endpoints=%s put %s init_dir",
-            auth, host, key)
-    end
-
-    local res, err = util.execute_cmd(cmd)
-    return res, err
-end
-
-
-local function prepare_dirs_via_grpc(yaml_conf, args, index, host)
-    local is_success = true
-
-    local errmsg
-    local dirs = {}
-    for name in pairs(constants.HTTP_ETCD_DIRECTORY) do
-        dirs[name] = true
-    end
-    for name in pairs(constants.STREAM_ETCD_DIRECTORY) do
-        dirs[name] = true
-    end
-
-    for dir_name in pairs(dirs) do
-        local key =  (yaml_conf.etcd.prefix or "") .. dir_name .. "/"
-        local res, err
-        local retry_time = 0
-        while retry_time < 2 do
-            res, err = grpc_request(host, yaml_conf, key)
-            retry_time = retry_time + 1
-            if res then
-                break
-            end
-            print(str_format("Warning! Request etcd endpoint \'%s\' error, %s, retry time=%s",
-                             host, err, retry_time))
-        end
-
-        if not res then
-            errmsg = str_format("request etcd endpoint \"%s\" error, %s\n", host, err)
-            util.die(errmsg)
-        end
-
-        if args and args["verbose"] then
-            print(res)
-        end
-    end
-
-    return is_success
-end
-
-
-local function prepare_dirs(use_grpc, yaml_conf, args, index, host, host_count)
-    if use_grpc then
-        return prepare_dirs_via_grpc(yaml_conf, args, index, host)
-    end
-
+local function prepare_dirs(yaml_conf, args, index, host, host_count)
     return prepare_dirs_via_http(yaml_conf, args, index, host, host_count)
 end
 
@@ -400,8 +304,6 @@ function _M.init(env, args)
         util.die("failed to read `etcd` field from yaml file when init etcd")
     end
 
-    local etcd_conf = yaml_conf.etcd
-
     -- convert old single etcd config to multiple etcd config
     if type(yaml_conf.etcd.host) == "string" then
         yaml_conf.etcd.host = {yaml_conf.etcd.host}
@@ -477,22 +379,9 @@ function _M.init(env, args)
         util.die("the etcd cluster needs at least 50% and above healthy nodes\n")
     end
 
-    if etcd_conf.use_grpc and not env.use_apisix_base then
-        io_stderr:write("'use_grpc: true' in the etcd configuration " ..
-                        "is not supported by vanilla OpenResty\n")
-    end
-
-    local use_grpc = etcd_conf.use_grpc and env.use_apisix_base
-    if use_grpc then
-        local ok, err = util.execute_cmd("command -v etcdctl")
-        if not ok then
-            util.die("can't find etcdctl: ", err, "\n")
-        end
-    end
-
     local etcd_ok = false
     for index, host in ipairs(etcd_healthy_hosts) do
-        if prepare_dirs(use_grpc, yaml_conf, args, index, host, host_count) then
+        if prepare_dirs(yaml_conf, args, index, host, host_count) then
             etcd_ok = true
             break
         end

apisix/cli/schema.lua

@@ -62,11 +62,6 @@ local etcd_schema = {
             minimum = 1,
             description = "etcd connection timeout in seconds",
         },
-        use_grpc = {
-            type = "boolean",
-            -- TODO: set true by default in v3.2
-            default = false,
-        },
     },
     required = {"prefix", "host"}
 }

apisix/core/config_etcd.lua

@@ -362,40 +362,6 @@ local function readdir(etcd_cli, key, formatter)
 end
 
 
-local function grpc_waitdir(self, etcd_cli, key, modified_index, timeout)
-    local watching_stream = self.watching_stream
-    if not watching_stream then
-        local attr = {}
-        attr.start_revision = modified_index
-        local opts = {}
-        opts.timeout = timeout
-
-        local st, err = etcd_cli:create_grpc_watch_stream(key, attr, opts)
-        if not st then
-            log.error("create watch stream failed: ", err)
-            return nil, err
-        end
-
-        log.info("create watch stream for key: ", key, ", modified_index: ", modified_index)
-
-        self.watching_stream = st
-        watching_stream = st
-    end
-
-    return etcd_cli:read_grpc_watch_stream(watching_stream)
-end
-
-
-local function flush_watching_streams(self)
-    local etcd_cli = self.etcd_cli
-    if not etcd_cli.use_grpc then
-        return
-    end
-
-    self.watching_stream = nil
-end
-
-
 local function http_waitdir(self, etcd_cli, key, modified_index, timeout)
     if not watch_ctx.idx[key] then
         watch_ctx.idx[key] = 1
@@ -470,12 +436,7 @@ local function waitdir(self)
         return nil, "not inited"
     end
 
-    local res, err
-    if etcd_cli.use_grpc then
-        res, err = grpc_waitdir(self, etcd_cli, key, modified_index, timeout)
-    else
-        res, err = http_waitdir(self, etcd_cli, key, modified_index, timeout)
-    end
+    local res, err = http_waitdir(self, etcd_cli, key, modified_index, timeout)
 
     if not res then
         -- log.error("failed to get key from etcd: ", err)
@@ -620,13 +581,9 @@ local function sync_data(self)
         return nil, "missing 'key' arguments"
     end
 
-    if not self.etcd_cli.use_grpc then
-        init_watch_ctx(self.key)
-    end
+    init_watch_ctx(self.key)
 
     if self.need_reload then
-        flush_watching_streams(self)
-
         local res, err = readdir(self.etcd_cli, self.key)
         if not res then
             return false, err
@@ -916,7 +873,6 @@ local function _automatic_fetch(premature, self)
     end
 
     if not exiting() and self.running then
-        flush_watching_streams(self)
         ngx_timer_at(0, _automatic_fetch, self)
     end
 end
@@ -1118,10 +1074,6 @@ function _M.init()
         return true
     end
 
-    if local_conf.etcd.use_grpc then
-        return true
-    end
-
     -- don't go through proxy during start because the proxy is not available
     local etcd_cli, prefix, err = etcd_apisix.new_without_proxy()
     if not etcd_cli then
@@ -1147,21 +1099,6 @@ function _M.init_worker()
         return true
     end
 
-    if not local_conf.etcd.use_grpc then
-        return true
-    end
-
-    -- don't go through proxy during start because the proxy is not available
-    local etcd_cli, prefix, err = etcd_apisix.new_without_proxy()
-    if not etcd_cli then
-        return nil, "failed to start a etcd instance: " .. err
-    end
-
-    local res, err = readdir(etcd_cli, prefix, create_formatter(prefix))
-    if not res then
-        return nil, err
-    end
-
     return true
 end