change: move etcd conf under deployment (#7860)

apisix/cli/snippet.lua

@@ -39,8 +39,7 @@ function _M.generate_conf_server(env, conf)
     if servers[1]:find(prefix, 1, true) then
         enable_https = true
     end
-    -- there is not a compatible way to verify upstream TLS like the one we do in cosocket
-    -- so here we just ignore it as the verification is already done in the init phase
+
     for i, s in ipairs(servers) do
         if (s:find(prefix, 1, true) ~= nil) ~= enable_https then
             return nil, "all nodes in the etcd cluster should enable/disable TLS together"
@@ -113,6 +112,11 @@ function _M.generate_conf_server(env, conf)
             proxy_ssl_protocols TLSv1.2 TLSv1.3;
             proxy_ssl_server_name on;
 
+            {% if etcd_tls_verify then %}
+            proxy_ssl_verify on;
+            proxy_ssl_trusted_certificate {* ssl_trusted_certificate *};
+            {% end %}
+
             {% if sni then %}
             proxy_ssl_name {* sni *};
             {% else %}
@@ -144,9 +148,21 @@ function _M.generate_conf_server(env, conf)
     local tls = etcd.tls
     local client_cert
     local client_cert_key
-    if tls and tls.cert then
-        client_cert = pl_path.abspath(tls.cert)
-        client_cert_key = pl_path.abspath(tls.key)
+    local ssl_trusted_certificate
+    local etcd_tls_verify
+    if tls then
+        if tls.cert then
+            client_cert = pl_path.abspath(tls.cert)
+            client_cert_key = pl_path.abspath(tls.key)
+        end
+
+        etcd_tls_verify = tls.verify
+        if enable_https and etcd_tls_verify then
+            if not conf.apisix.ssl.ssl_trusted_certificate then
+                return nil, "should set ssl_trusted_certificate if etcd tls verify is enabled"
+            end
+            ssl_trusted_certificate = pl_path.abspath(conf.apisix.ssl.ssl_trusted_certificate)
+        end
     end
 
     return conf_render({
@@ -157,6 +173,8 @@ function _M.generate_conf_server(env, conf)
         client_cert = client_cert,
         client_cert_key = client_cert_key,
         trusted_ca_cert = trusted_ca_cert,
+        etcd_tls_verify = etcd_tls_verify,
+        ssl_trusted_certificate = ssl_trusted_certificate,
     })
 end
 

apisix/core/config_etcd.lua

@@ -507,7 +507,7 @@ do
         end
 
         local err
-        etcd_cli, err = etcd_apisix.new()
+        etcd_cli, err = etcd_apisix.switch_proxy()
         return etcd_cli, err
     end
 end

apisix/core/etcd.lua

@@ -31,6 +31,7 @@ local string            = string
 local tonumber          = tonumber
 local ngx_config_prefix = ngx.config.prefix()
 local ngx_socket_tcp    = ngx.socket.tcp
+local ngx_get_phase     = ngx.get_phase
 
 
 local is_http = ngx.config.subsystem == "http"
@@ -157,7 +158,7 @@ _M.new = new
 -- @treturn table|nil the etcd client, or nil if failed.
 -- @treturn string|nil the configured prefix of etcd keys, or nil if failed.
 -- @treturn nil|string the error message.
-function _M.new_without_proxy()
+local function new_without_proxy()
     local local_conf, err = fetch_local_conf()
     if not local_conf then
         return nil, nil, err
@@ -166,8 +167,32 @@ function _M.new_without_proxy()
     local etcd_conf = clone_tab(local_conf.etcd)
     return _new(etcd_conf)
 end
+_M.new_without_proxy = new_without_proxy
 
 
+local function switch_proxy()
+    if ngx_get_phase() == "init" or ngx_get_phase() == "init_worker" then
+        return new_without_proxy()
+    end
+
+    local etcd_cli, prefix, err = new()
+    if not etcd_cli or err then
+        return etcd_cli, prefix, err
+    end
+
+    if not etcd_cli.unix_socket_proxy then
+        return etcd_cli, prefix, err
+    end
+    local sock = ngx_socket_tcp()
+    local ok = sock:connect(etcd_cli.unix_socket_proxy)
+    if not ok then
+        return new_without_proxy()
+    end
+
+    return etcd_cli, prefix, err
+end
+_M.switch_proxy = switch_proxy
+
 -- convert ETCD v3 entry to v2 one
 local function kvs_to_node(kvs)
     local node = {}
@@ -281,7 +306,7 @@ end
 
 
 function _M.get(key, is_dir)
-    local etcd_cli, prefix, err = new()
+    local etcd_cli, prefix, err = switch_proxy()
     if not etcd_cli then
         return nil, err
     end
@@ -300,7 +325,7 @@ end
 
 
 local function set(key, value, ttl)
-    local etcd_cli, prefix, err = new()
+    local etcd_cli, prefix, err = switch_proxy()
     if not etcd_cli then
         return nil, err
     end
@@ -344,7 +369,7 @@ _M.set = set
 
 
 function _M.atomic_set(key, value, ttl, mod_revision)
-    local etcd_cli, prefix, err = new()
+    local etcd_cli, prefix, err = switch_proxy()
     if not etcd_cli then
         return nil, err
     end
@@ -403,7 +428,7 @@ end
 
 
 function _M.push(key, value, ttl)
-    local etcd_cli, _, err = new()
+    local etcd_cli, _, err = switch_proxy()
     if not etcd_cli then
         return nil, err
     end
@@ -435,7 +460,7 @@ end
 
 
 function _M.delete(key)
-    local etcd_cli, prefix, err = new()
+    local etcd_cli, prefix, err = switch_proxy()
     if not etcd_cli then
         return nil, err
     end
@@ -473,7 +498,7 @@ end
 -- --   etcdserver = "3.5.0"
 -- -- }
 function _M.server_version()
-    local etcd_cli, _, err = new()
+    local etcd_cli, _, err = switch_proxy()
     if not etcd_cli then
         return nil, err
     end
@@ -483,7 +508,7 @@ end
 
 
 function _M.keepalive(id)
-    local etcd_cli, _, err = new()
+    local etcd_cli, _, err = switch_proxy()
     if not etcd_cli then
         return nil, err
     end

conf/config-default.yaml

@@ -275,27 +275,6 @@ nginx_config:                     # config for render the template to generate n
       kubernetes: 1m
       tars: 1m
 
-etcd:
-  host:                           # it's possible to define multiple etcd hosts addresses of the same etcd cluster.
-    - "http://127.0.0.1:2379"     # multiple etcd address, if your etcd cluster enables TLS, please use https scheme,
-                                  # e.g. https://127.0.0.1:2379.
-  prefix: /apisix                 # apisix configurations prefix
-  #timeout: 30                    # 30 seconds
-  #resync_delay: 5                # when sync failed and a rest is needed, resync after the configured seconds plus 50% random jitter
-  #health_check_timeout: 10       # etcd retry the unhealthy nodes after the configured seconds
-  startup_retry: 2                # the number of retry to etcd during the startup, default to 2
-  #user: root                     # root username for etcd
-  #password: 5tHkHhYkjr6cQY       # root password for etcd
-  tls:
-    # To enable etcd client certificate you need to build APISIX-Base, see
-    # https://apisix.apache.org/docs/apisix/FAQ#how-do-i-build-the-apisix-base-environment
-    #cert: /path/to/cert          # path of certificate used by the etcd client
-    #key: /path/to/key            # path of key used by the etcd client
-
-    verify: true                  # whether to verify the etcd endpoint certificate when setup a TLS connection to etcd,
-                                  # the default value is true, e.g. the certificate will be verified strictly.
-    #sni:                         # the SNI for etcd TLS requests. If missed, the host part of the URL will be used.
-
 # HashiCorp Vault storage backend for sensitive data retrieval. The config shows an example of what APISIX expects if you
 # wish to integrate Vault for secret (sensetive string, public private keys etc.) retrieval. APISIX communicates with Vault
 # server HTTP APIs. By default, APISIX doesn't need this configuration.
@@ -560,13 +539,27 @@ plugin_attr:
 #  redirect:
 #    https_port: 8443   # the default port for use by HTTP redirects to HTTPS
 
-#deployment:
-#    role: traditional
-#    role_traditional:
-#        config_provider: etcd
-#    etcd:
-#        host:                           # it's possible to define multiple etcd hosts addresses of the same etcd cluster.
-#          - "http://127.0.0.1:2379"     # multiple etcd address, if your etcd cluster enables TLS, please use https scheme,
-#                                        # e.g. https://127.0.0.1:2379.
-#        prefix: /apisix                 # configuration prefix in etcd
-#        timeout: 30                     # 30 seconds
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:                           # it's possible to define multiple etcd hosts addresses of the same etcd cluster.
+      - "http://127.0.0.1:2379"     # multiple etcd address, if your etcd cluster enables TLS, please use https scheme,
+                                    # e.g. https://127.0.0.1:2379.
+    prefix: /apisix                 # configuration prefix in etcd
+    timeout: 30                     # 30 seconds
+    #resync_delay: 5                # when sync failed and a rest is needed, resync after the configured seconds plus 50% random jitter
+    #health_check_timeout: 10       # etcd retry the unhealthy nodes after the configured seconds
+    startup_retry: 2                # the number of retry to etcd during the startup, default to 2
+    #user: root                     # root username for etcd
+    #password: 5tHkHhYkjr6cQY       # root password for etcd
+    tls:
+      # To enable etcd client certificate you need to build APISIX-Base, see
+      # https://apisix.apache.org/docs/apisix/FAQ#how-do-i-build-the-apisix-base-environment
+      #cert: /path/to/cert          # path of certificate used by the etcd client
+      #key: /path/to/key            # path of key used by the etcd client
+
+      verify: true                  # whether to verify the etcd endpoint certificate when setup a TLS connection to etcd,
+                                    # the default value is true, e.g. the certificate will be verified strictly.
+      #sni:                         # the SNI for etcd TLS requests. If missed, the host part of the URL will be used.

conf/config.yaml

@@ -17,13 +17,21 @@
 # If you want to set the specified configuration value, you can set the new
 # in this file. For example if you want to specify the etcd address:
 #
-# etcd:
+# deployment:
+#   role: traditional
+#   role_traditional:
+#     config_provider: etcd
+#   etcd:
 #     host:
 #       - http://127.0.0.1:2379
 #
 # To configure via environment variables, you can use `${{VAR}}` syntax. For instance:
 #
-# etcd:
+# deployment:
+#   role: traditional
+#   role_traditional:
+#     config_provider: etcd
+#   etcd:
 #     host:
 #       - http://${{ETCD_HOST}}:2379
 #
@@ -34,7 +42,11 @@
 # Also, If you want to use default value when the environment variable not set,
 # Use `${{VAR:=default_value}}` instead. For instance:
 #
-# etcd:
+# deployment:
+#   role: traditional
+#   role_traditional:
+#     config_provider: etcd
+#   etcd:
 #     host:
 #       - http://${{ETCD_HOST:=localhost}}:2379
 #

docs/en/latest/certificate.md

@@ -246,15 +246,19 @@ apisix:
   ssl:
     ssl_trusted_certificate: /path/to/apisix.ca-bundle
 
-etcd:
-  host:
-    - "https://127.0.0.1:12379"
-    - "https://127.0.0.1:22379"
-    - "https://127.0.0.1:32379"
-  tls:
-    cert: /path/to/bar_apisix.crt
-    key: /path/to/bar_apisix.key
-    sni: etcd.cluster.dev
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "https://127.0.0.1:12379"
+      - "https://127.0.0.1:22379"
+      - "https://127.0.0.1:32379"
+    tls:
+      cert: /path/to/bar_apisix.crt
+      key: /path/to/bar_apisix.key
+      sni: etcd.cluster.dev
 ```
 
 4. Test APISIX Admin API

docs/en/latest/installation-guide.md

@@ -240,8 +240,13 @@ Now, if you decide you want to change the etcd address to `http://foo:2379`, you
 apisix:
   node_listen: 8000
 
-etcd:
-  host: "http://foo:2379"
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "http://foo:2379"
 ```
 
 :::warning

docs/en/latest/mtls.md

@@ -71,10 +71,14 @@ curl --cacert /data/certs/mtls_ca.crt --key /data/certs/mtls_client.key --cert /
 You need to build [APISIX-Base](./FAQ.md#how-do-i-build-the-apisix-base-environment) and configure `etcd.tls` section if you want APISIX to work on an etcd cluster with mTLS enabled.
 
 ```yaml
-etcd:
-  tls:
-    cert: /data/certs/etcd_client.pem       # path of certificate used by the etcd client
-    key: /data/certs/etcd_client.key        # path of key used by the etcd client
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    tls:
+      cert: /data/certs/etcd_client.pem       # path of certificate used by the etcd client
+      key: /data/certs/etcd_client.key        # path of key used by the etcd client
 ```
 
 If APISIX does not trust the CA certificate that used by etcd server, we need to set up the CA certificate.

docs/zh/latest/certificate.md

@@ -243,15 +243,19 @@ apisix:
   ssl:
     ssl_trusted_certificate: /path/to/apisix.ca-bundle
 
-etcd:
-  host:
-    - "https://127.0.0.1:12379"
-    - "https://127.0.0.1:22379"
-    - "https://127.0.0.1:32379"
-  tls:
-    cert: /path/to/bar_apisix.crt
-    key: /path/to/bar_apisix.key
-    sni: etcd.cluster.dev
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "https://127.0.0.1:12379"
+      - "https://127.0.0.1:22379"
+      - "https://127.0.0.1:32379"
+    tls:
+      cert: /path/to/bar_apisix.crt
+      key: /path/to/bar_apisix.key
+      sni: etcd.cluster.dev
 ```
 
 4. 测试 Admin API

docs/zh/latest/installation-guide.md

@@ -234,8 +234,13 @@ apisix:
 apisix:
   node_listen: 8000 # APISIX listening port
 
-etcd:
-  host: "http://foo:2379" # etcd address
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "http://foo:2379"
 ```
 
 :::warning