feat(L4): support TLS over TCP upstream

Signed-off-by: spacewander <[email protected]>

.github/workflows/centos7-ci.yml

@@ -44,7 +44,7 @@ jobs:
       run: |
         export VERSION=${{ steps.branch_env.outputs.version }}
         sudo gem install --no-document fpm
-        git clone -b v2.6.0 https://github.com/api7/apisix-build-tools.git
+        git clone -b v2.7.0 https://github.com/api7/apisix-build-tools.git
 
         # move codes under build tool
         mkdir ./apisix-build-tools/apisix

apisix/cli/ngx_tpl.lua

@@ -148,6 +148,12 @@ stream {
 
         proxy_pass apisix_backend;
 
+        {% if use_apisix_openresty then %}
+        set $upstream_sni "apisix_backend";
+        proxy_ssl_server_name on;
+        proxy_ssl_name $upstream_sni;
+        {% end %}
+
         log_by_lua_block {
             apisix.stream_log_phase()
         }

apisix/schema_def.lua

@@ -452,7 +452,10 @@ local upstream_schema = {
         },
         scheme = {
             default = "http",
-            enum = {"grpc", "grpcs", "http", "https"}
+            enum = {"grpc", "grpcs", "http", "https", "tcp", "tls", "udp"},
+            description = "The scheme of the upstream." ..
+                " For L7 proxy, it can be one of grpc/grpcs/http/https." ..
+                " For L4 proxy, it can be one of tcp/tls/udp."
         },
         labels = labels_def,
         discovery_type = {

apisix/upstream.lua

@@ -24,6 +24,7 @@ local error = error
 local tostring = tostring
 local ipairs = ipairs
 local pairs = pairs
+local ngx_var = ngx.var
 local is_http = ngx.config.subsystem == "http"
 local upstreams
 local healthcheck
@@ -39,6 +40,19 @@ else
     end
 end
 
+local set_stream_upstream_tls
+if not is_http then
+    local ok, apisix_ngx_stream_upstream = pcall(require, "resty.apisix.stream.upstream")
+    if ok then
+        set_stream_upstream_tls = apisix_ngx_stream_upstream.set_tls
+    else
+        set_stream_upstream_tls = function ()
+            return nil, "need to build APISIX-OpenResty to support TLS over TCP upstream"
+        end
+    end
+end
+
+
 
 local HTTP_CODE_UPSTREAM_UNAVAILABLE = 503
 local _M = {}
@@ -286,6 +300,19 @@ function _M.set_by_route(route, api_ctx)
             return 503, err
         end
 
+        local scheme = up_conf.scheme
+        if scheme == "tls" then
+            local ok, err = set_stream_upstream_tls()
+            if not ok then
+                return 503, err
+            end
+
+            local sni = apisix_ssl.server_name()
+            if sni then
+                ngx_var.upstream_sni = sni
+            end
+        end
+
         return
     end
 
@@ -451,6 +478,11 @@ local function filter_upstream(value, parent)
 
     value.parent = parent
 
+    if not is_http and value.scheme == "http" then
+        -- For L4 proxy, the default scheme is "tcp"
+        value.scheme = "tcp"
+    end
+
     if not value.nodes then
         return
     end

docs/en/latest/admin-api.md

@@ -554,7 +554,7 @@ In addition to the basic complex equalization algorithm selection, APISIX's Upst
 |desc            |optional|upstream usage scenarios, and more.||
 |pass_host       |optional| `host` option when the request is sent to the upstream, can be one of [`pass`, `node`, `rewrite`], the default option is `pass`. `pass`: Pass the client's host transparently to the upstream; `node`: Use the host configured in the node of `upstream`; `rewrite`: Use the value of the configuration `upstream_host`.||
 |upstream_host   |optional|Specify the host of the upstream request. This option is only valid if the `pass_host` is `rewrite`.||
-|scheme          |optional |The scheme used when talk with the upstream. The value is one of ['http', 'https', 'grpc', 'grpcs'], default to 'http'.||
+|scheme          |optional |The scheme used when talk with the upstream. For L7 proxy, the value is one of ['http', 'https', 'grpc', 'grpcs']. For L7 proxy, the value is one of ['tcp', 'udp', 'tls']. Default to 'http'. See below for more details.||
 |labels          |optional |Key/value pairs to specify attributes|{"version":"v2","build":"16","env":"production"}|
 |create_time     |optional| epoch timestamp in second, like `1602883670`, will be created automatically if missing|1602883670|
 |update_time     |optional| epoch timestamp in second, like `1602883670`, will be created automatically if missing|1602883670|
@@ -581,13 +581,15 @@ In addition to the basic complex equalization algorithm selection, APISIX's Upst
 1. When it is `vars_combinations`, the `key` is required. The `key` can be any [Nginx builtin variables](http://nginx.org/en/docs/varindex.html) combinations, such as `$request_uri$remote_addr`.
 1. If there is no value for either `hash_on` or `key`, `remote_addr` will be used as key.
 
+Features below require APISIX to run on [APISIX-OpenResty](./how-to-build.md#step-6-build-openresty-for-apache-apisix):
+
+The `scheme` can be set to `tls`, which actually means "TLS over TCP".
+
 `tls.client_cert/key` can be used to communicate with upstream via mTLS.
 Their formats are the same as SSL's `cert` and `key` fields.
-This feature requires APISIX to run on [APISIX-OpenResty](./how-to-build.md#step-6-build-openresty-for-apache-apisix).
 
 `keepalive_pool` allows the upstream to have its separate connection pool.
 Its children fields, like `requests`, can be used to configure the upstream keepalive options.
-This feature requires APISIX to run on [APISIX-OpenResty](./how-to-build.md#step-6-build-openresty-for-apache-apisix).
 
 **Config Example:**
 

docs/en/latest/stream-proxy.md

@@ -166,9 +166,9 @@ Let's take another real world example:
 
 Read [Admin API's Stream Route section](./admin-api.md#stream-route) for the complete options list.
 
-## Accept TLS over TCP
+## Accept TLS over TCP connection
 
-APISIX can accept TLS over TCP.
+APISIX can accept TLS over TCP connection.
 
 First of all, we need to enable TLS for the TCP address:
 
@@ -189,7 +189,6 @@ Third, we need to configure a stream route to match and proxy it to the upstream
 ```shell
 curl http://127.0.0.1:9080/apisix/admin/stream_routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
 {
-    "remote_addr": "127.0.0.1",
     "upstream": {
         "nodes": {
             "127.0.0.1:1995": 1
@@ -215,3 +214,24 @@ curl http://127.0.0.1:9080/apisix/admin/stream_routes/1 -H 'X-API-KEY: edd1c9f03
 ```
 
 In this case, a connection handshaked with SNI `a.test.com` will be proxied to `127.0.0.1:5991`.
+
+## Proxy to TLS over TCP upstream
+
+APISIX also supports proxying to TLS over TCP upstream.
+
+```shell
+curl http://127.0.0.1:9080/apisix/admin/stream_routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+    "upstream": {
+        "scheme": "tls",
+        "nodes": {
+            "127.0.0.1:1995": 1
+        },
+        "type": "roundrobin"
+    }
+}'
+```
+
+By setting the `scheme` to "tls", APISIX will do TLS handshake with the upstream.
+
+When the client is also speaking TLS over TCP, the SNI from the client will pass through to the upstream. Otherwise, a dummy SNI "apisix_backend" will be used.

docs/zh/latest/admin-api.md

@@ -559,7 +559,7 @@ APISIX 的 Upstream 除了基本的负载均衡算法选择外，还支持对上
 | desc           | 可选                               | 辅助           | 上游服务描述、使用场景等。                                                                                                                                                                                                                                                                                                                                  |                                                  |
 | pass_host      | 可选                               | 枚举           | 请求发给上游时的 host 设置选型。 [`pass`，`node`，`rewrite`] 之一，默认是`pass`。`pass`: 将客户端的 host 透传给上游； `node`: 使用 `upstream`  node 中配置的 host； `rewrite`: 使用配置项 `upstream_host` 的值。                                                                                                                                                                        |                                                  |
 | upstream_host  | 可选                               | 辅助           | 指定上游请求的 host，只在 `pass_host` 配置为 `rewrite` 时有效。                                                                                                                                                                                                                                                                                                                  |                                                  |
-| scheme         | 可选                               | 辅助           | 跟上游通信时使用的 scheme。需要是 ['http', 'https', 'grpc', 'grpcs'] 其中的一个，默认是 'http'。                                                                                                                                                                                                                                                            |
+| scheme         | 可选                               | 辅助           | 跟上游通信时使用的 scheme。对于 7 层代理，需要是 ['http', 'https', 'grpc', 'grpcs'] 其中的一个。对于 4 层代理，需要是 ['tcp', 'udp', 'tls'] 其中的一个。默认是 'http'。细节见下文。                                                                                                                                                                                                                                                           |
 | labels         | 可选                               | 匹配规则       | 标识附加属性的键值对                                                                                                                                                                                                                                                                                                                                        | {"version":"v2","build":"16","env":"production"} |
 | create_time    | 可选                               | 辅助           | 单位为秒的 epoch 时间戳，如果不指定则自动创建                                                                                                                                                                                                                                                                                                               | 1602883670                                       |
 | update_time    | 可选                               | 辅助           | 单位为秒的 epoch 时间戳，如果不指定则自动创建                                                                                                                                                                                                                                                                                                               | 1602883670                                       |
@@ -585,9 +585,12 @@ APISIX 的 Upstream 除了基本的负载均衡算法选择外，还支持对上
 4. 设为 `consumer` 时，`key` 不需要设置。此时哈希算法采用的 `key` 为认证通过的 `consumer_name`。
 5. 如果指定的 `hash_on` 和 `key` 获取不到值时，就是用默认值：`remote_addr`。
 
+以下特性需要 APISIX 运行于 [APISIX-OpenResty](./how-to-build.md#步骤6：为-Apache-APISIX-构建-OpenResty)：
+
+`scheme` 可以设置成 `tls`，表示 "TLS over TCP"。
+
 `tls.client_cert/key` 可以用来跟上游进行 mTLS 通信。
 他们的格式和 SSL 对象的 `cert` 和 `key` 一样。
-这个特性需要 APISIX 运行于 [APISIX-OpenResty](./how-to-build.md#步骤6：为-Apache-APISIX-构建-OpenResty)。
 
 `keepalive_pool` 允许 upstream 对象有自己单独的连接池。
 它下属的字段，比如 `requests`，可以用了配置上游连接保持的参数。

docs/zh/latest/stream-proxy.md

@@ -163,9 +163,9 @@ curl http://127.0.0.1:9080/apisix/admin/stream_routes/1 -H 'X-API-KEY: edd1c9f03
 
 完整的匹配选项列表参见 [Admin API 的 Stream Route](./admin-api.md#stream-route)。
 
-## 接收 TLS over TCP
+## 接收 TLS over TCP 连接
 
-APISIX 支持接收 TLS over TCP。
+APISIX 支持接收 TLS over TCP 连接。
 
 首先，我们需要给对应的 TCP 地址启用 TLS：
 
@@ -186,7 +186,6 @@ mTLS 也是支持的，参考 [保护路由](./mtls.md#保护路由)。
 ```shell
 curl http://127.0.0.1:9080/apisix/admin/stream_routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
 {
-    "remote_addr": "127.0.0.1",
     "upstream": {
         "nodes": {
             "127.0.0.1:1995": 1
@@ -212,3 +211,24 @@ curl http://127.0.0.1:9080/apisix/admin/stream_routes/1 -H 'X-API-KEY: edd1c9f03
 ```
 
 在这里，握手时发送 SNI `a.test.com` 的连接会被代理到 `127.0.0.1:5991`。
+
+## 代理到 TLS over TCP 上游
+
+APISIX 还支持代理到 TLS over TCP 上游。
+
+```shell
+curl http://127.0.0.1:9080/apisix/admin/stream_routes/1 -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1' -X PUT -d '
+{
+    "upstream": {
+        "scheme": "tls",
+        "nodes": {
+            "127.0.0.1:1995": 1
+        },
+        "type": "roundrobin"
+    }
+}'
+```
+
+通过设置 `scheme` 为 tls，APISIX 将与上游进行 TLS 握手。
+
+当客户端也使用 TLS over TCP，客户端发送的 SNI 将传递给上游。否则，将使用一个假的 SNI "apisix_backend"。

t/APISIX.pm

@@ -251,6 +251,8 @@ _EOC_
     if ($stream_tls_request) {
         # generate a springboard to send tls stream request
         $block->set_value("stream_conf_enable", 1);
+        # avoid conflict with stream_enable
+        $block->set_value("stream_enable");
         $block->set_value("request", "GET /stream_tls_request");
 
         my $sni = "nil";
@@ -414,7 +416,17 @@ _EOC_
     }
 
     proxy_pass apisix_backend;
+_EOC_
+
+    if ($version =~ m/\/apisix-nginx-module/) {
+        $stream_server_config .= <<_EOC_;
+    proxy_ssl_server_name on;
+    proxy_ssl_name \$upstream_sni;
+    set \$upstream_sni "apisix_backend";
+_EOC_
+    }
 
+    $stream_server_config .= <<_EOC_;
     log_by_lua_block {
         apisix.stream_log_phase()
     }