From ced897eba95b3f4501133df5a8edf6f5401dc434 Mon Sep 17 00:00:00 2001 From: spacewander Date: Wed, 24 Mar 2021 18:29:57 +0800 Subject: [PATCH 1/3] feat: support mTLS with etcd Signed-off-by: spacewander --- apisix/core/config_etcd.lua | 11 +- apisix/core/etcd.lua | 13 +- apisix/patch.lua | 18 ++- rockspec/apisix-master-0.rockspec | 2 +- t/APISIX.pm | 9 +- t/core/etcd-mtls.t | 239 ++++++++++++++++++++++++++++++ 6 files changed, 284 insertions(+), 8 deletions(-) create mode 100644 t/core/etcd-mtls.t diff --git a/apisix/core/config_etcd.lua b/apisix/core/config_etcd.lua index 10d54f364f29..a9888a3f4bc3 100644 --- a/apisix/core/config_etcd.lua +++ b/apisix/core/config_etcd.lua @@ -506,8 +506,15 @@ do -- default to verify etcd cluster certificate etcd_conf.ssl_verify = true - if etcd_conf.tls and etcd_conf.tls.verify == false then - etcd_conf.ssl_verify = false + if etcd_conf.tls then + if etcd_conf.tls.verify == false then + etcd_conf.ssl_verify = false + end + + if etcd_conf.tls.cert then + etcd_conf.ssl_cert_path = etcd_conf.tls.cert + etcd_conf.ssl_key_path = etcd_conf.tls.key + end end local err diff --git a/apisix/core/etcd.lua b/apisix/core/etcd.lua index 6a3072853e78..6ce2742a9b61 100644 --- a/apisix/core/etcd.lua +++ b/apisix/core/etcd.lua @@ -24,6 +24,7 @@ local tonumber = tonumber local _M = {} +-- this function create the etcd client instance used in the Admin API local function new() local local_conf, err = fetch_local_conf() if not local_conf then @@ -40,8 +41,16 @@ local function new() etcd_conf.ssl_verify = true -- default to verify etcd cluster certificate - if etcd_conf.tls and etcd_conf.tls.verify == false then - etcd_conf.ssl_verify = false + etcd_conf.ssl_verify = true + if etcd_conf.tls then + if etcd_conf.tls.verify == false then + etcd_conf.ssl_verify = false + end + + if etcd_conf.tls.cert then + etcd_conf.ssl_cert_path = etcd_conf.tls.cert + etcd_conf.ssl_key_path = etcd_conf.tls.key + end end local etcd_cli diff --git a/apisix/patch.lua b/apisix/patch.lua index e310bcae5049..62f546c250fe 100644 --- a/apisix/patch.lua +++ b/apisix/patch.lua @@ -119,7 +119,12 @@ local luasocket_wrapper = { return self.sock:settimeout(time) end, - sslhandshake = function (self, reused_session, server_name, verify, send_status_req) + tlshandshake = function (self, options) + local reused_session = options.reused_session + local server_name = options.server_name + local verify = options.verify + local send_status_req = options.ocsp_status_req + if reused_session then log(WARN, "reused_session is not supported yet") end @@ -132,6 +137,8 @@ local luasocket_wrapper = { mode = "client", protocol = "any", verify = verify and "peer" or "none", + certificate = options.client_cert_path, + key = options.client_priv_key_path, options = { "all", "no_sslv2", @@ -157,6 +164,15 @@ local luasocket_wrapper = { self.sock = sec_sock return true + end, + + sslhandshake = function (self, reused_session, server_name, verify, send_status_req) + return self:tlshandshake({ + reused_session = reused_session, + server_name = server_name, + verify = verify, + ocsp_status_req = send_status_req, + }) end } diff --git a/rockspec/apisix-master-0.rockspec b/rockspec/apisix-master-0.rockspec index 52d559c47fd4..9e149a8c0f73 100644 --- a/rockspec/apisix-master-0.rockspec +++ b/rockspec/apisix-master-0.rockspec @@ -34,7 +34,7 @@ dependencies = { "lua-resty-ctxdump = 0.1-0", "lua-resty-dns-client = 5.2.0", "lua-resty-template = 1.9", - "lua-resty-etcd = 1.4.3", + "lua-resty-etcd = 1.5.0", "lua-resty-balancer = 0.02rc5", "lua-resty-ngxvar = 0.5.2", "lua-resty-jit-uuid = 0.0.7", diff --git a/t/APISIX.pm b/t/APISIX.pm index 9898f014a1e3..060394eea18c 100644 --- a/t/APISIX.pm +++ b/t/APISIX.pm @@ -298,8 +298,9 @@ _EOC_ apisix.stream_balancer_phase() } } +_EOC_ - init_by_lua_block { + my $stream_init_by_lua_block = $block->stream_init_by_lua_block // <<_EOC_; if os.getenv("APISIX_ENABLE_LUACOV") == "1" then require("luacov.runner")("t/apisix.luacov") jit.off() @@ -309,8 +310,12 @@ _EOC_ apisix = require("apisix") apisix.stream_init() - } +_EOC_ + $stream_config .= <<_EOC_; + init_by_lua_block { + $stream_init_by_lua_block + } init_worker_by_lua_block { apisix.stream_init_worker() } diff --git a/t/core/etcd-mtls.t b/t/core/etcd-mtls.t new file mode 100644 index 000000000000..e6a46b4571b5 --- /dev/null +++ b/t/core/etcd-mtls.t @@ -0,0 +1,239 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX; + +my $out = eval { `resty -e "local s=ngx.socket.tcp();print(s.tlshandshake)"` }; + +if ($out !~ m/function:/) { + plan(skip_all => "tlshandshake not patched"); +} else { + plan('no_plan'); +} + + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!$block->no_error_log && !$block->error_log) { + $block->set_value("no_error_log", "[error]\n[alert]"); + } +}); + +run_tests; + +__DATA__ + +=== TEST 1: run etcd in init phase +--- yaml_config +etcd: + host: + - "https://127.0.0.1:22379" + prefix: "/apisix" + tls: + cert: t/certs/mtls_client.crt + key: t/certs/mtls_client.key + verify: false +--- init_by_lua_block + local apisix = require("apisix") + apisix.http_init() + local etcd = require("apisix.core.etcd") + assert(etcd.set("/a", "ab")) + + local res, err = etcd.get("/a") + if not res then + ngx.log(ngx.ERR, err) + return + end + ngx.log(ngx.WARN, res.body.node.value) + + local res, err = etcd.delete("/a") + if not res then + ngx.log(ngx.ERR, err) + return + end + ngx.log(ngx.WARN, res.status) + + local res, err = etcd.get("/a") + if not res then + ngx.log(ngx.ERR, err) + return + end + ngx.log(ngx.WARN, res.status) +--- config + location /t { + return 200; + } +--- request +GET /t +--- grep_error_log eval +qr/init_by_lua:\d+: \S+/ +--- grep_error_log_out +init_by_lua:12: ab +init_by_lua:19: 200 +init_by_lua:26: 404 + + + +=== TEST 2: run etcd in init phase (stream) +--- yaml_config +etcd: + host: + - "https://127.0.0.1:22379" + prefix: "/apisix" + tls: + cert: t/certs/mtls_client.crt + key: t/certs/mtls_client.key + verify: false +--- stream_init_by_lua_block + apisix = require("apisix") + apisix.stream_init() + local etcd = require("apisix.core.etcd") + assert(etcd.set("/a", "ab")) + + local res, err = etcd.get("/a") + if not res then + ngx.log(ngx.ERR, err) + return + end + ngx.log(ngx.WARN, res.body.node.value) + + local res, err = etcd.delete("/a") + if not res then + ngx.log(ngx.ERR, err) + return + end + ngx.log(ngx.WARN, res.status) + + local res, err = etcd.get("/a") + if not res then + ngx.log(ngx.ERR, err) + return + end + ngx.log(ngx.WARN, res.status) +--- stream_server_config + content_by_lua_block { + ngx.say("ok") + } +--- stream_enable +--- grep_error_log eval +qr/init_by_lua:\d+: \S+/ +--- grep_error_log_out +init_by_lua:12: ab +init_by_lua:19: 200 +init_by_lua:26: 404 + + + +=== TEST 3: sync +--- extra_yaml_config +etcd: + host: + - "https://127.0.0.1:22379" + prefix: "/apisix" + tls: + cert: t/certs/mtls_client.crt + key: t/certs/mtls_client.key + verify: false +--- config + location /t { + content_by_lua_block { + local core = require("apisix.core") + local t = require("lib.test_admin").test + + local consumers, _ = core.config.new("/consumers", { + automatic = true, + item_schema = core.schema.consumer, + }) + + ngx.sleep(0.6) + local idx = consumers.prev_index + + local code, body = t('/apisix/admin/consumers', + ngx.HTTP_PUT, + [[{ + "username": "jobs", + "plugins": { + "basic-auth": { + "username": "jobs", + "password": "678901" + } + } + }]]) + + ngx.sleep(2) + local new_idx = consumers.prev_index + if new_idx > idx then + ngx.say("prev_index updated") + else + ngx.say("prev_index not update") + end + } + } +--- request +GET /t +--- response_body +prev_index updated +--- no_error_log +[error] +--- error_log +waitdir key + + + +=== TEST 4: sync (stream) +--- extra_yaml_config +etcd: + host: + - "https://127.0.0.1:22379" + prefix: "/apisix" + tls: + cert: t/certs/mtls_client.crt + key: t/certs/mtls_client.key + verify: false +--- stream_server_config + content_by_lua_block { + local core = require("apisix.core") + + local sr, _ = core.config.new("/stream_routes", { + automatic = true, + item_schema = core.schema.stream_routes, + }) + + ngx.sleep(0.6) + local idx = sr.prev_index + + assert(core.etcd.set("/stream_routes/1", + { + plugins = { + } + })) + + ngx.sleep(2) + local new_idx = sr.prev_index + if new_idx > idx then + ngx.say("prev_index updated") + else + ngx.say("prev_index not update") + end + } +--- stream_enable +--- stream_response +prev_index updated +--- no_error_log +[error] +--- error_log +waitdir key From 1069a08686e9692191c5ec6a89fc1e5555470332 Mon Sep 17 00:00:00 2001 From: spacewander Date: Thu, 25 Mar 2021 10:40:14 +0800 Subject: [PATCH 2/3] feat: handle ssl_trusted_certificate Signed-off-by: spacewander --- apisix/patch.lua | 20 ++++++++++++++++++++ t/core/etcd-mtls.t | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/apisix/patch.lua b/apisix/patch.lua index 62f546c250fe..1bd8d5361a53 100644 --- a/apisix/patch.lua +++ b/apisix/patch.lua @@ -30,9 +30,19 @@ local setmetatable = setmetatable local type = type +local config_local local _M = {} +local function get_local_conf() + if not config_local then + config_local = require("apisix.core.config_local") + end + + return config_local.local_conf() +end + + local function flatten(args) local buf = new_tab(#args, 0) for i, v in ipairs(args) do @@ -147,6 +157,16 @@ local luasocket_wrapper = { } } + local local_conf, err = get_local_conf() + if not local_conf then + return nil, err + end + + local apisix_ssl = local_conf.apisix.ssl + if apisix_ssl and apisix_ssl.ssl_trusted_certificate then + params.cafile = apisix_ssl.ssl_trusted_certificate + end + local sec_sock, err = ssl.wrap(self.sock, params) if not sec_sock then return false, err diff --git a/t/core/etcd-mtls.t b/t/core/etcd-mtls.t index e6a46b4571b5..a004aef04711 100644 --- a/t/core/etcd-mtls.t +++ b/t/core/etcd-mtls.t @@ -237,3 +237,37 @@ prev_index updated [error] --- error_log waitdir key + + + +=== TEST 5: ssl_trusted_certificate +--- yaml_config +apisix: + ssl: + ssl_trusted_certificate: t/certs/mtls_ca.crt +etcd: + host: + - "https://127.0.0.1:22379" + prefix: "/apisix" + tls: + cert: t/certs/mtls_client.crt + key: t/certs/mtls_client.key +--- init_by_lua_block + local apisix = require("apisix") + apisix.http_init() + local etcd = require("apisix.core.etcd") + assert(etcd.set("/a", "ab")) + local res, err = etcd.get("/a") + if not res then + ngx.log(ngx.ERR, err) + return + end + ngx.log(ngx.WARN, res.body.node.value) +--- config + location /t { + return 200; + } +--- request +GET /t +--- error_log +init_by_lua:11: ab From f95a0d8a4f15de9c82917b199d297e34d9f138c2 Mon Sep 17 00:00:00 2001 From: spacewander Date: Thu, 25 Mar 2021 10:47:20 +0800 Subject: [PATCH 3/3] lint Signed-off-by: spacewander --- apisix/patch.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/apisix/patch.lua b/apisix/patch.lua index 1bd8d5361a53..850c6b112d2b 100644 --- a/apisix/patch.lua +++ b/apisix/patch.lua @@ -14,6 +14,7 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- +local require = require local socket = require("socket") local unix_socket = require("socket.unix") local ssl = require("ssl")