Skip to content

Commit

Permalink
feat(ext-plugin): avoid sending conf request more times (apache#5183)
Browse files Browse the repository at this point in the history
  • Loading branch information
tzssangglass authored and bzp2010 committed Dec 31, 2021
1 parent 19c1c2f commit 58e70e7
Show file tree
Hide file tree
Showing 6 changed files with 201 additions and 2 deletions.
3 changes: 3 additions & 0 deletions apisix/cli/ngx_tpl.lua
Original file line number Diff line number Diff line change
Expand Up @@ -191,6 +191,9 @@ http {
# for authz-keycloak
lua_shared_dict access-tokens {* http.lua_shared_dict["access-tokens"] *}; # cache for service account access tokens
# for ext-plugin
lua_shared_dict ext-plugin {* http.lua_shared_dict["ext-plugin"] *}; # cache for ext-plugin
# for custom shared dict
{% if http.custom_lua_shared_dict then %}
{% for cache_key, cache_size in pairs(http.custom_lua_shared_dict) do %}
Expand Down
51 changes: 49 additions & 2 deletions apisix/plugins/ext-plugin/init.lua
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ local ipairs = ipairs
local pairs = pairs
local tostring = tostring
local type = type
local dict = ngx.shared["ext-plugin"]


local events_list
Expand Down Expand Up @@ -293,6 +294,44 @@ local function handle_extra_info(ctx, input)
end


local function fetch_token(key)
if dict then
return dict:get(key)
else
core.log.error('shm "ext-plugin" not found')
return nil
end
end


local function store_token(key, token)
if dict then
local exp = helper.get_conf_token_cache_time()
-- early expiry, lrucache in critical state sends prepare_conf_req as original behaviour
exp = exp * 0.9
local success, err, forcible = dict:set(key, token, exp)
if not success then
core.log.error("ext-plugin:failed to set conf token, err: ", err)
end
if forcible then
core.log.warn("ext-plugin:set valid items forcibly overwritten")
end
else
core.log.error('shm "ext-plugin" not found')
end
end


local function flush_token()
if dict then
core.log.warn("flush conf token in shared dict")
dict:flush_all()
else
core.log.error('shm "ext-plugin" not found')
end
end


local rpc_call
local rpc_handlers = {
nil,
Expand All @@ -301,6 +340,12 @@ local rpc_handlers = {

local key = builder:CreateString(unique_key)

local token = fetch_token(key)
if token then
core.log.info("fetch token from shared dict, token: ", token)
return token
end

local conf_vec
if conf.conf then
local len = #conf.conf
Expand Down Expand Up @@ -345,9 +390,10 @@ local rpc_handlers = {

local buf = flatbuffers.binaryArray.New(resp)
local pcr = prepare_conf_resp.GetRootAsResp(buf, 0)
local token = pcr:ConfToken()
token = pcr:ConfToken()

core.log.notice("get conf token: ", token, " conf: ", core.json.delay_encode(conf.conf))
store_token(key, token)
return token
end,
function (conf, ctx, sock, entry)
Expand Down Expand Up @@ -471,7 +517,6 @@ local rpc_handlers = {
local buf = flatbuffers.binaryArray.New(resp)
local call_resp = http_req_call_resp.GetRootAsResp(buf, 0)
local action_type = call_resp:ActionType()

if action_type == http_req_call_action.Stop then
local action = call_resp:Action()
local stop = http_req_call_stop.New()
Expand Down Expand Up @@ -589,6 +634,8 @@ end


local function create_lrucache()
flush_token()

if lrucache then
core.log.warn("flush conf token lrucache")
end
Expand Down
1 change: 1 addition & 0 deletions conf/config-default.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,7 @@ nginx_config: # config for render the template to generate n
jwks: 1m
introspection: 10m
access-tokens: 1m
ext-plugin: 1m

etcd:
host: # it's possible to define multiple etcd hosts addresses of the same etcd cluster.
Expand Down
1 change: 1 addition & 0 deletions t/APISIX.pm
Original file line number Diff line number Diff line change
Expand Up @@ -445,6 +445,7 @@ _EOC_
lua_shared_dict plugin-api-breaker 10m;
lua_capture_error_log 1m; # plugin error-log-logger
lua_shared_dict etcd-cluster-health-check 10m; # etcd health check
lua_shared_dict ext-plugin 1m;
proxy_ssl_name \$upstream_host;
proxy_ssl_server_name on;
Expand Down
145 changes: 145 additions & 0 deletions t/plugin/ext-plugin/conf_token.t
Original file line number Diff line number Diff line change
@@ -0,0 +1,145 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
use t::APISIX 'no_plan';

workers(8);
repeat_each(1);
no_long_string();
no_root_location();
no_shuffle();
log_level("info");
worker_connections(10240);

$ENV{"PATH"} = $ENV{PATH} . ":" . $ENV{TEST_NGINX_HTML_DIR};

add_block_preprocessor(sub {
my ($block) = @_;

$block->set_value("stream_conf_enable", 1);

if (!defined $block->extra_stream_config) {
my $stream_config = <<_EOC_;
server {
listen unix:\$TEST_NGINX_HTML_DIR/nginx.sock;
content_by_lua_block {
local ext = require("lib.ext-plugin")
ext.go({})
}
}
_EOC_
$block->set_value("extra_stream_config", $stream_config);
}

my $unix_socket_path = $ENV{"TEST_NGINX_HTML_DIR"} . "/nginx.sock";
my $orig_extra_yaml_config = $block->extra_yaml_config // "";
my $cmd = $block->ext_plugin_cmd // "['sleep', '5s']";
my $extra_yaml_config = <<_EOC_;
ext-plugin:
path_for_test: $unix_socket_path
cmd: $cmd
_EOC_
$extra_yaml_config = $extra_yaml_config . $orig_extra_yaml_config;

$block->set_value("extra_yaml_config", $extra_yaml_config);

if (!$block->request) {
$block->set_value("request", "GET /t");
}

if (!$block->error_log) {
$block->set_value("no_error_log", "[error]\n[alert]");
}
});

run_tests;

__DATA__
=== TEST 1: sanity
--- config
location /t {
content_by_lua_block {
local json = require("toolkit.json")
local t = require("lib.test_admin")
local code, message, res = t.test('/apisix/admin/routes/1',
ngx.HTTP_PUT,
[[{
"uri": "/hello",
"plugins": {
"ext-plugin-pre-req": {"a":"b"}
},
"upstream": {
"nodes": {
"127.0.0.1:1980": 1
},
"type": "roundrobin"
}
}]]
)
if code >= 300 then
ngx.status = code
ngx.say(message)
return
end
ngx.say(message)
}
}
--- response_body
passed
=== TEST 2: share conf token in different workers
--- ext_plugin_cmd
["t/plugin/ext-plugin/runner.sh", "3600"]
--- config
location /t {
content_by_lua_block {
local http = require "resty.http"
local uri = "http://127.0.0.1:" .. ngx.var.server_port .. "/hello"
local t = {}
for i = 1, 180 do
local th = assert(ngx.thread.spawn(function(i)
local httpc = http.new()
local res, err = httpc:request_uri(uri)
if not res then
ngx.log(ngx.ERR, err)
return
end
end, i))
table.insert(t, th)
end
for i, th in ipairs(t) do
ngx.thread.wait(th)
end
ngx.say("done")
}
}
--- response_body
done
--- grep_error_log eval
qr/fetch token from shared dict, token: 233/
--- grep_error_log_out eval
qr/(fetch token from shared dict, token: 233){1,}/
--- no_error_log
[error]
2 changes: 2 additions & 0 deletions t/plugin/ext-plugin/sanity.t
Original file line number Diff line number Diff line change
Expand Up @@ -270,6 +270,7 @@ sending rpc type: 1 data length:
receiving rpc type: 1 data length:
--- error_log
flush conf token lrucache
flush conf token in shared dict
--- no_error_log
[error]
Expand Down Expand Up @@ -382,6 +383,7 @@ hello world
}
--- error_log
refresh cache and try again
flush conf token in shared dict
--- no_error_log
[error]
Expand Down

0 comments on commit 58e70e7

Please sign in to comment.