From 7e070d97cb13f44a5a5b10d246fdcbfb6ba00f92 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=BD=97=E6=B3=BD=E8=BD=A9?= Date: Fri, 16 Sep 2022 16:36:32 +0800 Subject: [PATCH] feat: integrate grpc-client-nginx-module in APISIX (#7917) --- apisix/cli/ngx_tpl.lua | 2 + apisix/core/grpc.lua | 27 +++++ apisix/init.lua | 4 + rockspec/apisix-master-0.rockspec | 1 + t/APISIX.pm | 2 + t/core/grpc.t | 177 ++++++++++++++++++++++++++++++ 6 files changed, 213 insertions(+) create mode 100644 apisix/core/grpc.lua create mode 100644 t/core/grpc.t diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua index c69255c23fc3..a4d2d30a2084 100644 --- a/apisix/cli/ngx_tpl.lua +++ b/apisix/cli/ngx_tpl.lua @@ -58,6 +58,8 @@ env {*name*}; {% end %} {% if use_apisix_openresty then %} +thread_pool grpc-client-nginx-module threads=1; + lua { {% if enabled_stream_plugins["prometheus"] then %} lua_shared_dict prometheus-metrics {* meta.lua_shared_dict["prometheus-metrics"] *}; diff --git a/apisix/core/grpc.lua b/apisix/core/grpc.lua new file mode 100644 index 000000000000..9cf7d747a667 --- /dev/null +++ b/apisix/core/grpc.lua @@ -0,0 +1,27 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local pcall = pcall + +local ok, mod = pcall(require, "resty.grpc") +if not ok then + -- vanilla OpenResty doesn't have grpc-client-nginx-module + return nil +end + +-- Reimport the `resty.grpc` as `core.grpc`. For the doc of the gRPC API, +-- see https://github.com/api7/grpc-client-nginx-module +return mod diff --git a/apisix/init.lua b/apisix/init.lua index 7f29fbc6b5b2..b2e0b706b5f2 100644 --- a/apisix/init.lua +++ b/apisix/init.lua @@ -109,6 +109,10 @@ function _M.http_init_worker() -- for testing only core.log.info("random test in [1, 10000]: ", math.random(1, 10000)) + -- Because go's scheduler doesn't work after fork, we have to load the gRPC module + -- in each worker. + core.grpc = require("apisix.core.grpc") + local we = require("resty.worker.events") local ok, err = we.configure({shm = "worker-events", interval = 0.1}) if not ok then diff --git a/rockspec/apisix-master-0.rockspec b/rockspec/apisix-master-0.rockspec index a42d5207e6c6..da9991485d14 100644 --- a/rockspec/apisix-master-0.rockspec +++ b/rockspec/apisix-master-0.rockspec @@ -47,6 +47,7 @@ dependencies = { "opentracing-openresty = 0.1", "lua-resty-radixtree = 2.8.2", "lua-protobuf = 0.3.4", + "grpc-client-nginx-module = 0.2.2", "lua-resty-openidc = 1.7.5", "luafilesystem = 1.7.0-2", "api7-lua-tinyyaml = 0.4.2", diff --git a/t/APISIX.pm b/t/APISIX.pm index e95ad46cd346..9f964b8898a4 100644 --- a/t/APISIX.pm +++ b/t/APISIX.pm @@ -256,6 +256,8 @@ _EOC_ if ($version =~ m/\/apisix-nginx-module/) { $main_config .= <<_EOC_; +thread_pool grpc-client-nginx-module threads=1; + lua { lua_shared_dict prometheus-metrics 15m; } diff --git a/t/core/grpc.t b/t/core/grpc.t new file mode 100644 index 000000000000..c6d4d7be16a3 --- /dev/null +++ b/t/core/grpc.t @@ -0,0 +1,177 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX; + +my $nginx_binary = $ENV{'TEST_NGINX_BINARY'} || 'nginx'; +my $version = eval { `$nginx_binary -V 2>&1` }; + +if ($version !~ m/\/apisix-nginx-module/) { + plan(skip_all => "apisix-nginx-module not installed"); +} else { + plan('no_plan'); +} + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!$block->request) { + $block->set_value("request", "GET /t"); + } + + if ((!defined $block->error_log) && (!defined $block->no_error_log)) { + $block->set_value("no_error_log", "[error]"); + } +}); + +run_tests; + +__DATA__ + +=== TEST 1: unary +--- config + location /t { + content_by_lua_block { + local core = require "apisix.core" + local gcli = core.grpc + assert(gcli.load("t/grpc_server_example/proto/helloworld.proto")) + local conn = assert(gcli.connect("127.0.0.1:50051")) + local res, err = conn:call("helloworld.Greeter", "SayHello", { + name = "apisix" }) + conn:close() + if not res then + ngx.status = 503 + ngx.say(err) + return + end + ngx.say(res.message) + } + } +--- response_body +Hello apisix + + + +=== TEST 2: server stream +--- config + location /t { + content_by_lua_block { + local core = require "apisix.core" + local gcli = core.grpc + assert(gcli.load("t/grpc_server_example/proto/helloworld.proto")) + local conn = assert(gcli.connect("127.0.0.1:50051")) + local st, err = conn:new_server_stream("helloworld.Greeter", + "SayHelloServerStream", { name = "apisix" }) + if not st then + ngx.status = 503 + ngx.say(err) + return + end + + for i = 1, 5 do + local res, err = st:recv() + if not res then + ngx.status = 503 + ngx.say(err) + return + end + ngx.say(res.message) + end + } + } +--- response_body eval +"Hello apisix\n" x 5 + + + +=== TEST 3: client stream +--- config + location /t { + content_by_lua_block { + local core = require "apisix.core" + local gcli = core.grpc + assert(gcli.load("t/grpc_server_example/proto/helloworld.proto")) + local conn = assert(gcli.connect("127.0.0.1:50051")) + local st, err = conn:new_client_stream("helloworld.Greeter", + "SayHelloClientStream", { name = "apisix" }) + if not st then + ngx.status = 503 + ngx.say(err) + return + end + + for i = 1, 3 do + local ok, err = st:send({ name = "apisix" }) + if not ok then + ngx.status = 503 + ngx.say(err) + return + end + end + + local res, err = st:recv_close() + if not res then + ngx.status = 503 + ngx.say(err) + return + end + ngx.say(res.message) + } + } +--- response_body +Hello apisix!Hello apisix!Hello apisix!Hello apisix! + + + +=== TEST 4: bidirectional stream +--- config + location /t { + content_by_lua_block { + local core = require "apisix.core" + local gcli = core.grpc + assert(gcli.load("t/grpc_server_example/proto/helloworld.proto")) + local conn = assert(gcli.connect("127.0.0.1:50051")) + local st, err = conn:new_bidirectional_stream("helloworld.Greeter", + "SayHelloBidirectionalStream", { name = "apisix" }) + if not st then + ngx.status = 503 + ngx.say(err) + return + end + + for i = 1, 3 do + local ok, err = st:send({ name = "apisix" }) + if not ok then + ngx.status = 503 + ngx.say(err) + return + end + end + + assert(st:close_send()) + for i = 1, 5 do + local res, err = st:recv() + if not res then + ngx.status = 503 + ngx.say(err) + return + end + ngx.say(res.message) + end + } + } +--- response_body eval +"Hello apisix\n" x 4 . "stream ended\n"