From bfff298376ef5a12438a7106031ddf365571736c Mon Sep 17 00:00:00 2001 From: zhendongcmss Date: Thu, 21 Jul 2022 17:57:39 +0800 Subject: [PATCH 1/4] feat(clickhouse-logger): support multi endpoints --- apisix/plugins/clickhouse-logger.lua | 10 +-- docs/en/latest/plugins/clickhouse-logger.md | 5 +- docs/zh/latest/plugins/clickhouse-logger.md | 21 +++--- t/plugin/clickhouse-logger.t | 75 +++++++++++++++++++-- 4 files changed, 91 insertions(+), 20 deletions(-) diff --git a/apisix/plugins/clickhouse-logger.lua b/apisix/plugins/clickhouse-logger.lua index f7b734645334..923adb28cb40 100644 --- a/apisix/plugins/clickhouse-logger.lua +++ b/apisix/plugins/clickhouse-logger.lua @@ -21,6 +21,7 @@ local core = require("apisix.core") local http = require("resty.http") local url = require("net.url") local plugin = require("apisix.plugin") +local math_random = math.random local ngx = ngx local tostring = tostring @@ -31,7 +32,7 @@ local batch_processor_manager = bp_manager_mod.new(plugin_name) local schema = { type = "object", properties = { - endpoint_addr = core.schema.uri_def, + endpoint_addrs = {items = core.schema.uri_def, type = "array", minItems = 1}, user = {type = "string", default = ""}, password = {type = "string", default = ""}, database = {type = "string", default = ""}, @@ -40,7 +41,7 @@ local schema = { name = {type = "string", default = "clickhouse logger"}, ssl_verify = {type = "boolean", default = true}, }, - required = {"endpoint_addr", "user", "password", "database", "logtable"} + required = {"endpoint_addrs", "user", "password", "database", "logtable"} } @@ -72,11 +73,12 @@ end local function send_http_data(conf, log_message) local err_msg local res = true - local url_decoded = url.parse(conf.endpoint_addr) + local selected_endpoint_addr = conf.endpoint_addrs[math_random(#conf.endpoint_addrs)] + local url_decoded = url.parse(selected_endpoint_addr) local host = url_decoded.host local port = url_decoded.port - core.log.info("sending a batch logs to ", conf.endpoint_addr) + core.log.info("sending a batch logs to ", selected_endpoint_addr) if not port then if url_decoded.scheme == "https" then diff --git a/docs/en/latest/plugins/clickhouse-logger.md b/docs/en/latest/plugins/clickhouse-logger.md index 5eb8f77f930e..18733aca3356 100644 --- a/docs/en/latest/plugins/clickhouse-logger.md +++ b/docs/en/latest/plugins/clickhouse-logger.md @@ -35,7 +35,7 @@ The `clickhouse-logger` Plugin is used to push logs to [ClickHouse](https://clic | Name | Type | Required | Default | Valid values | Description | |---------------|---------|----------|---------------------|--------------|----------------------------------------------------------------| -| endpoint_addr | string | True | | | ClickHouse endpoint. | +| endpoint_addrs | array | True | | | ClickHouse endpoints. | | database | string | True | | | Name of the database to store the logs. | | logtable | string | True | | | Table name to store the logs. | | user | string | True | | | ClickHouse username. | @@ -96,6 +96,7 @@ Now, if you run `select * from default.test;`, you will get the following row: ## Enabling the Plugin +If multiple endpoints are configured, they will be written randomly. The example below shows how you can enable the Plugin on a specific Route: ```shell @@ -107,7 +108,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13 "password": "a", "database": "default", "logtable": "test", - "endpoint_addr": "http://127.0.0.1:8123" + "endpoint_addrs": ["http://127.0.0.1:8123"] } }, "upstream": { diff --git a/docs/zh/latest/plugins/clickhouse-logger.md b/docs/zh/latest/plugins/clickhouse-logger.md index c704893b1c93..27cf821c457e 100644 --- a/docs/zh/latest/plugins/clickhouse-logger.md +++ b/docs/zh/latest/plugins/clickhouse-logger.md @@ -27,21 +27,22 @@ title: clickhouse-logger ## 属性 -| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 | +| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 | | ---------------- | ------- | ------ | ------------- | ------- | ------------------------------------------------ | -| endpoint_addr | string | 必须 | | | `clickhouse` 服务器的 endpoint。 | -| database | string | 必须 | | | 使用的数据库。 | -| logtable | string | 必须 | | | 写入的表名 。 | -| user | string | 必须 | | | clickhouse 的用户。 | -| password | string | 必须 | | | clickhouse 的密码 。 | -| timeout | integer | 可选 | 3 | [1,...] | 发送请求后保持连接活动的时间。 | -| name | string | 可选 | "clickhouse logger" | | 标识 logger 的唯一标识符。 | -| ssl_verify | boolean | 可选 | true | [true,false] | 验证证书。 | +| endpoint_addrs | 数组 | 必须 | | | `clickhouse` 服务器的 endpoints。 | +| database | string | 必须 | | | 使用的数据库。 | +| logtable | string | 必须 | | | 写入的表名 。 | +| user | string | 必须 | | | clickhouse 的用户。 | +| password | string | 必须 | | | clickhouse 的密码 。 | +| timeout | integer | 可选 | 3 | [1,...] | 发送请求后保持连接活动的时间。 | +| name | string | 可选 | "clickhouse logger" | | 标识 logger 的唯一标识符。 | +| ssl_verify | boolean | 可选 | true | [true,false] | 验证证书。 | 本插件支持使用批处理器来聚合并批量处理条目（日志/数据）。这样可以避免插件频繁地提交数据，默认设置情况下批处理器会每 `5` 秒钟或队列中的数据达到 `1000` 条时提交数据，如需了解或自定义批处理器相关参数设置，请参考 [Batch-Processor](../batch-processor.md#配置) 配置部分。 ## 如何开启 +如果配置多个 endpoints 将随机写入。 这是有关如何为特定路由启用 `clickhouse-logger` 插件的示例。 ```shell @@ -53,7 +54,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13 "password": "a", "database": "default", "logtable": "test", - "endpoint_addr": "http://127.0.0.1:8123" + "endpoint_addrs": ["http://127.0.0.1:8123"] } }, "upstream": { diff --git a/t/plugin/clickhouse-logger.t b/t/plugin/clickhouse-logger.t index 5426ce028489..4e572cfb4bc6 100644 --- a/t/plugin/clickhouse-logger.t +++ b/t/plugin/clickhouse-logger.t @@ -48,6 +48,18 @@ add_block_preprocessor(sub { ngx.say("ok") } } + location /clickhouse-logger/test1 { + content_by_lua_block { + ngx.req.read_body() + local data = ngx.req.get_body_data() + local headers = ngx.req.get_headers() + ngx.log(ngx.WARN, "clickhouse body: ", data) + for k, v in pairs(headers) do + ngx.log(ngx.WARN, "clickhouse headers: " .. k .. ":" .. v) + end + ngx.say("ok") + } + } } _EOC_ @@ -70,7 +82,7 @@ __DATA__ password = "a", database = "default", logtable = "t", - endpoint_addr = "http://127.0.0.1:10420/clickhouse-logger/test", + endpoint_addrs = {"http://127.0.0.1:10420/clickhouse-logger/test"}, max_retry_count = 1, name = "clickhouse logger", ssl_verify = false @@ -97,7 +109,7 @@ passed password = "a", database = "default", logtable = "t", - endpoint_addr = "http://127.0.0.1:10420/clickhouse-logger/test" + endpoint_addrs = {"http://127.0.0.1:10420/clickhouse-logger/test"} }) if not ok then @@ -131,7 +143,7 @@ passed } } --- response_body -property "endpoint_addr" is required +property "endpoint_addrs" is required @@ -149,7 +161,7 @@ property "endpoint_addr" is required "password": "a", "database": "default", "logtable": "t", - "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", + "endpoint_addrs": ["http://127.0.0.1:10420/clickhouse-logger/test"], "batch_max_size":1, "inactive_timeout":1 } @@ -186,3 +198,58 @@ clickhouse headers: x-clickhouse-key:a clickhouse headers: x-clickhouse-user:default clickhouse headers: x-clickhouse-database:default --- wait: 5 + + + +=== TEST 6: add plugin on routes using multi clickhouse-logger +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "plugins": { + "clickhouse-logger": { + "user": "default", + "password": "a", + "database": "default", + "logtable": "t", + "endpoint_addrs": ["http://127.0.0.1:10420/clickhouse-logger/test", + "http://127.0.0.1:10420/clickhouse-logger/test1"], + "batch_max_size":1, + "inactive_timeout":1 + } + }, + "upstream": { + "nodes": { + "127.0.0.1:1982": 1 + }, + "type": "roundrobin" + }, + "uri": "/opentracing" + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } + } +--- response_body +passed + + + +=== TEST 7: access local server +--- request +GET /opentracing +--- response_body +opentracing +--- error_log +clickhouse body: INSERT INTO t FORMAT JSONEachRow +clickhouse headers: x-clickhouse-key:a +clickhouse headers: x-clickhouse-user:default +clickhouse headers: x-clickhouse-database:default +--- wait: 5 From 1daf0ae8815f84e890d3070b81cf730616f99331 Mon Sep 17 00:00:00 2001 From: zhendongcmss Date: Mon, 25 Jul 2022 10:45:25 +0800 Subject: [PATCH 2/4] add compatibility test cases --- apisix/plugins/clickhouse-logger.lua | 2 +- t/plugin/clickhouse-logger.t | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/apisix/plugins/clickhouse-logger.lua b/apisix/plugins/clickhouse-logger.lua index 923adb28cb40..71450927a07a 100644 --- a/apisix/plugins/clickhouse-logger.lua +++ b/apisix/plugins/clickhouse-logger.lua @@ -32,7 +32,7 @@ local batch_processor_manager = bp_manager_mod.new(plugin_name) local schema = { type = "object", properties = { - endpoint_addrs = {items = core.schema.uri_def, type = "array", minItems = 1}, + endpoint_addrs = {items = core.schema.uri_def, type = "array", minItems = 1}, -- deprecated, use "host" instead user = {type = "string", default = ""}, password = {type = "string", default = ""}, database = {type = "string", default = ""}, diff --git a/t/plugin/clickhouse-logger.t b/t/plugin/clickhouse-logger.t index 4e572cfb4bc6..70ce0a4f2cfd 100644 --- a/t/plugin/clickhouse-logger.t +++ b/t/plugin/clickhouse-logger.t @@ -161,7 +161,7 @@ property "endpoint_addrs" is required "password": "a", "database": "default", "logtable": "t", - "endpoint_addrs": ["http://127.0.0.1:10420/clickhouse-logger/test"], + "endpoint_addrs": "http://127.0.0.1:10420/clickhouse-logger/test", "batch_max_size":1, "inactive_timeout":1 } @@ -183,7 +183,7 @@ property "endpoint_addrs" is required } } --- response_body -passed +property "endpoint_addrs" validation failed: wrong type: expected array, got string From b6706768fdf2fecc790375596d8fa2fa92f7363a Mon Sep 17 00:00:00 2001 From: zhendongcmss Date: Mon, 25 Jul 2022 17:18:11 +0800 Subject: [PATCH 3/4] deprecated addr with host --- apisix/plugins/clickhouse-logger.lua | 16 ++++++++--- docs/en/latest/plugins/clickhouse-logger.md | 3 ++- docs/zh/latest/plugins/clickhouse-logger.md | 11 ++++---- t/plugin/clickhouse-logger.t | 30 ++++++--------------- 4 files changed, 29 insertions(+), 31 deletions(-) diff --git a/apisix/plugins/clickhouse-logger.lua b/apisix/plugins/clickhouse-logger.lua index 71450927a07a..026f0cfa93da 100644 --- a/apisix/plugins/clickhouse-logger.lua +++ b/apisix/plugins/clickhouse-logger.lua @@ -32,7 +32,9 @@ local batch_processor_manager = bp_manager_mod.new(plugin_name) local schema = { type = "object", properties = { - endpoint_addrs = {items = core.schema.uri_def, type = "array", minItems = 1}, -- deprecated, use "host" instead + -- deprecated, use "endpoint_addrs" instead + endpoint_addr = core.schema.uri_def, + endpoint_addrs = {items = core.schema.uri_def, type = "array", minItems = 1}, user = {type = "string", default = ""}, password = {type = "string", default = ""}, database = {type = "string", default = ""}, @@ -41,7 +43,10 @@ local schema = { name = {type = "string", default = "clickhouse logger"}, ssl_verify = {type = "boolean", default = true}, }, - required = {"endpoint_addrs", "user", "password", "database", "logtable"} + oneOf = { + {required = {"endpoint_addr", "user", "password", "database", "logtable"}}, + {required = {"endpoint_addrs", "user", "password", "database", "logtable"}} + }, } @@ -73,7 +78,12 @@ end local function send_http_data(conf, log_message) local err_msg local res = true - local selected_endpoint_addr = conf.endpoint_addrs[math_random(#conf.endpoint_addrs)] + local selected_endpoint_addr + if conf.endpoint_addr then + selected_endpoint_addr = conf.endpoint_addr + else + selected_endpoint_addr = conf.endpoint_addrs[math_random(#conf.endpoint_addrs)] + end local url_decoded = url.parse(selected_endpoint_addr) local host = url_decoded.host local port = url_decoded.port diff --git a/docs/en/latest/plugins/clickhouse-logger.md b/docs/en/latest/plugins/clickhouse-logger.md index 18733aca3356..f0808b19b30f 100644 --- a/docs/en/latest/plugins/clickhouse-logger.md +++ b/docs/en/latest/plugins/clickhouse-logger.md @@ -35,7 +35,8 @@ The `clickhouse-logger` Plugin is used to push logs to [ClickHouse](https://clic | Name | Type | Required | Default | Valid values | Description | |---------------|---------|----------|---------------------|--------------|----------------------------------------------------------------| -| endpoint_addrs | array | True | | | ClickHouse endpoints. | +| endpoint_addr | Deprecated | True | | | Use `endpoint_addrs` instead. ClickHouse endpoints. | +| endpoint_addrs | array | True | | | ClickHouse endpoints. | | database | string | True | | | Name of the database to store the logs. | | logtable | string | True | | | Table name to store the logs. | | user | string | True | | | ClickHouse username. | diff --git a/docs/zh/latest/plugins/clickhouse-logger.md b/docs/zh/latest/plugins/clickhouse-logger.md index 27cf821c457e..c967b3d6e1bc 100644 --- a/docs/zh/latest/plugins/clickhouse-logger.md +++ b/docs/zh/latest/plugins/clickhouse-logger.md @@ -27,11 +27,12 @@ title: clickhouse-logger ## 属性 -| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 | -| ---------------- | ------- | ------ | ------------- | ------- | ------------------------------------------------ | -| endpoint_addrs | 数组 | 必须 | | | `clickhouse` 服务器的 endpoints。 | -| database | string | 必须 | | | 使用的数据库。 | -| logtable | string | 必须 | | | 写入的表名 。 | +| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 | +| ---------------- | ------- | ------ | ------------- | ------- | ------------------------------------------------ | +| endpoint_addr | 废弃 | 必须 | | | 推荐使用 `endpoint_addrs` 代替。`clickhouse` 服务器的 endpoints。| +| endpoint_addrs | array | 必须 | | | `clickhouse` 服务器的 endpoints。| +| database | string | 必须 | | | 使用的数据库。 | +| logtable | string | 必须 | | | 写入的表名 。 | | user | string | 必须 | | | clickhouse 的用户。 | | password | string | 必须 | | | clickhouse 的密码 。 | | timeout | integer | 可选 | 3 | [1,...] | 发送请求后保持连接活动的时间。 | diff --git a/t/plugin/clickhouse-logger.t b/t/plugin/clickhouse-logger.t index 70ce0a4f2cfd..5b255012cdf3 100644 --- a/t/plugin/clickhouse-logger.t +++ b/t/plugin/clickhouse-logger.t @@ -82,7 +82,7 @@ __DATA__ password = "a", database = "default", logtable = "t", - endpoint_addrs = {"http://127.0.0.1:10420/clickhouse-logger/test"}, + endpoint_addr = {"http://127.0.0.1:10420/clickhouse-logger/test"}, max_retry_count = 1, name = "clickhouse logger", ssl_verify = false @@ -109,7 +109,7 @@ passed password = "a", database = "default", logtable = "t", - endpoint_addrs = {"http://127.0.0.1:10420/clickhouse-logger/test"} + endpoint_addr = {"http://127.0.0.1:10420/clickhouse-logger/test"} }) if not ok then @@ -143,7 +143,7 @@ passed } } --- response_body -property "endpoint_addrs" is required +property "endpoint_addr" is required @@ -161,7 +161,7 @@ property "endpoint_addrs" is required "password": "a", "database": "default", "logtable": "t", - "endpoint_addrs": "http://127.0.0.1:10420/clickhouse-logger/test", + "endpoint_addr": "http://127.0.0.1:10420/clickhouse-logger/test", "batch_max_size":1, "inactive_timeout":1 } @@ -183,25 +183,11 @@ property "endpoint_addrs" is required } } --- response_body -property "endpoint_addrs" validation failed: wrong type: expected array, got string - - - -=== TEST 5: access local server ---- request -GET /opentracing ---- response_body -opentracing ---- error_log -clickhouse body: INSERT INTO t FORMAT JSONEachRow -clickhouse headers: x-clickhouse-key:a -clickhouse headers: x-clickhouse-user:default -clickhouse headers: x-clickhouse-database:default ---- wait: 5 +passed -=== TEST 6: add plugin on routes using multi clickhouse-logger +=== TEST 5: add plugin on routes using multi clickhouse-logger --- config location /t { content_by_lua_block { @@ -215,7 +201,7 @@ clickhouse headers: x-clickhouse-database:default "password": "a", "database": "default", "logtable": "t", - "endpoint_addrs": ["http://127.0.0.1:10420/clickhouse-logger/test", + "endpoint_addr": ["http://127.0.0.1:10420/clickhouse-logger/test", "http://127.0.0.1:10420/clickhouse-logger/test1"], "batch_max_size":1, "inactive_timeout":1 @@ -242,7 +228,7 @@ passed -=== TEST 7: access local server +=== TEST 6: access local server --- request GET /opentracing --- response_body From f181116c8da4318b66cb9e614b0836f2b649fd86 Mon Sep 17 00:00:00 2001 From: zhendongcmss Date: Wed, 27 Jul 2022 10:03:13 +0800 Subject: [PATCH 4/4] update test cases --- t/plugin/clickhouse-logger.t | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/t/plugin/clickhouse-logger.t b/t/plugin/clickhouse-logger.t index 5b255012cdf3..ccb0be11ad91 100644 --- a/t/plugin/clickhouse-logger.t +++ b/t/plugin/clickhouse-logger.t @@ -82,7 +82,7 @@ __DATA__ password = "a", database = "default", logtable = "t", - endpoint_addr = {"http://127.0.0.1:10420/clickhouse-logger/test"}, + endpoint_addr = "http://127.0.0.1:10420/clickhouse-logger/test", max_retry_count = 1, name = "clickhouse logger", ssl_verify = false @@ -109,7 +109,7 @@ passed password = "a", database = "default", logtable = "t", - endpoint_addr = {"http://127.0.0.1:10420/clickhouse-logger/test"} + endpoint_addr = "http://127.0.0.1:10420/clickhouse-logger/test" }) if not ok then @@ -143,7 +143,7 @@ passed } } --- response_body -property "endpoint_addr" is required +value should match only one schema, but matches none @@ -201,7 +201,7 @@ passed "password": "a", "database": "default", "logtable": "t", - "endpoint_addr": ["http://127.0.0.1:10420/clickhouse-logger/test", + "endpoint_addrs": ["http://127.0.0.1:10420/clickhouse-logger/test", "http://127.0.0.1:10420/clickhouse-logger/test1"], "batch_max_size":1, "inactive_timeout":1 @@ -223,6 +223,7 @@ passed ngx.say(body) } } +--- error_code: 200 --- response_body passed