Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(clickhouse-logger): support multi clickhouse endpoints #7517

Merged
merged 4 commits into from
Jul 28, 2022
Merged

feat(clickhouse-logger): support multi clickhouse endpoints #7517

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
bfff298
feat(clickhouse-logger): support multi endpoints
zhendongcmss Jul 21, 2022
1daf0ae
add compatibility test cases
zhendongcmss Jul 25, 2022
b670676
deprecated addr with host
zhendongcmss Jul 25, 2022
f181116
update test cases
zhendongcmss Jul 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 15 additions & 3 deletions apisix/plugins/clickhouse-logger.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ local core = require("apisix.core")
local http = require("resty.http")
local url = require("net.url")
local plugin = require("apisix.plugin")
local math_random = math.random

local ngx = ngx
local tostring = tostring
Expand All @@ -31,7 +32,9 @@ local batch_processor_manager = bp_manager_mod.new(plugin_name)
local schema = {
type = "object",
properties = {
-- deprecated, use "endpoint_addrs" instead
endpoint_addr = core.schema.uri_def,
zhendongcmss marked this conversation as resolved.
Show resolved Hide resolved
endpoint_addrs = {items = core.schema.uri_def, type = "array", minItems = 1},
user = {type = "string", default = ""},
password = {type = "string", default = ""},
database = {type = "string", default = ""},
Expand All @@ -40,7 +43,10 @@ local schema = {
name = {type = "string", default = "clickhouse logger"},
ssl_verify = {type = "boolean", default = true},
},
required = {"endpoint_addr", "user", "password", "database", "logtable"}
oneOf = {
{required = {"endpoint_addr", "user", "password", "database", "logtable"}},
{required = {"endpoint_addrs", "user", "password", "database", "logtable"}}
},
}


Expand Down Expand Up @@ -72,11 +78,17 @@ end
local function send_http_data(conf, log_message)
local err_msg
local res = true
local url_decoded = url.parse(conf.endpoint_addr)
local selected_endpoint_addr
if conf.endpoint_addr then
selected_endpoint_addr = conf.endpoint_addr
else
selected_endpoint_addr = conf.endpoint_addrs[math_random(#conf.endpoint_addrs)]
end
local url_decoded = url.parse(selected_endpoint_addr)
local host = url_decoded.host
local port = url_decoded.port

core.log.info("sending a batch logs to ", conf.endpoint_addr)
core.log.info("sending a batch logs to ", selected_endpoint_addr)
zhendongcmss marked this conversation as resolved.
Show resolved Hide resolved

if not port then
if url_decoded.scheme == "https" then
Expand Down
6 changes: 4 additions & 2 deletions docs/en/latest/plugins/clickhouse-logger.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ The `clickhouse-logger` Plugin is used to push logs to [ClickHouse](https://clic

| Name | Type | Required | Default | Valid values | Description |
|---------------|---------|----------|---------------------|--------------|----------------------------------------------------------------|
| endpoint_addr | string | True | | | ClickHouse endpoint. |
zhendongcmss marked this conversation as resolved.
Show resolved Hide resolved
zhendongcmss marked this conversation as resolved.
Show resolved Hide resolved
| endpoint_addr | Deprecated | True | | | Use `endpoint_addrs` instead. ClickHouse endpoints. |
| endpoint_addrs | array | True | | | ClickHouse endpoints. |
| database | string | True | | | Name of the database to store the logs. |
| logtable | string | True | | | Table name to store the logs. |
| user | string | True | | | ClickHouse username. |
Expand Down Expand Up @@ -96,6 +97,7 @@ Now, if you run `select * from default.test;`, you will get the following row:

## Enabling the Plugin

If multiple endpoints are configured, they will be written randomly.
The example below shows how you can enable the Plugin on a specific Route:

```shell
Expand All @@ -107,7 +109,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13
"password": "a",
"database": "default",
"logtable": "test",
"endpoint_addr": "http://127.0.0.1:8123"
"endpoint_addrs": ["http://127.0.0.1:8123"]
}
},
"upstream": {
Expand Down
24 changes: 13 additions & 11 deletions docs/zh/latest/plugins/clickhouse-logger.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,21 +27,23 @@ title: clickhouse-logger

## 属性

| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 |
| ---------------- | ------- | ------ | ------------- | ------- | ------------------------------------------------ |
| endpoint_addr | string | 必须 | | | `clickhouse` 服务器的 endpoint。 |
| database | string | 必须 | | | 使用的数据库。 |
| logtable | string | 必须 | | | 写入的表名 。 |
| user | string | 必须 | | | clickhouse 的用户。 |
| password | string | 必须 | | | clickhouse 的密码 。 |
| timeout | integer | 可选 | 3 | [1,...] | 发送请求后保持连接活动的时间。 |
| name | string | 可选 | "clickhouse logger" | | 标识 logger 的唯一标识符。 |
| ssl_verify | boolean | 可选 | true | [true,false] | 验证证书。 |
| 名称 | 类型 | 必选项 | 默认值 | 有效值 | 描述 |
| ---------------- | ------- | ------ | ------------- | ------- | ------------------------------------------------ |
| endpoint_addr | 废弃 | 必须 | | | 推荐使用 `endpoint_addrs` 代替。`clickhouse` 服务器的 endpoints。|
| endpoint_addrs | array | 必须 | | | `clickhouse` 服务器的 endpoints。|
| database | string | 必须 | | | 使用的数据库。 |
| logtable | string | 必须 | | | 写入的表名 。 |
| user | string | 必须 | | | clickhouse 的用户。 |
| password | string | 必须 | | | clickhouse 的密码 。 |
| timeout | integer | 可选 | 3 | [1,...] | 发送请求后保持连接活动的时间。 |
| name | string | 可选 | "clickhouse logger" | | 标识 logger 的唯一标识符。 |
| ssl_verify | boolean | 可选 | true | [true,false] | 验证证书。 |

本插件支持使用批处理器来聚合并批量处理条目（日志/数据）。这样可以避免插件频繁地提交数据，默认设置情况下批处理器会每 `5` 秒钟或队列中的数据达到 `1000` 条时提交数据，如需了解或自定义批处理器相关参数设置，请参考 [Batch-Processor](../batch-processor.md#配置) 配置部分。

## 如何开启

如果配置多个 endpoints 将随机写入。
这是有关如何为特定路由启用 `clickhouse-logger` 插件的示例。

```shell
Expand All @@ -53,7 +55,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13
"password": "a",
"database": "default",
"logtable": "test",
"endpoint_addr": "http://127.0.0.1:8123"
"endpoint_addrs": ["http://127.0.0.1:8123"]
}
},
"upstream": {
Expand Down
58 changes: 56 additions & 2 deletions t/plugin/clickhouse-logger.t
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,18 @@ add_block_preprocessor(sub {
ngx.say("ok")
}
}
location /clickhouse-logger/test1 {
content_by_lua_block {
ngx.req.read_body()
local data = ngx.req.get_body_data()
local headers = ngx.req.get_headers()
ngx.log(ngx.WARN, "clickhouse body: ", data)
for k, v in pairs(headers) do
ngx.log(ngx.WARN, "clickhouse headers: " .. k .. ":" .. v)
end
ngx.say("ok")
}
}
}
_EOC_

Expand Down Expand Up @@ -131,7 +143,7 @@ passed
}
}
--- response_body
property "endpoint_addr" is required
value should match only one schema, but matches none



Expand Down Expand Up @@ -175,7 +187,49 @@ passed



=== TEST 5: access local server
=== TEST 5: add plugin on routes using multi clickhouse-logger
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin").test
local code, body = t('/apisix/admin/routes/1',
ngx.HTTP_PUT,
[[{
"plugins": {
"clickhouse-logger": {
"user": "default",
"password": "a",
"database": "default",
"logtable": "t",
"endpoint_addrs": ["http://127.0.0.1:10420/clickhouse-logger/test",
"http://127.0.0.1:10420/clickhouse-logger/test1"],
"batch_max_size":1,
"inactive_timeout":1
}
},
"upstream": {
"nodes": {
"127.0.0.1:1982": 1
},
"type": "roundrobin"
},
"uri": "/opentracing"
}]]
)

if code >= 300 then
ngx.status = code
end
ngx.say(body)
}
}
--- error_code: 200
--- response_body
passed



=== TEST 6: access local server
--- request
GET /opentracing
--- response_body
Expand Down