feat: content-moderation plugin #11541
Conversation
dosubot
bot
added
size:XL
|
|
||
| if result.Toxicity > conf.toxicity_level then | ||
| return 400, "request body exceeds toxicity threshold" | ||
| end |
There was a problem hiding this comment.
response body also need filter by content moderation service.
dosubot
bot
added
size:XXL
apisix/core/request.lua
Outdated
| @@ -334,6 +335,26 @@ function _M.get_body(max_size, ctx) | |||
| end | |||
|
|
|||
|
|
|||
| function _M.get_body_table() | |||
There was a problem hiding this comment.
ai-proxy PR also has this code so later we can merge from master after ai-proxy is merged.
There was a problem hiding this comment.
note that the name of the method there changed :D
| function _M.rewrite(conf, ctx) | ||
| conf = fetch_secrets(conf, true, conf, "") | ||
| if not conf then | ||
| return 500, "failed to retrieve secrets from conf" |
There was a problem hiding this comment.
use Ngx predefined constants?
| end | ||
|
|
||
| local msgs = body.messages | ||
| if not msgs or type(msgs) ~= "table" or #msgs < 1 then |
There was a problem hiding this comment.
| if not msgs or type(msgs) ~= "table" or #msgs < 1 then | |
| if type(msgs) ~= "table" or #msgs < 1 then |
There was a problem hiding this comment.
#msgs < 1 -> core.table.isempty(msgs)
|
|
||
| local body, err = core.request.get_body_table() | ||
| if not body then | ||
| return 400, err |
There was a problem hiding this comment.
ditto and the following the same
apisix/core/request.lua
Outdated
| @@ -334,6 +335,26 @@ function _M.get_body(max_size, ctx) | |||
| end | |||
|
|
|||
|
|
|||
| function _M.get_body_table() | |||
There was a problem hiding this comment.
note that the name of the method there changed :D
| local type = type | ||
| local ipairs = ipairs | ||
| local require = require | ||
| local internal_server_error = ngx.HTTP_INTERNAL_SERVER_ERROR |
There was a problem hiding this comment.
| local internal_server_error = ngx.HTTP_INTERNAL_SERVER_ERROR | |
| local INTERNAL_SERVER_ERROR = ngx.HTTP_INTERNAL_SERVER_ERROR |
const var. sugg. this style
| return bad_request, "messages not found in request body" | ||
| end | ||
|
|
||
| local provider = conf.provider[next(conf.provider)] |
There was a problem hiding this comment.
What if conf.provider has multiple properties?
| end | ||
|
|
||
| local results = res.body and res.body.ResultList | ||
| if not results or type(results) ~= "table" or #results < 1 then |
There was a problem hiding this comment.
| if not results or type(results) ~= "table" or #results < 1 then | |
| if type(results) ~= "table" or core.table.isempty(results) then |
Description
The
content-moderationplugin processes the request body to check for toxicity and rejects the request if it exceeds the configured threshold.In later PRs, other plugins like ai-prompt-decorator and ai-prompt-template can use function from this plugin to ensure content moderation in requests proxying LLMs.
Checklist