diff --git a/apisix/cli/config.lua b/apisix/cli/config.lua index 7f15542b1d7e..6ab10c9256cd 100644 --- a/apisix/cli/config.lua +++ b/apisix/cli/config.lua @@ -214,6 +214,7 @@ local _M = { "proxy-cache", "body-transformer", "ai-prompt-template", + "ai-prompt-decorator", "proxy-mirror", "proxy-rewrite", "workflow", diff --git a/apisix/plugins/ai-prompt-decorator.lua b/apisix/plugins/ai-prompt-decorator.lua new file mode 100644 index 000000000000..10b36e82cd1d --- /dev/null +++ b/apisix/plugins/ai-prompt-decorator.lua @@ -0,0 +1,117 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local ngx = ngx +local pairs = pairs +local EMPTY = {} + +local prompt_schema = { + properties = { + role = { + type = "string", + enum = { "system", "user", "assistant" } + }, + content = { + type = "string", + minLength = 1, + } + }, + required = { "role", "content" } +} + +local prompts = { + type = "array", + items = prompt_schema +} + +local schema = { + type = "object", + properties = { + prepend = prompts, + append = prompts, + }, + anyOf = { + { required = { "prepend" } }, + { required = { "append" } }, + { required = { "append", "prepend" } }, + }, +} + + +local _M = { + version = 0.1, + priority = 1070, + name = "ai-prompt-decorator", + schema = schema, +} + + +function _M.check_schema(conf) + return core.schema.check(schema, conf) +end + + +local function get_request_body_table() + local body, err = core.request.get_body() + if not body then + return nil, { message = "could not get body: " .. err } + end + + local body_tab, err = core.json.decode(body) + if not body_tab then + return nil, { message = "could not get parse JSON request body: " .. err } + end + + return body_tab +end + + +local function decorate(conf, body_tab) + local new_messages = conf.prepend or EMPTY + for _, message in pairs(body_tab.messages) do + core.table.insert_tail(new_messages, message) + end + + for _, message in pairs(conf.append or EMPTY) do + core.table.insert_tail(new_messages, message) + end + + body_tab.messages = new_messages +end + + +function _M.rewrite(conf, ctx) + local body_tab, err = get_request_body_table() + if not body_tab then + return 400, err + end + + if not body_tab.messages then + return 400, "messages missing from request body" + end + decorate(conf, body_tab) -- will decorate body_tab in place + + local new_jbody, err = core.json.encode(body_tab) + if not new_jbody then + return 500, { message = "failed to parse modified JSON request body: " .. err } + end + + ngx.req.set_body_data(new_jbody) +end + + +return _M diff --git a/apisix/plugins/ai-prompt-template.lua b/apisix/plugins/ai-prompt-template.lua index 0a092c3f77c0..d2c36693c8a5 100644 --- a/apisix/plugins/ai-prompt-template.lua +++ b/apisix/plugins/ai-prompt-template.lua @@ -72,7 +72,7 @@ local schema = { local _M = { version = 0.1, - priority = 1060, + priority = 1071, name = "ai-prompt-template", schema = schema, } diff --git a/conf/config.yaml.example b/conf/config.yaml.example index 5d22418caeb5..17b385216a88 100644 --- a/conf/config.yaml.example +++ b/conf/config.yaml.example @@ -476,7 +476,8 @@ plugins: # plugin list (sorted by priority) #- error-log-logger # priority: 1091 - proxy-cache # priority: 1085 - body-transformer # priority: 1080 - - ai-prompt-template # priority: 1060 + - ai-prompt-template # priority: 1071 + - ai-prompt-decorator # priority: 1070 - proxy-mirror # priority: 1010 - proxy-rewrite # priority: 1008 - workflow # priority: 1006 diff --git a/docs/en/latest/config.json b/docs/en/latest/config.json index 0998ec730cd1..2195688a365c 100644 --- a/docs/en/latest/config.json +++ b/docs/en/latest/config.json @@ -87,6 +87,7 @@ "type": "category", "label": "Transformation", "items": [ + "plugins/ai-prompt-decorator", "plugins/response-rewrite", "plugins/proxy-rewrite", "plugins/grpc-transcode", diff --git a/docs/en/latest/plugins/ai-prompt-decorator.md b/docs/en/latest/plugins/ai-prompt-decorator.md new file mode 100644 index 000000000000..44ee59e74bde --- /dev/null +++ b/docs/en/latest/plugins/ai-prompt-decorator.md @@ -0,0 +1,109 @@ +--- +title: ai-prompt-decorator +keywords: + - Apache APISIX + - API Gateway + - Plugin + - ai-prompt-decorator +description: This document contains information about the Apache APISIX ai-prompt-decorator Plugin. +--- + + + +## Description + +The `ai-prompt-decorator` plugin simplifies access to LLM providers, such as OpenAI and Anthropic, and their models by appending or prepending prompts into the request. + +## Plugin Attributes + +| **Field** | **Required** | **Type** | **Description** | +| ----------------- | --------------- | -------- | --------------------------------------------------- | +| `prepend` | Conditionally\* | Array | An array of prompt objects to be prepended | +| `prepend.role` | Yes | String | Role of the message (`system`, `user`, `assistant`) | +| `prepend.content` | Yes | String | Content of the message. Minimum length: 1 | +| `append` | Conditionally\* | Array | An array of prompt objects to be appended | +| `append.role` | Yes | String | Role of the message (`system`, `user`, `assistant`) | +| `append.content` | Yes | String | Content of the message. Minimum length: 1 | + +\* **Conditionally Required**: At least one of `prepend` or `append` must be provided. + +## Example usage + +Create a route with the `ai-prompt-decorator` plugin like so: + +```shell +curl "http://127.0.0.1:9180/apisix/admin/routes/1" -X PUT \ + -H "X-API-KEY: ${ADMIN_API_KEY}" \ + -d '{ + "uri": "/v1/chat/completions", + "plugins": { + "ai-prompt-decorator": { + "prepend":[ + { + "role": "system", + "content": "I have exams tomorrow so explain conceptually and briefly" + } + ], + "append":[ + { + "role": "system", + "content": "End the response with an analogy." + } + ] + } + }, + "upstream": { + "type": "roundrobin", + "nodes": { + "api.openai.com:443": 1 + }, + "pass_host": "node", + "scheme": "https" + } + }' +``` + +Now send a request: + +```shell +curl http://127.0.0.1:9080/v1/chat/completions -i -XPOST -H 'Content-Type: application/json' -d '{ + "model": "gpt-4", + "messages": [{ "role": "user", "content": "What is TLS Handshake?" }] +}' -H "Authorization: Bearer " +``` + +Then the request body will be modified to something like this: + +```json +{ + "model": "gpt-4", + "messages": [ + { + "role": "system", + "content": "I have exams tomorrow so explain conceptually and briefly" + }, + { "role": "user", "content": "What is TLS Handshake?" }, + { + "role": "system", + "content": "End the response with an analogy." + } + ] +} +``` diff --git a/t/admin/plugins.t b/t/admin/plugins.t index 547b1a316d56..ef43ea9f3965 100644 --- a/t/admin/plugins.t +++ b/t/admin/plugins.t @@ -94,6 +94,7 @@ authz-keycloak proxy-cache body-transformer ai-prompt-template +ai-prompt-decorator proxy-mirror proxy-rewrite workflow diff --git a/t/plugin/ai-prompt-decorator.t b/t/plugin/ai-prompt-decorator.t new file mode 100644 index 000000000000..15f40eeeda10 --- /dev/null +++ b/t/plugin/ai-prompt-decorator.t @@ -0,0 +1,293 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +use t::APISIX 'no_plan'; + +repeat_each(1); +log_level('info'); +no_root_location(); +no_shuffle(); + +add_block_preprocessor(sub { + my ($block) = @_; + + if (!$block->request) { + $block->set_value("request", "GET /t"); + } + +}); + +run_tests(); + +__DATA__ + +=== TEST 1: sanity: configure prepend only +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/echo", + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + }, + "plugins": { + "ai-prompt-decorator": { + "prepend":[ + { + "role": "system", + "content": "some content" + } + ] + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } +} +--- response_body +passed + + + +=== TEST 2: test prepend +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body, actual_resp = t('/echo', + ngx.HTTP_POST, + [[{ + "messages": [ + { "role": "system", "content": "You are a mathematician" }, + { "role": "user", "content": "What is 1+1?" } + ] + }]], + [[{ + "messages": [ + { "role": "system", "content": "some content" }, + { "role": "system", "content": "You are a mathematician" }, + { "role": "user", "content": "What is 1+1?" } + ] + }]] + ) + if code >= 300 then + ngx.status = code + ngx.say("failed") + return + end + ngx.say("passed") + } + } +--- response_body +passed + + + +=== TEST 3: sanity: configure append only +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/echo", + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + }, + "plugins": { + "ai-prompt-decorator": { + "append":[ + { + "role": "system", + "content": "some content" + } + ] + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } +} +--- response_body +passed + + + +=== TEST 4: test append +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body, actual_resp = t('/echo', + ngx.HTTP_POST, + [[{ + "messages": [ + { "role": "system", "content": "You are a mathematician" }, + { "role": "user", "content": "What is 1+1?" } + ] + }]], + [[{ + "messages": [ + { "role": "system", "content": "You are a mathematician" }, + { "role": "user", "content": "What is 1+1?" }, + { "role": "system", "content": "some content" } + ] + }]] + ) + if code >= 300 then + ngx.status = code + ngx.say("failed") + return + end + ngx.say("passed") + } + } +--- response_body +passed + + + +=== TEST 5: sanity: configure append and prepend both +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/echo", + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + }, + "plugins": { + "ai-prompt-decorator": { + "append":[ + { + "role": "system", + "content": "some append" + } + ], + "prepend":[ + { + "role": "system", + "content": "some prepend" + } + ] + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } +} +--- response_body +passed + + + +=== TEST 6: test append +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body, actual_resp = t('/echo', + ngx.HTTP_POST, + [[{ + "messages": [ + { "role": "system", "content": "You are a mathematician" }, + { "role": "user", "content": "What is 1+1?" } + ] + }]], + [[{ + "messages": [ + { "role": "system", "content": "some prepend" }, + { "role": "system", "content": "You are a mathematician" }, + { "role": "user", "content": "What is 1+1?" }, + { "role": "system", "content": "some append" } + ] + }]] + ) + if code >= 300 then + ngx.status = code + ngx.say("failed") + return + end + ngx.say("passed") + } + } +--- response_body +passed + + + +=== TEST 7: sanity: configure neither append nor prepend should fail +--- config + location /t { + content_by_lua_block { + local t = require("lib.test_admin").test + local code, body = t('/apisix/admin/routes/1', + ngx.HTTP_PUT, + [[{ + "uri": "/echo", + "upstream": { + "type": "roundrobin", + "nodes": { + "127.0.0.1:1980": 1 + } + }, + "plugins": { + "ai-prompt-decorator": { + } + } + }]] + ) + + if code >= 300 then + ngx.status = code + end + ngx.say(body) + } +} +--- response_body_eval +qr/.*failed to check the configuration of plugin ai-prompt-decorator err.*/ +--- error_code: 400