diff --git a/.env.example b/.env.example index 0c9d303d6cc..22b7743e0d0 100644 --- a/.env.example +++ b/.env.example @@ -188,6 +188,10 @@ MEILI_MASTER_KEY=DrhYf7zENyR6AlUCKmnz0eYASOQdl6zxH7s7MKFSfFCt # Moderation # #========================# +OPENAI_MODERATION=false +OPENAI_MODERATION_API_KEY= +# OPENAI_MODERATION_REVERSE_PROXY=not working with some reverse proxys + BAN_VIOLATIONS=true BAN_DURATION=1000 * 60 * 60 * 2 BAN_INTERVAL=20 diff --git a/api/server/middleware/index.js b/api/server/middleware/index.js index 0d878d4c902..77afd971650 100644 --- a/api/server/middleware/index.js +++ b/api/server/middleware/index.js @@ -12,6 +12,7 @@ const concurrentLimiter = require('./concurrentLimiter'); const validateMessageReq = require('./validateMessageReq'); const buildEndpointOption = require('./buildEndpointOption'); const validateRegistration = require('./validateRegistration'); +const moderateText = require('./moderateText'); const noIndex = require('./noIndex'); module.exports = { @@ -29,5 +30,6 @@ module.exports = { validateMessageReq, buildEndpointOption, validateRegistration, + moderateText, noIndex, }; diff --git a/api/server/middleware/moderateText.js b/api/server/middleware/moderateText.js new file mode 100644 index 00000000000..c4bfd8a13ae --- /dev/null +++ b/api/server/middleware/moderateText.js @@ -0,0 +1,39 @@ +const axios = require('axios'); +const denyRequest = require('./denyRequest'); + +async function moderateText(req, res, next) { + if (process.env.OPENAI_MODERATION === 'true') { + try { + const { text } = req.body; + + const response = await axios.post( + process.env.OPENAI_MODERATION_REVERSE_PROXY || 'https://api.openai.com/v1/moderations', + { + input: text, + }, + { + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${process.env.OPENAI_MODERATION_API_KEY}`, + }, + }, + ); + + const results = response.data.results; + const flagged = results.some((result) => result.flagged); + + if (flagged) { + const type = 'moderation'; + const errorMessage = { type }; + return await denyRequest(req, res, errorMessage); + } + } catch (error) { + console.error('Error in moderateText:', error); + const errorMessage = 'error in moderation check'; + return await denyRequest(req, res, errorMessage); + } + } + next(); +} + +module.exports = moderateText; diff --git a/api/server/routes/ask/gptPlugins.js b/api/server/routes/ask/gptPlugins.js index bab60ee23a7..85616cd1b31 100644 --- a/api/server/routes/ask/gptPlugins.js +++ b/api/server/routes/ask/gptPlugins.js @@ -13,9 +13,11 @@ const { setHeaders, validateEndpoint, buildEndpointOption, + moderateText, } = require('~/server/middleware'); const { logger } = require('~/config'); +router.use(moderateText); router.post('/abort', handleAbort()); router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req, res) => { diff --git a/api/server/routes/ask/openAI.js b/api/server/routes/ask/openAI.js index 180ee27f299..31b3111077f 100644 --- a/api/server/routes/ask/openAI.js +++ b/api/server/routes/ask/openAI.js @@ -6,10 +6,11 @@ const { setHeaders, validateEndpoint, buildEndpointOption, + moderateText, } = require('~/server/middleware'); const router = express.Router(); - +router.use(moderateText); router.post('/abort', handleAbort()); router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req, res, next) => { diff --git a/api/server/routes/edit/gptPlugins.js b/api/server/routes/edit/gptPlugins.js index cd61b53237c..8ddf92c2507 100644 --- a/api/server/routes/edit/gptPlugins.js +++ b/api/server/routes/edit/gptPlugins.js @@ -12,9 +12,11 @@ const { setHeaders, validateEndpoint, buildEndpointOption, + moderateText, } = require('~/server/middleware'); const { logger } = require('~/config'); +router.use(moderateText); router.post('/abort', handleAbort()); router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req, res) => { diff --git a/api/server/routes/edit/openAI.js b/api/server/routes/edit/openAI.js index 47f36d6cb4d..e54881148dc 100644 --- a/api/server/routes/edit/openAI.js +++ b/api/server/routes/edit/openAI.js @@ -6,10 +6,11 @@ const { setHeaders, validateEndpoint, buildEndpointOption, + moderateText, } = require('~/server/middleware'); const router = express.Router(); - +router.use(moderateText); router.post('/abort', handleAbort()); router.post('/', validateEndpoint, buildEndpointOption, setHeaders, async (req, res, next) => { diff --git a/client/src/components/Messages/Content/Error.tsx b/client/src/components/Messages/Content/Error.tsx index 7876bce48c2..bf2f2b9a241 100644 --- a/client/src/components/Messages/Content/Error.tsx +++ b/client/src/components/Messages/Content/Error.tsx @@ -29,6 +29,8 @@ const errorMessages = { 'Invalid API key. Please check your API key and try again. You can do this by clicking on the model logo in the left corner of the textbox and selecting "Set Token" for the current selected endpoint. Thank you for your understanding.', insufficient_quota: 'We apologize for any inconvenience caused. The default API key has reached its limit. To continue using this service, please set up your own API key. You can do this by clicking on the model logo in the left corner of the textbox and selecting "Set Token" for the current selected endpoint. Thank you for your understanding.', + moderation: + 'It appears that the content submitted has been flagged by our moderation system for not aligning with our community guidelines. We\'re unable to proceed with this specific topic. If you have any other questions or topics you\'d like to explore, please edit your message, or create a new conversation.', concurrent: (json: TConcurrent) => { const { limit } = json; const plural = limit > 1 ? 's' : ''; diff --git a/docs/features/mod_system.md b/docs/features/mod_system.md index 2775f879c3c..099c5cb3a45 100644 --- a/docs/features/mod_system.md +++ b/docs/features/mod_system.md @@ -69,4 +69,29 @@ MESSAGE_IP_WINDOW=1 # in minutes, determines the window of time for MESSAGE_IP_M LIMIT_MESSAGE_USER=false # Whether to limit the amount of messages an IP can send per MESSAGE_USER_WINDOW MESSAGE_USER_MAX=40 # The max amount of messages an IP can send per MESSAGE_USER_WINDOW MESSAGE_USER_WINDOW=1 # in minutes, determines the window of time for MESSAGE_USER_MAX messages -``` \ No newline at end of file +``` + +## OpenAI moderation text + +### OPENAI_MODERATION +enable or disable OpenAI moderation + +Values: +`true`: OpenAI moderation is enabled +`false`: OpenAI moderation is disabled + +### OPENAI_MODERATION_API_KEY +Specify your OpenAI moderation API key here + +### OPENAI_MODERATION_REVERSE_PROXY +enable or disable reverse proxy compatibility for OpenAI moderation. Note that it may not work with some reverse proxies + +Values: +`true`: Enable reverse proxy compatibility +`false`: Disable reverse proxy compatibility + +```bash +OPENAI_MODERATION=true +OPENAI_MODERATION_API_KEY=sk-1234 +# OPENAI_MODERATION_REVERSE_PROXY=false +```