👮feat: moderation text

.env.example

@@ -188,6 +188,10 @@ MEILI_MASTER_KEY=DrhYf7zENyR6AlUCKmnz0eYASOQdl6zxH7s7MKFSfFCt
 # Moderation             #
 #========================#
 
+OPENAI_MODERATION=false
+OPENAI_MODERATION_API_KEY=
+# OPENAI_MODERATION_REVERSE_PROXY=not working with some reverse proxys
+
 BAN_VIOLATIONS=true
 BAN_DURATION=1000 * 60 * 60 * 2
 BAN_INTERVAL=20

api/package.json

@@ -35,6 +35,7 @@
     "axios": "^1.3.4",
     "bcryptjs": "^2.4.3",
     "cheerio": "^1.0.0-rc.12",
+    "circular-json-es6": "^2.0.2",
     "cohere-ai": "^6.0.0",
     "connect-redis": "^7.1.0",
     "cookie": "^0.5.0",

api/server/middleware/index.js

@@ -12,6 +12,7 @@ const concurrentLimiter = require('./concurrentLimiter');
 const validateMessageReq = require('./validateMessageReq');
 const buildEndpointOption = require('./buildEndpointOption');
 const validateRegistration = require('./validateRegistration');
+const moderateText = require('./moderateText');
 const noIndex = require('./noIndex');
 
 module.exports = {
@@ -29,5 +30,6 @@ module.exports = {
   validateMessageReq,
   buildEndpointOption,
   validateRegistration,
+  moderateText,
   noIndex,
 };

api/server/middleware/moderateText.js

@@ -0,0 +1,41 @@
+const axios = require('axios');
+const CircularJSON = require('circular-json-es6');
+const denyRequest = require('./denyRequest');
+
+async function moderateText(req, res, next) {
+  if (process.env.OPENAI_MODERATION === 'true') {
+    try {
+      const { text } = req.body;
+      const textWithoutCircular = CircularJSON.stringify(text);
+
+      const response = await axios.post(
+        process.env.OPENAI_MODERATION_REVERSE_PROXY || 'https://api.openai.com/v1/moderations',
+        {
+          input: textWithoutCircular,
+        },
+        {
+          headers: {
+            'Content-Type': 'application/json',
+            Authorization: `Bearer ${process.env.OPENAI_MODERATION_API_KEY}`,
+          },
+        },
+      );
+
+      const results = response.data.results;
+      const flagged = results.some((result) => result.flagged);
+
+      if (flagged) {
+        const type = 'moderation';
+        const errorMessage = { type };
+        return await denyRequest(req, res, errorMessage);
+      }
+    } catch (error) {
+      console.error('Error in moderateText:', error);
+      const errorMessage = 'error in moderation check';
+      return await denyRequest(req, res, errorMessage);
+    }
+  }
+  next();
+}
+
+module.exports = moderateText;