feat: add documentsTransformer to plugin config

admin/src/components/RelationsSelect/index.js

@@ -12,6 +12,7 @@ const RelationsSelect = ({ onChange, collectionRelations, relations = [] }) => (
         disabled={relations.length === 0}
         onChange={onChange}
         value={collectionRelations}
+        withTags
       >
         {relations.map((relation, i) => (
           <MultiSelectOption key={relation.value + i} value={relation.value}>

admin/src/components/SchemaMapper/index.js

@@ -1,16 +1,50 @@
 import React from 'react'
-import { Box, Checkbox, Flex, Switch, Table, Thead, Tbody, Tr, Th, Td, Typography } from '@strapi/design-system'
+import {
+  Box,
+  Checkbox,
+  Flex,
+  Switch,
+  Table,
+  Thead,
+  Tbody,
+  Tr,
+  Th,
+  Td,
+  Tooltip,
+  Typography
+} from '@strapi/design-system'
+import WarningIcon from '../WarningIcon'
 import { getSchemaFromAttributes, getSelectedAttributesFromSchema } from '../../../../utils/schema'
 
+const isCollection = (value) => Array.isArray(value) && value.length > 0 && typeof value[0] === 'object'
+
+const handleObjectField = (acc, fieldKey, fieldValue, relations) => {
+  if (relations.includes(fieldKey)) {
+    Object.keys(fieldValue).forEach((key) => acc.push(`${fieldKey}.${key}`))
+  }
+}
+
+const handleCollectionField = (acc, fieldKey, fieldValue, relations) => {
+  if (relations.includes(fieldKey)) {
+    acc.push(fieldKey)
+  }
+}
+
 const generateSelectableAttributesFromSchema = ({ schema, relations }) => {
+  const handlers = {
+    object: handleObjectField,
+    collection: handleCollectionField
+  }
+
   return Object.entries(schema).reduce((acc, [fieldKey, fieldValue]) => {
-    if (typeof fieldValue === 'object') {
-      if (relations.includes(fieldKey)) {
-        Object.keys(fieldValue).forEach((key) => acc.push(`${fieldKey}.${key}`))
-      }
-    } else {
+    const fieldType = fieldValue === 'collection' ? 'collection' : typeof fieldValue
+
+    if (fieldType in handlers) {
+      handlers[fieldType](acc, fieldKey, fieldValue, relations)
+    } else if (!isCollection(fieldValue)) {
       acc.push(fieldKey)
     }
+
     return acc
   }, [])
 }
@@ -120,8 +154,20 @@ const SchemaMapper = ({ collection, contentTypeSchema, onSchemaChange }) => {
                 <Td>
                   <Checkbox checked={isChecked(field)} onChange={() => handleCheck(field)} />
                 </Td>
-                <Td onClick={() => handleCheck(field)} style={{ cursor: 'pointer' }}>
+                <Td
+                  onClick={() => handleCheck(field)}
+                  style={{ cursor: 'pointer', display: 'flex', alignItems: 'center' }}
+                >
                   <Typography textColor="neutral800">{field}</Typography>
+                  {contentTypeSchema[field] === 'collection' && (
+                    <>
+                      <Tooltip label="This attribute needs work through a documentsTransformer">
+                        <a href="https://docs.orama.com/cloud/data-sources/native-integrations/strapi" target="_blank">
+                          <WarningIcon size={16} fill="#ddaa00" style={{ margin: '5px 0 0 5px' }} />
+                        </a>
+                      </Tooltip>
+                    </>
+                  )}
                 </Td>
                 <Td>
                   <Flex justifyContent="flex-end">

admin/src/components/WarningIcon/index.js

@@ -0,0 +1,7 @@
+import React from 'react'
+
+export default ({ size = 24, fill, ...rest }) => (
+  <svg xmlns="http://www.w3.org/2000/svg" width={size} height={size} fill={fill} {...rest} viewBox="0 0 256 256">
+    <path d="M236.8,188.09,149.35,36.22h0a24.76,24.76,0,0,0-42.7,0L19.2,188.09a23.51,23.51,0,0,0,0,23.72A24.35,24.35,0,0,0,40.55,224h174.9a24.35,24.35,0,0,0,21.33-12.19A23.51,23.51,0,0,0,236.8,188.09ZM222.93,203.8a8.5,8.5,0,0,1-7.48,4.2H40.55a8.5,8.5,0,0,1-7.48-4.2,7.59,7.59,0,0,1,0-7.72L120.52,44.21a8.75,8.75,0,0,1,15,0l87.45,151.87A7.59,7.59,0,0,1,222.93,203.8ZM120,144V104a8,8,0,0,1,16,0v40a8,8,0,0,1-16,0Zm20,36a12,12,0,1,1-12-12A12,12,0,0,1,140,180Z"></path>
+  </svg>
+)

admin/src/pages/PluginSettings/index.js

@@ -90,9 +90,16 @@ const HomePage = () => {
   }, [get])
 
   useEffect(() => {
-    if (currentCollection && !currentContentType) {
+    if (currentCollection) {
       const contentType = contentTypes.find((ct) => ct.value === currentCollection.entity)
       setCurrentContentType(contentType)
+
+      if (contentType?.availableRelations.length === 0) {
+        setCurrentCollection({
+          ...currentCollection,
+          includedRelations: []
+        })
+      }
     }
   }, [currentCollection])
 

server/controllers/content-types.js

@@ -3,19 +3,13 @@
 module.exports = ({ strapi }) => {
   return {
     async getContentTypes(ctx) {
-      const contentTypes = strapi.plugin('orama-cloud').service('contentTypesService').getContentTypes()
-
-      return contentTypes
+      return strapi.plugin('orama-cloud').service('contentTypesService').getContentTypes()
     },
 
     getAvailableRelations(ctx) {
       const { id } = ctx.params
-      const relations = strapi
-        .plugin('orama-cloud')
-        .service('contentTypesService')
-        .getAvailableRelations({ contentTypeId: id })
 
-      return relations
+      return strapi.plugin('orama-cloud').service('contentTypesService').getAvailableRelations({ contentTypeId: id })
     },
 
     async getContentTypesSchema(ctx) {
@@ -24,12 +18,10 @@ module.exports = ({ strapi }) => {
 
       const includedRelationsArray = includedRelations?.split(',') || []
 
-      const schema = strapi.plugin('orama-cloud').service('contentTypesService').getContentTypeSchema({
+      return strapi.plugin('orama-cloud').service('contentTypesService').getContentTypeSchema({
         contentTypeId: id,
         includedRelations: includedRelationsArray
       })
-
-      return schema
     }
   }
 }

server/services/content-types.js

@@ -14,9 +14,12 @@ const OramaTypesMap = {
   date: 'string',
   time: 'string',
   datetime: 'string',
-  enumeration: 'enum'
+  enumeration: 'enum',
+  collection: 'collection'
 }
 
+const arrayRelations = ['oneToMany', 'manyToMany']
+
 const filterContentTypesAPIs = ({ contentTypes }) => {
   return Object.keys(contentTypes).reduce((sanitized, contentType) => {
     if (contentType.startsWith('api::')) {
@@ -49,8 +52,14 @@ const shouldAttributeBeIncluded = (attribute, includedRelations) => {
 const getSelectedRelations = ({ schema, relations }) => {
   return relations.reduce((acc, relation) => {
     if (relation in schema) {
-      acc[relation] = {
-        select: Object.keys(schema[relation]).map((key) => key)
+      if (schema[relation] === 'collection') {
+        acc[relation] = {
+          select: '*'
+        }
+      } else {
+        acc[relation] = {
+          select: Object.keys(schema[relation]).map((key) => key)
+        }
       }
     }
 
@@ -59,7 +68,10 @@ const getSelectedRelations = ({ schema, relations }) => {
 }
 
 const getSelectedFieldsConfigObj = (schema) =>
-  Object.entries(schema).reduce((acc, [key, value]) => (typeof value === 'object' ? acc : [...acc, key]), ['id'])
+  Object.entries(schema).reduce(
+    (acc, [key, value]) => (typeof value === 'object' || value === 'collection' ? acc : [...acc, key]),
+    ['id']
+  )
 
 module.exports = ({ strapi }) => {
   return {
@@ -120,10 +132,14 @@ module.exports = ({ strapi }) => {
 
     getType(attribute) {
       if (attribute.type === 'relation') {
-        return this.getContentTypeSchema({
-          contentTypeId: attribute.target,
-          includedRelations: []
-        })
+        if (arrayRelations.includes(attribute.relation)) {
+          return OramaTypesMap.collection
+        } else {
+          return this.getContentTypeSchema({
+            contentTypeId: attribute.target,
+            includedRelations: []
+          })
+        }
       }
 
       return OramaTypesMap[attribute.type]

server/services/orama-manager.js

@@ -1,14 +1,15 @@
 'use strict'
 
 const { CloudManager } = require('@oramacloud/client')
-const { getSchemaFromAttributes } = require('../../utils/schema')
+const { getSchemaFromEntryStructure, getSchemaFromAttributes } = require('../../utils/schema')
 
 class OramaManager {
   constructor({ strapi }) {
     this.strapi = strapi
     this.contentTypesService = strapi.plugin('orama-cloud').service('contentTypesService')
     this.collectionService = strapi.plugin('orama-cloud').service('collectionsService')
     this.privateApiKey = strapi.config.get('plugin.orama-cloud.privateApiKey')
+    this.collectionSettings = strapi.config.get('plugin.orama-cloud.collectionSettings')
 
     this.oramaCloudManager = new CloudManager({ api_key: this.privateApiKey })
     this.DocumentActionsMap = {
@@ -46,6 +47,31 @@ class OramaManager {
     return true
   }
 
+  filterOutNonSearchableAttributes(schema, searchableAttributes) {
+    return Object.entries(schema).reduce((acc, [key, value]) => {
+      if (searchableAttributes.includes(key)) {
+        acc[key] = value
+      }
+      return acc
+    })
+  }
+
+  documentsTransformer(indexId, entries) {
+    const transformerFnMap = this.collectionSettings[indexId]?.documentsTransformer
+
+    if (!transformerFnMap) {
+      return entries
+    }
+
+    return entries.map((entry) => {
+      return Object.entries(entry)
+        .map(([key, value]) => ({
+          [key]: transformerFnMap[key]?.(value) ?? value
+        }))
+        .reduce((acc, curr) => ({ ...acc, ...curr }), {})
+    })
+  }
+
   async setOutdated(collection) {
     return await this.collectionService.updateWithoutHooks(collection.id, {
       status: 'outdated'
@@ -80,6 +106,11 @@ class OramaManager {
     return await index.snapshot([])
   }
 
+  /*
+   * Processes all entries from a collection and inserts them into the index
+   * Bulk insert is done recursively to avoid memory issues
+   * Bulk dispatches 50 entries at a time
+   * */
   async bulkInsert(collection, offset = 0) {
     const entries = await this.contentTypesService.getEntries({
       contentType: collection.entity,
@@ -89,6 +120,19 @@ class OramaManager {
     })
 
     if (entries.length > 0) {
+      if (offset === 0) {
+        const transformedEntries = this.documentsTransformer(collection.indexId, entries)
+        const filteredEntry = this.filterOutNonSearchableAttributes(
+          transformedEntries[0],
+          collection.searchableAttributes
+        )
+
+        await this.oramaUpdateSchema({
+          indexId: collection.indexId,
+          schema: getSchemaFromEntryStructure(filteredEntry)
+        })
+      }
+
       await this.oramaInsert({
         indexId: collection.indexId,
         entries
@@ -107,18 +151,32 @@ class OramaManager {
 
   async oramaInsert({ indexId, entries }) {
     const index = this.oramaCloudManager.index(indexId)
-    const result = await index.insert(entries)
+    const formattedData = this.documentsTransformer(indexId, entries)
 
-    this.strapi.log.info(`INSERT: documents with id ${entries.map(({ id }) => id)} into index ${indexId}`)
+    if (!formattedData) {
+      this.strapi.log.error(`ERROR: documentsTransformer needs a return value`)
+      return false
+    }
+
+    const result = await index.insert(formattedData)
+
+    this.strapi.log.info(`INSERT: documents with id ${formattedData.map(({ id }) => id)} into index ${indexId}`)
 
     return result
   }
 
   async oramaUpdate({ indexId, entries }) {
     const index = this.oramaCloudManager.index(indexId)
-    const result = await index.update(entries)
+    const formattedData = this.documentsTransformer?.(entries) || entries
+
+    if (!formattedData) {
+      this.strapi.log.error(`ERROR: documentsTransformer needs a return value`)
+      return false
+    }
+
+    const result = await index.update(formattedData)
 
-    this.strapi.log.info(`UPDATE: document with id ${entries.map(({ id }) => id)} into index ${indexId}`)
+    this.strapi.log.info(`UPDATE: document with id ${formattedData.map(({ id }) => id)} into index ${indexId}`)
 
     return result
   }
@@ -149,20 +207,10 @@ class OramaManager {
       return
     }
 
-    const oramaSchema = getSchemaFromAttributes({
-      attributes: collection.searchableAttributes,
-      schema: collection.schema
-    })
-
     await this.updatingStarted(collection)
 
     await this.resetIndex(collection)
 
-    await this.oramaUpdateSchema({
-      indexId: collection.indexId,
-      schema: oramaSchema
-    })
-
     const { documents_count } = await this.bulkInsert(collection)
 
     await this.oramaDeployIndex(collection)