refactor: update transformer structure

oramasearch · Oct 28, 2024 · 642a4eb · 642a4eb
1 parent 789ff70
commit 642a4eb
Show file tree

Hide file tree

Showing 6 changed files with 90 additions and 158 deletions.
diff --git a/admin/src/components/SchemaMapper/index.js b/admin/src/components/SchemaMapper/index.js
@@ -1,97 +1,20 @@
-import React from "react"
-import {
-  Box,
-  Checkbox,
-  Flex,
-  Switch,
-  Table,
-  Thead,
-  Tbody,
-  Tr,
-  Th,
-  Td,
-  Tooltip,
-  Typography,
-  Status
-} from "@strapi/design-system"
-import WarningIcon from "../WarningIcon"
-import { getSelectedPropsFromObj, getSelectedAttributesFromSchema } from "../../../../utils/schema"
-
-const isCollection = (value) => Array.isArray(value) && value.length > 0 && typeof value[0] === "object"
-
-const handleObjectField = (acc, fieldKey, fieldValue, relations) => {
-  if (relations.includes(fieldKey)) {
-    Object.keys(fieldValue).forEach((key) => acc.push(`${fieldKey}.${key}`))
-  }
-}
-
-const handleCollectionField = (acc, fieldKey, fieldValue, relations) => {
-  if (relations.includes(fieldKey)) {
-    acc.push(fieldKey)
-  }
-}
+import React from 'react'
+import { Box, Checkbox, Flex, Switch, Table, Thead, Tbody, Tr, Th, Td, Typography } from '@strapi/design-system'
+import { getSelectedPropsFromObj, getSelectedAttributesFromSchema } from '../../../../utils'
 
 const generateSelectableAttributesFromSchema = ({ schema, relations }) => {
-  const handlers = {
-    object: handleObjectField,
-    collection: handleCollectionField
-  }
-
   return Object.entries(schema).reduce((acc, [fieldKey, fieldValue]) => {
-    const fieldType = fieldValue === "collection" ? "collection" : typeof fieldValue
-
-    if (fieldType in handlers) {
-      handlers[fieldType](acc, fieldKey, fieldValue, relations)
-    } else if (!isCollection(fieldValue)) {
+    if (typeof fieldValue === 'object') {
+      if (relations.includes(fieldKey)) {
+        Object.keys(fieldValue).forEach((key) => acc.push(`${fieldKey}.${key}`))
+      }
+    } else {
       acc.push(fieldKey)
     }
-
     return acc
   }, [])
 }
 
-const AttributeRow = ({ contentTypeSchema, field, isChecked, onCheck, isSearchableSelected, onSearchableSwitch, needsTransformation }) => (
-  <Tr key={field}>
-    <Td>
-      <Checkbox checked={isChecked(field)} onChange={() => onCheck(field)} />
-    </Td>
-    <Td
-      onClick={() => onCheck(field)}
-      style={{ cursor: "pointer", display: "flex", alignItems: "center" }}
-    >
-      <Typography textColor="neutral800">{field}</Typography>
-      {contentTypeSchema[field] === "collection" && needsTransformation && (
-        <>
-          <Tooltip
-            position="right"
-            label="You need to transform this attribute's data. Click for more info."
-          >
-            <Status variant="primary" size="S" showBullet={false} style={{ marginLeft: 10 }}>
-              <div
-                style={{
-                  display: "flex",
-                  flexDirection: "row",
-                  alignItems: "center",
-                  justifyContent: "center",
-                  columnGap: "5px"
-                }}
-              >
-                <WarningIcon size={12} fill="#ddaa00" />
-                <Typography variant="pi">Action required</Typography>
-              </div>
-            </Status>
-          </Tooltip>
-        </>
-      )}
-    </Td>
-    <Td>
-      <Flex justifyContent="flex-end">
-        <Switch selected={isSearchableSelected(field)} onChange={() => onSearchableSwitch(field)} />
-      </Flex>
-    </Td>
-  </Tr>
-)
-
 const SchemaMapper = ({ collection, contentTypeSchema, onSchemaChange }) => {
   const [selectedAttributes, setSelectedAttributes] = React.useState(
     getSelectedAttributesFromSchema({
@@ -154,20 +77,24 @@ const SchemaMapper = ({ collection, contentTypeSchema, onSchemaChange }) => {
     }
   }
 
-  const collectionDocumentsTransformers = collection?.documentsTransformers || {}
-
   return (
     <Box marginBottom={2} width="100%">
       <Typography variant="beta" fontWeight="bold">
-        Attributes Mapping <b style={{ color: "#ee5e52" }}>*</b>
+        Attributes Mapping<b style={{ color: '#ee5e52' }}>*</b>
       </Typography>
       <Flex style={{ marginBottom: 16, marginTop: 4 }}>
         <Typography variant="gamma" color="grey-600">
           Select the attributes you want to include in your Orama Cloud index document.
         </Typography>
       </Flex>
       <Box>
-        <Table colCount={3} rowCount={schemaAttributes.length}>
+        {/*TODO: style this*/}
+        {collection.hasSettings && (
+          <Typography variant="gamma" color="grey-600">
+            The schema is handled by the Orama Cloud plugin settings.
+          </Typography>
+        )}
+        {!collection.hasSettings && (<Table colCount={3} rowCount={schemaAttributes.length}>
           <Thead>
             <Tr>
               <Th>
@@ -177,15 +104,15 @@ const SchemaMapper = ({ collection, contentTypeSchema, onSchemaChange }) => {
                   onChange={() => selectAllAttributes()}
                 />
               </Th>
-              <Th style={{ minWidth: "300px" }}>
+              <Th style={{ minWidth: '300px' }}>
                 <Typography variant="sigma">Attribute</Typography>
               </Th>
               <Th>
                 <div
                   style={{
-                    display: "flex",
-                    justifyContent: "flex-end",
-                    width: "100%"
+                    display: 'flex',
+                    justifyContent: 'flex-end',
+                    width: '100%'
                   }}
                 >
                   <Typography variant="sigma">Searchable</Typography>
@@ -194,22 +121,23 @@ const SchemaMapper = ({ collection, contentTypeSchema, onSchemaChange }) => {
             </Tr>
           </Thead>
           <Tbody>
-            {schemaAttributes.map(
-              (field) => (
-                <AttributeRow
-                  key={field}
-                  needsTransformation={!collectionDocumentsTransformers[field]}
-                  contentTypeSchema={contentTypeSchema}
-                  field={field}
-                  isChecked={isChecked}
-                  onCheck={handleCheck}
-                  isSearchableSelected={isSearchableSelected}
-                  onSearchableSwitch={handleSearchable}
-                />
-              )
-            )}
+            {schemaAttributes.map((field) => (
+              <Tr key={field}>
+                <Td>
+                  <Checkbox checked={isChecked(field)} onChange={() => handleCheck(field)} />
+                </Td>
+                <Td onClick={() => handleCheck(field)} style={{ cursor: 'pointer' }}>
+                  <Typography textColor="neutral800">{field}</Typography>
+                </Td>
+                <Td>
+                  <Flex justifyContent="flex-end">
+                    <Switch selected={isSearchableSelected(field)} onChange={() => handleSearchable(field)} />
+                  </Flex>
+                </Td>
+              </Tr>
+            ))}
           </Tbody>
-        </Table>
+        </Table>)}
       </Box>
     </Box>
   )

diff --git a/server/content-types/collection/index.js b/server/content-types/collection/index.js
@@ -1,7 +1,7 @@
 // @ts-nocheck
 'use strict'
 
-const schema = require('./schema')
+const schema = require('./schema.json')
 const lifecycles = require('./lifecycles')
 
 module.exports = {

diff --git a/server/services/collections.js b/server/services/collections.js
@@ -12,26 +12,25 @@ module.exports = ({ strapi }) => {
     async find() {
       const collections = await strapi.entityService.findMany(ENTITY_NAME)
 
-      const result =  collectionSettings ? collections.map(collection => {
+      return collectionSettings ? collections.reduce((acc, collection) => {
         const settings = collectionSettings[collection.indexId]
-        const hasTransformers = !!settings?.documentsTransformer
-
-        if (hasTransformers) {
-          const documentsTransformers = Object.keys(settings.documentsTransformer).reduce((acc, key) => {
-            acc[key] = true
-            return acc
-          }, {})
-
-          return {
-            ...collection,
-            documentsTransformers,
+        const hasValidSettings = settings?.schema && settings?.transformer
+
+        if (settings) {
+          if (hasValidSettings) {
+            acc.push({
+              ...collection,
+              hasSettings: true,
+            })
+          } else {
+            strapi.log.warn(`Collection with indexId ${collection.indexId} has settings but no schema or transformer`)
           }
+        } else {
+          acc.push(collection)
         }
 
-        return collection
-      }) : collections
-
-      return result
+        return acc
+      }, []) : collections
     },
 
     /**

diff --git a/server/services/orama-manager.js b/server/services/orama-manager.js
@@ -1,7 +1,7 @@
 'use strict'
 
 const { CloudManager } = require('@oramacloud/client')
-const { getSchemaFromEntryStructure, getSelectedPropsFromObj } = require('../../utils/schema')
+const { getSelectedPropsFromObj } = require('../../utils')
 
 class OramaManager {
   constructor({ strapi }) {
@@ -26,6 +26,8 @@ class OramaManager {
    * @param {Object} collection - Collection object
    * */
   validate(collection) {
+    const collectionSettings = this.collectionSettings?.[collection.indexId]
+
     if (!collection) {
       this.strapi.log.error(`Collection not found`)
       return false
@@ -50,6 +52,13 @@ class OramaManager {
       return false
     }
 
+    if ((collectionSettings.schema && !collectionSettings.transformer) || (!collectionSettings.schema && collectionSettings.transformer)) {
+      this.strapi.log.error(
+        `ERROR: Both schema and transformer are required in the collection settings`
+      )
+      return false
+    }
+
     return true
   }
 
@@ -61,19 +70,13 @@ class OramaManager {
    * @param {Array} entries - Array of entries
    * */
   documentsTransformer(indexId, entries) {
-    const transformerFnMap = this.collectionSettings?.[indexId]?.documentsTransformer
+    const transformer = this.collectionSettings?.[indexId]?.transformer
 
-    if (!transformerFnMap) {
+    if (!transformer) {
       return entries
     }
 
-    return entries.map((entry) => {
-      return Object.entries(entry)
-        .map(([key, value]) => ({
-          [key]: transformerFnMap[key]?.(value) ?? value
-        }))
-        .reduce((acc, curr) => ({ ...acc, ...curr }), {})
-    })
+    return entries.map(transformer)
   }
 
   /*
@@ -148,19 +151,6 @@ class OramaManager {
     })
 
     if (entries.length > 0) {
-      if (offset === 0) {
-        const transformedEntries = this.documentsTransformer(collection.indexId, entries)
-        const filteredEntry = getSelectedPropsFromObj({
-          props: collection.searchableAttributes,
-          obj: transformedEntries[0]
-        })
-
-        await this.oramaUpdateSchema({
-          indexId: collection.indexId,
-          schema: getSchemaFromEntryStructure(filteredEntry)
-        })
-      }
-
       await this.oramaInsert({
         indexId: collection.indexId,
         entries
@@ -190,16 +180,16 @@ class OramaManager {
    * */
   async oramaInsert({ indexId, entries }) {
     const index = this.oramaCloudManager.index(indexId)
-    const formattedData = this.documentsTransformer(indexId, entries)
+    const transformedData = this.documentsTransformer(indexId, entries)
 
-    if (!formattedData) {
+    if (!transformedData) {
       this.strapi.log.error(`ERROR: documentsTransformer needs a return value`)
       return false
     }
 
-    const result = await index.insert(formattedData)
+    const result = await index.insert(transformedData)
 
-    this.strapi.log.info(`INSERT: documents with id ${formattedData.map(({ id }) => id)} into index ${indexId}`)
+    this.strapi.log.info(`INSERT: documents with id ${transformedData.map(({ id }) => id)} into index ${indexId}`)
 
     return result
   }
@@ -212,16 +202,16 @@ class OramaManager {
    * */
   async oramaUpdate({ indexId, entries }) {
     const index = this.oramaCloudManager.index(indexId)
-    const formattedData = this.documentsTransformer(entries) || entries
+    const transformedData = this.documentsTransformer(indexId, entries)
 
-    if (!formattedData) {
+    if (!transformedData) {
       this.strapi.log.error(`ERROR: documentsTransformer needs a return value`)
       return false
     }
 
-    const result = await index.update(formattedData)
+    const result = await index.update(transformedData)
 
-    this.strapi.log.info(`UPDATE: document with id ${formattedData.map(({ id }) => id)} into index ${indexId}`)
+    this.strapi.log.info(`UPDATE: document with id ${transformedData.map(({ id }) => id)} into index ${indexId}`)
 
     return result
   }
@@ -269,13 +259,27 @@ class OramaManager {
       return
     }
 
+    const customSchema = this.collectionSettings?.[collection.indexId]?.schema
+
+    const oramaSchema = customSchema ?? getSelectedPropsFromObj({
+      props: collection.searchableAttributes,
+      obj: collection.schema
+    })
+
     await this.updatingStarted(collection)
 
     await this.resetIndex(collection)
 
+    await this.oramaUpdateSchema({
+      indexId: collection.indexId,
+      schema: oramaSchema
+    })
+
     const { documents_count } = await this.bulkInsert(collection)
 
-    await this.oramaDeployIndex(collection)
+    if (documents_count > 0) {
+      await this.oramaDeployIndex(collection)
+    }
 
     await this.updatingCompleted(collection, documents_count)
   }

diff --git a/utils/schema.js → utils/index.js b/utils/schema.js → utils/index.js
@@ -1,3 +1,4 @@
+const { collection } = require("../server/content-types")
 const getSelectedPropsFromObj = ({ props, obj }) => {
   return props.reduce((acc, field) => {
     if (field.includes('.')) {