diff --git a/admin/src/components/RelationsSelect/index.js b/admin/src/components/RelationsSelect/index.js index 90892e8..482e465 100644 --- a/admin/src/components/RelationsSelect/index.js +++ b/admin/src/components/RelationsSelect/index.js @@ -12,6 +12,7 @@ const RelationsSelect = ({ onChange, collectionRelations, relations = [] }) => ( disabled={relations.length === 0} onChange={onChange} value={collectionRelations} + withTags > {relations.map((relation, i) => ( diff --git a/admin/src/components/SchemaMapper/index.js b/admin/src/components/SchemaMapper/index.js index 043e959..ba325b2 100644 --- a/admin/src/components/SchemaMapper/index.js +++ b/admin/src/components/SchemaMapper/index.js @@ -1,16 +1,50 @@ import React from 'react' -import { Box, Checkbox, Flex, Switch, Table, Thead, Tbody, Tr, Th, Td, Typography } from '@strapi/design-system' +import { + Box, + Checkbox, + Flex, + Switch, + Table, + Thead, + Tbody, + Tr, + Th, + Td, + Tooltip, + Typography +} from '@strapi/design-system' +import WarningIcon from '../WarningIcon' import { getSchemaFromAttributes, getSelectedAttributesFromSchema } from '../../../../utils/schema' +const isCollection = (value) => Array.isArray(value) && value.length > 0 && typeof value[0] === 'object' + +const handleObjectField = (acc, fieldKey, fieldValue, relations) => { + if (relations.includes(fieldKey)) { + Object.keys(fieldValue).forEach((key) => acc.push(`${fieldKey}.${key}`)) + } +} + +const handleCollectionField = (acc, fieldKey, fieldValue, relations) => { + if (relations.includes(fieldKey)) { + acc.push(fieldKey) + } +} + const generateSelectableAttributesFromSchema = ({ schema, relations }) => { + const handlers = { + object: handleObjectField, + collection: handleCollectionField + } + return Object.entries(schema).reduce((acc, [fieldKey, fieldValue]) => { - if (typeof fieldValue === 'object') { - if (relations.includes(fieldKey)) { - Object.keys(fieldValue).forEach((key) => acc.push(`${fieldKey}.${key}`)) - } - } else { + const fieldType = fieldValue === 'collection' ? 'collection' : typeof fieldValue + + if (fieldType in handlers) { + handlers[fieldType](acc, fieldKey, fieldValue, relations) + } else if (!isCollection(fieldValue)) { acc.push(fieldKey) } + return acc }, []) } @@ -120,8 +154,20 @@ const SchemaMapper = ({ collection, contentTypeSchema, onSchemaChange }) => { handleCheck(field)} /> - handleCheck(field)} style={{ cursor: 'pointer' }}> + handleCheck(field)} + style={{ cursor: 'pointer', display: 'flex', alignItems: 'center' }} + > {field} + {contentTypeSchema[field] === 'collection' && ( + <> + + + + + + + )} diff --git a/admin/src/components/WarningIcon/index.js b/admin/src/components/WarningIcon/index.js new file mode 100644 index 0000000..30ef647 --- /dev/null +++ b/admin/src/components/WarningIcon/index.js @@ -0,0 +1,7 @@ +import React from 'react' + +export default ({ size = 24, fill, ...rest }) => ( + + + +) diff --git a/admin/src/pages/PluginSettings/index.js b/admin/src/pages/PluginSettings/index.js index bfc25c4..580e657 100644 --- a/admin/src/pages/PluginSettings/index.js +++ b/admin/src/pages/PluginSettings/index.js @@ -90,9 +90,16 @@ const HomePage = () => { }, [get]) useEffect(() => { - if (currentCollection && !currentContentType) { + if (currentCollection) { const contentType = contentTypes.find((ct) => ct.value === currentCollection.entity) setCurrentContentType(contentType) + + if (contentType?.availableRelations.length === 0) { + setCurrentCollection({ + ...currentCollection, + includedRelations: [] + }) + } } }, [currentCollection]) diff --git a/server/controllers/content-types.js b/server/controllers/content-types.js index 2b6b81f..674dafc 100644 --- a/server/controllers/content-types.js +++ b/server/controllers/content-types.js @@ -3,19 +3,13 @@ module.exports = ({ strapi }) => { return { async getContentTypes(ctx) { - const contentTypes = strapi.plugin('orama-cloud').service('contentTypesService').getContentTypes() - - return contentTypes + return strapi.plugin('orama-cloud').service('contentTypesService').getContentTypes() }, getAvailableRelations(ctx) { const { id } = ctx.params - const relations = strapi - .plugin('orama-cloud') - .service('contentTypesService') - .getAvailableRelations({ contentTypeId: id }) - return relations + return strapi.plugin('orama-cloud').service('contentTypesService').getAvailableRelations({ contentTypeId: id }) }, async getContentTypesSchema(ctx) { @@ -24,12 +18,10 @@ module.exports = ({ strapi }) => { const includedRelationsArray = includedRelations?.split(',') || [] - const schema = strapi.plugin('orama-cloud').service('contentTypesService').getContentTypeSchema({ + return strapi.plugin('orama-cloud').service('contentTypesService').getContentTypeSchema({ contentTypeId: id, includedRelations: includedRelationsArray }) - - return schema } } } diff --git a/server/services/content-types.js b/server/services/content-types.js index cc887d3..2b05c8c 100644 --- a/server/services/content-types.js +++ b/server/services/content-types.js @@ -14,9 +14,12 @@ const OramaTypesMap = { date: 'string', time: 'string', datetime: 'string', - enumeration: 'enum' + enumeration: 'enum', + collection: 'collection' } +const arrayRelations = ['oneToMany', 'manyToMany'] + const filterContentTypesAPIs = ({ contentTypes }) => { return Object.keys(contentTypes).reduce((sanitized, contentType) => { if (contentType.startsWith('api::')) { @@ -49,8 +52,14 @@ const shouldAttributeBeIncluded = (attribute, includedRelations) => { const getSelectedRelations = ({ schema, relations }) => { return relations.reduce((acc, relation) => { if (relation in schema) { - acc[relation] = { - select: Object.keys(schema[relation]).map((key) => key) + if (schema[relation] === 'collection') { + acc[relation] = { + select: '*' + } + } else { + acc[relation] = { + select: Object.keys(schema[relation]).map((key) => key) + } } } @@ -59,7 +68,10 @@ const getSelectedRelations = ({ schema, relations }) => { } const getSelectedFieldsConfigObj = (schema) => - Object.entries(schema).reduce((acc, [key, value]) => (typeof value === 'object' ? acc : [...acc, key]), ['id']) + Object.entries(schema).reduce( + (acc, [key, value]) => (typeof value === 'object' || value === 'collection' ? acc : [...acc, key]), + ['id'] + ) module.exports = ({ strapi }) => { return { @@ -120,10 +132,14 @@ module.exports = ({ strapi }) => { getType(attribute) { if (attribute.type === 'relation') { - return this.getContentTypeSchema({ - contentTypeId: attribute.target, - includedRelations: [] - }) + if (arrayRelations.includes(attribute.relation)) { + return OramaTypesMap.collection + } else { + return this.getContentTypeSchema({ + contentTypeId: attribute.target, + includedRelations: [] + }) + } } return OramaTypesMap[attribute.type] diff --git a/server/services/orama-manager.js b/server/services/orama-manager.js index f2f5ad3..4588670 100644 --- a/server/services/orama-manager.js +++ b/server/services/orama-manager.js @@ -1,7 +1,7 @@ 'use strict' const { CloudManager } = require('@oramacloud/client') -const { getSchemaFromAttributes } = require('../../utils/schema') +const { getSchemaFromEntryStructure, getSchemaFromAttributes } = require('../../utils/schema') class OramaManager { constructor({ strapi }) { @@ -9,6 +9,7 @@ class OramaManager { this.contentTypesService = strapi.plugin('orama-cloud').service('contentTypesService') this.collectionService = strapi.plugin('orama-cloud').service('collectionsService') this.privateApiKey = strapi.config.get('plugin.orama-cloud.privateApiKey') + this.collectionSettings = strapi.config.get('plugin.orama-cloud.collectionSettings') this.oramaCloudManager = new CloudManager({ api_key: this.privateApiKey }) this.DocumentActionsMap = { @@ -46,6 +47,31 @@ class OramaManager { return true } + filterOutNonSearchableAttributes(schema, searchableAttributes) { + return Object.entries(schema).reduce((acc, [key, value]) => { + if (searchableAttributes.includes(key)) { + acc[key] = value + } + return acc + }) + } + + documentsTransformer(indexId, entries) { + const transformerFnMap = this.collectionSettings[indexId]?.documentsTransformer + + if (!transformerFnMap) { + return entries + } + + return entries.map((entry) => { + return Object.entries(entry) + .map(([key, value]) => ({ + [key]: transformerFnMap[key]?.(value) ?? value + })) + .reduce((acc, curr) => ({ ...acc, ...curr }), {}) + }) + } + async setOutdated(collection) { return await this.collectionService.updateWithoutHooks(collection.id, { status: 'outdated' @@ -80,6 +106,11 @@ class OramaManager { return await index.snapshot([]) } + /* + * Processes all entries from a collection and inserts them into the index + * Bulk insert is done recursively to avoid memory issues + * Bulk dispatches 50 entries at a time + * */ async bulkInsert(collection, offset = 0) { const entries = await this.contentTypesService.getEntries({ contentType: collection.entity, @@ -89,6 +120,19 @@ class OramaManager { }) if (entries.length > 0) { + if (offset === 0) { + const transformedEntries = this.documentsTransformer(collection.indexId, entries) + const filteredEntry = this.filterOutNonSearchableAttributes( + transformedEntries[0], + collection.searchableAttributes + ) + + await this.oramaUpdateSchema({ + indexId: collection.indexId, + schema: getSchemaFromEntryStructure(filteredEntry) + }) + } + await this.oramaInsert({ indexId: collection.indexId, entries @@ -107,18 +151,32 @@ class OramaManager { async oramaInsert({ indexId, entries }) { const index = this.oramaCloudManager.index(indexId) - const result = await index.insert(entries) + const formattedData = this.documentsTransformer(indexId, entries) - this.strapi.log.info(`INSERT: documents with id ${entries.map(({ id }) => id)} into index ${indexId}`) + if (!formattedData) { + this.strapi.log.error(`ERROR: documentsTransformer needs a return value`) + return false + } + + const result = await index.insert(formattedData) + + this.strapi.log.info(`INSERT: documents with id ${formattedData.map(({ id }) => id)} into index ${indexId}`) return result } async oramaUpdate({ indexId, entries }) { const index = this.oramaCloudManager.index(indexId) - const result = await index.update(entries) + const formattedData = this.documentsTransformer?.(entries) || entries + + if (!formattedData) { + this.strapi.log.error(`ERROR: documentsTransformer needs a return value`) + return false + } + + const result = await index.update(formattedData) - this.strapi.log.info(`UPDATE: document with id ${entries.map(({ id }) => id)} into index ${indexId}`) + this.strapi.log.info(`UPDATE: document with id ${formattedData.map(({ id }) => id)} into index ${indexId}`) return result } @@ -149,20 +207,10 @@ class OramaManager { return } - const oramaSchema = getSchemaFromAttributes({ - attributes: collection.searchableAttributes, - schema: collection.schema - }) - await this.updatingStarted(collection) await this.resetIndex(collection) - await this.oramaUpdateSchema({ - indexId: collection.indexId, - schema: oramaSchema - }) - const { documents_count } = await this.bulkInsert(collection) await this.oramaDeployIndex(collection) diff --git a/server/services/orama-manager.test.js b/server/services/orama-manager.test.js index 8b7ab56..783ec9d 100644 --- a/server/services/orama-manager.test.js +++ b/server/services/orama-manager.test.js @@ -3,11 +3,16 @@ const { CloudManager } = require('@oramacloud/client') const { mockCollection, mockNotValidCollection } = require('../__mocks__/collection') const { mockedTestRecord } = require('../__mocks__/content-types') +const mockedDocumentTransformer = jest.fn((documents) => documents) + const strapi = { plugin: jest.fn().mockReturnThis(), service: jest.fn().mockReturnThis(), config: { - get: jest.fn().mockReturnValue('mockPrivateApiKey') + get: jest.fn((string) => { + if (string === 'plugin.orama-cloud.privateApiKey') return 'mockPrivateApiKey' + if (string === 'plugin.orama-cloud.documentsTransformer') return null + }) }, log: { error: jest.fn(), @@ -61,7 +66,10 @@ describe('OramaManager', () => { describe('validate', () => { afterEach(() => { - jest.spyOn(strapi.config, 'get').mockReturnValue('mockPrivateApiKey') + strapi.config.get = jest.fn((string) => { + if (string === 'plugin.orama-cloud.privateApiKey') return 'mockPrivateApiKey' + if (string === 'plugin.orama-cloud.documentsTransformer') return null + }) }) it('should return false if collection is not found', () => { @@ -72,7 +80,10 @@ describe('OramaManager', () => { }) it('should return false if privateApiKey is not found', () => { - jest.spyOn(strapi.config, 'get').mockReturnValueOnce() + strapi.config.get = jest.fn((string) => { + if (string === 'plugin.orama-cloud.privateApiKey') return null + if (string === 'plugin.orama-cloud.documentsTransformer') return null + }) oramaManager = new OramaManager({ strapi }) const result = oramaManager.validate(mockCollection) @@ -193,6 +204,26 @@ describe('OramaManager', () => { }) expect(insert).toHaveBeenCalledWith([{ id: 1, title: 'Test Entry' }]) + expect(mockedDocumentTransformer).not.toHaveBeenCalledWith([{ id: 1, title: 'Test Entry' }]) + }) + + it('should call documentsFormatter fn if declared in plugin config', async () => { + const { insert } = new CloudManager({ strapi }).index() + + await oramaManager.oramaInsert({ + indexId: mockCollection.indexId, + entries: [{ id: 1, title: 'Test Entry' }] + }) + + expect(insert).toHaveBeenCalledWith([{ id: 1, title: 'Test Entry' }]) + expect(mockedDocumentTransformer).toHaveBeenCalledWith([{ id: 1, title: 'Test Entry' }]) + }) + + afterEach(() => { + strapi.config.get = jest.fn((string) => { + if (string === 'plugin.orama-cloud.privateApiKey') return 'mockPrivateApiKey' + if (string === 'plugin.orama-cloud.documentsTransformer') return mockedDocumentTransformer + }) }) }) diff --git a/utils/schema.js b/utils/schema.js index dd8512f..ba46371 100644 --- a/utils/schema.js +++ b/utils/schema.js @@ -25,4 +25,24 @@ const getSelectedAttributesFromSchema = ({ schema }) => { }, []) } -module.exports = { getSchemaFromAttributes, getSelectedAttributesFromSchema } +const getSchemaFromEntryStructure = (entry) => { + return Object.entries(entry).reduce((acc, [key, value]) => { + if (Array.isArray(value)) { + const firstValue = value[0] + if (['string', 'number', 'boolean'].includes(typeof firstValue)) { + acc[key] = `${typeof firstValue}[]` + } + } else if (typeof value === 'object') { + acc[key] = getSchemaFromEntryStructure(value) + } else { + acc[key] = typeof value + } + return acc + }, {}) +} + +module.exports = { + getSchemaFromAttributes, + getSelectedAttributesFromSchema, + getSchemaFromEntryStructure +} diff --git a/utils/schema.test.js b/utils/schema.test.js index c28d8a9..fb910ea 100644 --- a/utils/schema.test.js +++ b/utils/schema.test.js @@ -1,4 +1,4 @@ -const { getSchemaFromAttributes, getSelectedAttributesFromSchema } = require('./schema') +const { getSchemaFromAttributes, getSelectedAttributesFromSchema, getSchemaFromEntryStructure } = require('./schema') describe('Schema Utils', () => { describe('getSchemaFromAttributes', () => { @@ -56,4 +56,27 @@ describe('Schema Utils', () => { expect(result).toEqual(['address.city', 'address.zip']) }) }) + + describe('getSchemaFromEntryStructure', () => { + it('should return the correct schema', () => { + const entry = { + potato: 'hello', + apple: 5, + watermelon: { + seeds: { + many: true + } + }, + banana: ['yellow', 'green'] + } + const result = getSchemaFromEntryStructure(entry) + + expect(result).toEqual({ + potato: 'string', + apple: 'number', + watermelon: { seeds: { many: 'boolean' } }, + banana: 'string[]' + }) + }) + }) })