Skip to content

Commit

Permalink
fix: records not being filtered by schema on live updates
Browse files Browse the repository at this point in the history
  • Loading branch information
@raiindev
raiindev committed Dec 13, 2024
1 parent e18d3ee commit 4e023cd
Show file tree
Hide file tree
Showing 5 changed files with 105 additions and 66 deletions.
4 changes: 2 additions & 2 deletions server/src/__mocks__/collection.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ const mockCollection = {
entity: 'testEntity',
indexId: 'testIndexId',
status: 'outdated',
searchableAttributes: ['name', 'description'],
searchableAttributes: ['title', 'description'],
schema: {
name: { type: 'string' },
title: { type: 'string' },
description: { type: 'string' }
},
includedRelations: ['relation1', 'relation2']
Expand Down
3 changes: 2 additions & 1 deletion server/src/services/content-types.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,13 +93,14 @@ module.exports = ({ strapi }) => {
}))
},

async getEntries({ contentType, relations = [], schema, offset = 0, limit = 50 }) {
async getEntries({ contentType, relations = [], schema, offset = 0, limit = 50, where }) {
const selectedRelations = getSelectedRelations({ schema, relations })
const selectedFields = getSelectedFieldsConfigObj(schema)

return await strapi.query(contentType).findMany({
populate: selectedRelations,
select: selectedFields,
...(where ? { where } : {}),
limit,
offset
})
Expand Down
2 changes: 1 addition & 1 deletion server/src/services/live-updates.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ module.exports = ({ strapi }) => {
hookManagerService.unregisterHooks(collection)
hookManagerService.registerHooks(collection, {
async afterCreate(event) {
await handleLiveUpdates(event, 'create')
await handleLiveUpdates(event, 'insert')
},
async afterUpdate(event) {
await handleLiveUpdates(event, 'update')
Expand Down
90 changes: 56 additions & 34 deletions server/src/services/orama-manager.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,16 @@
const { CloudManager } = require('@oramacloud/client')
const { getSelectedPropsFromObj } = require('../../../utils')

const getAssociatedActionName = (action) => {
const actionMap = {
insert: 'upsert',
update: 'upsert',
delete: 'delete'
}

return actionMap[action]
}

class OramaManager {
constructor({ strapi }) {
this.strapi = strapi
Expand All @@ -13,8 +23,7 @@ class OramaManager {

this.oramaCloudManager = new CloudManager({ api_key: this.privateApiKey })
this.DocumentActionsMap = {
create: this.oramaInsert.bind(this),
update: this.oramaUpdate.bind(this),
upsert: this.oramaUpsert.bind(this),
delete: this.oramaDelete.bind(this)
}
}
Expand Down Expand Up @@ -153,9 +162,11 @@ class OramaManager {
})

if (entries.length > 0) {
await this.oramaInsert({
indexId: collection.indexId,
entries
await this.oramaUpsert({
collection,
entries,
action: 'insert',
isFromBulk: true
})

return await this.bulkInsert(collection, offset + entries.length)
Expand All @@ -175,45 +186,38 @@ class OramaManager {
}

/*
* Inserts new documents into the index in the Orama Cloud using the OramaCloud SDK
* Updates documents of the specified index in the Orama Cloud using the OramaCloud SDK
* Formats data before insertion using the documentsTransformer function, if provided
* @param {string} indexId - Index ID
* @param {Array} entries - Array of entries
* */
async oramaInsert({ indexId, entries }) {
async oramaUpsert({ collection: { indexId, entity, schema, includedRelations }, action, entries, isFromBulk }) {
let filteredEntries = entries
const index = this.oramaCloudManager.index(indexId)
const transformedData = this.documentsTransformer(indexId, entries)

if (!transformedData) {
this.strapi.log.error(`ERROR: documentsTransformer needs a return value`)
return false
if (!isFromBulk) {
filteredEntries = await this.contentTypesService.getEntries({
contentType: entity,
relations: includedRelations,
schema: schema,
where: {
id: {
$in: entries.map(({ id }) => id)
}
}
})
}

const result = await index.insert(transformedData)

this.strapi.log.info(`INSERT: documents with id ${transformedData.map(({ id }) => id)} into index ${indexId}`)

return result
}

/*
* Updates documents of the specified index in the Orama Cloud using the OramaCloud SDK
* Formats data before insertion using the documentsTransformer function, if provided
* @param {string} indexId - Index ID
* @param {Array} entries - Array of entries
* */
async oramaUpdate({ indexId, entries }) {
const index = this.oramaCloudManager.index(indexId)
const transformedData = this.documentsTransformer(indexId, entries)
const transformedData = this.documentsTransformer(indexId, filteredEntries)

if (!transformedData) {
this.strapi.log.error(`ERROR: documentsTransformer needs a return value`)
return false
}

const result = await index.update(transformedData)
const result = await index[action](transformedData)

this.strapi.log.info(`UPDATE: document with id ${transformedData.map(({ id }) => id)} into index ${indexId}`)
this.strapi.log.info(`${action.toUpperCase()}: document with id ${transformedData.map(({ id }) => id)} into index ${indexId}`)

return result
}
Expand All @@ -223,7 +227,7 @@ class OramaManager {
* @param {string} indexId - Index ID
* @param {Array} entries - Array of entries
* */
async oramaDelete({ indexId, entries }) {
async oramaDelete({ collection: { indexId }, entries }) {
const index = this.oramaCloudManager.index(indexId)
const result = await index.delete(entries.map(({ id }) => id))

Expand All @@ -239,14 +243,18 @@ class OramaManager {
* @param {Object} record - Record object
* @param {string} action - Action to perform (insert, update, delete)
* */
async handleDocument({ indexId, record, action }) {
if (!action || !record || !this.DocumentActionsMap[action]) {
async handleDocument({ collection, record, action }) {
const associatedActionName = getAssociatedActionName(action)

if (!action || !record || !this.DocumentActionsMap[associatedActionName]) {
this.strapi.log.warn(`Action ${action} not found. Skipping...`)

return false
}

const { createdBy, updatedBy, ...rest } = record

return await this.DocumentActionsMap[action]({ indexId, entries: [{ ...rest, id: rest.id.toString() }] })
return await this.DocumentActionsMap[associatedActionName]({ collection, action, entries: [{ ...rest, id: rest.id.toString() }] })
}

/*
Expand Down Expand Up @@ -302,8 +310,22 @@ class OramaManager {
return
}

const customSchema = this.collectionSettings?.[collection.indexId]?.schema

const oramaSchema =
customSchema ??
getSelectedPropsFromObj({
props: collection.searchableAttributes,
obj: collection.schema
})

await this.updatingStarted(collection)

await this.oramaUpdateSchema({
indexId: collection.indexId,
schema: oramaSchema
})

await this.oramaDeployIndex(collection)

await this.updatingCompleted(collection)
Expand All @@ -328,7 +350,7 @@ class OramaManager {
await this.updatingStarted(collection)

const handleDocumentResult = await this.handleDocument({
indexId: collection.indexId,
collection,
record,
action
})
Expand Down
72 changes: 44 additions & 28 deletions server/src/services/orama-manager.test.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ const strapi = {
log: {
error: jest.fn(),
debug: jest.fn(),
warn: jest.fn(),
info: jest.fn()
}
}
Expand Down Expand Up @@ -242,15 +243,17 @@ describe('OramaManager', () => {

describe('bulkInsert', () => {
it('should insert entries', async () => {
const oramaInsertSpy = jest.spyOn(oramaManager, 'oramaInsert').mockResolvedValue()
const oramaUpsertSpy = jest.spyOn(oramaManager, 'oramaUpsert').mockResolvedValue()
const bulkInsertSpy = jest.spyOn(oramaManager, 'bulkInsert')

await oramaManager.bulkInsert(mockCollection)

expect(bulkInsertSpy).toHaveBeenNthCalledWith(1, mockCollection)
expect(oramaInsertSpy).toHaveBeenCalledWith({
indexId: mockCollection.indexId,
entries: [{ id: 1, title: 'Test Entry' }]
expect(oramaUpsertSpy).toHaveBeenCalledWith({
collection: mockCollection,
entries: [{ id: 1, title: 'Test Entry' }],
action: 'insert',
isFromBulk: true
})
expect(bulkInsertSpy).toHaveBeenLastCalledWith(mockCollection, 1)
})
Expand All @@ -266,14 +269,16 @@ describe('OramaManager', () => {
})
})

describe('oramaInsert', () => {
describe('oramaUpsert - insert', () => {
it('should insert entries', async () => {
const documentsTransformerSpy = jest.spyOn(oramaManager, 'documentsTransformer')
const { insert } = new CloudManager({ strapi }).index()

await oramaManager.oramaInsert({
indexId: mockCollection.indexId,
entries: [{ id: 1, title: 'Test Entry' }]
await oramaManager.oramaUpsert({
collection: mockCollection,
entries: [{ id: 1, title: 'Test Entry' }],
action: 'insert',
isFromBulk: true
})

expect(insert).toHaveBeenCalledWith([{ id: 1, title: 'Test Entry' }])
Expand All @@ -296,9 +301,11 @@ describe('OramaManager', () => {
const documentsTransformerSpy = jest.spyOn(oramaManager, 'documentsTransformer')
const { insert } = new CloudManager({ strapi }).index()

await oramaManager.oramaInsert({
indexId: mockCollection.indexId,
entries: [{ id: 1, title: 'Test Entry' }]
await oramaManager.oramaUpsert({
collection: mockCollection,
entries: [{ id: 1, title: 'Test Entry' }],
action: 'insert',
isFromBulk: true
})

expect(documentsTransformerSpy).toHaveBeenCalledWith(mockCollection.indexId, [{ id: 1, title: 'Test Entry' }])
Expand All @@ -314,21 +321,30 @@ describe('OramaManager', () => {
})
})

describe('oramaUpdate', () => {
describe('oramaUpsert - update', () => {
beforeEach(() => {
contentTypesService.getEntries.mockResolvedValue([{ id: 1, title: 'Test Entry' }])
})

it('should update entries', async () => {
const { update } = new CloudManager({ strapi }).index()

await oramaManager.oramaUpdate({
indexId: mockCollection.indexId,
entries: [{ id: 1, title: 'Test Entry' }]
await oramaManager.oramaUpsert({
collection: mockCollection,
entries: [{ id: 1, title: 'Test Entry' }],
action: 'update'
})

expect(update).toHaveBeenCalledWith([{ id: 1, title: 'Test Entry' }])
})
})

describe('handleDocument', () => {
it('should return if action is not found', async () => {
beforeEach(() => {
contentTypesService.getEntries.mockResolvedValue([{ id: 1, title: 'Test Entry' }])
})

it('should return false if action is not found', async () => {
const { insert } = new CloudManager({ strapi }).index()

await oramaManager.handleDocument({
Expand Down Expand Up @@ -358,36 +374,36 @@ describe('OramaManager', () => {
const { insert } = new CloudManager({ strapi }).index()

await oramaManager.handleDocument({
indexId: mockCollection.indexId,
collection: mockCollection,
record: mockedTestRecord,
action: 'create'
action: 'insert',
})

expect(insert).toHaveBeenCalledWith([{ id: '1', title: 'Test Entry' }])
expect(insert).toHaveBeenCalledWith([{ id: 1, title: 'Test Entry' }])
})

it('should return if action is update', async () => {
const { update } = new CloudManager({ strapi }).index()

await oramaManager.handleDocument({
indexId: mockCollection.indexId,
collection: mockCollection,
record: mockedTestRecord,
action: 'update'
})

expect(update).toHaveBeenCalledWith([{ id: '1', title: 'Test Entry' }])
expect(update).toHaveBeenCalledWith([{ id: 1, title: 'Test Entry' }])
})

it('should return if action is delete', async () => {
const { delete: deleteFn } = new CloudManager({ strapi }).index()

await oramaManager.handleDocument({
indexId: mockCollection.indexId,
collection: mockCollection,
record: mockedTestRecord,
action: 'delete'
})

expect(deleteFn).toHaveBeenCalledWith(['1'])
expect(deleteFn).toHaveBeenCalledWith(["1"])
})
})

Expand Down Expand Up @@ -465,22 +481,22 @@ describe('OramaManager', () => {

describe('processLiveUpdate', () => {
it('should process live update', async () => {
await oramaManager.processLiveUpdate(mockCollection, mockedTestRecord, 'create')
await oramaManager.processLiveUpdate(mockCollection, mockedTestRecord, 'insert')

expect(oramaManager.validate).toHaveBeenCalledWith(mockCollection)
expect(updatingStartedSpy).toHaveBeenCalledWith(mockCollection)
expect(handleDocumentSpy).toHaveBeenCalledWith({
indexId: mockCollection.indexId,
collection: mockCollection,
record: mockedTestRecord,
action: 'create'
action: 'insert'
})
expect(setOutdatedSpy).toHaveBeenCalledWith(mockCollection)
})

it('should not process live update if collection is not valid', async () => {
collectionService.findOne.mockReturnValueOnce(mockNotValidCollection)

await oramaManager.processLiveUpdate(mockNotValidCollection, mockedTestRecord, 'create')
await oramaManager.processLiveUpdate(mockNotValidCollection, mockedTestRecord, 'insert')

expect(oramaManager.validate).toHaveBeenCalledWith(mockNotValidCollection)
expect(updatingStartedSpy).not.toHaveBeenCalled()
Expand All @@ -494,7 +510,7 @@ describe('OramaManager', () => {
//Force handleDocument to reject
handleDocumentSpy.mockResolvedValueOnce(false)

await oramaManager.processLiveUpdate(mockCollection, mockedTestRecord, 'create')
await oramaManager.processLiveUpdate(mockCollection, mockedTestRecord, 'insert')

expect(oramaManager.validate).toHaveBeenCalledWith(mockCollection)
expect(updatingStartedSpy).toHaveBeenCalled()
Expand Down

0 comments on commit 4e023cd

Please sign in to comment.