From 1f86e20562222e6c0dabe71748c3238fa3ff440b Mon Sep 17 00:00:00 2001 From: Rakib Ansary <rakib.saikot@colorado.edu> Date: Mon, 31 Jan 2022 00:09:26 +0600 Subject: [PATCH 1/3] feat: lookup from dynamodb if member is not found in ES if member is not found in ES lookup from DynamoDB and on success emit profile created event to let member-processor-es index the member in ES --- app-constants.js | 1 + src/services/MemberService.js | 17 ++++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/app-constants.js b/app-constants.js index 3c40c7e..27bcf46 100644 --- a/app-constants.js +++ b/app-constants.js @@ -8,6 +8,7 @@ const EVENT_ORIGINATOR = 'topcoder-member-api' const EVENT_MIME_TYPE = 'application/json' const TOPICS = { + MemberCreated: 'member.action.profile.create', MemberUpdated: 'member.action.profile.update', EmailChanged: 'member.action.email.profile.emailchange.verification', MemberTraitCreated: 'member.action.profile.trait.create', diff --git a/src/services/MemberService.js b/src/services/MemberService.js index bbf3a59..186cb06 100644 --- a/src/services/MemberService.js +++ b/src/services/MemberService.js @@ -64,6 +64,7 @@ function omitMemberAttributes (currentUser, mb) { async function getMember (currentUser, handle, query) { // validate and parse query parameter const selectFields = helper.parseCommaSeparatedString(query.fields, MEMBER_FIELDS) || MEMBER_FIELDS + // query member from Elasticsearch const esQuery = { index: config.ES.MEMBER_PROFILE_ES_INDEX, @@ -80,11 +81,25 @@ async function getMember (currentUser, handle, query) { } // Search with constructed query let members = await esClient.search(esQuery) + if (members.hits.total === 0) { - throw new errors.NotFoundError(`Member with handle: "${handle}" doesn't exist`) + logger.debug(`Member ${handle} not found in ES. Lookup in DynamoDB...`) + try { + // Check if the member handle exists in DynamoDB + members = [ await helper.getMemberByHandle(handle) ] + // Memember was found in DynamoDB but not ES. Send message to member-processor-es + // to index the member in ES. It's safe to use the "create" topic since the processor + // will only create a new item of the item doesn't exist, otherwise it'll perform an update operation. + helper.postBusEvent(constants.TOPICS.MemberCreated, members[0].originalItem()) + } + catch (e) { + logger.debug(`Member ${handle} not found in DynamoDB.`) + throw new errors.NotFoundError(`Member with handle: "${handle}" doesn't exist`) + } } else { members = _.map(members.hits.hits, '_source') } + // get the 'maxRating' from stats if (_.includes(selectFields, 'maxRating')) { for (let i = 0; i < members.length; i += 1) { From ef1148b85a5e6f2550e9055757ee3bb2c4ac9cb4 Mon Sep 17 00:00:00 2001 From: Rakib Ansary <rakib.saikot@colorado.edu> Date: Mon, 31 Jan 2022 15:51:10 +0600 Subject: [PATCH 2/3] feat: lookup traits from dynamodb if not found in ES --- src/services/MemberTraitService.js | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/services/MemberTraitService.js b/src/services/MemberTraitService.js index 49cc037..fdd44f3 100644 --- a/src/services/MemberTraitService.js +++ b/src/services/MemberTraitService.js @@ -51,6 +51,29 @@ async function getTraits (currentUser, handle, query) { // Search with constructed query const docs = await esClient.search(esQuery) let result = _.map(docs.hits.hits, (item) => item._source) + + if (result.length == 0) { + logger.debug(`MemberTraits for member ${handle} not found in ES. Lookup in DynamoDB...`) + const resultDynamo = await helper.query('MemberTrait', { userId: { eq: member.userId } }) + result = resultDynamo.map(traits => { + traits = traits.originalItem(); + traits.traits = JSON.parse(traits.traits); + + if (traits.createdAt != null) { + traits.createdAt = new Date(traits.createdAt).getTime(); + } + + if (traits.updatedAt != null) { + traits.updatedAt = new Date(traits.createdAt).getTime(); + } + + // index in ES so subsequent API calls pull data from ES + helper.postBusEvent(constants.TOPICS.MemberTraitUpdated, traits) + + return traits; + }) + } + // keep only those of given trait ids if (traitIds) { result = _.filter(result, (item) => _.includes(traitIds, item.traitId)) From 346647da0a4eaea73eefd21d8b0b0e789878e3ce Mon Sep 17 00:00:00 2001 From: Rakib Ansary <rakib.saikot@colorado.edu> Date: Mon, 31 Jan 2022 15:52:50 +0600 Subject: [PATCH 3/3] lint: fix lint issues --- src/services/MemberService.js | 3 +-- src/services/MemberTraitService.js | 18 +++++++++--------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/services/MemberService.js b/src/services/MemberService.js index 186cb06..2ff4d53 100644 --- a/src/services/MemberService.js +++ b/src/services/MemberService.js @@ -91,8 +91,7 @@ async function getMember (currentUser, handle, query) { // to index the member in ES. It's safe to use the "create" topic since the processor // will only create a new item of the item doesn't exist, otherwise it'll perform an update operation. helper.postBusEvent(constants.TOPICS.MemberCreated, members[0].originalItem()) - } - catch (e) { + } catch (e) { logger.debug(`Member ${handle} not found in DynamoDB.`) throw new errors.NotFoundError(`Member with handle: "${handle}" doesn't exist`) } diff --git a/src/services/MemberTraitService.js b/src/services/MemberTraitService.js index fdd44f3..f70e1bc 100644 --- a/src/services/MemberTraitService.js +++ b/src/services/MemberTraitService.js @@ -51,27 +51,27 @@ async function getTraits (currentUser, handle, query) { // Search with constructed query const docs = await esClient.search(esQuery) let result = _.map(docs.hits.hits, (item) => item._source) - + if (result.length == 0) { logger.debug(`MemberTraits for member ${handle} not found in ES. Lookup in DynamoDB...`) const resultDynamo = await helper.query('MemberTrait', { userId: { eq: member.userId } }) result = resultDynamo.map(traits => { - traits = traits.originalItem(); - traits.traits = JSON.parse(traits.traits); - + traits = traits.originalItem() + traits.traits = JSON.parse(traits.traits) + if (traits.createdAt != null) { - traits.createdAt = new Date(traits.createdAt).getTime(); + traits.createdAt = new Date(traits.createdAt).getTime() } - + if (traits.updatedAt != null) { - traits.updatedAt = new Date(traits.createdAt).getTime(); + traits.updatedAt = new Date(traits.createdAt).getTime() } // index in ES so subsequent API calls pull data from ES helper.postBusEvent(constants.TOPICS.MemberTraitUpdated, traits) - return traits; - }) + return traits + }) } // keep only those of given trait ids