Radar api enhancements and optimisation

desci-server/src/controllers/attestations/claims.ts

@@ -7,6 +7,7 @@ import { AuthFailureError, NotFoundError } from '../../core/ApiError.js';
 import { SuccessMessageResponse, SuccessResponse } from '../../core/ApiResponse.js';
 import { logger } from '../../logger.js';
 import { RequestWithUser } from '../../middleware/authorisation.js';
+import { delFromCache } from '../../redisClient.js';
 import { removeClaimSchema } from '../../routes/v1/attestations/schema.js';
 import { attestationService } from '../../services/Attestation.js';
 import { communityService } from '../../services/Communities.js';
@@ -41,6 +42,11 @@ export const claimAttestation = async (req: RequestWithUser, res: Response, _nex
     await saveInteraction(req, ActionType.CLAIM_ATTESTATION, { ...body, claimId: reclaimed.id });
     // trigger update radar entry
     await communityService.addToRadar(reclaimed.desciCommunityId, reclaimed.nodeUuid);
+    // invalidate radar and curated feed count cache
+    await delFromCache(`radar-${reclaimed.desciCommunityId}-count`);
+    await delFromCache(`curated-${reclaimed.desciCommunityId}-count`);
+    await delFromCache(`all-communities-curated-count`);
+
     new SuccessResponse(reclaimed).send(res);
     return;
   }
@@ -53,6 +59,10 @@ export const claimAttestation = async (req: RequestWithUser, res: Response, _nex
   });
   // trigger update radar entry
   await communityService.addToRadar(nodeClaim.desciCommunityId, nodeClaim.nodeUuid);
+  // invalidate radar and curated feed count cache
+  await delFromCache(`radar-${nodeClaim.desciCommunityId}-count`);
+  await delFromCache(`curated-${nodeClaim.desciCommunityId}-count`);
+  await delFromCache(`all-communities-curated-count`);
 
   await saveInteraction(req, ActionType.CLAIM_ATTESTATION, { ...body, claimId: nodeClaim.id });
 
@@ -136,6 +146,11 @@ export const removeClaim = async (req: RequestWithUser, res: Response, _next: Ne
   // trigger update radar entry
   await communityService.removeFromRadar(claim.desciCommunityId, claim.nodeUuid);
 
+  // invalidate radar and curated feed count cache
+  await delFromCache(`radar-${claim.desciCommunityId}-count`);
+  await delFromCache(`curated-${claim.desciCommunityId}-count`);
+  await delFromCache(`all-communities-curated-count`);
+
   await saveInteraction(req, ActionType.REVOKE_CLAIM, body);
 
   logger.info({ removeOrRevoke, totalSignal, claimSignal }, 'Claim Removed|Revoked');
@@ -193,6 +208,10 @@ export const claimEntryRequirements = async (req: Request, res: Response, _next:
   });
   // trigger update radar entry
   await communityService.addToRadar(communityId, uuid);
+  // invalidate radar and curated feed count cache
+  await delFromCache(`radar-${communityId}-count`);
+  await delFromCache(`curated-${communityId}-count`);
+  await delFromCache(`all-communities-curated-count`);
 
   await saveInteraction(req, ActionType.CLAIM_ENTRY_ATTESTATIONS, {
     communityId,

desci-server/src/controllers/attestations/verification.ts

@@ -7,6 +7,7 @@ import { prisma } from '../../client.js';
 import { ForbiddenError } from '../../core/ApiError.js';
 import { SuccessMessageResponse, SuccessResponse } from '../../core/ApiResponse.js';
 import { logger as parentLogger } from '../../logger.js';
+import { delFromCache } from '../../redisClient.js';
 import { attestationService } from '../../services/Attestation.js';
 import { getTargetDpidUrl } from '../../services/fixDpid.js';
 import { doiService } from '../../services/index.js';
@@ -60,6 +61,11 @@ export const removeVerification = async (
     const claim = await attestationService.findClaimById(verification.nodeAttestationId);
     const attestation = await attestationService.findAttestationById(claim.attestationId);
 
+    // invalidate radar and curated feed count cache
+    await delFromCache(`radar-${claim.desciCommunityId}-count`);
+    await delFromCache(`curated-${claim.desciCommunityId}-count`);
+    await delFromCache(`all-communities-curated-count`);
+
     if (attestation.protected) {
       /**
        * Update ORCID Profile
@@ -103,6 +109,11 @@ export const addVerification = async (
 
   new SuccessMessageResponse().send(res);
 
+  // invalidate radar and curated feed count cache
+  await delFromCache(`radar-${claim.desciCommunityId}-count`);
+  await delFromCache(`curated-${claim.desciCommunityId}-count`);
+  await delFromCache(`all-communities-curated-count`);
+
   const attestation = await attestationService.findAttestationById(claim.attestationId);
   if (attestation.protected) {
     /**

desci-server/src/controllers/communities/feed.ts

@@ -5,7 +5,8 @@ import z from 'zod';
 import { NotFoundError } from '../../core/ApiError.js';
 import { SuccessResponse } from '../../core/ApiResponse.js';
 import { logger as parentLogger } from '../../logger.js';
-import { getCommunityFeedSchema } from '../../routes/v1/communities/schema.js';
+import { getFromCache, setToCache } from '../../redisClient.js';
+import { getAllCommunitiesFeedSchema, getCommunityFeedSchema } from '../../routes/v1/communities/schema.js';
 import { attestationService } from '../../services/Attestation.js';
 import { communityService } from '../../services/Communities.js';
 import { asyncMap } from '../../utils.js';
@@ -51,15 +52,22 @@ export const getCommunityFeed = async (req: Request, res: Response, next: NextFu
 export const listCommunityFeed = async (req: Request, res: Response, next: NextFunction) => {
   const { query, params } = await getCommunityFeedSchema.parseAsync(req);
   const limit = 20;
-  const page = Math.max(Math.max((query.cursor ?? 0) - 1, 0), 0);
+  const page = Math.max(Math.max((query.page ?? 0) - 1, 0), 0);
   const offset = limit * page;
 
+  let totalCount = await getFromCache<number>(`curated-${params.communityId}-count`);
+  if (!totalCount) {
+    totalCount = await communityService.countCommunityCuratedFeed(parseInt(params.communityId.toString()));
+    logger.trace({ totalCount }, 'FeedCount');
+    setToCache(`curated-${params.communityId}-count`, totalCount);
+  }
+
   const curatedNodes = await communityService.listCommunityCuratedFeed({
     communityId: parseInt(params.communityId.toString()),
     offset,
     limit,
   });
-  logger.trace({ offset, page, cursor: query.cursor }, 'Feed');
+  logger.trace({ offset, page }, 'Feed');
   // THIS is necessary because the engagement signal returned from getcuratedNodes
   // accounts for only engagements on community selected attestations
   const entries = await asyncMap(curatedNodes, async (entry) => {
@@ -77,7 +85,13 @@ export const listCommunityFeed = async (req: Request, res: Response, next: NextF
 
   const data = await Promise.all(entries.map(getCommunityNodeDetails));
   // logger.info({ count: data.length, page: offset }, 'listCommunityFeed');
-  return new SuccessResponse({ count: data.length, cursor: page + 1, data }).send(res);
+  return new SuccessResponse({
+    data,
+    page: page + 1,
+    count: totalCount,
+    nextPage: data.length === limit ? page + 2 : undefined,
+    communityId: params.communityId,
+  }).send(res);
 };
 
 export const getCommunityDetails = async (req: Request, res: Response, next: NextFunction) => {
@@ -135,3 +149,48 @@ export const getAllFeeds = async (req: Request, res: Response, next: NextFunctio
 
   return new SuccessResponse(data).send(res);
 };
+
+export const listAllCommunityCuratedFeeds = async (req: Request, res: Response, next: NextFunction) => {
+  const { query } = await getAllCommunitiesFeedSchema.parseAsync(req);
+  const limit = 20;
+  const page = Math.max(Math.max((query.page ?? 0) - 1, 0), 0);
+  const offset = limit * page;
+
+  let totalCount = await getFromCache<number>(`all-communities-curated-count`);
+  if (!totalCount) {
+    totalCount = await communityService.countAllCommunityCuratedFeeds();
+    logger.trace({ totalCount }, 'countAllCommunityCuratedFeeds');
+    setToCache(`all-communities-curated-count`, totalCount);
+  }
+
+  const curatedNodes = await communityService.listAllCommunityCuratedFeeds({
+    // communityId: parseInt(params.communityId.toString()),
+    offset,
+    limit,
+  });
+  logger.trace({ offset, page }, 'countAllCommunityCuratedFeeds');
+  // THIS is necessary because the engagement signal returned from getcuratedNodes
+  // accounts for only engagements on community selected attestations
+  const entries = await asyncMap(curatedNodes, async (entry) => {
+    const engagements = await attestationService.getNodeEngagementSignalsByUuid(entry.nodeUuid);
+    return {
+      ...entry,
+      engagements,
+      verifiedEngagements: {
+        reactions: entry.reactions,
+        annotations: entry.annotations,
+        verifications: entry.verifications,
+      },
+    };
+  });
+
+  const data = await Promise.all(entries.map(getCommunityNodeDetails));
+  // logger.info({ count: data.length, page: offset }, 'listCommunityFeed');
+  return new SuccessResponse({
+    data,
+    page: page + 1,
+    count: totalCount,
+    nextPage: data.length === limit ? page + 2 : undefined,
+    // communityId: params.communityId,
+  }).send(res);
+};

desci-server/src/controllers/communities/radar.ts

@@ -4,6 +4,7 @@ import _ from 'lodash';
 
 import { SuccessResponse } from '../../core/ApiResponse.js';
 import { logger as parentLogger } from '../../logger.js';
+import { getFromCache, redisClient, setToCache } from '../../redisClient.js';
 import { getCommunityFeedSchema } from '../../routes/v1/communities/schema.js';
 import { attestationService } from '../../services/Attestation.js';
 import { communityService } from '../../services/Communities.js';
@@ -65,15 +66,23 @@ export const getCommunityRadar = async (req: Request, res: Response, next: NextF
 export const listCommunityRadar = async (req: Request, res: Response, next: NextFunction) => {
   const { query, params } = await getCommunityFeedSchema.parseAsync(req);
   const limit = 20;
-  const page = Math.max(Math.max((query.cursor ?? 0) - 1, 0), 0);
+  const page = Math.max(Math.max((query.page ?? 0) - 1, 0), 0);
   const offset = limit * page;
 
+  let totalCount = await getFromCache<number>(`radar-${params.communityId}-count`);
+  if (!totalCount) {
+    totalCount = await communityService.countCommunityRadar(parseInt(params.communityId.toString()));
+    logger.trace({ totalCount }, 'RadarCount');
+    setToCache(`radar-${params.communityId}-count`, totalCount);
+  }
+
   const communityRadar = await communityService.listCommunityRadar({
     communityId: parseInt(params.communityId.toString()),
     offset,
     limit,
   });
-  logger.trace({ offset, page, cursor: query.cursor }, 'Radar');
+
+  logger.trace({ offset, page, cursor: query.page }, 'Radar');
   // THIS is necessary because the engagement signal returned from getCommunityRadar
   // accounts for only engagements on community selected attestations
   const entries = await asyncMap(communityRadar, async (entry) => {
@@ -92,5 +101,12 @@ export const listCommunityRadar = async (req: Request, res: Response, next: Next
   // rank nodes by sum of sum of verified and non verified signals
   const data = await Promise.all(entries.map(getCommunityNodeDetails));
   // logger.trace({ count: data.length }, 'listCommunityRadar');
-  return new SuccessResponse({ count: data.length, cursor: page + 1, data }).send(res);
+
+  return new SuccessResponse({
+    data,
+    count: totalCount,
+    page: page + 1,
+    nextPage: data.length === limit ? page + 2 : undefined,
+    communityId: params.communityId,
+  }).send(res);
 };

desci-server/src/controllers/communities/util.ts

@@ -71,7 +71,7 @@ export const resolveLatestNode = async (radar: Partial<NodeRadar>) => {
 };
 
 export const getCommunityNodeDetails = async (
-  radar: RadarEntry & { node?: Partial<Node & { versions: number }>; manifest?: ResearchObjectV1 },
+  radar: RadarEntry & { node?: Partial<Node & { versions: number; dpid?: number }>; manifest?: ResearchObjectV1 },
 ) => {
   const uuid = ensureUuidEndsWithDot(radar.nodeUuid);
 
@@ -95,8 +95,8 @@ export const getCommunityNodeDetails = async (
     logger.warn({ uuid }, 'uuid not found');
   }
 
-  const selectAttributes: (keyof typeof discovery)[] = ['ownerId', 'NodeCover'];
-  const node: Partial<Node & { versions: number }> = _.pick(discovery, selectAttributes);
+  const selectAttributes: (keyof typeof discovery)[] = ['ownerId', 'NodeCover', 'dpidAlias'];
+  const node: Partial<Node & { versions: number; dpid?: number }> = _.pick(discovery, selectAttributes);
   const publishedVersions =
     (await prisma.$queryRaw`SELECT * from "NodeVersion" where "nodeId" = ${discovery.id} AND ("transactionId" IS NOT NULL or "commitId" IS NOT NULL) ORDER BY "createdAt" DESC`) as NodeVersion[];
 
@@ -106,6 +106,8 @@ export const getCommunityNodeDetails = async (
   node['publishedDate'] = publishedVersions[0].createdAt;
   node.manifestUrl = publishedVersions[0].manifestUrl;
   radar.node = node;
+  radar.node.dpid = node.dpidAlias;
+  delete radar.node.dpidAlias;
 
   let gatewayUrl = publishedVersions[0].manifestUrl;
 

desci-server/src/controllers/nodes/feed.ts

@@ -1,5 +1,5 @@
 import { NodeFeedItem } from '@prisma/client';
-import { Sql } from '@prisma/client/runtime/library.js';
+import { Sql } from '@prisma/client/runtime/data-proxy.js';
 import { Request, Response } from 'express';
 
 import { prisma } from '../../client.js';

desci-server/src/routes/v1/communities/index.ts

@@ -8,6 +8,7 @@ import {
   getAllFeeds,
   getCommunityDetails,
   getCommunityFeed,
+  listAllCommunityCuratedFeeds,
   listCommunityFeed,
 } from '../../../controllers/communities/feed.js';
 import { checkMemberGuard } from '../../../controllers/communities/guard.js';
@@ -17,13 +18,18 @@ import { ensureUser } from '../../../middleware/permissions.js';
 import { validate } from '../../../middleware/validator.js';
 import { asyncHandler } from '../../../utils/asyncHandler.js';
 
-import { getCommunityDetailsSchema, getCommunityFeedSchema, memberGuardSchema } from './schema.js';
+import {
+  getAllCommunitiesFeedSchema,
+  getCommunityDetailsSchema,
+  getCommunityFeedSchema,
+  memberGuardSchema,
+} from './schema.js';
 
 const router = Router();
 
 // list all communities and curated nodes()
 router.get('/list', [], asyncHandler(listCommunities));
-router.get('/feeds', [], asyncHandler(getAllFeeds));
+router.get('/feeds', [validate(getAllCommunitiesFeedSchema)], asyncHandler(listAllCommunityCuratedFeeds));
 router.get('/:communityName', [validate(getCommunityDetailsSchema)], asyncHandler(getCommunityDetails));
 router.get(
   '/:communityName/attestations',

desci-server/src/routes/v1/communities/schema.ts

@@ -11,7 +11,13 @@ export const getCommunityFeedSchema = z.object({
     communityId: z.coerce.number(),
   }),
   query: z.object({
-    cursor: z.coerce.number().optional().default(0),
+    page: z.coerce.number().optional().default(0),
+  }),
+});
+
+export const getAllCommunitiesFeedSchema = z.object({
+  query: z.object({
+    page: z.coerce.number().optional().default(0),
   }),
 });