[backend] Internal cache performance improvements (#9805)

opencti-platform/opencti-graphql/src/database/cache.ts

@@ -56,6 +56,20 @@ export const writeCacheForEntity = (entityType: string, data: unknown) => {
   cache[entityType] = data;
 };
 
+export const partialRefreshCacheForEntity = async (instance: StoreEntity | StoreRelation) => {
+  const instanceCache = cache[instance.entity_type];
+  if (instanceCache && instanceCache.values && instanceCache.refresh) {
+    logApp.debug('Partial refresh cache for entity', { type: instance.entity_type, id: instance.id });
+    const instanceToStore = await instanceCache.refresh(instance.id);
+    const cacheWithoutInstance = instanceCache.values.filter((v: any) => v.id !== instanceToStore.id);
+    if (cacheWithoutInstance) {
+      instanceCache.values = [...cacheWithoutInstance, instanceToStore];
+    } else { // instance not found (deleted)
+      instanceCache.values = [...cacheWithoutInstance];
+    }
+  }
+};
+
 export const resetCacheForEntity = (entityType: string) => {
   const types = [entityType, ...(STORE_ENTITIES_LINKS[entityType] ?? [])];
   types.forEach((type) => {
@@ -68,6 +82,18 @@ export const resetCacheForEntity = (entityType: string) => {
   });
 };
 
+export const refreshCacheForEntity = async (instance: StoreEntity | StoreRelation) => {
+  // called by manager
+  const entityType = instance.entity_type;
+  const entityCache = cache[entityType];
+  if (entityCache && entityCache.values && entityCache.refresh && !STORE_ENTITIES_LINKS[entityType]) {
+    // try to partial reset cache if we have the instance and there are no linked cache
+    await partialRefreshCacheForEntity(instance);
+  } else {
+    resetCacheForEntity(entityType);
+  }
+};
+
 export const dynamicCacheUpdater = (instance: StoreEntity | StoreRelation) => {
   // Dynamic update of filtering cache
   const currentFiltersValues = cache[ENTITY_TYPE_RESOLVED_FILTERS]?.values;

opencti-platform/opencti-graphql/src/domain/user.js

@@ -1663,6 +1663,7 @@ const buildCompleteUserFromCacheOrDb = async (context, user, userToLoad, cachedU
     // we need groups and capabilities to compute user effective confidence level, which are accurate in cache.
     completeUser = {
       ...userToLoad,
+      effective_confidence_level: cachedUser.effective_confidence_level, // already in cache (buildCompleteUser)
       groups: cachedUser.groups,
       capabilities: cachedUser.capabilities,
     };

opencti-platform/opencti-graphql/src/manager/cacheManager.ts

@@ -1,7 +1,7 @@
 import { Promise as Bluebird } from 'bluebird';
 import * as R from 'ramda';
 import { logApp, TOPIC_PREFIX } from '../config/conf';
-import { dynamicCacheUpdater, resetCacheForEntity, writeCacheForEntity } from '../database/cache';
+import { dynamicCacheUpdater, refreshCacheForEntity, writeCacheForEntity } from '../database/cache';
 import type { AuthContext } from '../types/user';
 import { ENTITY_TYPE_RESOLVED_FILTERS } from '../schema/stixDomainObject';
 import { ENTITY_TYPE_ENTITY_SETTING } from '../modules/entitySetting/entitySetting-types';
@@ -139,12 +139,15 @@ const platformRunningPlaybooks = (context: AuthContext) => {
   return { values: null, fn: reloadPlaybooks };
 };
 const platformUsers = (context: AuthContext) => {
+  const loadUser = async (userId: string) => {
+    return resolveUserById(context, userId);
+  };
   const reloadUsers = async () => {
     const users = await listAllEntities(context, SYSTEM_USER, [ENTITY_TYPE_USER], { connectionFormat: false });
     const allUserIds = users.map((user) => user.internal_id);
     return Bluebird.map(allUserIds, (userId: string) => resolveUserById(context, userId), { concurrency: ES_MAX_CONCURRENCY }).filter((u) => u != null);
   };
-  return { values: null, fn: reloadUsers };
+  return { values: null, fn: reloadUsers, refresh: loadUser };
 };
 const platformSettings = (context: AuthContext) => {
   const reloadSettings = async () => {
@@ -240,7 +243,7 @@ const initCacheManager = () => {
   const resetCacheContent = async (event: { instance: StoreEntity | StoreRelation }) => {
     const { instance } = event;
     // Invalid cache if any entity has changed.
-    resetCacheForEntity(instance.entity_type);
+    await refreshCacheForEntity(instance);
     // Smart dynamic cache loading (for filtering ...)
     dynamicCacheUpdater(instance);
   };

opencti-platform/opencti-graphql/src/utils/confidence-level.ts

@@ -24,6 +24,8 @@ type ConfidenceOverride = {
 };
 
 export const computeUserEffectiveConfidenceLevel = (user: AuthUser) => {
+  // already computed (in buildCompleteUser)
+  if (user.effective_confidence_level) { return user.effective_confidence_level; }
   // if a user has BYPASS capability, we consider a level 100
   if (isBypassUser(user)) {
     return {