[backend] Introducing profiling capability with pyroscope

opencti-platform/opencti-dev/docker-compose.yml

@@ -118,15 +118,28 @@ services:
     ports:
       - 5672:5672
       - 15672:15672
-  opencti-dev-jaegertracing:
+
+  ## Telemetry + tracing dependencies
+  ## docker compose --profile telemetry up -d (launch everything + telemetry)
+  opencti-dev-telemetry-pyroscope:
+    container_name: opencti-pyroscope
+    image: grafana/pyroscope
+    restart: unless-stopped
+    ports:
+      - "4040:4040"
+    profiles:
+      - "telemetry"
+  opencti-dev-telemetry-jaegertracing:
     container_name: opencti-dev-jaegertracing
     image: jaegertracing/all-in-one:latest
     environment:
       COLLECTOR_OTLP_ENABLED: true
     ports:
       - "16686:16686"
       - "4318:4318"
-  opencti-dev-telemetry-otlp: ## docker compose --profile telemetry up -d (launch everything + telemetry)
+    profiles:
+      - "telemetry"
+  opencti-dev-telemetry-otlp:
     container_name: opencti-telemetry-otlp
     image: otel/opentelemetry-collector-contrib:0.116.1
     restart: unless-stopped

opencti-platform/opencti-graphql/builder/dev/dev.js

@@ -2,11 +2,22 @@ const esbuild = require('esbuild');
 const {default: importGlobPlugin} = require('esbuild-plugin-import-glob');
 const {default: graphqlLoaderPlugin} = require('@luckycatfactory/esbuild-graphql-loader');
 const nativeNodePlugin = require("../plugin/native.node.plugin");
+const {copy} = require('esbuild-plugin-copy');
 
 esbuild.build({
     logLevel: 'info',
     define: {'process.env.NODE_ENV': '\"development\"'},
-    plugins: [importGlobPlugin(), graphqlLoaderPlugin(), nativeNodePlugin()],
+    plugins: [
+        importGlobPlugin(),
+        graphqlLoaderPlugin(),
+        nativeNodePlugin(),
+        copy({
+            assets: {
+                from: ['./node_modules/@datadog/pprof/prebuilds/**/*'],
+                to: ['./prebuilds'],
+            }
+        }),
+    ],
     entryPoints: [
         'src/back.js',
         'script/script-clean-relations.js',
@@ -19,7 +30,7 @@ esbuild.build({
     platform: 'node',
     target: ['node14'],
     minify: false,
-    keepNames: true,
+    keepNames: false,
     sourcemap: 'inline',
     outdir: 'build',
 });

opencti-platform/opencti-graphql/builder/prod/prod.js

@@ -2,11 +2,22 @@ const esbuild = require('esbuild');
 const {default: importGlobPlugin} = require('esbuild-plugin-import-glob');
 const {default: graphqlLoaderPlugin} = require('@luckycatfactory/esbuild-graphql-loader');
 const nativeNodePlugin = require("../plugin/native.node.plugin");
+const {copy} = require("esbuild-plugin-copy");
 
 esbuild.build({
     logLevel: 'info',
     define: {'process.env.NODE_ENV': '\"production\"'},
-    plugins: [importGlobPlugin(), graphqlLoaderPlugin(), nativeNodePlugin()],
+    plugins: [
+        importGlobPlugin(),
+        graphqlLoaderPlugin(),
+        nativeNodePlugin(),
+        copy({
+            assets: {
+                from: ['./node_modules/@datadog/pprof/prebuilds/**/*'],
+                to: ['./prebuilds'],
+            }
+        }),
+    ],
     entryPoints: [
         'src/back.js',
         'script/script-clean-relations.js'
@@ -16,8 +27,10 @@ esbuild.build({
     loader: {'.js': 'jsx'},
     platform: 'node',
     target: ['node14'],
-    minify: true,
-    keepNames: true,
+    minifyWhitespace: true,
+    minifyIdentifiers: false,
+    minifySyntax: true,
+    keepNames: false,
     sourcemap: true,
     sourceRoot: 'src',
     sourcesContent: false,

opencti-platform/opencti-graphql/config/default.json

@@ -56,6 +56,11 @@
       }
     },
     "telemetry": {
+      "pyroscope": {
+        "enabled": false,
+        "identifier": "OpenCTI",
+        "exporter": ""
+      },
       "tracing": {
         "enabled": false,
         "exporter_otlp": "",

opencti-platform/opencti-graphql/package.json

@@ -47,6 +47,7 @@
     "@aws-sdk/credential-provider-node": "3.679.0",
     "@aws-sdk/lib-storage": "3.679.0",
     "@aws-sdk/property-provider": "3.374.0",
+    "@datadog/pprof": "5.4.1",
     "@elastic/elasticsearch": "8.17.0",
     "@escape.tech/graphql-armor": "3.1.2",
     "@graphql-tools/import": "7.0.1",
@@ -73,6 +74,7 @@
     "@opentelemetry/sdk-trace-base": "1.25.1",
     "@opentelemetry/sdk-trace-node": "1.25.1",
     "@opentelemetry/semantic-conventions": "1.25.1",
+    "@pyroscope/nodejs": "0.4.3",
     "ajv": "8.17.1",
     "amqplib": "0.10.5",
     "antlr4": "4.13.2",
@@ -186,6 +188,7 @@
     "apollo-server-errors": "3.3.1",
     "cross-env": "7.0.3",
     "esbuild": "0.24.2",
+    "esbuild-plugin-copy": "2.1.1",
     "esbuild-plugin-import-glob": "0.1.1",
     "eslint": "8.57.1",
     "eslint-config-airbnb": "19.0.4",
@@ -204,6 +207,7 @@
   },
   "resolutions": {
     "axios": "1.7.9",
+    "chokidar": "4.0.3",
     "cross-spawn": "7.0.6",
     "body-parser": "1.20.3",
     "cookie": "0.7.2",
@@ -218,6 +222,7 @@
     "domino": "patch:domino@2.1.6#./patch/domino-2.1.6.patch",
     "graphql": "patch:graphql@16.9.0#./patch/graphql-16.8.1.patch",
     "openid-client": "patch:openid-client@5.6.5#./patch/openid-client-5.6.5.patch",
+    "@datadog/pprof": "patch:@datadog/pprof@5.4.1#./patch/@datadog-pprof-5.4.1.patch",
     "path-to-regexp": "0.1.12"
   },
   "packageManager": "yarn@4.5.3"

opencti-platform/opencti-graphql/patch/@datadog-pprof-5.4.1.patch

@@ -0,0 +1,26 @@
+diff --git a/out/src/time-profiler-bindings.js b/out/src/time-profiler-bindings.js
+index d6d3f23..b7da0d7 100644
+--- a/out/src/time-profiler-bindings.js
++++ b/out/src/time-profiler-bindings.js
+@@ -18,7 +18,7 @@ exports.getNativeThreadId = exports.constants = exports.TimeProfiler = void 0;
+  */
+ const path_1 = require("path");
+ const findBinding = require('node-gyp-build');
+-const profiler = findBinding((0, path_1.join)(__dirname, '..', '..'));
++const profiler = findBinding((0, path_1.join)(__dirname, '..', 'build'));
+ exports.TimeProfiler = profiler.TimeProfiler;
+ exports.constants = profiler.constants;
+ exports.getNativeThreadId = profiler.getNativeThreadId;
+diff --git a/out/src/heap-profiler-bindings.js b/out/src/heap-profiler-bindings.js
+index 25157fb..8838bec 100644
+--- a/out/src/heap-profiler-bindings.js
++++ b/out/src/heap-profiler-bindings.js
+@@ -41,7 +41,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
+ exports.monitorOutOfMemory = exports.getAllocationProfile = exports.stopSamplingHeapProfiler = exports.startSamplingHeapProfiler = void 0;
+ const path = __importStar(require("path"));
+ const findBinding = require('node-gyp-build');
+-const profiler = findBinding(path.join(__dirname, '..', '..'));
++const profiler = findBinding(path.join(__dirname, '..', 'build'));
+ // Wrappers around native heap profiler functions.
+ function startSamplingHeapProfiler(heapIntervalBytes, heapStackDepth) {
+     profiler.heapProfiler.startSamplingHeapProfiler(heapIntervalBytes, heapStackDepth);

opencti-platform/opencti-graphql/src/back.js

@@ -1,3 +1,5 @@
+import './instrumentation';
+
 import 'source-map-support/register';
 import blocked from 'blocked-at';
 import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';

opencti-platform/opencti-graphql/src/config/tracing.ts

@@ -1,7 +1,7 @@
 import { SEMATTRS_ENDUSER_ID } from '@opentelemetry/semantic-conventions';
 import { MeterProvider, MetricReader, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
 import { ValueType } from '@opentelemetry/api-metrics';
-import type { Counter } from '@opentelemetry/api-metrics/build/src/types/Metric';
+import type { Counter, Histogram } from '@opentelemetry/api-metrics/build/src/types/Metric';
 // eslint-disable-next-line @typescript-eslint/ban-ts-comment
 // @ts-ignore
 import nodeMetrics from 'opentelemetry-node-metrics';
@@ -20,7 +20,7 @@ class MeterManager {
 
   private errors: Counter | null = null;
 
-  private latencyGauge: Gauge | null = null;
+  private latencyHistogram: Histogram | null = null;
 
   private directBulkGauge: Gauge | null = null;
 
@@ -39,7 +39,7 @@ class MeterManager {
   }
 
   latency(val: number, attributes: any) {
-    this.latencyGauge?.record(val, attributes);
+    this.latencyHistogram?.record(val, attributes);
   }
 
   directBulk(val: number, attributes: any) {
@@ -61,11 +61,13 @@ class MeterManager {
       valueType: ValueType.INT,
       description: 'Counts total number of errors'
     });
-    // - Gauges
-    this.latencyGauge = meter.createGauge('opencti_api_latency', {
+    // - Histograms
+    this.latencyHistogram = meter.createHistogram('opencti_api_latency', {
       valueType: ValueType.INT,
-      description: 'Latency computing per query'
+      description: 'Latency computing per query',
+      advice: { explicitBucketBoundaries: [0, 100, 500, 2000, 5000] }
     });
+    // - Gauges
     this.directBulkGauge = meter.createGauge('opencti_api_direct_bulk', {
       valueType: ValueType.INT,
       description: 'Size of bulks for direct absorption'

opencti-platform/opencti-graphql/src/graphql/graphql.js

@@ -9,10 +9,11 @@ import { constraintDirectiveDocumentation } from 'graphql-constraint-directive';
 import { GraphQLError } from 'graphql/error';
 import { createApollo4QueryValidationPlugin } from 'graphql-constraint-directive/apollo4';
 import createSchema from './schema';
-import conf, { basePath, DEV_MODE, ENABLED_TRACING, GRAPHQL_ARMOR_DISABLED, PLAYGROUND_ENABLED, PLAYGROUND_INTROSPECTION_DISABLED } from '../config/conf';
+import conf, { basePath, DEV_MODE, ENABLED_METRICS, ENABLED_TRACING, GRAPHQL_ARMOR_DISABLED, PLAYGROUND_ENABLED, PLAYGROUND_INTROSPECTION_DISABLED } from '../config/conf';
 import { ForbiddenAccess, ValidationError } from '../config/errors';
 import loggerPlugin from './loggerPlugin';
 import telemetryPlugin from './telemetryPlugin';
+import tracingPlugin from './tracingPlugin';
 import httpResponsePlugin from './httpResponsePlugin';
 
 const createApolloServer = () => {
@@ -93,6 +94,9 @@ const createApolloServer = () => {
   };
   apolloPlugins.push(secureIntrospectionPlugin);
   if (ENABLED_TRACING) {
+    apolloPlugins.push(tracingPlugin);
+  }
+  if (ENABLED_METRICS) {
     apolloPlugins.push(telemetryPlugin);
   }
   const apolloServer = new ApolloServer({

opencti-platform/opencti-graphql/src/graphql/telemetryPlugin.js

@@ -1,5 +1,4 @@
 import { head, includes } from 'ramda';
-import { SEMATTRS_DB_OPERATION, SEMATTRS_ENDUSER_ID, SEMATTRS_MESSAGING_MESSAGE_PAYLOAD_COMPRESSED_SIZE_BYTES } from '@opentelemetry/semantic-conventions';
 import { meterManager } from '../config/tracing';
 import { AUTH_FAILURE, AUTH_REQUIRED, FORBIDDEN_ACCESS } from '../config/errors';
 import { isEmptyField } from '../database/utils';
@@ -21,55 +20,26 @@ const getRequestError = (context) => {
 // noinspection JSUnusedGlobalSymbols
 export default {
   requestDidStart: /* v8 ignore next */ () => {
-    let tracingSpan;
     const start = Date.now();
     return {
-      didResolveOperation: (resolveContext) => {
-        const isWrite = resolveContext.operation && resolveContext.operation.operation === 'mutation';
-        const operationType = `${isWrite ? 'INSERT' : 'SELECT'}`;
-        const { contextValue: context } = resolveContext;
-        const endUserId = context.user?.origin?.user_id ?? 'anonymous';
-        tracingSpan = context.tracing.getTracer().startSpan(`${operationType} ${resolveContext.operationName}`, {
-          attributes: {
-            'enduser.type': context.source,
-            [SEMATTRS_DB_OPERATION]: operationType,
-            [SEMATTRS_ENDUSER_ID]: endUserId,
-          },
-          kind: 1,
-        });
-        context.tracing.setCurrentCtx(tracingSpan);
-      },
       willSendResponse: async (sendContext) => {
         const requestError = getRequestError(sendContext);
-        const payloadSize = Buffer.byteLength(JSON.stringify(sendContext.request.variables || {}));
-        // Tracing span can be null for invalid operations
-        if (tracingSpan) {
-          tracingSpan.setAttribute(SEMATTRS_MESSAGING_MESSAGE_PAYLOAD_COMPRESSED_SIZE_BYTES, payloadSize);
-        }
+        let operationAttributes;
         if (requestError) {
           const operation = sendContext.request.query.startsWith('mutation') ? 'mutation' : 'query';
-          const { operationName } = sendContext.request;
+          const operationName = sendContext.request.operationName ?? 'Unspecified';
           const type = sendContext.response.body.singleResult.errors.at(0)?.name ?? requestError.name;
-          const operationAttributes = { operation, name: operationName, type };
+          operationAttributes = { operation, name: operationName, type };
           meterManager.error(operationAttributes);
-          if (tracingSpan) {
-            tracingSpan.setStatus({ code: 2, message: requestError.name });
-          }
         } else {
           const operation = sendContext.operation?.operation ?? 'query';
           const operationName = sendContext.operationName ?? 'Unspecified';
-          const operationAttributes = { operation, name: operationName };
+          operationAttributes = { operation, name: operationName };
           meterManager.request(operationAttributes);
-          const stop = Date.now();
-          const elapsed = stop - start;
-          meterManager.latency(elapsed, operationAttributes);
-          if (tracingSpan) {
-            tracingSpan.setStatus({ code: 1 });
-          }
-        }
-        if (tracingSpan) {
-          tracingSpan.end();
         }
+        const stop = Date.now();
+        const elapsed = stop - start;
+        meterManager.latency(elapsed, operationAttributes);
       },
     };
   },

opencti-platform/opencti-graphql/src/graphql/tracingPlugin.js

@@ -0,0 +1,55 @@
+import { head, includes } from 'ramda';
+import { SEMATTRS_DB_OPERATION, SEMATTRS_ENDUSER_ID, SEMATTRS_MESSAGING_MESSAGE_PAYLOAD_COMPRESSED_SIZE_BYTES } from '@opentelemetry/semantic-conventions';
+import { AUTH_FAILURE, AUTH_REQUIRED, FORBIDDEN_ACCESS } from '../config/errors';
+import { isEmptyField } from '../database/utils';
+
+const getRequestError = (context) => {
+  const isSuccess = isEmptyField(context.errors) || context.errors.length === 0;
+  if (isSuccess) {
+    return undefined;
+  }
+  const currentError = head(context.errors);
+  const callError = currentError.originalError ? currentError.originalError : currentError;
+  const isAuthenticationCall = includes(callError.name, [AUTH_REQUIRED, AUTH_FAILURE, FORBIDDEN_ACCESS]);
+  if (isAuthenticationCall) {
+    return undefined;
+  }
+  return callError;
+};
+
+// noinspection JSUnusedGlobalSymbols
+export default {
+  requestDidStart: /* v8 ignore next */ () => {
+    let tracingSpan;
+    return {
+      didResolveOperation: (resolveContext) => {
+        const isWrite = resolveContext.operation && resolveContext.operation.operation === 'mutation';
+        const operationType = `${isWrite ? 'INSERT' : 'SELECT'}`;
+        const { contextValue: context } = resolveContext;
+        const endUserId = context.user?.origin?.user_id ?? 'anonymous';
+        tracingSpan = context.tracing.getTracer().startSpan(`${operationType} ${resolveContext.operationName}`, {
+          attributes: {
+            'enduser.type': context.source,
+            [SEMATTRS_DB_OPERATION]: operationType,
+            [SEMATTRS_ENDUSER_ID]: endUserId,
+          },
+          kind: 1,
+        });
+        context.tracing.setCurrentCtx(tracingSpan);
+      },
+      willSendResponse: async (sendContext) => {
+        if (tracingSpan) { // Tracing span can be null for invalid operations
+          const requestError = getRequestError(sendContext);
+          const payloadSize = Buffer.byteLength(JSON.stringify(sendContext.request.variables || {}));
+          tracingSpan.setAttribute(SEMATTRS_MESSAGING_MESSAGE_PAYLOAD_COMPRESSED_SIZE_BYTES, payloadSize);
+          if (requestError) {
+            tracingSpan.setStatus({ code: 2, message: requestError.name });
+          } else {
+            tracingSpan.setStatus({ code: 1 });
+          }
+          tracingSpan.end();
+        }
+      },
+    };
+  },
+};

opencti-platform/opencti-graphql/src/http/httpPlatform.js

@@ -493,8 +493,9 @@ const createApp = async (app) => {
       res.set('Expires', '-1');
       res.set('Pragma', 'no-cache');
       res.send(withOptionValued);
+    } else {
+      res.status(503).send({ status: 'error', error: 'Interface is disabled by configuration' });
     }
-    res.status(503).send({ status: 'error', error: 'Interface is disabled by configuration' });
   });
 
   // Any random unexpected request not GET