ISW - Input Diagnosis (#21184)

graylog2-web-interface/src/components/inputs/InputDiagnosis/NetworkStats.tsx

@@ -0,0 +1,83 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+import React from 'react';
+import styled, { css } from 'styled-components';
+
+import NumberUtils from 'util/NumberUtils';
+import { Icon } from 'components/common';
+
+const InputIO = styled.span(({ theme }) => css`
+  margin-left: -5px;
+
+  .total {
+    color: ${theme.colors.gray[70]};
+  }
+
+  .value {
+    font-family: ${theme.fonts.family.monospace};
+  }
+
+  .persec {
+    margin-left: 3px;
+  }
+
+  .channel-direction {
+    position: relative;
+    left: -1px;
+  }
+
+  .channel-direction-down {
+    position: relative;
+    top: 1px;
+  }
+
+  .channel-direction-up {
+    position: relative;
+    top: -1px;
+  }
+`);
+
+type Props = {
+  writtenBytes1Sec: number,
+  writtenBytesTotal: number,
+  readBytes1Sec: number,
+  readBytesTotal: number,
+}
+
+const NetworkStats = ({ writtenBytes1Sec, writtenBytesTotal, readBytes1Sec, readBytesTotal }: Props) => (
+  <InputIO>
+    <span className="persec">
+      <Icon name="arrow_drop_down" className="channel-direction channel-direction-down" />
+      <span className="rx value">{NumberUtils.formatBytes(readBytes1Sec)} </span>
+
+      <Icon name="arrow_drop_up" className="channel-direction channel-direction-up" />
+      <span className="tx value">{NumberUtils.formatBytes(writtenBytes1Sec)}</span>
+    </span>
+
+    <span className="total">
+      <span> (total: </span>
+      <Icon name="arrow_drop_down" className="channel-direction channel-direction-down" />
+      <span className="rx value">{NumberUtils.formatBytes(readBytesTotal)} </span>
+
+      <Icon name="arrow_drop_up" className="channel-direction channel-direction-up" />
+      <span className="tx value">{NumberUtils.formatBytes(writtenBytesTotal)}</span>
+      <span>)</span>
+    </span>
+  </InputIO>
+);
+
+export default NetworkStats;

graylog2-web-interface/src/components/inputs/InputDiagnosis/ShowReceivedMessagesButton.tsx

@@ -0,0 +1,61 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+import React from 'react';
+
+import { Button } from 'components/bootstrap';
+import { isPermitted } from 'util/PermissionsMixin';
+import type { Input } from 'components/messageloaders/Types';
+import { LinkContainer } from 'components/common/router';
+import { TELEMETRY_EVENT_TYPE } from 'logic/telemetry/Constants';
+import recentMessagesTimeRange from 'util/TimeRangeHelper';
+import { getPathnameWithoutId } from 'util/URLUtils';
+import useCurrentUser from 'hooks/useCurrentUser';
+import Routes from 'routing/Routes';
+import useSendTelemetry from 'logic/telemetry/useSendTelemetry';
+import useLocation from 'routing/useLocation';
+
+type Props = {
+  input: Input,
+}
+
+const ShowReceivedMessagesButton = ({ input }: Props) => {
+  const currentUser = useCurrentUser();
+  const sendTelemetry = useSendTelemetry();
+  const { pathname } = useLocation();
+
+  const queryField = (input?.type === 'org.graylog.plugins.forwarder.input.ForwarderServiceInput') ? 'gl2_forwarder_input' : 'gl2_source_input';
+
+  if (input?.id && isPermitted(currentUser.permissions, ['searches:relative'])) {
+    return (
+      <LinkContainer key={`received-messages-${input.id}`}
+                     to={Routes.search(`${queryField}:${input.id}`, recentMessagesTimeRange())}>
+        <Button onClick={() => {
+          sendTelemetry(TELEMETRY_EVENT_TYPE.INPUTS.SHOW_RECEIVED_MESSAGES_CLICKED, {
+            app_pathname: getPathnameWithoutId(pathname),
+            app_action_value: 'show-received-messages',
+          });
+        }}>
+          Show received messages
+        </Button>
+      </LinkContainer>
+    );
+  }
+
+  return null;
+};
+
+export default ShowReceivedMessagesButton;

graylog2-web-interface/src/components/inputs/InputDiagnosis/useInputDiagnosis.ts

@@ -0,0 +1,153 @@
+/*
+ * Copyright (C) 2020 Graylog, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the Server Side Public License, version 1,
+ * as published by MongoDB, Inc.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * Server Side Public License for more details.
+ *
+ * You should have received a copy of the Server Side Public License
+ * along with this program. If not, see
+ * <http://www.mongodb.com/licensing/server-side-public-license>.
+ */
+import { useEffect, useMemo } from 'react';
+import { useQuery } from '@tanstack/react-query';
+
+import { useStore } from 'stores/connect';
+import InputStatesStore from 'stores/inputs/InputStatesStore';
+import { InputsStore, InputsActions } from 'stores/inputs/InputsStore';
+import { MetricsStore, MetricsActions } from 'stores/metrics/MetricsStore';
+import type { InputStateByNode, InputStates } from 'stores/inputs/InputStatesStore';
+import type { Input } from 'components/messageloaders/Types';
+import type { CounterMetric, GaugeMetric, Rate } from 'stores/metrics/MetricsStore';
+import { qualifyUrl } from 'util/URLUtils';
+import fetch from 'logic/rest/FetchProvider';
+import { defaultOnError } from 'util/conditional/onError';
+
+export type InputDiagnosisMetrics = {
+  incomingMessagesTotal: number;
+  emptyMessages: number;
+  open_connections: number;
+  total_connections: number;
+  read_bytes_1sec: number;
+  read_bytes_total: number;
+  write_bytes_1sec: number;
+  write_bytes_total: number;
+  failures_indexing: any;
+  failures_processing: any;
+  failures_inputs_codecs: any;
+  stream_message_count: [string, number][];
+}
+
+export type InputNodeStateInfo = {
+  detailed_message: string,
+  node_id: string,
+}
+
+export type InputNodeStates = {
+  states: {
+    'RUNNING'?: InputNodeStateInfo[],
+    'FAILED'?: InputNodeStateInfo[],
+    'STOPPED'?: InputNodeStateInfo[],
+    'STARTING'?: InputNodeStateInfo[],
+    'FAILING'?: InputNodeStateInfo[],
+    'SETUP'?: InputNodeStateInfo[],
+  }
+  total: number;
+}
+
+export type InputDiagnostics = {
+  stream_message_count: {
+    [streamName: string]: number,
+  },
+}
+
+export const metricWithPrefix = (input: Input, metric: string) => `${input?.type}.${input?.id}.${metric}`;
+
+export const fetchInputDiagnostics = (inputId: string): Promise<InputDiagnostics> => fetch<InputDiagnostics>('GET', qualifyUrl(`system/inputs/diagnostics/${inputId}`));
+
+const useInputDiagnosis = (inputId: string): {
+  input: Input,
+  inputNodeStates: InputNodeStates,
+  inputMetrics: InputDiagnosisMetrics,
+} => {
+  const { input } = useStore(InputsStore);
+
+  useEffect(() => {
+    InputsActions.get(inputId);
+  }, [inputId]);
+
+  const { data: messageCountByStream } = useQuery<InputDiagnostics, Error>(
+    ['input-diagnostics', inputId],
+    () => defaultOnError(fetchInputDiagnostics(inputId), 'Fetching Input Diagnostics failed with status', 'Could not fetch Input Diagnostics'),
+    { refetchInterval: 5000 },
+  );
+
+  const { inputStates } = useStore(InputStatesStore) as { inputStates: InputStates };
+  const inputStateByNode = inputStates ? inputStates[inputId] || {} : {} as InputStateByNode;
+  const inputNodeStates = { total: Object.keys(inputStateByNode).length, states: {} };
+
+  Object.values(inputStateByNode).forEach(({ state, detailed_message, message_input: { node: node_id } }) => {
+    if (!inputNodeStates.states[state]) {
+      inputNodeStates.states[state] = [{ detailed_message, node_id }];
+    } else if (Array.isArray(inputNodeStates.states[state])) {
+      inputNodeStates.states[state].push({ detailed_message, node_id });
+    }
+  });
+
+  const failures_indexing = `org.graylog2.${inputId}.failures.indexing`;
+  const failures_processing = `org.graylog2.${inputId}.failures.processing`;
+  const failures_inputs_codecs = `org.graylog2.inputs.codecs.*.${inputId}.failures`;
+
+  const InputDiagnosisMetricNames = useMemo(() => ([
+    metricWithPrefix(input, 'incomingMessages'),
+    metricWithPrefix(input, 'emptyMessages'),
+    metricWithPrefix(input, 'open_connections'),
+    metricWithPrefix(input, 'total_connections'),
+    metricWithPrefix(input, 'written_bytes_1sec'),
+    metricWithPrefix(input, 'written_bytes_total'),
+    metricWithPrefix(input, 'read_bytes_1sec'),
+    metricWithPrefix(input, 'read_bytes_total'),
+    metricWithPrefix(input, 'failures.indexing'),
+    metricWithPrefix(input, 'failures.processing'),
+    failures_indexing,
+    failures_processing,
+    failures_inputs_codecs,
+  ]), [input, failures_indexing, failures_processing, failures_inputs_codecs]);
+
+  const { metrics: metricsByNode } = useStore(MetricsStore);
+  const nodeMetrics = (metricsByNode && input?.node) ? metricsByNode[input?.node] : {};
+
+  useEffect(() => {
+    InputDiagnosisMetricNames.forEach((metricName) => MetricsActions.addGlobal(metricName));
+
+    return () => {
+      InputDiagnosisMetricNames.forEach((metricName) => MetricsActions.removeGlobal(metricName));
+    };
+  }, [InputDiagnosisMetricNames]);
+
+  return {
+    input,
+    inputNodeStates,
+    inputMetrics: {
+      incomingMessagesTotal: (nodeMetrics[metricWithPrefix(input, 'incomingMessages')]?.metric as Rate)?.rate?.total || 0,
+      emptyMessages: (nodeMetrics[metricWithPrefix(input, 'emptyMessages')] as CounterMetric)?.metric?.count || 0,
+      open_connections: (nodeMetrics[metricWithPrefix(input, 'open_connections')] as GaugeMetric)?.metric?.value,
+      total_connections: (nodeMetrics[metricWithPrefix(input, 'total_connections')] as GaugeMetric)?.metric?.value,
+      read_bytes_1sec: (nodeMetrics[metricWithPrefix(input, 'read_bytes_1sec')] as GaugeMetric)?.metric?.value,
+      read_bytes_total: (nodeMetrics[metricWithPrefix(input, 'read_bytes_total')] as GaugeMetric)?.metric?.value,
+      write_bytes_1sec: (nodeMetrics[metricWithPrefix(input, 'write_bytes_1sec')] as GaugeMetric)?.metric?.value,
+      write_bytes_total: (nodeMetrics[metricWithPrefix(input, 'write_bytes_total')] as GaugeMetric)?.metric?.value,
+      failures_indexing: (nodeMetrics[failures_indexing]?.metric as Rate)?.rate?.fifteen_minute || 0,
+      failures_processing: (nodeMetrics[failures_processing]?.metric as Rate)?.rate?.fifteen_minute || 0,
+      failures_inputs_codecs: (nodeMetrics[failures_inputs_codecs]?.metric as Rate)?.rate?.fifteen_minute || 0,
+      stream_message_count: Object.entries(messageCountByStream?.stream_message_count || {}),
+    },
+  };
+};
+
+export default useInputDiagnosis;

graylog2-web-interface/src/components/inputs/InputListItem.tsx

@@ -210,6 +210,19 @@ const InputListItem = ({ input, currentNode, permissions }: Props) => {
                   disabled={definition === undefined}>
           Edit input
         </MenuItem>
+
+        <LinkContainer to={Routes.SYSTEM.INPUT_DIAGNOSIS(input.id)}>
+          <MenuItem key={`input-diagnosis-${input.id}`}
+                    onClick={() => {
+                      sendTelemetry(TELEMETRY_EVENT_TYPE.INPUTS.INPUT_DIAGNOSIS_CLICKED, {
+                        app_pathname: getPathnameWithoutId(pathname),
+                        app_action_value: 'input-diagnosis',
+                      });
+                    }}>
+            Input Diagnosis
+          </MenuItem>
+        </LinkContainer>
+
         {inputSetupFeatureFlagIsEnabled && (
           isInputInSetupMode(inputStates, input.id) ? (
             <MenuItem key={`remove-setup-mode-${input.id}`}
@@ -229,7 +242,7 @@ const InputListItem = ({ input, currentNode, permissions }: Props) => {
         )}
       </IfPermitted>
 
-      {input.global && (
+      {input.global && input.node && (
         <LinkContainer to={Routes.filtered_metrics(input.node, input.id)}>
           <MenuItem key={`show-metrics-${input.id}`}
                     onClick={() => {

graylog2-web-interface/src/components/inputs/InputSetupWizard/Wizard.tsx

@@ -90,15 +90,15 @@ const Wizard = ({ show, input, onClose }: Props) => {
           </>
         ),
         component: (
-          <InputDiagnosisStep />
+          <InputDiagnosisStep onClose={() => onClose()} />
         ),
         disabled: !getStepConfigOrData(stepsConfig, INPUT_WIZARD_STEPS.INPUT_DIAGNOSIS, 'enabled'),
       },
     };
     if (enterpriseSteps) return { ...defaultSteps, ...enterpriseSteps };
 
     return defaultSteps;
-  }, [enterpriseSteps, stepsConfig]);
+  }, [enterpriseSteps, stepsConfig, onClose]);
 
   const determineFirstStep = useCallback(() => {
     setActiveStep(INPUT_WIZARD_STEPS.SETUP_ROUTING);