Add datanode.log to the support bundle

changelog/unreleased/issue-20919.toml

@@ -0,0 +1,5 @@
+type = "a"
+message = "Add datanode.log to support bundle."
+
+pulls = ["21553"]
+issues = ["20919"]

data-node/src/main/java/org/graylog/datanode/rest/LogsController.java

@@ -16,6 +16,14 @@
  */
 package org.graylog.datanode.rest;
 
+import jakarta.ws.rs.InternalServerErrorException;
+import jakarta.ws.rs.NotFoundException;
+import jakarta.ws.rs.core.Response;
+import jakarta.ws.rs.core.StreamingOutput;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.core.Appender;
+import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.config.Configuration;
 import org.graylog.datanode.opensearch.OpensearchProcess;
 
 import jakarta.inject.Inject;
@@ -24,12 +32,17 @@
 import jakarta.ws.rs.Path;
 import jakarta.ws.rs.Produces;
 import jakarta.ws.rs.core.MediaType;
+import org.graylog.datanode.rest.config.OnlyInSecuredNode;
+import org.graylog2.log4j.MemoryAppender;
 
 import java.util.List;
 
 @Path("/logs")
 @Produces(MediaType.APPLICATION_JSON)
 public class LogsController {
+
+    private static final String MEMORY_APPENDER_NAME = "datanode-internal-logs";
+
     private final OpensearchProcess managedOpensearch;
 
     @Inject
@@ -48,4 +61,32 @@ public List<String> getOpensearchStdout() {
     public List<String> getOpensearchStderr() {
         return managedOpensearch.stdErrLogs();
     }
+
+    @GET
+    @OnlyInSecuredNode
+    @Produces(MediaType.TEXT_PLAIN)
+    @Path("/internal")
+    public Response getOpensearchInternal() {
+        final Appender appender = getAppender(MEMORY_APPENDER_NAME);
+        if (appender == null) {
+            throw new NotFoundException("Memory appender is disabled. Please refer to the example log4j.xml file.");
+        }
+
+        if (!(appender instanceof MemoryAppender memoryAppender)) {
+            throw new InternalServerErrorException("Memory appender is not an instance of MemoryAppender. Please refer to the example log4j.xml file.");
+        }
+        var mediaType = MediaType.valueOf(MediaType.TEXT_PLAIN);
+
+        StreamingOutput streamingOutput = outputStream -> memoryAppender.streamFormattedLogMessages(outputStream, 0);
+        Response.ResponseBuilder response = Response.ok(streamingOutput, mediaType);
+
+        return response.build();
+    }
+
+
+    private Appender getAppender(final String appenderName) {
+        final LoggerContext loggerContext = (LoggerContext) LogManager.getContext(false);
+        final Configuration configuration = loggerContext.getConfiguration();
+        return configuration.getAppender(appenderName);
+    }
 }

data-node/src/main/resources/log4j2.xml

@@ -4,13 +4,18 @@
         <Console name="STDOUT" target="SYSTEM_OUT">
             <PatternLayout pattern="%d %-5p: %c - %m%n"/>
         </Console>
+        <!-- Internal Graylog log appender. Please do not disable. This makes internal log messages available via REST calls. -->
+        <Memory name="datanode-internal-logs" bufferSizeBytes="10MB">
+            <PatternLayout pattern="%d{yyyy-MM-dd'T'HH:mm:ss.SSSXXX} %-5p [%c{1}] %m%n"/>
+        </Memory>
     </Appenders>
     <Loggers>
         <!-- Application Loggers -->
         <Logger name="org.graylog.datanode" level="info"/>
         <Logger name="org.graylog.datanode.opensearch.statemachine.tracer.StateMachineTransitionLogger" level="debug"/>
         <Root level="info">
             <AppenderRef ref="STDOUT"/>
+            <AppenderRef ref="datanode-internal-logs"/>
         </Root>
     </Loggers>
 </Configuration>

graylog2-server/src/main/java/org/graylog2/rest/resources/system/debug/bundle/SupportBundleService.java

@@ -175,7 +175,7 @@ public void buildBundle(HttpHeaders httpHeaders, Subject currentSubject) {
                     nodeManifests.entrySet().stream().map(entry ->
                             CompletableFuture.runAsync(() -> fetchNodeInfos(proxiedResourceHelper, entry.getKey(), entry.getValue(), finalSpoolDir), executor)),
                     datanodeService.allActive().values().stream().map(datanode ->
-                            CompletableFuture.runAsync(() -> fetchDataNodeInfos(proxiedResourceHelper, datanode, dataNodeDir), executor))
+                            CompletableFuture.runAsync(() -> fetchDataNodeInfos(datanode, dataNodeDir), executor))
             ).toList();
             for (CompletableFuture<Void> f : futures) {
                 f.get();
@@ -386,11 +386,11 @@ private void fetchNodeInfo(ProxiedResourceHelper proxiedResourceHelper, String n
     }
 
 
-    private void fetchDataNodeInfos(ProxiedResourceHelper proxiedResourceHelper, DataNodeDto datanode, Path dataNodeDir) {
+    private void fetchDataNodeInfos(DataNodeDto datanode, Path dataNodeDir) {
         final Path nodeDir = dataNodeDir.resolve(Objects.requireNonNull(datanode.getHostname()));
         var ignored = nodeDir.toFile().mkdirs();
 
-        fetchDataNodeLogs(proxiedResourceHelper, datanode, nodeDir);
+        fetchDataNodeLogs(datanode, nodeDir);
         try (var certificatesFile = new FileOutputStream(nodeDir.resolve("certificates.json").toFile())) {
 
             Map<String, Map<String, KeyStoreDto>> certificates = datanodeProxy.remoteInterface(datanode.getHostname(), RemoteCertificatesResource.class, RemoteCertificatesResource::certificates);
@@ -402,32 +402,22 @@ private void fetchDataNodeInfos(ProxiedResourceHelper proxiedResourceHelper, Dat
         }
     }
 
-    private void fetchDataNodeLogs(ProxiedResourceHelper proxiedResourceHelper, DataNodeDto datanode, Path nodeDir) {
-        getProxiedLog(datanode, nodeDir, "opensearch.log", RemoteDataNodeStatusResource::opensearchStdOut);
-        getProxiedLog(datanode, nodeDir, "opensearch.err", RemoteDataNodeStatusResource::opensearchStdErr);
+    private void fetchDataNodeLogs(DataNodeDto datanode, Path nodeDir) {
+        getProxiedLog(datanode, nodeDir, "datanode.log", RemoteDataNodeStatusResource::datanodeInternalLogs);
     }
 
-    private void getProxiedLog(DataNodeDto datanode, Path nodeDir, String logfile, Function<RemoteDataNodeStatusResource, Call<List<String>>> function) {
+    private void getProxiedLog(DataNodeDto datanode, Path nodeDir, String logfile, Function<RemoteDataNodeStatusResource, Call<ResponseBody>> function) {
         try (var opensearchLog = new FileOutputStream(nodeDir.resolve(logfile).toFile())) {
-
-            Map<String, List<String>> opensearchOut = datanodeProxy
-                    .remoteInterface(datanode.getHostname(), RemoteDataNodeStatusResource.class, function);
+            Map<String, ResponseBody> opensearchOut = datanodeProxy.remoteInterface(datanode.getHostname(), RemoteDataNodeStatusResource.class, function);
             if (opensearchOut.containsKey(datanode.getHostname())) {
-                opensearchOut.get(datanode.getHostname()).stream()
-                        .map(line -> line + System.lineSeparator())
-                        .forEach(line -> {
-                            try {
-                                opensearchLog.write(line.getBytes(StandardCharsets.UTF_8));
-                            } catch (IOException e) {
-                                LOG.warn("Failed to write line <{}>", line, e);
-                            }
-                        });
+                opensearchLog.write(opensearchOut.get(datanode.getHostname()).bytes());
             }
         } catch (Exception e) {
             LOG.warn("Failed to get logs from data node <{}>", datanode.getHostname(), e);
         }
     }
 
+
     private void fetchDataNodeMigrationInfos(Path dataNodeDir) {
         var ignored = dataNodeDir.toFile().mkdirs();
         migrationService.getLatestMigrationId()

graylog2-server/src/main/java/org/graylog2/shared/rest/resources/system/RemoteDataNodeStatusResource.java

@@ -17,8 +17,11 @@
 package org.graylog2.shared.rest.resources.system;
 
 import com.fasterxml.jackson.databind.JsonNode;
+import okhttp3.ResponseBody;
 import retrofit2.Call;
 import retrofit2.http.GET;
+import retrofit2.http.Headers;
+import retrofit2.http.Streaming;
 
 import java.util.List;
 
@@ -33,4 +36,9 @@ public interface RemoteDataNodeStatusResource {
     @GET("/logs/stderr")
     Call<List<String>> opensearchStdErr();
 
+    @Streaming
+    @Headers({"Accept: */*"})
+    @GET("/logs/internal")
+    Call<ResponseBody> datanodeInternalLogs();
+
 }