add per server metadata api

apache · Oct 29, 2024 · a0d35af · a0d35af
1 parent dbc583e
commit a0d35af
Show file tree

Hide file tree

Showing 5 changed files with 91 additions and 5 deletions.
diff --git a/.../src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java b/.../src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java
@@ -810,6 +810,7 @@ private void deleteSegmentsInternal(String tableNameWithType, List<String> segme
     }
   }
 
+  @Deprecated
   @GET
   @Path("segments/{tableName}/metadata")
   @Authorize(targetType = TargetType.TABLE, paramName = "tableName", action = Actions.Table.GET_METADATA)
@@ -840,6 +841,36 @@ public String getServerMetadata(
     return segmentsMetadata;
   }
 
+  @GET
+  @Path("segments/{tableName}/metadataV2")
+  @Authorize(targetType = TargetType.TABLE, paramName = "tableName", action = Actions.Table.GET_METADATA)
+  @Produces(MediaType.APPLICATION_JSON)
+  @ApiOperation(value = "Get the server metadata for all table segments",
+      notes = "Get the server metadata for all table segments")
+  public String getServerSegmentsMetadata(
+      @ApiParam(value = "Name of the table", required = true) @PathParam("tableName") String tableName,
+      @ApiParam(value = "OFFLINE|REALTIME") @QueryParam("type") String tableTypeStr,
+      @ApiParam(value = "Columns name", allowMultiple = true) @QueryParam("columns") @DefaultValue("")
+      List<String> columns, @Context HttpHeaders headers) {
+    tableName = DatabaseUtils.translateTableName(tableName, headers);
+    LOGGER.info("Received a request to fetch metadata for all segments for table {}", tableName);
+    TableType tableType = Constants.validateTableType(tableTypeStr);
+
+    String tableNameWithType =
+        ResourceUtils.getExistingTableNamesWithType(_pinotHelixResourceManager, tableName, tableType, LOGGER).get(0);
+    String segmentsMetadata;
+    try {
+      JsonNode segmentsMetadataJson = getSegmentsMetadataFromServer(tableNameWithType, columns);
+      segmentsMetadata = JsonUtils.objectToPrettyString(segmentsMetadataJson);
+    } catch (InvalidConfigException e) {
+      throw new ControllerApplicationException(LOGGER, e.getMessage(), Status.BAD_REQUEST);
+    } catch (IOException ioe) {
+      throw new ControllerApplicationException(LOGGER, "Error parsing Pinot server response: " + ioe.getMessage(),
+          Status.INTERNAL_SERVER_ERROR, ioe);
+    }
+    return segmentsMetadata;
+  }
+
   @GET
   @Path("segments/{tableNameWithType}/needReload")
   @Authorize(targetType = TargetType.TABLE, paramName = "tableNameWithType", action = Actions.Table.GET_METADATA)

diff --git a/...ontroller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java b/...ontroller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java
@@ -417,7 +417,7 @@ public String generateTableMetadataServerURL(String tableNameWithType, List<Stri
     tableNameWithType = URLEncoder.encode(tableNameWithType, StandardCharsets.UTF_8);
     String paramsStr = generateColumnsParam(columns)
             + generateSegmentsParam(segmentsToInclude);
-    return String.format("%s/tables/%s/metadata?%s", endpoint, tableNameWithType, paramsStr);
+    return String.format("%s/tables/%s/segments/metadata?%s", endpoint, tableNameWithType, paramsStr);
   }
 
   private String generateCheckReloadSegmentsServerURL(String tableNameWithType, String endpoint) {

diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableMetadataReader.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/TableMetadataReader.java
@@ -119,9 +119,7 @@ private JsonNode getSegmentsMetadataInternal(String tableNameWithType, List<Stri
       Iterator<Map.Entry<String, JsonNode>> fields = responseJson.fields();
       while (fields.hasNext()) {
         Map.Entry<String, JsonNode> field = fields.next();
-        String segmentName = field.getKey();
-        JsonNode segmentJson = field.getValue();
-        response.put(segmentName, segmentJson);
+        response.put(field.getKey(), field.getValue());
       }
     }
     return JsonUtils.objectToJsonNode(response);

diff --git a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java
@@ -18,6 +18,7 @@
  */
 package org.apache.pinot.server.api.resources;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.base.Preconditions;
 import io.swagger.annotations.Api;
 import io.swagger.annotations.ApiKeyAuthDefinition;
@@ -412,6 +413,62 @@ public String getSegmentMetadata(
     }
   }
 
+  @GET
+  @Encoded
+  @Path("/tables/{tableName}/segments/metadata")
+  @Produces(MediaType.APPLICATION_JSON)
+  @ApiOperation(value = "Provide segments metadata", notes = "Provide segments metadata for the segments on server")
+  @ApiResponses(value = {
+      @ApiResponse(code = 200, message = "Success"),
+      @ApiResponse(code = 500, message = "Internal server error", response = ErrorInfo.class),
+      @ApiResponse(code = 404, message = "Table or segment not found", response = ErrorInfo.class)
+  })
+  public String getSegmentsMetadata(
+      @ApiParam(value = "Table name including type", required = true, example = "myTable_OFFLINE")
+      @PathParam("tableName") String tableName,
+      @ApiParam(value = "Segment names to include", allowMultiple = true) @QueryParam("segmentsToInclude")
+      @DefaultValue("") List<String> segmentsToInclude,
+      @ApiParam(value = "Column name", allowMultiple = true) @QueryParam("columns") @DefaultValue("")
+      List<String> columns, @Context HttpHeaders headers) {
+    tableName = DatabaseUtils.translateTableName(tableName, headers);
+    TableDataManager tableDataManager = ServerResourceUtils.checkGetTableDataManager(_serverInstance, tableName);
+    // decode columns and segments
+    List<String> decodedSegments = new ArrayList<>();
+    if (segmentsToInclude != null && !segmentsToInclude.isEmpty()) {
+      for (String segment : segmentsToInclude) {
+        if (!segment.isEmpty()) {
+          decodedSegments.add(URIUtils.decode(segment));
+        }
+      }
+    }
+    List<SegmentDataManager> segmentDataManagers;
+    if (!decodedSegments.isEmpty()) {
+      segmentDataManagers = tableDataManager.acquireSegments(decodedSegments, new ArrayList<>());
+    } else {
+      segmentDataManagers = tableDataManager.acquireAllSegments();
+    }
+    for (int i = 0; i < columns.size(); i++) {
+      columns.set(i, URIUtils.decode(columns.get(i)));
+    }
+    // get metadata for every segment in the list
+    Map<String, JsonNode> response = new HashMap<>();
+    for (SegmentDataManager segmentDataManager: segmentDataManagers) {
+      String segmentName = segmentDataManager.getSegmentName();
+      try {
+        String segmentMetadata = SegmentMetadataFetcher.getSegmentMetadata(segmentDataManager, columns);
+        JsonNode segmentMetadataJson = JsonUtils.stringToJsonNode(segmentMetadata);
+        response.put(segmentName, segmentMetadataJson);
+      } catch (Exception e) {
+        LOGGER.error("Failed to convert table {} segment {} to json", tableName, segmentName);
+        throw new WebApplicationException("Failed to convert segment metadata to json",
+            Response.Status.INTERNAL_SERVER_ERROR);
+      } finally {
+        tableDataManager.releaseSegment(segmentDataManager);
+      }
+    }
+    return ResourceUtils.convertToJsonString(response);
+  }
+
   @GET
   @Path("/tables/{tableName}/segments/crc")
   @Produces(MediaType.APPLICATION_JSON)

diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java
@@ -406,7 +406,7 @@ public String forSegmentsMetadataFromServer(String tableName, @Nullable String c
   }
 
   public String forSegmentsMetadataFromServer(String tableName, @Nullable List<String> columns) {
-    return StringUtil.join("/", _baseUrl, "segments", tableName, "metadata") + constructColumnsParameter(columns);
+    return StringUtil.join("/", _baseUrl, "segments", tableName, "metadataV2") + constructColumnsParameter(columns);
   }
 
   public String forSegmentMetadata(String tableName, String segmentName) {