From a0d35af6e1507da17a32d4ef85f530cb05411922 Mon Sep 17 00:00:00 2001 From: Shreyaa Sharma Date: Wed, 30 Oct 2024 00:52:39 +0530 Subject: [PATCH] add per server metadata api --- .../PinotSegmentRestletResource.java | 31 ++++++++++ .../util/ServerSegmentMetadataReader.java | 2 +- .../controller/util/TableMetadataReader.java | 4 +- .../server/api/resources/TablesResource.java | 57 +++++++++++++++++++ .../builder/ControllerRequestURLBuilder.java | 2 +- 5 files changed, 91 insertions(+), 5 deletions(-) diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java index 2f0d565590d0..a9b12157bc36 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotSegmentRestletResource.java @@ -810,6 +810,7 @@ private void deleteSegmentsInternal(String tableNameWithType, List segme } } + @Deprecated @GET @Path("segments/{tableName}/metadata") @Authorize(targetType = TargetType.TABLE, paramName = "tableName", action = Actions.Table.GET_METADATA) @@ -840,6 +841,36 @@ public String getServerMetadata( return segmentsMetadata; } + @GET + @Path("segments/{tableName}/metadataV2") + @Authorize(targetType = TargetType.TABLE, paramName = "tableName", action = Actions.Table.GET_METADATA) + @Produces(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Get the server metadata for all table segments", + notes = "Get the server metadata for all table segments") + public String getServerSegmentsMetadata( + @ApiParam(value = "Name of the table", required = true) @PathParam("tableName") String tableName, + @ApiParam(value = "OFFLINE|REALTIME") @QueryParam("type") String tableTypeStr, + @ApiParam(value = "Columns name", allowMultiple = true) @QueryParam("columns") @DefaultValue("") + List columns, @Context HttpHeaders headers) { + tableName = DatabaseUtils.translateTableName(tableName, headers); + LOGGER.info("Received a request to fetch metadata for all segments for table {}", tableName); + TableType tableType = Constants.validateTableType(tableTypeStr); + + String tableNameWithType = + ResourceUtils.getExistingTableNamesWithType(_pinotHelixResourceManager, tableName, tableType, LOGGER).get(0); + String segmentsMetadata; + try { + JsonNode segmentsMetadataJson = getSegmentsMetadataFromServer(tableNameWithType, columns); + segmentsMetadata = JsonUtils.objectToPrettyString(segmentsMetadataJson); + } catch (InvalidConfigException e) { + throw new ControllerApplicationException(LOGGER, e.getMessage(), Status.BAD_REQUEST); + } catch (IOException ioe) { + throw new ControllerApplicationException(LOGGER, "Error parsing Pinot server response: " + ioe.getMessage(), + Status.INTERNAL_SERVER_ERROR, ioe); + } + return segmentsMetadata; + } + @GET @Path("segments/{tableNameWithType}/needReload") @Authorize(targetType = TargetType.TABLE, paramName = "tableNameWithType", action = Actions.Table.GET_METADATA) diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java b/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java index 43d712f8848e..ba7b51d917b6 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/util/ServerSegmentMetadataReader.java @@ -417,7 +417,7 @@ public String generateTableMetadataServerURL(String tableNameWithType, List> fields = responseJson.fields(); while (fields.hasNext()) { Map.Entry field = fields.next(); - String segmentName = field.getKey(); - JsonNode segmentJson = field.getValue(); - response.put(segmentName, segmentJson); + response.put(field.getKey(), field.getValue()); } } return JsonUtils.objectToJsonNode(response); diff --git a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java index 0ee36a37ddc8..0162302af0c3 100644 --- a/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java +++ b/pinot-server/src/main/java/org/apache/pinot/server/api/resources/TablesResource.java @@ -18,6 +18,7 @@ */ package org.apache.pinot.server.api.resources; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.base.Preconditions; import io.swagger.annotations.Api; import io.swagger.annotations.ApiKeyAuthDefinition; @@ -412,6 +413,62 @@ public String getSegmentMetadata( } } + @GET + @Encoded + @Path("/tables/{tableName}/segments/metadata") + @Produces(MediaType.APPLICATION_JSON) + @ApiOperation(value = "Provide segments metadata", notes = "Provide segments metadata for the segments on server") + @ApiResponses(value = { + @ApiResponse(code = 200, message = "Success"), + @ApiResponse(code = 500, message = "Internal server error", response = ErrorInfo.class), + @ApiResponse(code = 404, message = "Table or segment not found", response = ErrorInfo.class) + }) + public String getSegmentsMetadata( + @ApiParam(value = "Table name including type", required = true, example = "myTable_OFFLINE") + @PathParam("tableName") String tableName, + @ApiParam(value = "Segment names to include", allowMultiple = true) @QueryParam("segmentsToInclude") + @DefaultValue("") List segmentsToInclude, + @ApiParam(value = "Column name", allowMultiple = true) @QueryParam("columns") @DefaultValue("") + List columns, @Context HttpHeaders headers) { + tableName = DatabaseUtils.translateTableName(tableName, headers); + TableDataManager tableDataManager = ServerResourceUtils.checkGetTableDataManager(_serverInstance, tableName); + // decode columns and segments + List decodedSegments = new ArrayList<>(); + if (segmentsToInclude != null && !segmentsToInclude.isEmpty()) { + for (String segment : segmentsToInclude) { + if (!segment.isEmpty()) { + decodedSegments.add(URIUtils.decode(segment)); + } + } + } + List segmentDataManagers; + if (!decodedSegments.isEmpty()) { + segmentDataManagers = tableDataManager.acquireSegments(decodedSegments, new ArrayList<>()); + } else { + segmentDataManagers = tableDataManager.acquireAllSegments(); + } + for (int i = 0; i < columns.size(); i++) { + columns.set(i, URIUtils.decode(columns.get(i))); + } + // get metadata for every segment in the list + Map response = new HashMap<>(); + for (SegmentDataManager segmentDataManager: segmentDataManagers) { + String segmentName = segmentDataManager.getSegmentName(); + try { + String segmentMetadata = SegmentMetadataFetcher.getSegmentMetadata(segmentDataManager, columns); + JsonNode segmentMetadataJson = JsonUtils.stringToJsonNode(segmentMetadata); + response.put(segmentName, segmentMetadataJson); + } catch (Exception e) { + LOGGER.error("Failed to convert table {} segment {} to json", tableName, segmentName); + throw new WebApplicationException("Failed to convert segment metadata to json", + Response.Status.INTERNAL_SERVER_ERROR); + } finally { + tableDataManager.releaseSegment(segmentDataManager); + } + } + return ResourceUtils.convertToJsonString(response); + } + @GET @Path("/tables/{tableName}/segments/crc") @Produces(MediaType.APPLICATION_JSON) diff --git a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java index 98a2a5aefeb5..6ec653f87c62 100644 --- a/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java +++ b/pinot-spi/src/main/java/org/apache/pinot/spi/utils/builder/ControllerRequestURLBuilder.java @@ -406,7 +406,7 @@ public String forSegmentsMetadataFromServer(String tableName, @Nullable String c } public String forSegmentsMetadataFromServer(String tableName, @Nullable List columns) { - return StringUtil.join("/", _baseUrl, "segments", tableName, "metadata") + constructColumnsParameter(columns); + return StringUtil.join("/", _baseUrl, "segments", tableName, "metadataV2") + constructColumnsParameter(columns); } public String forSegmentMetadata(String tableName, String segmentName) {