From 60621a0757e6105b7583582b9f3f728822bb995f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jan 2025 19:04:31 -0800 Subject: [PATCH 1/9] Bump swagger.version from 1.6.14 to 1.6.15 (#14826) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 36c1cffa8d6f..8da2053e1d72 100644 --- a/pom.xml +++ b/pom.xml @@ -147,7 +147,7 @@ 3.0.1 2.45 2.6.1 - 1.6.14 + 1.6.15 5.18.2 3.4.1 2.9.0 From 8d707edc30451f81bc8799eb3da68e075c52bcd9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 16 Jan 2025 19:05:22 -0800 Subject: [PATCH 2/9] Bump com.google.cloud:libraries-bom from 26.52.0 to 26.53.0 (#14827) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 8da2053e1d72..60e618a1f93c 100644 --- a/pom.xml +++ b/pom.xml @@ -233,7 +233,7 @@ 3.25.5 1.69.0 - 26.52.0 + 26.53.0 1.1.1 1.8 2.36.0 From df71f20ac1a322fd375c4e49fa8b7f3bd732d9e1 Mon Sep 17 00:00:00 2001 From: Daniil Roman Date: Fri, 17 Jan 2025 04:06:42 +0100 Subject: [PATCH 3/9] [flaky-test] BrokerManagedAsyncExecutorProviderTest#testRejectHandler (#14814) --- ...rokerManagedAsyncExecutorProviderTest.java | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/pinot-broker/src/test/java/org/apache/pinot/broker/broker/BrokerManagedAsyncExecutorProviderTest.java b/pinot-broker/src/test/java/org/apache/pinot/broker/broker/BrokerManagedAsyncExecutorProviderTest.java index 2cab3985d5ea..89cd94f68506 100644 --- a/pinot-broker/src/test/java/org/apache/pinot/broker/broker/BrokerManagedAsyncExecutorProviderTest.java +++ b/pinot-broker/src/test/java/org/apache/pinot/broker/broker/BrokerManagedAsyncExecutorProviderTest.java @@ -26,6 +26,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.Phaser; import java.util.concurrent.RejectedExecutionHandler; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -135,20 +136,16 @@ public void testGet() } @Test(expectedExceptions = ServiceUnavailableException.class) - public void testRejectHandler() - throws InterruptedException { + public void testRejectHandler() { BrokerManagedAsyncExecutorProvider provider = new BrokerManagedAsyncExecutorProvider(1, 1, 1, _brokerMetrics); - ThreadPoolExecutor threadPoolExecutor = (ThreadPoolExecutor) provider.getExecutorService(); + ExecutorService threadPoolExecutor = provider.getExecutorService(); // test the rejection policy - AtomicInteger counter = new AtomicInteger(); - CountDownLatch latch = new CountDownLatch(10); - for (int i = 0; i < 10; i++) { - threadPoolExecutor.execute(() -> { - counter.incrementAndGet(); - latch.countDown(); - }); + int taskCount = 3; + Phaser phaser = new Phaser(taskCount); + for (int i = 0; i < taskCount; i++) { + threadPoolExecutor.execute(phaser::arriveAndAwaitAdvance); } - latch.await(); + phaser.arriveAndDeregister(); } } From 81822367548d6aa590dd9d38b03cfed3eaf565c5 Mon Sep 17 00:00:00 2001 From: cutiepie-10 Date: Fri, 17 Jan 2025 10:13:53 +0530 Subject: [PATCH 4/9] Adding error message in the SchemUtils.java (#14804) --- .../java/org/apache/pinot/segment/local/utils/SchemaUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SchemaUtils.java b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SchemaUtils.java index 9661923d30d9..63eef83214af 100644 --- a/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SchemaUtils.java +++ b/pinot-segment-local/src/main/java/org/apache/pinot/segment/local/utils/SchemaUtils.java @@ -176,7 +176,7 @@ private static void validateCompatibilityWithTableConfig(Schema schema, TableCon } catch (Exception e) { throw new IllegalStateException( "Schema is incompatible with tableConfig with name: " + tableConfig.getTableName() + " and type: " - + tableConfig.getTableType(), e); + + tableConfig.getTableType() + ", reason: " + e.getMessage(), e); } } From d1ac83e98020810560acb0c88a6fb7475c596252 Mon Sep 17 00:00:00 2001 From: Rajat Venkatesh <1638298+vrajat@users.noreply.github.com> Date: Fri, 17 Jan 2025 10:24:25 +0530 Subject: [PATCH 5/9] Add a GeoSpatial QuickStart (#14816) --- .../pinot/tools/GeoSpatialQuickStart.java | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 pinot-tools/src/main/java/org/apache/pinot/tools/GeoSpatialQuickStart.java diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/GeoSpatialQuickStart.java b/pinot-tools/src/main/java/org/apache/pinot/tools/GeoSpatialQuickStart.java new file mode 100644 index 000000000000..d3ba1e2ce5d4 --- /dev/null +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/GeoSpatialQuickStart.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.tools; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.apache.pinot.tools.admin.PinotAdministrator; + + +public class GeoSpatialQuickStart extends Quickstart { + private static final String QUICKSTART_IDENTIFIER = "GEOSPATIAL"; + private static final String[] DATA_DIRECTORIES = new String[]{ + "examples/batch/starbucksStores/", + }; + + @Override + public List types() { + return Collections.singletonList(QUICKSTART_IDENTIFIER); + } + + @Override + public String[] getDefaultBatchTableDirectories() { + return DATA_DIRECTORIES; + } + + public static void main(String[] args) + throws Exception { + List arguments = new ArrayList<>(); + arguments.addAll(Arrays.asList("QuickStart", "-type", QUICKSTART_IDENTIFIER)); + arguments.addAll(Arrays.asList(args)); + PinotAdministrator.main(arguments.toArray(new String[arguments.size()])); + } +} From 44b07b0510cd588a9ec5c36d43f6d1cbfc867c64 Mon Sep 17 00:00:00 2001 From: Shounak kulkarni Date: Fri, 17 Jan 2025 23:00:38 +0530 Subject: [PATCH 6/9] Avoid closing response stream while downloading instance logs (#14730) --- .../api/resources/PinotControllerLogger.java | 52 ++++++++++--------- 1 file changed, 27 insertions(+), 25 deletions(-) diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotControllerLogger.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotControllerLogger.java index 54c6e2fbfa4e..02e3d03add3b 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotControllerLogger.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotControllerLogger.java @@ -26,6 +26,7 @@ import io.swagger.annotations.SecurityDefinition; import io.swagger.annotations.SwaggerDefinition; import java.io.IOException; +import java.io.InputStream; import java.net.URI; import java.net.URISyntaxException; import java.util.Arrays; @@ -47,9 +48,9 @@ import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import javax.ws.rs.core.StreamingOutput; import javax.ws.rs.core.UriBuilder; import org.apache.commons.collections4.MapUtils; -import org.apache.commons.io.IOUtils; import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; import org.apache.hc.core5.http.HttpVersion; import org.apache.hc.core5.http.io.support.ClassicRequestBuilder; @@ -212,33 +213,34 @@ public Response downloadLogFileFromInstance( @ApiParam(value = "Instance Name", required = true) @PathParam("instanceName") String instanceName, @ApiParam(value = "Log file path", required = true) @QueryParam("filePath") String filePath, @Context Map headers) { - try { - URI uri = UriBuilder.fromUri(getInstanceBaseUri(instanceName)).path("/loggers/download") - .queryParam("filePath", filePath).build(); - ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.get(uri).setVersion(HttpVersion.HTTP_1_1); - if (MapUtils.isNotEmpty(headers)) { - for (Map.Entry header : headers.entrySet()) { - requestBuilder.addHeader(header.getKey(), header.getValue()); - } + URI uri = UriBuilder.fromUri(getInstanceBaseUri(instanceName)).path("/loggers/download") + .queryParam("filePath", filePath).build(); + ClassicRequestBuilder requestBuilder = ClassicRequestBuilder.get(uri).setVersion(HttpVersion.HTTP_1_1); + if (MapUtils.isNotEmpty(headers)) { + for (Map.Entry header : headers.entrySet()) { + requestBuilder.addHeader(header.getKey(), header.getValue()); } - if (authorization != null) { - requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, authorization); - } - try (CloseableHttpResponse httpResponse = _fileUploadDownloadClient.getHttpClient() - .execute(requestBuilder.build())) { - if (httpResponse.getCode() >= 400) { - throw new WebApplicationException(IOUtils.toString(httpResponse.getEntity().getContent(), "UTF-8"), - Response.Status.fromStatusCode(httpResponse.getCode())); + } + if (authorization != null) { + requestBuilder.addHeader(HttpHeaders.AUTHORIZATION, authorization); + } + + StreamingOutput streamingOutput = output -> { + try (CloseableHttpResponse response = _fileUploadDownloadClient.getHttpClient().execute(requestBuilder.build()); + InputStream inputStream = response.getEntity().getContent()) { + // Stream the data using a buffer + byte[] buffer = new byte[1024]; + int bytesRead; + while ((bytesRead = inputStream.read(buffer)) != -1) { + output.write(buffer, 0, bytesRead); } - Response.ResponseBuilder builder = Response.ok(); - builder.entity(httpResponse.getEntity().getContent()); - builder.contentLocation(uri); - builder.header(HttpHeaders.CONTENT_LENGTH, httpResponse.getEntity().getContentLength()); - return builder.build(); + output.flush(); } - } catch (IOException e) { - throw new WebApplicationException(e, Response.Status.INTERNAL_SERVER_ERROR); - } + }; + Response.ResponseBuilder builder = Response.ok(); + builder.entity(streamingOutput); + builder.contentLocation(uri); + return builder.build(); } private String getInstanceBaseUri(String instanceName) { From eb9c759344502969c80e3e9ec00fe67bd24d2965 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 17 Jan 2025 22:00:30 -0800 Subject: [PATCH 7/9] Bump io.grpc:grpc-bom from 1.69.0 to 1.69.1 (#14831) --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 60e618a1f93c..989ceb86b769 100644 --- a/pom.xml +++ b/pom.xml @@ -232,7 +232,7 @@ 3.25.5 - 1.69.0 + 1.69.1 26.53.0 1.1.1 1.8 From 4f41d2b7783759c11eb61fc30e8f9abde980cd19 Mon Sep 17 00:00:00 2001 From: Clay Johnson Date: Sun, 19 Jan 2025 10:31:57 -0600 Subject: [PATCH 8/9] Publish build scans to develocity.apache.org (#14813) * Publish build scans to develocity.apache.org * Update Develocity extension versions * Use `DEVELOCITY_ACCESS_KEY` to authenticate to `develocity.apache.org` * Add a projectId --- .github/workflows/pinot_tests.yml | 14 +++++++------- .gitignore | 1 + .../{gradle-enterprise.xml => develocity.xml} | 19 +++++++++---------- .mvn/extensions.xml | 4 ++-- 4 files changed, 19 insertions(+), 19 deletions(-) rename .mvn/{gradle-enterprise.xml => develocity.xml} (69%) diff --git a/.github/workflows/pinot_tests.yml b/.github/workflows/pinot_tests.yml index 93b85c1b15a8..93073e6f4802 100644 --- a/.github/workflows/pinot_tests.yml +++ b/.github/workflows/pinot_tests.yml @@ -58,7 +58,7 @@ jobs: cache: 'maven' - name: Linter Test env: - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} MAVEN_OPTS: > -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false @@ -112,7 +112,7 @@ jobs: env: RUN_INTEGRATION_TESTS: false RUN_TEST_SET: ${{ matrix.testset }} - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} MAVEN_OPTS: > -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false @@ -134,7 +134,7 @@ jobs: RUN_INTEGRATION_TESTS: false RUN_TEST_SET: ${{ matrix.testset }} PINOT_OFFHEAP_SKIP_BYTEBUFFER: ${{ matrix.skip_bytebuffer }} - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} MAVEN_OPTS: > -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false @@ -202,7 +202,7 @@ jobs: - name: Build Project env: RUN_INTEGRATION_TESTS: true - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} MAVEN_OPTS: > -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false @@ -219,7 +219,7 @@ jobs: RUN_INTEGRATION_TESTS: true RUN_TEST_SET: ${{ matrix.testset }} PINOT_OFFHEAP_SKIP_BYTEBUFFER: ${{ matrix.skip_bytebuffer }} - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} MAVEN_OPTS: > -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false @@ -245,7 +245,7 @@ jobs: env: RUN_INTEGRATION_TESTS: true RUN_TEST_SET: ${{ matrix.testset }} - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} MAVEN_OPTS: > -Xmx2G -DskipShade -DfailIfNoTests=false -Dmaven.wagon.httpconnectionManager.ttlSeconds=25 -Dmaven.wagon.http.retryHandler.count=30 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false @@ -407,7 +407,7 @@ jobs: - uses: actions/cache@v4 env: SEGMENT_DOWNLOAD_TIMEOUT_MINS: 10 - GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} + DEVELOCITY_ACCESS_KEY: ${{ secrets.DEVELOCITY_ACCESS_KEY }} with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} diff --git a/.gitignore b/.gitignore index 71aa91be6cb9..408b4aa87080 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,4 @@ kubernetes/helm/**/Chart.lock #Develocity .mvn/.gradle-enterprise/ +.mvn/.develocity/ diff --git a/.mvn/gradle-enterprise.xml b/.mvn/develocity.xml similarity index 69% rename from .mvn/gradle-enterprise.xml rename to .mvn/develocity.xml index cf1a9a0a9e88..8088748653a3 100644 --- a/.mvn/gradle-enterprise.xml +++ b/.mvn/develocity.xml @@ -19,20 +19,19 @@ under the License. --> - + + pinot - https://ge.apache.org + https://develocity.apache.org false - - true - true - true - #{isFalse(env['GITHUB_ACTIONS'])} - ALWAYS - true + + + + + #{{'0.0.0.0'}} @@ -45,4 +44,4 @@ false - + diff --git a/.mvn/extensions.xml b/.mvn/extensions.xml index b7a92d4ebd2e..0836fc47d010 100644 --- a/.mvn/extensions.xml +++ b/.mvn/extensions.xml @@ -22,8 +22,8 @@ com.gradle - gradle-enterprise-maven-extension - 1.20.1 + develocity-maven-extension + 1.22.2 com.gradle From fbbabd43f2d8ef6f8e4966d8f7f60dc88dfcede8 Mon Sep 17 00:00:00 2001 From: Shounak kulkarni Date: Mon, 20 Jan 2025 08:06:16 +0530 Subject: [PATCH 9/9] Skip task scheduling error entries while processing scheduled tasks (#14834) --- .../controller/api/resources/PinotTaskRestletResource.java | 6 ++++-- .../java/org/apache/pinot/tools/BootstrapTableTool.java | 6 ++++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTaskRestletResource.java b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTaskRestletResource.java index 29cf164f9246..24cd444dc592 100644 --- a/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTaskRestletResource.java +++ b/pinot-controller/src/main/java/org/apache/pinot/controller/api/resources/PinotTaskRestletResource.java @@ -141,6 +141,8 @@ public class PinotTaskRestletResource { private static final String TASK_QUEUE_STATE_STOP = "STOP"; private static final String TASK_QUEUE_STATE_RESUME = "RESUME"; + public static final String GENERATION_ERRORS_KEY = "generationErrors"; + public static final String SCHEDULING_ERRORS_KEY = "schedulingErrors"; @Inject PinotHelixTaskResourceManager _pinotHelixTaskResourceManager; @@ -667,8 +669,8 @@ public Map scheduleTasks( schedulingErrors.addAll(value.getSchedulingErrors()); }); } - response.put("generationErrors", String.join(",", generationErrors)); - response.put("schedulingErrors", String.join(",", schedulingErrors)); + response.put(GENERATION_ERRORS_KEY, String.join(",", generationErrors)); + response.put(SCHEDULING_ERRORS_KEY, String.join(",", schedulingErrors)); return response; } diff --git a/pinot-tools/src/main/java/org/apache/pinot/tools/BootstrapTableTool.java b/pinot-tools/src/main/java/org/apache/pinot/tools/BootstrapTableTool.java index 83f6232b95da..7bc803b03e93 100644 --- a/pinot-tools/src/main/java/org/apache/pinot/tools/BootstrapTableTool.java +++ b/pinot-tools/src/main/java/org/apache/pinot/tools/BootstrapTableTool.java @@ -35,6 +35,7 @@ import org.apache.pinot.common.auth.AuthProviderUtils; import org.apache.pinot.common.minion.MinionClient; import org.apache.pinot.common.utils.tls.TlsUtils; +import org.apache.pinot.controller.api.resources.PinotTaskRestletResource; import org.apache.pinot.core.common.MinionConstants; import org.apache.pinot.spi.auth.AuthProvider; import org.apache.pinot.spi.config.table.TableConfig; @@ -308,6 +309,11 @@ private boolean waitForMinionTaskToFinish(Map scheduledTasks, lo try { boolean allCompleted = true; for (String taskType : scheduledTasks.keySet()) { + // ignore the error message entries + if (taskType.equals(PinotTaskRestletResource.GENERATION_ERRORS_KEY) + || taskType.equals(PinotTaskRestletResource.SCHEDULING_ERRORS_KEY)) { + continue; + } String taskName = scheduledTasks.get(taskType); String taskState = _minionClient.getTaskState(taskName); if (!COMPLETED.equalsIgnoreCase(taskState)) {