-
Notifications
You must be signed in to change notification settings - Fork 1.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add tests that verify timestamp indexes can be used in MSE
- Loading branch information
Showing
5 changed files
with
232 additions
and
56 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
57 changes: 57 additions & 0 deletions
57
...n-tests/src/test/java/org/apache/pinot/integration/tests/ExplainIntegrationTestTrait.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| package org.apache.pinot.integration.tests; | ||
|
|
||
| import com.fasterxml.jackson.databind.JsonNode; | ||
| import org.intellij.lang.annotations.Language; | ||
| import org.testng.Assert; | ||
|
|
||
|
|
||
| public interface ExplainIntegrationTestTrait { | ||
|
|
||
| JsonNode postQuery(@Language("sql") String query) | ||
| throws Exception; | ||
|
|
||
| default void explainLogical(@Language("sql") String query, String expected) { | ||
| try { | ||
| JsonNode jsonNode = postQuery("explain plan without implementation for " + query); | ||
| JsonNode plan = jsonNode.get("resultTable").get("rows").get(0).get(1); | ||
|
|
||
| Assert.assertEquals(plan.asText(), expected); | ||
| } catch (RuntimeException e) { | ||
| throw e; | ||
| } catch (Exception e) { | ||
| throw new RuntimeException(e); | ||
| } | ||
| } | ||
|
|
||
| default void explain(@Language("sql") String query, String expected) { | ||
| try { | ||
| JsonNode jsonNode = postQuery("explain plan for " + query); | ||
| JsonNode plan = jsonNode.get("resultTable").get("rows").get(0).get(1); | ||
|
|
||
| Assert.assertEquals(plan.asText(), expected); | ||
| } catch (RuntimeException e) { | ||
| throw e; | ||
| } catch (Exception e) { | ||
| throw new RuntimeException(e); | ||
| } | ||
| } | ||
|
|
||
| default void explainVerbose(@Language("sql") String query, String expected) { | ||
| try { | ||
| JsonNode jsonNode = postQuery("set explainPlanVerbose=true; explain plan for " + query); | ||
| JsonNode plan = jsonNode.get("resultTable").get("rows").get(0).get(1); | ||
|
|
||
| String actual = plan.asText() | ||
| .replaceAll("numDocs=\\[[^\\]]*]", "numDocs=[any]") | ||
| .replaceAll("segment=\\[[^\\]]*]", "segment=[any]") | ||
| .replaceAll("totalDocs=\\[[^\\]]*]", "totalDocs=[any]"); | ||
|
|
||
|
|
||
| Assert.assertEquals(actual, expected); | ||
| } catch (RuntimeException e) { | ||
| throw e; | ||
| } catch (Exception e) { | ||
| throw new RuntimeException(e); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
165 changes: 165 additions & 0 deletions
165
...ion-tests/src/test/java/org/apache/pinot/integration/tests/custom/TimestampIndexTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,165 @@ | ||
| package org.apache.pinot.integration.tests.custom; | ||
|
|
||
| import java.io.File; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
| import org.apache.pinot.integration.tests.BaseClusterIntegrationTest; | ||
| import org.apache.pinot.integration.tests.ClusterIntegrationTestUtils; | ||
| import org.apache.pinot.integration.tests.ExplainIntegrationTestTrait; | ||
| import org.apache.pinot.spi.config.table.FieldConfig; | ||
| import org.apache.pinot.spi.config.table.TableConfig; | ||
| import org.apache.pinot.spi.config.table.TimestampConfig; | ||
| import org.apache.pinot.spi.config.table.TimestampIndexGranularity; | ||
| import org.apache.pinot.spi.data.Schema; | ||
| import org.apache.pinot.spi.env.PinotConfiguration; | ||
| import org.apache.pinot.spi.utils.CommonConstants; | ||
| import org.apache.pinot.util.TestUtils; | ||
| import org.testng.annotations.BeforeClass; | ||
| import org.testng.annotations.Test; | ||
|
|
||
|
|
||
| public class TimestampIndexTest extends BaseClusterIntegrationTest implements ExplainIntegrationTestTrait { | ||
| @BeforeClass | ||
| public void setUp() | ||
| throws Exception { | ||
| TestUtils.ensureDirectoriesExistAndEmpty(_tempDir, _segmentDir, _tarDir); | ||
|
|
||
| // Start the Pinot cluster | ||
| startZk(); | ||
| startController(); | ||
| startBroker(); | ||
| startServers(2); | ||
|
|
||
| // Create and upload the schema and table config | ||
| Schema schema = createSchema(); | ||
| addSchema(schema); | ||
| TableConfig tableConfig = createOfflineTableConfig(); | ||
| addTableConfig(tableConfig); | ||
|
|
||
| // Unpack the Avro files | ||
| List<File> avroFiles = unpackAvroData(_tempDir); | ||
|
|
||
| // Create and upload segments | ||
| ClusterIntegrationTestUtils.buildSegmentsFromAvro(avroFiles, tableConfig, schema, 0, _segmentDir, _tarDir); | ||
| uploadSegments(getTableName(), _tarDir); | ||
|
|
||
| // Wait for all documents loaded | ||
| waitForAllDocsLoaded(600_000L); | ||
| } | ||
|
|
||
| protected void overrideBrokerConf(PinotConfiguration brokerConf) { | ||
| String property = CommonConstants.MultiStageQueryRunner.KEY_OF_MULTISTAGE_EXPLAIN_INCLUDE_SEGMENT_PLAN; | ||
| brokerConf.setProperty(property, "true"); | ||
| } | ||
|
|
||
| @Test | ||
| public void timestampIndexSubstitutedInProjectionsMSE() { | ||
| setUseMultiStageQueryEngine(true); | ||
| explain("SELECT datetrunc('SECOND',ArrTime) FROM mytable", | ||
| "Execution Plan\n" | ||
| + "PinotLogicalExchange(distribution=[broadcast])\n" | ||
| + " LeafStageCombineOperator(table=[mytable])\n" | ||
| + " StreamingInstanceResponse\n" | ||
| + " StreamingCombineSelect\n" | ||
| + " SelectStreaming(table=[mytable], totalDocs=[115545])\n" | ||
| + " Project(columns=[[$ArrTime$SECOND]])\n" | ||
| + " DocIdSet(maxDocs=[120000])\n" | ||
| + " FilterMatchEntireSegment(numDocs=[115545])\n"); | ||
| } | ||
|
|
||
| @Test | ||
| public void timestampIndexSubstitutedInFiltersMSE() { | ||
| setUseMultiStageQueryEngine(true); | ||
| explain("SELECT 1 FROM mytable where datetrunc('SECOND',ArrTime) > 1", | ||
| "Execution Plan\n" | ||
| + "PinotLogicalExchange(distribution=[broadcast])\n" | ||
| + " LeafStageCombineOperator(table=[mytable])\n" | ||
| + " StreamingInstanceResponse\n" | ||
| + " StreamingCombineSelect\n" | ||
| + " SelectStreaming(table=[mytable], totalDocs=[115545])\n" | ||
| + " Transform(expressions=[['1']])\n" | ||
| + " Project(columns=[[]])\n" | ||
| + " DocIdSet(maxDocs=[120000])\n" | ||
| + " FilterRangeIndex(predicate=[$ArrTime$SECOND > '1'], " | ||
| + "indexLookUp=[range_index], operator=[RANGE])\n"); | ||
| } | ||
|
|
||
| @Test | ||
| public void timestampIndexSubstitutedInAggregatesMSE() { | ||
| setUseMultiStageQueryEngine(true); | ||
| explain("SELECT sum(case when datetrunc('SECOND',ArrTime) > 1 then 2 else 0 end) FROM mytable", | ||
| "Execution Plan\n" | ||
| + "LogicalProject(EXPR$0=[CASE(=($1, 0), null:BIGINT, $0)])\n" | ||
| + " PinotLogicalAggregate(group=[{}], agg#0=[$SUM0($0)], agg#1=[COUNT($1)], aggType=[FINAL])\n" | ||
| + " PinotLogicalExchange(distribution=[hash])\n" | ||
| + " LeafStageCombineOperator(table=[mytable])\n" | ||
| + " StreamingInstanceResponse\n" | ||
| + " CombineAggregate\n" | ||
| + " AggregateFiltered(aggregations=[[sum('2'), count(*)]])\n" | ||
| + " Transform(expressions=[['2']])\n" | ||
| + " Project(columns=[[]])\n" | ||
| + " DocIdSet(maxDocs=[120000])\n" | ||
| + " FilterRangeIndex(predicate=[$ArrTime$SECOND > '1'], indexLookUp=[range_index], " | ||
| + "operator=[RANGE])\n" | ||
| + " Project(columns=[[]])\n" | ||
| + " DocIdSet(maxDocs=[120000])\n" | ||
| + " FilterMatchEntireSegment(numDocs=[115545])\n"); | ||
| } | ||
|
|
||
| @Test | ||
| public void timestampIndexSubstitutedInJOinMSE() { | ||
| setUseMultiStageQueryEngine(true); | ||
| explain("SELECT 1 " | ||
| + "FROM mytable as a1 " | ||
| + "join mytable as a2 " | ||
| + "on datetrunc('SECOND',a1.ArrTime) = datetrunc('DAY',a2.ArrTime)", | ||
| "Execution Plan\n" | ||
| + "LogicalProject(EXPR$0=[1])\n" | ||
| + " LogicalJoin(condition=[=($0, $1)], joinType=[inner])\n" | ||
| + " PinotLogicalExchange(distribution=[hash[0]])\n" | ||
| + " LeafStageCombineOperator(table=[mytable])\n" | ||
| + " StreamingInstanceResponse\n" | ||
| + " StreamingCombineSelect\n" | ||
| + " SelectStreaming(table=[mytable], totalDocs=[115545])\n" | ||
| + " Project(columns=[[$ArrTime$SECOND]])\n" // substituted because we have SECOND granularity | ||
| + " DocIdSet(maxDocs=[120000])\n" | ||
| + " FilterMatchEntireSegment(numDocs=[115545])\n" | ||
| + " PinotLogicalExchange(distribution=[hash[0]])\n" | ||
| + " LeafStageCombineOperator(table=[mytable])\n" | ||
| + " StreamingInstanceResponse\n" | ||
| + " StreamingCombineSelect\n" | ||
| + " SelectStreaming(table=[mytable], totalDocs=[115545])\n" | ||
| + " Transform(expressions=[[datetrunc('DAY',ArrTime)]])\n" // we don't set the DAY granularity | ||
| + " Project(columns=[[ArrTime]])\n" | ||
| + " DocIdSet(maxDocs=[120000])\n" | ||
| + " FilterMatchEntireSegment(numDocs=[115545])\n"); | ||
| } | ||
|
|
||
|
|
||
| protected TableConfig createOfflineTableConfig() { | ||
| String colName = "ArrTime"; | ||
|
|
||
| TableConfig tableConfig = super.createOfflineTableConfig(); | ||
| List<FieldConfig> fieldConfigList = tableConfig.getFieldConfigList(); | ||
| if (fieldConfigList == null) { | ||
| fieldConfigList = new ArrayList<>(); | ||
| tableConfig.setFieldConfigList(fieldConfigList); | ||
| } else { | ||
| fieldConfigList.stream() | ||
| .filter(fieldConfig -> fieldConfig.getName().equals(colName)) | ||
| .findFirst() | ||
| .ifPresent( | ||
| fieldConfig -> { | ||
| throw new IllegalStateException("Time column already exists in the field config list"); | ||
| } | ||
| ); | ||
| } | ||
| FieldConfig newTimeFieldConfig = new FieldConfig.Builder(colName) | ||
| .withTimestampConfig( | ||
| new TimestampConfig(List.of(TimestampIndexGranularity.SECOND)) | ||
| ) | ||
| .build(); | ||
| fieldConfigList.add(newTimeFieldConfig); | ||
| return tableConfig; | ||
| } | ||
| } |