From c321d1ccb8e8b47a047369833c3ef03e94f10f55 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Tue, 12 Dec 2023 23:41:52 -0800 Subject: [PATCH] add support for ColumnConfig skipValueRangeIndexScale and skipValueRangeIndexScale to traditional string columns --- .../DictionaryEncodedColumnPartSerde.java | 19 ++++++++++++++++++- .../serde/StringUtf8ColumnIndexSupplier.java | 6 +++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java b/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java index 41aef13f576c..d46452450691 100644 --- a/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java +++ b/processing/src/main/java/org/apache/druid/segment/serde/DictionaryEncodedColumnPartSerde.java @@ -319,12 +319,27 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo final WritableSupplier rSingleValuedColumn; final WritableSupplier rMultiValuedColumn; + final int numRows; if (hasMultipleValues) { rMultiValuedColumn = readMultiValuedColumn(rVersion, buffer, rFlags); rSingleValuedColumn = null; + // wtb: better way to have number of rows in segment, maybe just passed into all the read methods + try (ColumnarMultiInts throwAway = rMultiValuedColumn.get()) { + numRows = throwAway.size(); + } + catch (IOException e) { + throw new RuntimeException(e); + } } else { rSingleValuedColumn = readSingleValuedColumn(rVersion, buffer); rMultiValuedColumn = null; + // wtb: better way to have number of rows in segment, maybe just passed into all the read methods + try (ColumnarInts throwAway = rSingleValuedColumn.get()) { + numRows = throwAway.size(); + } + catch (IOException e) { + throw new RuntimeException(e); + } } final boolean hasNulls = dictionarySupplier.get().get(0) == null; @@ -360,7 +375,9 @@ public void read(ByteBuffer buffer, ColumnBuilder builder, ColumnConfig columnCo bitmapSerdeFactory.getBitmapFactory(), dictionarySupplier, rBitmaps, - rSpatialIndex + rSpatialIndex, + columnConfig, + numRows ), rBitmaps != null, rSpatialIndex != null diff --git a/processing/src/main/java/org/apache/druid/segment/serde/StringUtf8ColumnIndexSupplier.java b/processing/src/main/java/org/apache/druid/segment/serde/StringUtf8ColumnIndexSupplier.java index e80ca2595a36..20c20a71c7d2 100644 --- a/processing/src/main/java/org/apache/druid/segment/serde/StringUtf8ColumnIndexSupplier.java +++ b/processing/src/main/java/org/apache/druid/segment/serde/StringUtf8ColumnIndexSupplier.java @@ -62,6 +62,10 @@ public class StringUtf8ColumnIndexSupplier> private final ColumnConfig columnConfig; private final int numRows; + /** + * Legacy constructor which always uses bitmap indexes, sparing no expense (ignores the configurable range and + * predicate indexes thresholds on {@link ColumnConfig}, in favor of using {@link ColumnConfig#ALWAYS_USE_INDEXES}). + */ public StringUtf8ColumnIndexSupplier( BitmapFactory bitmapFactory, Supplier utf8Dictionary, @@ -77,7 +81,7 @@ public StringUtf8ColumnIndexSupplier( Supplier utf8Dictionary, @Nullable GenericIndexed bitmaps, @Nullable ImmutableRTree indexedTree, - @Nullable ColumnConfig columnConfig, + ColumnConfig columnConfig, int numRows ) {