diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java index 6ff0cb5c2f..e147e51b2d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java @@ -15,6 +15,8 @@ import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; +import com.bakdata.conquery.sql.conversion.model.select.ConceptColumnSelectConverter; +import com.bakdata.conquery.sql.conversion.model.select.SelectConverter; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; import lombok.Data; @@ -67,4 +69,8 @@ public ResultType getResultType() { return new ResultType.ListT<>(ResultType.StringT.INSTANCE); } + @Override + public SelectConverter createConverter() { + return new ConceptColumnSelectConverter(); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java index 351432d17c..de9d5217d3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java @@ -42,7 +42,7 @@ public ConversionContext convert(CQExternal external, ConversionContext context) .toList(); Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 converted resolved row when converting a CQExternal"); - QueryStep externalStep = QueryStep.createUnionStep(unions, CQ_EXTERNAL_CTE_NAME, Collections.emptyList()); + QueryStep externalStep = QueryStep.createUnionAllStep(unions, CQ_EXTERNAL_CTE_NAME, Collections.emptyList()); return context.withQueryStep(externalStep); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 27b626f93e..6cac5ba991 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -12,6 +12,7 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.models.datasets.concepts.Connector; +import com.bakdata.conquery.models.datasets.concepts.select.concept.ConceptColumnSelect; import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeChild; import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeNode; import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; @@ -153,9 +154,10 @@ private CQTableContext createTableContext(TablePath tablePath, CQConcept cqConce // convert selects SelectContext selectContext = SelectContext.create(cqTable, ids, tablesValidityDate, connectorTables, conversionContext); - List allSelectsForTable = cqTable.getSelects().stream() - .map(select -> select.createConverter().connectorSelect(select, selectContext)) - .toList(); + List allSelectsForTable = new ArrayList<>(); + ConnectorSqlSelects conceptColumnSelect = createConceptColumnConnectorSqlSelects(cqConcept, selectContext); + allSelectsForTable.add(conceptColumnSelect); + cqTable.getSelects().stream().map(select -> select.createConverter().connectorSelect(select, selectContext)).forEach(allSelectsForTable::add); return CQTableContext.builder() .ids(ids) @@ -275,4 +277,12 @@ private static Optional getDateRestriction(ConversionContext context )); } + private static ConnectorSqlSelects createConceptColumnConnectorSqlSelects(CQConcept cqConcept, SelectContext selectContext) { + return cqConcept.getSelects().stream() + .filter(select -> select instanceof ConceptColumnSelect) + .findFirst() + .map(select -> select.createConverter().connectorSelect(select, selectContext)) + .orElse(ConnectorSqlSelects.none()); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterCte.java index bc13891a0c..209c516a1a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterCte.java @@ -58,15 +58,18 @@ private Selects collectSelects(CQTableContext tableContext) { } /** - * Collects the columns required in {@link ConceptCteStep#AGGREGATION_SELECT}, but also columns additional tables require (like the ones created by the - * {@link SumSqlAggregator}) when distinct-by columns are present. An additional predecessor can contain an N-ary tree of predecessors itself - * (like all {@link QueryStep}s), so we want to look for the deepest preceding QueryStep leafs and collect their {@link ConnectorSqlSelects}, - * because they expect this CTE to contain all their {@link SqlSelect#requiredColumns()}. + * Collects the columns required in {@link ConceptCteStep#AGGREGATION_SELECT}, the optional connector column, but also columns additional tables require + * (like the ones created by the {@link SumSqlAggregator}) when distinct-by columns are present. An additional predecessor can contain an N-ary tree of + * predecessors itself (like all {@link QueryStep}s), so we want to look for the deepest preceding QueryStep leafs and collect their + * {@link ConnectorSqlSelects}, because they expect this CTE to contain all their {@link SqlSelect#requiredColumns()}. */ private static List collectSelects(ConnectorSqlSelects sqlSelects) { return Stream.concat( - sqlSelects.getAggregationSelects().stream(), - sqlSelects.getAdditionalPredecessor().map(EventFilterCte::collectDeepestPredecessorsColumns).orElse(Stream.empty()) + sqlSelects.getConnectorColumn().stream(), + Stream.concat( + sqlSelects.getAggregationSelects().stream(), + sqlSelects.getAdditionalPredecessor().map(EventFilterCte::collectDeepestPredecessorsColumns).orElse(Stream.empty()) + ) ) .toList(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java index 62e1d8d66b..26a9c4eb57 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java @@ -5,23 +5,21 @@ import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Objects; -import java.util.stream.Collectors; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.ValidityDate; -import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import org.jooq.Condition; import org.jooq.DataType; import org.jooq.Field; -import org.jooq.Param; import org.jooq.Record; import org.jooq.Table; import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; public class HanaSqlFunctionProvider implements SqlFunctionProvider { @@ -43,6 +41,10 @@ public String getMaxDateExpression() { @Override public Field cast(Field field, DataType type) { + // HANA would require an explicit length param when using CAST with varchar type, TO_VARCHAR does not require this + if (type == SQLDataType.VARCHAR) { + return DSL.function("TO_VARCHAR", type.getType(), field); + } return DSL.function( "CAST", type.getType(), @@ -154,20 +156,29 @@ public QueryStep unnestValidityDate(QueryStep predecessor, String cteName) { } @Override - public Field daterangeStringAggregation(ColumnDateRange columnDateRange) { - - Field stringAggregation = DSL.field( + public Field stringAggregation(Field stringField, Field delimiter, List> orderByFields) { + return DSL.field( "{0}({1}, {2} {3})", String.class, DSL.keyword("STRING_AGG"), + stringField, + delimiter, + DSL.orderBy(orderByFields) + ); + } + + @Override + public Field daterangeStringAggregation(ColumnDateRange columnDateRange) { + + Field stringAggregation = stringAggregation( daterangeStringExpression(columnDateRange), DSL.toChar(DELIMITER), - DSL.orderBy(columnDateRange.getStart()) + List.of(columnDateRange.getStart()) ); // encapsulate all ranges (including empty ranges) within curly braces return DSL.when(stringAggregation.isNull(), DSL.val("{}")) - .otherwise(DSL.field("'{' || {0} || '}'", String.class, stringAggregation)); + .otherwise(encloseInCurlyBraces(stringAggregation)); } @Override @@ -180,9 +191,8 @@ public Field daterangeStringExpression(ColumnDateRange columnDateRange) Field startDate = columnDateRange.getStart(); Field endDate = columnDateRange.getEnd(); - Param dateLength = DSL.val(DEFAULT_DATE_FORMAT.length()); - Field startDateExpression = toVarcharField(startDate, dateLength); - Field endDateExpression = toVarcharField(endDate, dateLength); + Field startDateExpression = cast(startDate, SQLDataType.VARCHAR); + Field endDateExpression = cast(endDate, SQLDataType.VARCHAR); Field withMinDateReplaced = replace(startDateExpression, MIN_DATE_VALUE, MINUS_INFINITY_SIGN); Field withMaxDateReplaced = replace(endDateExpression, MAX_DATE_VALUE, INFINITY_SIGN); @@ -278,14 +288,6 @@ public Field yearQuarter(Field dateField) { return DSL.function("QUARTER", String.class, dateField); } - @Override - public Field asArray(List> fields) { - String arrayExpression = fields.stream() - .map(Field::toString) - .collect(Collectors.joining(", ", "array(", ")")); - return DSL.field(arrayExpression, Object[].class); - } - @Override public Field addDays(Field dateColumn, Field amountOfDays) { return DSL.function( @@ -296,17 +298,6 @@ public Field addDays(Field dateColumn, Field amountOfDays) ); } - private Field toVarcharField(Field startDate, Param dateExpressionLength) { - return DSL.field( - "{0}({1} {2}({3}))", - String.class, - DSL.keyword("CAST"), - startDate, - DSL.keyword("AS VARCHAR"), - dateExpressionLength - ); - } - private ColumnDateRange toColumnDateRange(CDateRange dateRestriction) { String startDateExpression = MIN_DATE_VALUE; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java index b68ab67c6b..23128c0fc4 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java @@ -23,7 +23,6 @@ import org.jooq.Record; import org.jooq.Table; import org.jooq.impl.DSL; -import com.google.common.base.Preconditions; import org.jooq.impl.SQLDataType; /** @@ -165,6 +164,18 @@ public QueryStep unnestValidityDate(QueryStep predecessor, String cteName) { .build(); } + @Override + public Field stringAggregation(Field stringField, Field delimiter, List> orderByFields) { + return DSL.field( + "{0}({1}, {2} {3})", + String.class, + DSL.keyword("string_agg"), + stringField, + delimiter, + DSL.orderBy(orderByFields) + ); + } + @Override public Field daterangeStringAggregation(ColumnDateRange columnDateRange) { Field asMultirange = rangeAgg(columnDateRange); @@ -246,14 +257,6 @@ public Field yearQuarter(Field dateField) { ); } - @Override - public Field asArray(List> fields) { - String arrayExpression = fields.stream() - .map(Field::toString) - .collect(Collectors.joining(", ", "array[", "]")); - return DSL.field(arrayExpression, Object[].class); - } - @Override public Field toDateField(String dateValue) { return DSL.field("{0}::{1}", Date.class, DSL.val(dateValue), DSL.keyword("date")); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index 91e05823bc..1d7e09644e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -3,6 +3,7 @@ import java.sql.Date; import java.time.temporal.ChronoUnit; import java.util.List; +import java.util.stream.Collectors; import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; import com.bakdata.conquery.models.common.CDateSet; @@ -11,6 +12,7 @@ import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; import org.jooq.Condition; import org.jooq.DataType; import org.jooq.Field; @@ -27,6 +29,7 @@ public interface SqlFunctionProvider { String DEFAULT_DATE_FORMAT = "yyyy-mm-dd"; String INFINITY_SIGN = "∞"; String MINUS_INFINITY_SIGN = "-∞"; + String SQL_UNIT_SEPARATOR = " || '%s' || ".formatted(ResultSetProcessor.UNIT_SEPARATOR); String getMinDateExpression(); @@ -95,6 +98,8 @@ public interface SqlFunctionProvider { */ QueryStep unnestValidityDate(QueryStep predecessor, String cteName); + Field stringAggregation(Field stringField, Field delimiter, List> orderByFields); + /** * Aggregates the start and end columns of the validity date of entries into one compound string expression. *

@@ -122,7 +127,7 @@ public interface SqlFunctionProvider { Field addDays(Field dateColumn, Field amountOfDays); - Field first(Field field, List> orderByColumn); + Field first(Field field, List> orderByColumn); Field last(Field column, List> orderByColumns); @@ -135,7 +140,15 @@ public interface SqlFunctionProvider { */ Field yearQuarter(Field dateField); - Field asArray(List> fields); + default Field concat(List> fields) { + String concatenated = fields.stream() + // if a field is null, the whole concatenation would be null - but we just want to skip this field in this case, + // thus concat an empty string + .map(field -> DSL.when(field.isNull(), DSL.val("")).otherwise(field)) + .map(Field::toString) + .collect(Collectors.joining(SQL_UNIT_SEPARATOR)); + return DSL.field(concatenated, String.class); + } default Field least(List> fields) { if (fields.isEmpty()) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java index d7915f52ba..404d5a030e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java @@ -154,7 +154,7 @@ private QueryStep createCompleteTable(QueryStep totalBoundsStep, RelativeFormQue QueryStep featureTable = form.getTimeCountBefore() > 0 ? createCompleteFeatureTable(predecessorSelects, interval, intRange, totalBoundsStep) : null; QueryStep outcomeTable = form.getTimeCountAfter() > 0 ? createCompleteOutcomeTable(predecessorSelects, interval, intRange, totalBoundsStep) : null; - return QueryStep.createUnionStep( + return QueryStep.createUnionAllStep( Stream.concat(Stream.ofNullable(outcomeTable), Stream.ofNullable(featureTable)).toList(), FormCteStep.COMPLETE.getSuffix(), Collections.emptyList() @@ -186,7 +186,7 @@ private QueryStep createIntervalTable(QueryStep totalBoundsStep, Resolution reso QueryStep timeBeforeStep = createFeatureTable(totalBoundsStep, interval, seriesIndex, bounds, ids); QueryStep timeAfterStep = createOutcomeTable(totalBoundsStep, interval, seriesIndex, bounds, ids); - return QueryStep.createUnionStep( + return QueryStep.createUnionAllStep( List.of(timeBeforeStep, timeAfterStep), FormCteStep.stratificationCte(resolution).getSuffix(), Collections.emptyList() diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java index c02cdb9f45..aaade2fa08 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java @@ -51,7 +51,7 @@ protected static QueryStep unionResolutionTables(List unionSteps, Lis .build()) .toList(); - return QueryStep.createUnionStep( + return QueryStep.createUnionAllStep( withQualifiedSelects, FormCteStep.FULL_STRATIFICATION.getSuffix(), Stream.concat(predecessors.stream(), unionSteps.stream()).toList() diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java index c6ea7da616..f2f9b80cac 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java @@ -3,7 +3,6 @@ import java.util.Collections; import java.util.List; -import com.bakdata.conquery.sql.conversion.model.select.ExistsSqlSelect; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; import lombok.Builder; import lombok.Singular; @@ -37,17 +36,31 @@ public class QueryStep { */ @Builder.Default List union = Collections.emptyList(); + /** + * Determines if this steps union steps should be unioned using a UNION ALL. Default is true. + */ + @Builder.Default + boolean unionAll = true; /** * All {@link QueryStep}'s that shall be converted before this {@link QueryStep}. */ @Singular List predecessors; + public static QueryStep createUnionAllStep(List unionSteps, String cteName, List predecessors) { + return createUnionStep(unionSteps, cteName, predecessors, true); + } + public static QueryStep createUnionStep(List unionSteps, String cteName, List predecessors) { + return createUnionStep(unionSteps, cteName, predecessors, false); + } + + private static QueryStep createUnionStep(List unionSteps, String cteName, List predecessors, boolean unionAll) { return unionSteps.get(0) .toBuilder() .cteName(cteName) .union(unionSteps.subList(1, unionSteps.size())) + .unionAll(unionAll) .predecessors(predecessors) .build(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java index 7058ce04e7..8690587890 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java @@ -48,7 +48,7 @@ public Select toSelectQuery(QueryStep queryStep) { if (!queryStep.isUnion()) { return ordered; } - return unionAll(queryStep, ordered); + return union(queryStep, ordered); } private List> constructPredecessorCteList(QueryStep queryStep) { @@ -86,15 +86,15 @@ private Select toSelectStep(QueryStep queryStep) { } if (queryStep.isUnion()) { - selectStep = unionAll(queryStep, selectStep); + selectStep = union(queryStep, selectStep); } return selectStep; } - private Select unionAll(QueryStep queryStep, Select base) { + private Select union(QueryStep queryStep, Select base) { for (QueryStep unionStep : queryStep.getUnion()) { - base = base.unionAll(toSelectStep(unionStep)); + base = queryStep.isUnionAll() ? base.unionAll(toSelectStep(unionStep)) : base.union(toSelectStep(unionStep)); } return base; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java index 6653d47f92..5c379e53f5 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java @@ -81,9 +81,9 @@ public ConnectorSqlSelects connectorSelect(FlagSelect flagSelect, SelectContext< Map> rootSelects = createFlagRootSelectMap(flagSelect, connectorTables.getRootTable()); String alias = selectContext.getNameGenerator().selectName(flagSelect); - FieldWrapper flagAggregation = createFlagSelect(alias, connectorTables, functionProvider, rootSelects); + FieldWrapper flagAggregation = createFlagSelect(alias, connectorTables, functionProvider, rootSelects); - ExtractingSqlSelect finalSelect = flagAggregation.qualify(connectorTables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); + ExtractingSqlSelect finalSelect = flagAggregation.qualify(connectorTables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)); return ConnectorSqlSelects.builder() .preprocessingSelects(rootSelects.values()) @@ -104,7 +104,7 @@ private static Map> createFlagRootSelectMap )); } - private static FieldWrapper createFlagSelect( + private static FieldWrapper createFlagSelect( String alias, SqlTables connectorTables, SqlFunctionProvider functionProvider, @@ -113,7 +113,7 @@ private static FieldWrapper createFlagSelect( Map> flagFieldsMap = createRootSelectReferences(connectorTables, flagRootSelectMap); // we first aggregate each flag column - List> flagAggregations = new ArrayList<>(); + List> flagAggregations = new ArrayList<>(); for (Map.Entry> entry : flagFieldsMap.entrySet()) { Field boolColumn = entry.getValue(); Condition anyTrue = DSL.max(functionProvider.cast(boolColumn, SQLDataType.INTEGER)) @@ -125,8 +125,7 @@ private static FieldWrapper createFlagSelect( } // and stuff them into 1 array field - Field flagsArray = functionProvider.asArray(flagAggregations).as(alias); - + Field flagsArray = functionProvider.concat(flagAggregations).as(alias); // we also need the references for all flag columns for the flag aggregation of multiple columns String[] requiredColumns = flagFieldsMap.values().stream().map(Field::getName).toArray(String[]::new); return new FieldWrapper<>(flagsArray, requiredColumns); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java new file mode 100644 index 0000000000..b91da43125 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConceptColumnSelectConverter.java @@ -0,0 +1,164 @@ +package com.bakdata.conquery.sql.conversion.model.select; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +import com.bakdata.conquery.models.datasets.concepts.Connector; +import com.bakdata.conquery.models.datasets.concepts.select.concept.ConceptColumnSelect; +import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; +import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptSqlTables; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorSqlTables; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.conversion.model.CteStep; +import com.bakdata.conquery.sql.conversion.model.NameGenerator; +import com.bakdata.conquery.sql.conversion.model.QualifyingUtil; +import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; +import com.bakdata.conquery.util.TablePrimaryColumnUtil; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; + +public class ConceptColumnSelectConverter implements SelectConverter { + + @Getter + @RequiredArgsConstructor + private enum CONCEPT_COLUMN_STEPS implements CteStep { + + UNIONED_COLUMNS("unioned_columns"), + STRING_AGG("concept_column_aggregated"); + + private final String suffix; + } + + @Override + public ConnectorSqlSelects connectorSelect(ConceptColumnSelect select, SelectContext selectContext) { + Connector connector = selectContext.getSelectHolder(); + if (connector.getColumn() == null) { + return ConnectorSqlSelects.none(); + } + ExtractingSqlSelect connectorColumn = new ExtractingSqlSelect<>(connector.getTable().getName(), connector.getColumn().getName(), Object.class); + ExtractingSqlSelect qualified = connectorColumn.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.EVENT_FILTER)); + return ConnectorSqlSelects.builder() + .preprocessingSelect(connectorColumn) + .connectorColumn(Optional.of(qualified)) + .build(); + } + + @Override + public ConceptSqlSelects conceptSelect(ConceptColumnSelect select, SelectContext selectContext) { + + // we will do a union distinct on all Connector tables + List connectors; + if (isSingleConnectorConcept(selectContext.getSelectHolder())) { + // we union the Connector table with itself if there is only 1 Connector + Connector connector = selectContext.getSelectHolder().getConcept().getConnectors().get(0); + connectors = List.of(connector, connector); + } + else { + connectors = selectContext.getSelectHolder().getConnectors(); + } + + NameGenerator nameGenerator = selectContext.getNameGenerator(); + String alias = nameGenerator.selectName(select); + QueryStep unionStep = createUnionConnectorConnectorsStep(connectors, alias, selectContext); + + FieldWrapper conceptColumnSelect = createConnectorColumnStringAgg(selectContext, unionStep, alias); + Selects unionStepSelects = unionStep.getQualifiedSelects(); + Selects selects = Selects.builder() + .ids(unionStepSelects.getIds()) + .sqlSelect(conceptColumnSelect) + .build(); + + String stringAggCteName = nameGenerator.cteStepName(CONCEPT_COLUMN_STEPS.STRING_AGG, alias); + QueryStep stringAggStep = QueryStep.builder() + .cteName(stringAggCteName) + .selects(selects) + .fromTable(QueryStep.toTableLike(unionStep.getCteName())) + .groupBy(unionStepSelects.getIds().toFields()) + .predecessor(unionStep) + .build(); + + ExtractingSqlSelect finalSelect = conceptColumnSelect.qualify(stringAggStep.getCteName()); + + return ConceptSqlSelects.builder() + .additionalPredecessor(Optional.of(stringAggStep)) + .finalSelect(finalSelect) + .build(); + } + + private static boolean isSingleConnectorConcept(TreeConcept treeConcept) { + return treeConcept.getConcept().getConnectors().size() == 1; + } + + private static QueryStep createUnionConnectorConnectorsStep( + List connectors, + String alias, + SelectContext selectContext + ) { + List unionSteps = selectContext.getTables() + .getConnectorTables() + .stream() + .map(tables -> createConnectorColumnSelectQuery(tables, connectors, alias, selectContext)) + .toList(); + + String unionedColumnsCteName = selectContext.getNameGenerator().cteStepName(CONCEPT_COLUMN_STEPS.UNIONED_COLUMNS, alias); + return QueryStep.createUnionStep(unionSteps, unionedColumnsCteName, Collections.emptyList()); + } + + private static QueryStep createConnectorColumnSelectQuery( + ConnectorSqlTables tables, + List connectors, + String alias, + SelectContext selectContext + ) { + Connector matchingConnector = + connectors.stream() + .filter(connector -> isMatchingConnector(tables, connector)) + .findFirst() + .orElseThrow(() -> new IllegalStateException("Could not find matching connector for ConnectorSqlTables %s".formatted(tables))); + + Table connectorTable = DSL.table(DSL.name(tables.cteName(ConceptCteStep.EVENT_FILTER))); + + Field primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(matchingConnector.getTable(), selectContext.getConversionContext().getConfig()); + Field qualifiedPrimaryColumn = QualifyingUtil.qualify(primaryColumn, connectorTable.getName()).as(SharedAliases.PRIMARY_COLUMN.getAlias()); + SqlIdColumns ids = new SqlIdColumns(qualifiedPrimaryColumn); + + Field connectorColumn = DSL.field(DSL.name(connectorTable.getName(), matchingConnector.getColumn().getName())); + Field casted = selectContext.getFunctionProvider().cast(connectorColumn, SQLDataType.VARCHAR).as(alias); + FieldWrapper connectorSelect = new FieldWrapper<>(casted); + + Selects selects = Selects.builder() + .ids(ids) + .sqlSelect(connectorSelect) + .build(); + + return QueryStep.builder() + .selects(selects) + .fromTable(connectorTable) + .build(); + } + + private static boolean isMatchingConnector(final ConnectorSqlTables tables, final Connector connector) { + return connector.getColumn() != null && (Objects.equals(tables.getRootTable(), connector.getTable().getName())); + } + + private static FieldWrapper createConnectorColumnStringAgg(SelectContext selectContext, QueryStep unionStep, String alias) { + SqlFunctionProvider functionProvider = selectContext.getFunctionProvider(); + Field unionedColumn = DSL.field(DSL.name(unionStep.getCteName(), alias), String.class); + return new FieldWrapper<>( + functionProvider.stringAggregation(unionedColumn, DSL.toChar(ResultSetProcessor.UNIT_SEPARATOR), List.of(unionedColumn)).as(alias) + ); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConnectorSqlSelects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConnectorSqlSelects.java index 4eaed8a4fc..57f9622b48 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConnectorSqlSelects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ConnectorSqlSelects.java @@ -15,6 +15,12 @@ public class ConnectorSqlSelects { @Singular List preprocessingSelects; + /** + * ConceptColumnSelect is an edge case which requires the connector column to be present in the event-filter step, but not afterward. + */ + @Builder.Default + Optional connectorColumn = Optional.empty(); + // Empty if only used in event filter @Singular List aggregationSelects; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java index 00c78c2c76..055bfcc262 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java @@ -6,6 +6,7 @@ import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.models.query.DateAggregationMode; import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; @@ -56,7 +57,7 @@ else if (preFinalSelects.getValidityDate().isEmpty()) { return preFinalSelects.withValidityDate(ColumnDateRange.empty()); } Field validityDateStringAggregation = functionProvider.daterangeStringAggregation(preFinalSelects.getValidityDate().get()); - return preFinalSelects.withValidityDate(ColumnDateRange.of(validityDateStringAggregation)); + return preFinalSelects.withValidityDate(ColumnDateRange.of(validityDateStringAggregation).as(SharedAliases.DATES_COLUMN.getAlias())); } private List> getFinalGroupBySelects(Selects preFinalSelects) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java index a576737695..ce433e53b9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java @@ -65,7 +65,7 @@ public ConversionContext convert(TableExportQuery tableExportQuery, ConversionCo ))) .toList(); - QueryStep unionedTables = QueryStep.createUnionStep( + QueryStep unionedTables = QueryStep.createUnionAllStep( convertedTables, null, // no CTE name required as this step will be the final select List.of(convertedPrerequisite) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java index ddb094bbee..62b430d9ed 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java @@ -2,13 +2,13 @@ import java.math.BigDecimal; import java.math.RoundingMode; -import java.sql.Array; import java.sql.Date; import java.sql.ResultSet; import java.sql.SQLException; import java.util.Arrays; +import java.util.Collections; import java.util.List; -import java.util.Objects; +import java.util.function.Predicate; import lombok.RequiredArgsConstructor; @@ -67,15 +67,13 @@ public List> getDateRangeList(ResultSet resultSet, int columnIndex @Override public List getStringList(ResultSet resultSet, int columnIndex) throws SQLException { - try { - Array result = resultSet.getArray(columnIndex); - // ResultSet does not provide a way to directly get an array of a specific type (see https://docs.oracle.com/javase/tutorial/jdbc/basics/array.html) - String[] casted = (String[]) result.getArray(); - return Arrays.stream(casted).filter(Objects::nonNull).toList(); - } - catch (ClassCastException exception) { - throw new SQLException("Expected an array of type String at column index %s in ResultSet %s.".formatted(columnIndex, resultSet)); + String arrayExpression = resultSet.getString(columnIndex); + if (arrayExpression == null) { + return Collections.emptyList(); } + return Arrays.stream(arrayExpression.split(String.valueOf(ResultSetProcessor.UNIT_SEPARATOR))) + .filter(Predicate.not(String::isBlank)) + .toList(); } @FunctionalInterface diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java index bd9d33abb5..9f63f58c36 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java @@ -7,6 +7,8 @@ public interface ResultSetProcessor { + char UNIT_SEPARATOR = (char) 31; // https://www.ascii-code.com/character/%E2%90%9F + String getString(ResultSet resultSet, int columnIndex) throws SQLException; Integer getInteger(ResultSet resultSet, int columnIndex) throws SQLException; diff --git a/backend/src/test/resources/tests/sql/selects/concept_values/concept_values.json b/backend/src/test/resources/tests/sql/selects/concept_values/concept_values.json new file mode 100644 index 0000000000..c98cc4559f --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/concept_values/concept_values.json @@ -0,0 +1,115 @@ +{ + "type": "QUERY_TEST", + "label": "CONCEPT_VALUES Test", + "expectedCsv": "tests/aggregator/CONCEPT_COLUMN_SELECTS/expected_raw.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "CONCEPT", + "selects": [ + "tree.select" + ], + "ids": [ + "tree" + ], + "tables": [ + { + "id": "tree.test_column" + }, + { + "id": "tree.test_column2" + } + ] + } + }, + "concepts": [ + { + "label": "tree", + "type": "TREE", + "selects": [ + { + "type": "CONCEPT_VALUES", + "name": "select", + "asIds": false + } + ], + "connectors": [ + { + "name": "test_column", + "column": "table.test_column", + "validityDates": { + "label": "datum", + "column": "table.datum" + } + }, + { + "label": "tree_label2", + "name": "test_column2", + "column": "table2.test_column", + "validityDates": { + "label": "datum", + "column": "table2.datum" + } + } + ], + "children": [ + { + "label": "test_child1", + "condition": { + "type": "PREFIX_LIST", + "prefixes": "A" + }, + "children": [] + }, + { + "label": "test_child2", + "condition": { + "type": "PREFIX_LIST", + "prefixes": "B" + }, + "children": [] + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/aggregator/CONCEPT_COLUMN_SELECTS/content.csv", + "name": "table", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + }, + { + "name": "test_column", + "type": "STRING" + } + ] + }, + { + "csv": "tests/aggregator/CONCEPT_COLUMN_SELECTS/content2.csv", + "name": "table2", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + }, + { + "name": "test_column", + "type": "STRING" + } + ] + } + ] + } +}