From ebfcb31600a7e675377ccc28943d7a524c23d633 Mon Sep 17 00:00:00 2001 From: Jonas Arnhold Date: Tue, 2 Jan 2024 20:28:40 +0100 Subject: [PATCH] Implement Flag select and filter conversion --- .../concepts/filters/specific/FlagFilter.java | 16 ++ .../select/connector/specific/FlagSelect.java | 10 + .../models/query/resultinfo/ResultInfo.java | 2 +- .../conquery/models/types/ResultType.java | 10 +- .../dialect/HanaSqlFunctionProvider.java | 15 ++ .../dialect/PostgreSqlFunctionProvider.java | 11 ++ .../dialect/SqlFunctionProvider.java | 5 + .../model/filter/FlagCondition.java | 29 +++ .../model/select/FlagSqlAggregator.java | 181 ++++++++++++++++++ .../execution/DefaultResultSetProcessor.java | 12 +- .../sql/execution/ResultSetProcessor.java | 68 +------ .../sql/execution/SqlExecutionService.java | 12 +- .../sql/execution/SqlStringListParser.java | 19 ++ .../tests/sql/filter/flag/content.csv | 10 + .../tests/sql/filter/flag/expected.csv | 6 + .../tests/sql/filter/flag/flag.spec.json | 80 ++++++++ .../tests/sql/selects/flag/content.csv | 9 + .../tests/sql/selects/flag/expected.csv | 9 + .../tests/sql/selects/flag/flag.spec.json | 71 +++++++ 19 files changed, 497 insertions(+), 78 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/FlagCondition.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FlagSqlAggregator.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/execution/SqlStringListParser.java create mode 100644 backend/src/test/resources/tests/sql/filter/flag/content.csv create mode 100644 backend/src/test/resources/tests/sql/filter/flag/expected.csv create mode 100644 backend/src/test/resources/tests/sql/filter/flag/flag.spec.json create mode 100644 backend/src/test/resources/tests/sql/selects/flag/content.csv create mode 100644 backend/src/test/resources/tests/sql/selects/flag/expected.csv create mode 100644 backend/src/test/resources/tests/sql/selects/flag/flag.spec.json diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java index 0a5c2598ba..7b6a46ed05 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java @@ -19,9 +19,14 @@ import com.bakdata.conquery.models.exceptions.ConceptConfigurationException; import com.bakdata.conquery.models.query.filter.event.FlagColumnsFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; +import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; +import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; +import com.bakdata.conquery.sql.conversion.model.select.FlagSqlAggregator; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; +import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; @@ -31,6 +36,7 @@ * * The selected flags are logically or-ed. */ +@Getter @CPSType(base = Filter.class, id = "FLAGS") @RequiredArgsConstructor(onConstructor_ = {@JsonCreator}) @ToString @@ -87,4 +93,14 @@ public boolean isAllColumnsOfSameTable() { public boolean isAllColumnsBoolean() { return flags.values().stream().map(Column::getType).allMatch(MajorTypeId.BOOLEAN::equals); } + + @Override + public SqlFilters convertToSqlFilter(FilterContext filterContext) { + return FlagSqlAggregator.create(this, filterContext).getSqlFilters(); + } + + @Override + public Set getRequiredSqlSteps() { + return ConceptCteStep.withOptionalSteps(ConceptCteStep.EVENT_FILTER); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java index 60cf25c430..b183047135 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java @@ -12,9 +12,13 @@ import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.FlagsAggregator; +import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; +import com.bakdata.conquery.sql.conversion.model.select.FlagSqlAggregator; +import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; +import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; @@ -24,6 +28,7 @@ * * The selected flags are logically or-ed. */ +@Getter @CPSType(base = Select.class, id = "FLAGS") @RequiredArgsConstructor(onConstructor_ = {@JsonCreator}) @ToString @@ -55,4 +60,9 @@ public boolean isAllColumnsOfSameTable() { public boolean isAllColumnsBoolean() { return flags.values().stream().map(Column::getType).allMatch(MajorTypeId.BOOLEAN::equals); } + + @Override + public SqlSelects convertToSqlSelects(SelectContext selectContext) { + return FlagSqlAggregator.create(this, selectContext).getSqlSelects(); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java index 2d31221c83..83a185fc17 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java @@ -33,7 +33,7 @@ public abstract class ResultInfo { public abstract String defaultColumnName(PrintSettings printSettings); @ToString.Include - public abstract ResultType getType(); + public abstract ResultType getType(); @ToString.Include public abstract Set getSemantics(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java b/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java index f88411142b..f4ede6725c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java +++ b/backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java @@ -244,6 +244,11 @@ protected String print(PrintSettings cfg, @NonNull Object f) { public String getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { return resultSetProcessor.getString(resultSet, columnIndex); } + + @Override + protected List getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { + return resultSetProcessor.getStringList(resultSet, columnIndex); + } } @CPSType(id = "MONEY", base = ResultType.class) @@ -277,6 +282,7 @@ public BigDecimal readIntermediateValue(PrintSettings cfg, Number f) { @Getter @EqualsAndHashCode(callSuper = false) public static class ListT extends ResultType> { + @NonNull private final ResultType elementType; @@ -307,10 +313,10 @@ public String typeInfo() { @Override public List getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException { - if (elementType instanceof DateRangeT) { + if (elementType.getClass() == DateRangeT.class || elementType.getClass() == StringT.class) { return elementType.getFromResultSetAsList(resultSet, columnIndex, resultSetProcessor); } - // TODO handle all other list types properly by + // TODO handle all other list types properly throw new UnsupportedOperationException("Other result type lists not supported for now."); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java index 9adef30865..474812f33e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java @@ -9,6 +9,7 @@ import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import org.jooq.Condition; +import org.jooq.DataType; import org.jooq.Field; import org.jooq.Name; import org.jooq.Param; @@ -30,6 +31,20 @@ public String getMaxDateExpression() { return MAX_DATE_VALUE; } + @Override + public Field cast(Field field, DataType type) { + return DSL.function( + "CAST", + type.getType(), + DSL.field("%s AS %s".formatted(field, type.getName())) + ); + } + + @Override + public Field toChar(int character) { + return DSL.function("char", String.class, DSL.val(character)); + } + @Override public Condition dateRestriction(ColumnDateRange dateRestriction, ColumnDateRange validityDate) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java index ebc2ebd4cb..e0b7a94a79 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java @@ -10,6 +10,7 @@ import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import org.jooq.Condition; +import org.jooq.DataType; import org.jooq.DatePart; import org.jooq.Field; import org.jooq.Name; @@ -31,6 +32,16 @@ public String getMaxDateExpression() { return INFINITY_DATE_VALUE; } + @Override + public Field cast(Field field, DataType type) { + return DSL.cast(field, type); + } + + @Override + public Field toChar(int character) { + return DSL.chr(character); + } + @Override public String getMinDateExpression() { return MINUS_INFINITY_DATE_VALUE; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index 59b6fd69f6..7ba2136e67 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -9,6 +9,7 @@ import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import org.jooq.Condition; +import org.jooq.DataType; import org.jooq.Field; import org.jooq.Name; import org.jooq.Record; @@ -29,6 +30,10 @@ public interface SqlFunctionProvider { String getMaxDateExpression(); + Field cast(Field field, DataType type); + + Field toChar(int character); + /** * A date restriction condition is true if holds: dateRestrictionStart <= validityDateEnd and dateRestrictionEnd >= validityDateStart */ diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/FlagCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/FlagCondition.java new file mode 100644 index 0000000000..01ebb4aeba --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/FlagCondition.java @@ -0,0 +1,29 @@ +package com.bakdata.conquery.sql.conversion.model.filter; + +import java.util.List; + +import lombok.RequiredArgsConstructor; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.impl.DSL; + +@RequiredArgsConstructor +public class FlagCondition implements WhereCondition { + + private final List> flagFields; + + @Override + public Condition condition() { + return flagFields.stream() + .map(DSL::condition) + .map(Field::isTrue) + .reduce(Condition::or) + .orElseThrow(() -> new IllegalArgumentException("Can't construct a FlagCondition with an empty flag field list.")); + } + + @Override + public ConditionType type() { + return ConditionType.EVENT; + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FlagSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FlagSqlAggregator.java new file mode 100644 index 0000000000..f24561c6b0 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FlagSqlAggregator.java @@ -0,0 +1,181 @@ +package com.bakdata.conquery.sql.conversion.model.select; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.datasets.concepts.filters.specific.FlagFilter; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.FlagSelect; +import com.bakdata.conquery.models.identifiable.NamedImpl; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; +import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; +import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.conversion.model.filter.FlagCondition; +import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; +import com.bakdata.conquery.sql.execution.ResultSetProcessor; +import lombok.Value; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.Param; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; + +/** + * {@link FlagSelect} conversion aggregates the keys of the flags of a {@link FlagSelect} into a string aggregation. + *

+ * If any value of the respective flag column is true, the flag key will be part of the string aggregation.
+ * If no value is true, an empty string will be added as value, because a {@code null} value would cause the whole string aggregation to be {@code null} too.
+ * Each value will be followed by the {@link ResultSetProcessor#UNIT_SEPARATOR}. + * + * 

+ * {@code
+ * "group_select" as (
+ * 		select
+ * 			"pid",
+ * 			(
+ * 				case when max(CAST("preprocessing"."a" AS integer)) = 1 then 'A' else '' end || char(31)
+ * 				|| case when max(CAST("preprocessing"."b" AS integer)) = 1 then 'B' else '' end || char(31)
+ * 				|| case when max(CAST("preprocessing"."c" AS integer)) = 1 then 'C' else '' end || char(31)
+ * 			) "flags_selects-1"
+ * 		from "preprocessing"
+ * 		group by "pid"
+ * )
+ * }
+ *
+ * + *
+ *

+ * {@link FlagFilter} conversion filters events if not at least 1 of the flag columns has a true value for the corresponding entry. + * + * 

+ * {@code
+ * "event_filter" as (
+ *		select "pid"
+ *		from "preprocessing"
+ *		where (
+ *			"preprocessing"."b" = true
+ *			or "preprocessing"."c" = true
+ *		)
+ * )
+ * }
+ *
+ */ +@Value +public class FlagSqlAggregator implements SqlAggregator { + + private static final Param NUMERIC_TRUE_VAL = DSL.val(1); + private static final Param NUMERIC_FALSE_VAL = DSL.val(0); + private static final Param EMPTY_STRING = DSL.val(""); + + SqlSelects sqlSelects; + WhereClauses whereClauses; + + public static FlagSqlAggregator create(FlagSelect flagSelect, SelectContext selectContext) { + + SqlFunctionProvider functionProvider = selectContext.getParentContext().getSqlDialect().getFunctionProvider(); + SqlTables conceptTables = selectContext.getConceptTables(); + + String rootTable = conceptTables.getPredecessor(ConceptCteStep.PREPROCESSING); + Map rootSelects = createFlagRootSelectMap(flagSelect, rootTable); + + String alias = selectContext.getNameGenerator().selectName(flagSelect); + FieldWrapper flagAggregation = createFlagSelect(alias, conceptTables, functionProvider, rootSelects); + + ExtractingSqlSelect finalSelect = flagAggregation.createAliasedReference(conceptTables.getPredecessor(ConceptCteStep.FINAL)); + + SqlSelects sqlSelects = SqlSelects.builder().preprocessingSelects(rootSelects.values()) + .aggregationSelect(flagAggregation) + .finalSelect(finalSelect) + .build(); + + return new FlagSqlAggregator(sqlSelects, WhereClauses.builder().build()); + } + + public static FlagSqlAggregator create(FlagFilter flagFilter, FilterContext filterContext) { + SqlTables conceptTables = filterContext.getConceptTables(); + String rootTable = conceptTables.getPredecessor(ConceptCteStep.PREPROCESSING); + + List rootSelects = + getRequiredColumnNames(flagFilter.getFlags(), filterContext.getValue()) + .stream() + .map(columnName -> new ExtractingSqlSelect<>(rootTable, columnName, Boolean.class)) + .collect(Collectors.toList()); + SqlSelects selects = SqlSelects.builder() + .preprocessingSelects(rootSelects) + .build(); + + List> flagFields = rootSelects.stream() + .map(sqlSelect -> conceptTables.qualifyOnPredecessor(ConceptCteStep.EVENT_FILTER, sqlSelect.aliased())) + .toList(); + FlagCondition flagCondition = new FlagCondition(flagFields); + WhereClauses whereClauses = WhereClauses.builder() + .eventFilter(flagCondition) + .build(); + + return new FlagSqlAggregator(selects, whereClauses); + } + + /** + * @return A mapping between a flags key and the corresponding {@link ExtractingSqlSelect} that will be created to reference the flag's column. + */ + private static Map createFlagRootSelectMap(FlagSelect flagSelect, String rootTable) { + return flagSelect.getFlags() + .entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> new ExtractingSqlSelect<>(rootTable, entry.getValue().getName(), Boolean.class) + )); + } + + private static FieldWrapper createFlagSelect( + String alias, + SqlTables conceptTables, + SqlFunctionProvider functionProvider, + Map flagRootSelectMap + ) { + Map> flagFieldsMap = createRootSelectReferences(conceptTables, flagRootSelectMap); + + List> flagAggregations = new ArrayList<>(); + for (Map.Entry> entry : flagFieldsMap.entrySet()) { + Field boolColumn = entry.getValue(); + Condition anyTrue = DSL.max(functionProvider.cast(boolColumn, SQLDataType.INTEGER)) + .eq(NUMERIC_TRUE_VAL); + // we have to prevent null values because then the whole String aggregation is null + String flagName = entry.getKey(); + Field flag = DSL.when(anyTrue, DSL.val(flagName)) + .otherwise(EMPTY_STRING); + // append separator + Field separator = functionProvider.toChar(ResultSetProcessor.UNIT_SEPARATOR); + Field withSeparator = DSL.field("%s || %s".formatted(flag, separator), String.class); + flagAggregations.add(withSeparator); + } + + return new FieldWrapper<>( + DSL.concat(flagAggregations.toArray(Field[]::new)).as(alias) + ); + } + + private static Map> createRootSelectReferences(SqlTables conceptTables, Map flagRootSelectMap) { + return flagRootSelectMap.entrySet().stream() + .collect(Collectors.toMap( + Map.Entry::getKey, + entry -> conceptTables.qualifyOnPredecessor(ConceptCteStep.AGGREGATION_SELECT, entry.getValue().aliased()) + )); + } + + /** + * @return Columns names of a given flags map that match the selected flags of the filter value. + */ + private static List getRequiredColumnNames(Map flags, String[] selectedFlags) { + return Arrays.stream(selectedFlags) + .map(flags::get) + .map(NamedImpl::getName) + .toList(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java index 17c8a3e3c1..97093dd214 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/DefaultResultSetProcessor.java @@ -7,14 +7,13 @@ import java.sql.SQLException; import java.util.List; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor class DefaultResultSetProcessor implements ResultSetProcessor { private final SqlCDateSetParser sqlCDateSetParser; - public DefaultResultSetProcessor(SqlCDateSetParser sqlCDateSetParser) { - this.sqlCDateSetParser = sqlCDateSetParser; - } - @Override public String getString(ResultSet resultSet, int columnIndex) throws SQLException { return resultSet.getString(columnIndex); @@ -63,6 +62,11 @@ public List> getDateRangeList(ResultSet resultSet, int columnIndex return this.sqlCDateSetParser.toEpochDayRangeList(resultSet.getString(columnIndex)); } + @Override + public List getStringList(ResultSet resultSet, int columnIndex) throws SQLException { + return SqlStringListParser.parse(resultSet.getString(columnIndex)); + } + @FunctionalInterface private interface Getter { Object getFromResultSet(int columnIndex) throws SQLException; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java index 10d809ec22..b1804be904 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/ResultSetProcessor.java @@ -1,23 +1,14 @@ package com.bakdata.conquery.sql.execution; -import static com.bakdata.conquery.models.types.ResultType.BooleanT; -import static com.bakdata.conquery.models.types.ResultType.DateRangeT; -import static com.bakdata.conquery.models.types.ResultType.DateT; -import static com.bakdata.conquery.models.types.ResultType.IntegerT; -import static com.bakdata.conquery.models.types.ResultType.ListT; -import static com.bakdata.conquery.models.types.ResultType.MoneyT; -import static com.bakdata.conquery.models.types.ResultType.NumericT; - import java.math.BigDecimal; import java.sql.ResultSet; import java.sql.SQLException; import java.util.List; -import com.bakdata.conquery.models.types.ResultType; -import com.bakdata.conquery.models.types.ResultType.StringT; - public interface ResultSetProcessor { + char UNIT_SEPARATOR = (char) 31; // see https://www.ascii-code.com/31 + String getString(ResultSet resultSet, int columnIndex) throws SQLException; Integer getInteger(ResultSet resultSet, int columnIndex) throws SQLException; @@ -34,58 +25,5 @@ public interface ResultSetProcessor { List> getDateRangeList(ResultSet resultSet, int columnIndex) throws SQLException; - @FunctionalInterface - interface ResultSetMapper { - Object getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException; - } - - static ResultSetMapper[] getMappers(List resultTypes) { - return resultTypes.stream() - .map(ResultSetProcessor::getMappers) - .toArray(ResultSetProcessor.ResultSetMapper[]::new); - } - - private static ResultSetMapper getMappers(ResultType resultType) { - - if (resultType instanceof ListT list) { - ResultType elementType = list.getElementType(); - if (elementType instanceof DateRangeT) { - return (resultSet, columnIndex, resultSetProcessor) -> resultSetProcessor.getDateRangeList(resultSet, columnIndex); - } - return getMappers(elementType); - } - - if (resultType instanceof StringT) { - // TODO mapping should probably be applied in query when using SQL-backend - return (resultSet, columnIndex, resultSetProcessor) -> resultSetProcessor.getString(resultSet, columnIndex); - } - - if (resultType instanceof IntegerT) { - return (resultSet, columnIndex, resultSetProcessor) -> resultSetProcessor.getInteger(resultSet, columnIndex); - } - - if (resultType instanceof NumericT) { - return (resultSet, columnIndex, resultSetProcessor) -> resultSetProcessor.getDouble(resultSet, columnIndex); - } - - if (resultType instanceof MoneyT) { - //TODO money needs formatting according to pretty printer? - return (resultSet, columnIndex, resultSetProcessor) -> resultSetProcessor.getMoney(resultSet, columnIndex); - } - - if (resultType instanceof BooleanT) { - return (resultSet, columnIndex, resultSetProcessor) -> resultSetProcessor.getBoolean(resultSet, columnIndex); - } - - if (resultType instanceof DateT) { - return (resultSet, columnIndex, resultSetProcessor) -> resultSetProcessor.getDate(resultSet, columnIndex); - } - - if (resultType instanceof DateRangeT) { - return (resultSet, columnIndex, resultSetProcessor) -> resultSetProcessor.getDateRange(resultSet, columnIndex); - } - - throw new IllegalArgumentException("Don't know how to handle result Type %s".formatted(resultType)); - } - + List getStringList(ResultSet resultSet, int columnIndex) throws SQLException; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java index 3e9f523a61..45534e32cf 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java @@ -6,6 +6,7 @@ import java.sql.Statement; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; import java.util.stream.IntStream; import com.bakdata.conquery.models.error.ConqueryError; @@ -39,7 +40,7 @@ public SqlExecutionResult execute(SqlManagedQuery sqlQuery) { private SqlExecutionResult createStatementAndExecute(SqlManagedQuery sqlQuery, Connection connection) { String sqlString = sqlQuery.getSqlQuery().getSql(); - List resultTypes = sqlQuery.getSqlQuery().getResultInfos().stream().map(ResultInfo::getType).toList(); + List> resultTypes = sqlQuery.getSqlQuery().getResultInfos().stream().map(ResultInfo::getType).collect(Collectors.toList()); log.debug("Executing query: \n{}", sqlString); try (Statement statement = connection.createStatement(); @@ -59,11 +60,10 @@ private SqlExecutionResult createStatementAndExecute(SqlManagedQuery sqlQuery, C } } - private List createResultTable(ResultSet resultSet, List resultTypes, int columnCount) throws SQLException { + private List createResultTable(ResultSet resultSet, List> resultTypes, int columnCount) throws SQLException { List resultTable = new ArrayList<>(resultSet.getFetchSize()); - ResultSetProcessor.ResultSetMapper[] mappers = ResultSetProcessor.getMappers(resultTypes); while (resultSet.next()) { - SqlEntityResult resultRow = getResultRow(resultSet, mappers, columnCount); + SqlEntityResult resultRow = getResultRow(resultSet, resultTypes, columnCount); resultTable.add(resultRow); } return resultTable; @@ -85,7 +85,7 @@ private String getColumnName(ResultSet resultSet, int columnIndex) { } } - private SqlEntityResult getResultRow(ResultSet resultSet, ResultSetProcessor.ResultSetMapper[] mappers, int columnCount) throws SQLException { + private SqlEntityResult getResultRow(ResultSet resultSet, List> resultTypes, int columnCount) throws SQLException { int rowNumber = resultSet.getRow(); String id = resultSet.getString(PID_COLUMN_INDEX); @@ -93,7 +93,7 @@ private SqlEntityResult getResultRow(ResultSet resultSet, ResultSetProcessor.Res for (int resultSetIndex = VALUES_OFFSET_INDEX; resultSetIndex <= columnCount; resultSetIndex++) { int resultTypeIndex = resultSetIndex - VALUES_OFFSET_INDEX; - resultRow[resultTypeIndex] = mappers[resultTypeIndex].getFromResultSet(resultSet, resultSetIndex, this.resultSetProcessor); + resultRow[resultTypeIndex] = resultTypes.get(resultTypeIndex).getFromResultSet(resultSet, resultSetIndex, this.resultSetProcessor); } return new SqlEntityResult(rowNumber, id, resultRow); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlStringListParser.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlStringListParser.java new file mode 100644 index 0000000000..7e9ec703c8 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlStringListParser.java @@ -0,0 +1,19 @@ +package com.bakdata.conquery.sql.execution; + +import java.util.Arrays; +import java.util.List; +import java.util.regex.Pattern; + +import org.apache.logging.log4j.util.Strings; + +class SqlStringListParser { + + private static final Pattern UNIT_SEPARATOR = Pattern.compile(String.valueOf(ResultSetProcessor.UNIT_SEPARATOR)); + + public static List parse(String entry) { + return Arrays.stream(UNIT_SEPARATOR.split(entry)) + .filter(Strings::isNotEmpty) + .toList(); + } + +} diff --git a/backend/src/test/resources/tests/sql/filter/flag/content.csv b/backend/src/test/resources/tests/sql/filter/flag/content.csv new file mode 100644 index 0000000000..c66e0b3cb9 --- /dev/null +++ b/backend/src/test/resources/tests/sql/filter/flag/content.csv @@ -0,0 +1,10 @@ +pid,a,b,c +1,false,false,false +1,true,false,false +2,false,true,false +2,false,true,true +3,false,false,true +4,true,true,false +5,false,false,true +6,,, +7,,,true diff --git a/backend/src/test/resources/tests/sql/filter/flag/expected.csv b/backend/src/test/resources/tests/sql/filter/flag/expected.csv new file mode 100644 index 0000000000..1452989b9d --- /dev/null +++ b/backend/src/test/resources/tests/sql/filter/flag/expected.csv @@ -0,0 +1,6 @@ +result,dates +2,{} +3,{} +4,{} +5,{} +7,{} diff --git a/backend/src/test/resources/tests/sql/filter/flag/flag.spec.json b/backend/src/test/resources/tests/sql/filter/flag/flag.spec.json new file mode 100644 index 0000000000..fc091442dd --- /dev/null +++ b/backend/src/test/resources/tests/sql/filter/flag/flag.spec.json @@ -0,0 +1,80 @@ +{ + "label": "FLAGS filter", + "type": "SQL_QUERY_TEST", + "expectedCsv": "tests/sql/filter/flag/expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "ids": [ + "flags" + ], + "type": "CONCEPT", + "label": "flags", + "tables": [ + { + "id": "flags.flags_connector", + "filters": [ + { + "filter": "flags.flags_connector.flags_filter", + "type": "MULTI_SELECT", + "value": [ + "B", + "C" + ] + } + ] + } + ] + } + ] + } + }, + "concepts": [ + { + "label": "flags", + "type": "TREE", + "connectors": [ + { + "label": "flags_connector", + "table": "table1", + "filters": { + "type": "FLAGS", + "name": "flags_filter", + "flags": { + "A": "table1.a", + "B": "table1.b", + "C": "table1.c" + } + } + } + ] + } + ], + "content": { + "tables": { + "csv": "tests/sql/filter/flag/content.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "a", + "type": "BOOLEAN" + }, + { + "name": "b", + "type": "BOOLEAN" + }, + { + "name": "c", + "type": "BOOLEAN" + } + ] + } + } +} diff --git a/backend/src/test/resources/tests/sql/selects/flag/content.csv b/backend/src/test/resources/tests/sql/selects/flag/content.csv new file mode 100644 index 0000000000..bf9fb3b6b7 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/flag/content.csv @@ -0,0 +1,9 @@ +pid,a,b,c +1,true,false,false +2,false,true,false +3,false,false,true +4,true,true,false +5,false,false,true +6,,, +7,,,true +8,true,true,true diff --git a/backend/src/test/resources/tests/sql/selects/flag/expected.csv b/backend/src/test/resources/tests/sql/selects/flag/expected.csv new file mode 100644 index 0000000000..d8beda0260 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/flag/expected.csv @@ -0,0 +1,9 @@ +result,dates,flags flags_selects +1,{},{A} +2,{},{B} +3,{},{C} +4,{},"{A,B}" +5,{},{C} +6,{},{} +7,{},{C} +8,{},"{A,B,C}" diff --git a/backend/src/test/resources/tests/sql/selects/flag/flag.spec.json b/backend/src/test/resources/tests/sql/selects/flag/flag.spec.json new file mode 100644 index 0000000000..51b365b12d --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/flag/flag.spec.json @@ -0,0 +1,71 @@ +{ + "label": "FLAGS select", + "type": "SQL_QUERY_TEST", + "expectedCsv": "tests/sql/selects/flag/expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "ids": [ + "flags" + ], + "type": "CONCEPT", + "label": "flags", + "tables": [ + { + "id": "flags.flags_connector", + "selects": "flags.flags_connector.flags_selects" + } + ] + } + ] + } + }, + "concepts": [ + { + "label": "flags", + "type": "TREE", + "connectors": [ + { + "label": "flags_connector", + "table": "table1", + "selects": { + "type": "FLAGS", + "name": "flags_selects", + "flags": { + "A": "table1.a", + "B": "table1.b", + "C": "table1.c" + } + } + } + ] + } + ], + "content": { + "tables": { + "csv": "tests/sql/selects/flag/content.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "a", + "type": "BOOLEAN" + }, + { + "name": "b", + "type": "BOOLEAN" + }, + { + "name": "c", + "type": "BOOLEAN" + } + ] + } + } +}