Merge branch 'develop' into dependabot/maven/it.unimi.dsi-fastutil-8.…

…5.12

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java

@@ -19,9 +19,14 @@
 import com.bakdata.conquery.models.exceptions.ConceptConfigurationException;
 import com.bakdata.conquery.models.query.filter.event.FlagColumnsFilterNode;
 import com.bakdata.conquery.models.query.queryplan.filter.FilterNode;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext;
+import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters;
+import com.bakdata.conquery.sql.conversion.model.select.FlagSqlAggregator;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import io.dropwizard.validation.ValidationMethod;
+import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.ToString;
 
@@ -31,6 +36,7 @@
  *
  * The selected flags are logically or-ed.
  */
+@Getter
 @CPSType(base = Filter.class, id = "FLAGS")
 @RequiredArgsConstructor(onConstructor_ = {@JsonCreator})
 @ToString
@@ -87,4 +93,14 @@ public boolean isAllColumnsOfSameTable() {
 	public boolean isAllColumnsBoolean() {
 		return flags.values().stream().map(Column::getType).allMatch(MajorTypeId.BOOLEAN::equals);
 	}
+
+	@Override
+	public SqlFilters convertToSqlFilter(FilterContext<String[]> filterContext) {
+		return FlagSqlAggregator.create(this, filterContext).getSqlFilters();
+	}
+
+	@Override
+	public Set<ConceptCteStep> getRequiredSqlSteps() {
+		return ConceptCteStep.withOptionalSteps(ConceptCteStep.EVENT_FILTER);
+	}
 }

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java

@@ -12,9 +12,13 @@
 import com.bakdata.conquery.models.events.MajorTypeId;
 import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator;
 import com.bakdata.conquery.models.query.queryplan.aggregators.specific.FlagsAggregator;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext;
+import com.bakdata.conquery.sql.conversion.model.select.FlagSqlAggregator;
+import com.bakdata.conquery.sql.conversion.model.select.SqlSelects;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import io.dropwizard.validation.ValidationMethod;
+import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.ToString;
 
@@ -24,6 +28,7 @@
  *
  * The selected flags are logically or-ed.
  */
+@Getter
 @CPSType(base = Select.class, id = "FLAGS")
 @RequiredArgsConstructor(onConstructor_ = {@JsonCreator})
 @ToString
@@ -55,4 +60,9 @@ public boolean isAllColumnsOfSameTable() {
 	public boolean isAllColumnsBoolean() {
 		return flags.values().stream().map(Column::getType).allMatch(MajorTypeId.BOOLEAN::equals);
 	}
+
+	@Override
+	public SqlSelects convertToSqlSelects(SelectContext selectContext) {
+		return FlagSqlAggregator.create(this, selectContext).getSqlSelects();
+	}
 }

backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/ResultInfo.java

@@ -33,7 +33,7 @@ public abstract class ResultInfo {
 	public abstract String defaultColumnName(PrintSettings printSettings);
 
 	@ToString.Include
-	public abstract ResultType getType();
+	public abstract ResultType<?> getType();
 
 	@ToString.Include
 	public abstract Set<SemanticType> getSemantics();

backend/src/main/java/com/bakdata/conquery/models/types/ResultType.java

@@ -244,6 +244,11 @@ protected String print(PrintSettings cfg, @NonNull Object f) {
 		public String getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
 			return resultSetProcessor.getString(resultSet, columnIndex);
 		}
+
+		@Override
+		protected List<String> getFromResultSetAsList(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
+			return resultSetProcessor.getStringList(resultSet, columnIndex);
+		}
 	}
 
 	@CPSType(id = "MONEY", base = ResultType.class)
@@ -277,6 +282,7 @@ public BigDecimal readIntermediateValue(PrintSettings cfg, Number f) {
 	@Getter
 	@EqualsAndHashCode(callSuper = false)
 	public static class ListT<T> extends ResultType<List<T>> {
+
 		@NonNull
 		private final ResultType<T> elementType;
 
@@ -307,10 +313,10 @@ public String typeInfo() {
 
 		@Override
 		public List<T> getFromResultSet(ResultSet resultSet, int columnIndex, ResultSetProcessor resultSetProcessor) throws SQLException {
-			if (elementType instanceof DateRangeT) {
+			if (elementType.getClass() == DateRangeT.class || elementType.getClass() == StringT.class) {
 				return elementType.getFromResultSetAsList(resultSet, columnIndex, resultSetProcessor);
 			}
-			// TODO handle all other list types properly by
+			// TODO handle all other list types properly
 			throw new UnsupportedOperationException("Other result type lists not supported for now.");
 		}
 

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java

@@ -37,4 +37,18 @@ public boolean dateRestrictionActive() {
 		return this.dateRestrictionRange != null;
 	}
 
+	/**
+	 * Adds a query step to the list of {@link QueryStep} of this context.
+	 */
+	public ConversionContext withQueryStep(QueryStep queryStep) {
+		return this.toBuilder().queryStep(queryStep).build();
+	}
+
+	/**
+	 * @return Creates a child context that keeps the meta information of this context, but empties the list of already converted querySteps.
+	 */
+	public ConversionContext createChildContext() {
+		return this.toBuilder().clearQuerySteps().build();
+	}
+
 }

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java

@@ -70,9 +70,7 @@ public ConversionContext convert(CQConcept cqConcept, ConversionContext context)
 			conceptCteContext = conceptCteContext.withPrevious(lastQueryStep.get());
 		}
 
-		return context.toBuilder()
-					  .queryStep(lastQueryStep.orElseThrow(() -> new RuntimeException("No conversion for concept possible.")))
-					  .build();
+		return context.withQueryStep(lastQueryStep.orElseThrow(() -> new RuntimeException("No conversion for concept possible.")));
 	}
 
 	private ConceptCteContext createConceptCteContext(CQConcept cqConcept, ConversionContext context) {

backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java

@@ -3,12 +3,14 @@
 import java.sql.Date;
 import java.time.temporal.ChronoUnit;
 import java.util.List;
+import java.util.stream.Collectors;
 
 import com.bakdata.conquery.models.common.daterange.CDateRange;
 import com.bakdata.conquery.models.datasets.Column;
 import com.bakdata.conquery.models.datasets.concepts.ValidityDate;
 import com.bakdata.conquery.sql.conversion.model.ColumnDateRange;
 import org.jooq.Condition;
+import org.jooq.DataType;
 import org.jooq.Field;
 import org.jooq.Name;
 import org.jooq.Param;
@@ -30,6 +32,15 @@ public String getMaxDateExpression() {
 		return MAX_DATE_VALUE;
 	}
 
+	@Override
+	public <T> Field<T> cast(Field<?> field, DataType<T> type) {
+		return DSL.function(
+				"CAST",
+				type.getType(),
+				DSL.field("%s AS %s".formatted(field, type.getName()))
+		);
+	}
+
 	@Override
 	public Condition dateRestriction(ColumnDateRange dateRestriction, ColumnDateRange validityDate) {
 
@@ -216,6 +227,14 @@ public Condition likeRegex(Field<String> field, String pattern) {
 		return DSL.condition("{0} {1} {2}", field, DSL.keyword("LIKE_REGEXPR"), pattern);
 	}
 
+	@Override
+	public Field<Object[]> asArray(List<Field<?>> fields) {
+		String arrayExpression = fields.stream()
+									   .map(Field::toString)
+									   .collect(Collectors.joining(", ", "array(", ")"));
+		return DSL.field(arrayExpression, Object[].class);
+	}
+
 	@Override
 	public Field<Date> addDays(Field<Date> dateColumn, int amountOfDays) {
 		return DSL.function(

backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java

@@ -11,6 +11,7 @@
 import com.bakdata.conquery.sql.conversion.model.ColumnDateRange;
 import org.jooq.ArrayAggOrderByStep;
 import org.jooq.Condition;
+import org.jooq.DataType;
 import org.jooq.DatePart;
 import org.jooq.Field;
 import org.jooq.Name;
@@ -31,6 +32,11 @@ public String getMaxDateExpression() {
 		return INFINITY_DATE_VALUE;
 	}
 
+	@Override
+	public <T> Field<T> cast(Field<?> field, DataType<T> type) {
+		return DSL.cast(field, type);
+	}
+
 	@Override
 	public String getMinDateExpression() {
 		return MINUS_INFINITY_DATE_VALUE;
@@ -191,6 +197,14 @@ public Condition likeRegex(Field<String> field, String pattern) {
 		return field.similarTo(pattern);
 	}
 
+	@Override
+	public Field<Object[]> asArray(List<Field<?>> fields) {
+		String arrayExpression = fields.stream()
+									   .map(Field::toString)
+									   .collect(Collectors.joining(", ", "array[", "]"));
+		return DSL.field(arrayExpression, Object[].class);
+	}
+
 	private Field<?> daterange(Field<?> startColumn, Field<?> endColumn, String bounds) {
 		return DSL.function(
 				"daterange",

backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java

@@ -9,6 +9,7 @@
 import com.bakdata.conquery.sql.conversion.model.ColumnDateRange;
 import com.bakdata.conquery.sql.conversion.model.QueryStep;
 import org.jooq.Condition;
+import org.jooq.DataType;
 import org.jooq.Field;
 import org.jooq.Name;
 import org.jooq.Record;
@@ -29,6 +30,8 @@ public interface SqlFunctionProvider {
 
 	String getMaxDateExpression();
 
+	<T> Field<T> cast(Field<?> field, DataType<T> type);
+
 	/**
 	 * A date restriction condition is true if holds: dateRestrictionStart <= validityDateEnd and dateRestrictionEnd >= validityDateStart
 	 */
@@ -65,6 +68,8 @@ public interface SqlFunctionProvider {
 
 	Condition likeRegex(Field<String> field, String pattern);
 
+	Field<Object[]> asArray(List<Field<?>> fields);
+
 	default <T> Field<T> least(List<Field<T>> fields) {
 		if (fields.isEmpty()) {
 			return null;

backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java

@@ -1,7 +1,6 @@
 package com.bakdata.conquery.sql.conversion.model;
 
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
@@ -30,61 +29,39 @@ public static ConversionContext joinChildren(
 			LogicalOperation logicalOperation,
 			DateAggregationAction dateAggregationAction
 	) {
+		ConversionContext childrenContext = context.createChildContext();
 
-		ConversionContext childrenContext = context;
 		for (CQElement childNode : children) {
 			childrenContext = context.getNodeConversions().convert(childNode, childrenContext);
 		}
 
 		List<QueryStep> queriesToJoin = childrenContext.getQuerySteps();
+
+		String joinedCteName = constructJoinedQueryStepLabel(queriesToJoin, logicalOperation);
 		Field<Object> primaryColumn = coalescePrimaryColumns(queriesToJoin);
 		List<SqlSelect> mergedSelects = mergeSelects(queriesToJoin);
+		TableLike<Record> joinedTable = constructJoinedTable(queriesToJoin, logicalOperation, context);
 
-		QueryStep.QueryStepBuilder andQueryStep = QueryStep.builder()
-														   .cteName(constructJoinedQueryStepLabel(queriesToJoin, logicalOperation))
-														   .fromTable(constructJoinedTable(queriesToJoin, logicalOperation, context))
-														   .conditions(Collections.emptyList())
-														   .predecessors(queriesToJoin);
+		QueryStep joinedStep;
+		QueryStep.QueryStepBuilder joinedStepBuilder = QueryStep.builder()
+																.cteName(joinedCteName)
+																.fromTable(joinedTable)
+																.predecessors(queriesToJoin);
 
 		DateAggregationDates dateAggregationDates = DateAggregationDates.forSteps(queriesToJoin);
 		if (dateAggregationAction == DateAggregationAction.BLOCK || dateAggregationDates.dateAggregationImpossible()) {
-			Selects selects = Selects.builder()
-									 .primaryColumn(primaryColumn)
-									 .sqlSelects(mergedSelects)
-									 .build();
-			andQueryStep = andQueryStep.selects(selects);
-			return context.withQuerySteps(List.of(andQueryStep.build()));
+			joinedStep = buildJoinedStep(primaryColumn, mergedSelects, Optional.empty(), joinedStepBuilder);
 		}
 		// if there is only 1 child node containing a validity date, we just keep it as overall validity date for the joined node
 		else if (dateAggregationDates.getValidityDates().size() == 1) {
 			ColumnDateRange validityDate = dateAggregationDates.getValidityDates().get(0);
-			Selects selects = Selects.builder()
-									 .primaryColumn(primaryColumn)
-									 .validityDate(Optional.ofNullable(validityDate))
-									 .sqlSelects(mergedSelects)
-									 .build();
-			andQueryStep = andQueryStep.selects(selects);
-			return context.withQuerySteps(List.of(andQueryStep.build()));
+			joinedStep = buildJoinedStep(primaryColumn, mergedSelects, Optional.of(validityDate), joinedStepBuilder);
+		}
+		else {
+			joinedStep = buildStepAndAggregateDates(primaryColumn, mergedSelects, joinedStepBuilder, dateAggregationDates, dateAggregationAction, context);
 		}
 
-		List<SqlSelect> mergedSelectsWithAllValidityDates = new ArrayList<>(mergedSelects);
-		mergedSelectsWithAllValidityDates.addAll(dateAggregationDates.allStartsAndEnds());
-		Selects selects = Selects.builder()
-								 .primaryColumn(primaryColumn)
-								 .sqlSelects(mergedSelectsWithAllValidityDates)
-								 .build();
-		andQueryStep = andQueryStep.selects(selects);
-
-		SqlDateAggregator sqlDateAggregator = context.getSqlDialect().getDateAggregator();
-		QueryStep mergeIntervalsStep = sqlDateAggregator.apply(
-				andQueryStep.build(),
-				mergedSelects,
-				dateAggregationDates,
-				dateAggregationAction,
-				context.getNameGenerator()
-		);
-
-		return context.withQuerySteps(List.of(mergeIntervalsStep));
+		return context.withQueryStep(joinedStep);
 	}
 
 	public static TableLike<Record> constructJoinedTable(List<QueryStep> queriesToJoin, LogicalOperation logicalOperation, ConversionContext context) {
@@ -111,6 +88,16 @@ public static TableLike<Record> constructJoinedTable(List<QueryStep> queriesToJo
 		return joinedQuery;
 	}
 
+	@FunctionalInterface
+	private interface JoinType {
+		TableOnConditionStep<Record> join(
+				Table<Record> leftPartQueryBase,
+				QueryStep rightPartQS,
+				Field<Object> leftPartPrimaryColumn,
+				Field<Object> rightPartPrimaryColumn
+		);
+	}
+
 	public static Field<Object> coalescePrimaryColumns(List<QueryStep> querySteps) {
 		List<Field<?>> primaryColumns = querySteps.stream()
 												  .map(queryStep -> queryStep.getQualifiedSelects().getPrimaryColumn())
@@ -143,13 +130,39 @@ private static Table<Record> getIntitialJoinTable(List<QueryStep> queriesToJoin)
 		return DSL.table(DSL.name(queriesToJoin.get(0).getCteName()));
 	}
 
-	@FunctionalInterface
-	private interface JoinType {
-		TableOnConditionStep<Record> join(
-				Table<Record> leftPartQueryBase,
-				QueryStep rightPartQS,
-				Field<Object> leftPartPrimaryColumn,
-				Field<Object> rightPartPrimaryColumn
+	private static QueryStep buildJoinedStep(
+			Field<Object> primaryColumn,
+			List<SqlSelect> mergedSelects,
+			Optional<ColumnDateRange> validityDate,
+			QueryStep.QueryStepBuilder builder
+	) {
+		Selects selects = Selects.builder()
+								 .primaryColumn(primaryColumn)
+								 .sqlSelects(mergedSelects)
+								 .validityDate(validityDate)
+								 .build();
+		return builder.selects(selects).build();
+	}
+
+	private static QueryStep buildStepAndAggregateDates(
+			Field<Object> primaryColumn,
+			List<SqlSelect> mergedSelects,
+			QueryStep.QueryStepBuilder builder,
+			DateAggregationDates dateAggregationDates,
+			DateAggregationAction dateAggregationAction,
+			ConversionContext context
+	) {
+		List<SqlSelect> withAllValidityDates = new ArrayList<>(mergedSelects);
+		withAllValidityDates.addAll(dateAggregationDates.allStartsAndEnds());
+		QueryStep joinedStep = buildJoinedStep(primaryColumn, withAllValidityDates, Optional.empty(), builder);
+
+		SqlDateAggregator sqlDateAggregator = context.getSqlDialect().getDateAggregator();
+		return sqlDateAggregator.apply(
+				joinedStep,
+				mergedSelects,
+				dateAggregationDates,
+				dateAggregationAction,
+				context.getNameGenerator()
 		);
 	}