Merge branch 'develop' into fix/result_metrics_npe

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java

@@ -19,6 +19,11 @@
 import com.bakdata.conquery.models.identifiable.ids.specific.FilterId;
 import com.bakdata.conquery.models.query.QueryResolveContext;
 import com.bakdata.conquery.models.query.queryplan.filter.FilterNode;
+import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext;
+import com.bakdata.conquery.sql.conversion.model.SqlTables;
+import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.fasterxml.jackson.core.JsonParser;
@@ -66,6 +71,14 @@ public FilterNode<?> createNode() {
 		return getFilter().createFilterNode(getValue());
 	}
 
+	public SqlFilters convertToSqlFilter(ConversionContext context, SqlTables<ConceptCteStep> conceptTables) {
+		FilterContext<VALUE> filterContext = new FilterContext<>(value, context, conceptTables);
+		SqlFilters sqlFilters = filter.convertToSqlFilter(filterContext);
+		if (context.isNegation()) {
+			return new SqlFilters(sqlFilters.getSelects(), sqlFilters.getFilters().negated());
+		}
+		return sqlFilters;
+	}
 
 	@NoArgsConstructor
 	@CPSType(id = FrontendFilterType.Fields.MULTI_SELECT, base = FilterValue.class)
@@ -228,4 +241,4 @@ public GroupFilterValue deserialize(JsonParser p, DeserializationContext ctxt) t
 		}
 
 	}
-}
+}

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/Filter.java

@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.datasets.concepts.filters;
 
 import java.util.List;
+import java.util.Set;
 
 import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration;
 import com.bakdata.conquery.io.cps.CPSBase;
@@ -13,6 +14,9 @@
 import com.bakdata.conquery.models.identifiable.ids.NamespacedIdentifiable;
 import com.bakdata.conquery.models.identifiable.ids.specific.FilterId;
 import com.bakdata.conquery.models.query.queryplan.filter.FilterNode;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext;
+import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters;
 import com.fasterxml.jackson.annotation.JsonAlias;
 import com.fasterxml.jackson.annotation.JsonBackReference;
 import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -96,4 +100,15 @@ public boolean isForConnectorsTable() {
 
 		return valid;
 	}
+
+	@JsonIgnore
+	public SqlFilters convertToSqlFilter(FilterContext<FILTER_VALUE> filterContext) {
+		throw new UnsupportedOperationException("SQL conversion of filter %s not implemented yet.".formatted(getClass()));
+	}
+
+	@JsonIgnore
+	public Set<ConceptCteStep> getRequiredSqlSteps() {
+		return ConceptCteStep.MANDATORY_STEPS;
+	}
+
 }

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java

@@ -1,10 +1,16 @@
 package com.bakdata.conquery.models.datasets.concepts.filters.specific;
 
+import java.util.Set;
+
 import com.bakdata.conquery.apiv1.frontend.FrontendFilterType;
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.models.datasets.concepts.filters.Filter;
 import com.bakdata.conquery.models.query.filter.event.MultiSelectFilterNode;
 import com.bakdata.conquery.models.query.queryplan.filter.FilterNode;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectFilterUtil;
+import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import lombok.Getter;
 import lombok.Setter;
@@ -30,4 +36,15 @@ public String getFilterType() {
 	public FilterNode createFilterNode(String[] value) {
 		return new MultiSelectFilterNode(getColumn(), value);
 	}
+
+	@Override
+	public SqlFilters convertToSqlFilter(FilterContext<String[]> filterContext) {
+		return SelectFilterUtil.convert(this, filterContext, filterContext.getValue());
+	}
+
+	@Override
+	public Set<ConceptCteStep> getRequiredSqlSteps() {
+		return ConceptCteStep.withOptionalSteps(ConceptCteStep.EVENT_FILTER);
+	}
+
 }

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java

@@ -3,6 +3,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 import javax.validation.constraints.NotNull;
 
@@ -19,6 +20,10 @@
 import com.bakdata.conquery.models.query.queryplan.aggregators.DistinctValuesWrapperAggregator;
 import com.bakdata.conquery.models.query.queryplan.aggregators.specific.CountAggregator;
 import com.bakdata.conquery.models.query.queryplan.filter.FilterNode;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext;
+import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters;
+import com.bakdata.conquery.sql.conversion.model.select.CountSqlAggregator;
 import lombok.Data;
 import lombok.NoArgsConstructor;
 
@@ -68,4 +73,15 @@ public List<Column> getRequiredColumns() {
 
 		return out;
 	}
+
+	@Override
+	public SqlFilters convertToSqlFilter(FilterContext<Range.LongRange> filterContext) {
+		return CountSqlAggregator.create(this, filterContext).getSqlFilters();
+	}
+
+	@Override
+	public Set<ConceptCteStep> getRequiredSqlSteps() {
+		return ConceptCteStep.withOptionalSteps(ConceptCteStep.AGGREGATION_FILTER);
+	}
+
 }

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java

@@ -2,6 +2,7 @@
 
 import java.time.temporal.ChronoUnit;
 import java.util.EnumSet;
+import java.util.Set;
 
 import javax.validation.constraints.NotNull;
 
@@ -16,6 +17,10 @@
 import com.bakdata.conquery.models.exceptions.ConceptConfigurationException;
 import com.bakdata.conquery.models.query.filter.event.DateDistanceFilterNode;
 import com.bakdata.conquery.models.query.queryplan.filter.FilterNode;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext;
+import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters;
+import com.bakdata.conquery.sql.conversion.model.select.DateDistanceSqlAggregator;
 import lombok.Getter;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
@@ -48,4 +53,15 @@ public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig
 	public FilterNode createFilterNode(Range.LongRange value) {
 		return new DateDistanceFilterNode(getColumn(), timeUnit, value);
 	}
-}
+
+	@Override
+	public SqlFilters convertToSqlFilter(FilterContext<Range.LongRange> filterContext) {
+		return DateDistanceSqlAggregator.create(this, filterContext).getSqlFilters();
+	}
+
+	@Override
+	public Set<ConceptCteStep> getRequiredSqlSteps() {
+		return ConceptCteStep.withOptionalSteps(ConceptCteStep.EVENT_FILTER);
+	}
+
+}

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/MultiSelectFilter.java

@@ -1,10 +1,16 @@
 package com.bakdata.conquery.models.datasets.concepts.filters.specific;
 
+import java.util.Set;
+
 import com.bakdata.conquery.apiv1.frontend.FrontendFilterType;
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.models.datasets.concepts.filters.Filter;
 import com.bakdata.conquery.models.query.filter.event.MultiSelectFilterNode;
 import com.bakdata.conquery.models.query.queryplan.filter.FilterNode;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectFilterUtil;
+import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 
 /**
@@ -29,4 +35,15 @@ public String getFilterType() {
 	public FilterNode<?> createFilterNode(String[] value) {
 		return new MultiSelectFilterNode(getColumn(), value);
 	}
+
+	@Override
+	public SqlFilters convertToSqlFilter(FilterContext<String[]> filterContext) {
+		return SelectFilterUtil.convert(this, filterContext, filterContext.getValue());
+	}
+
+	@Override
+	public Set<ConceptCteStep> getRequiredSqlSteps() {
+		return ConceptCteStep.withOptionalSteps(ConceptCteStep.EVENT_FILTER);
+	}
+
 }

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java

@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.datasets.concepts.filters.specific;
 
 import java.math.BigDecimal;
+import java.util.Set;
 
 import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration;
 import com.bakdata.conquery.apiv1.frontend.FrontendFilterType;
@@ -16,6 +17,10 @@
 import com.bakdata.conquery.models.query.filter.event.number.MoneyFilterNode;
 import com.bakdata.conquery.models.query.filter.event.number.RealFilterNode;
 import com.bakdata.conquery.models.query.queryplan.filter.FilterNode;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext;
+import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters;
+import com.bakdata.conquery.sql.conversion.model.select.NumberSqlAggregator;
 import lombok.Getter;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
@@ -53,4 +58,15 @@ public FilterNode<?> createFilterNode(RANGE value) {
 			default -> throw new IllegalStateException(String.format("Column type %s may not be used (Assignment should not have been possible)", getColumn()));
 		};
 	}
+
+	@Override
+	public SqlFilters convertToSqlFilter(FilterContext<RANGE> filterContext) {
+		return NumberSqlAggregator.create(this, filterContext).getSqlFilters();
+	}
+
+	@Override
+	public Set<ConceptCteStep> getRequiredSqlSteps() {
+		return ConceptCteStep.withOptionalSteps(ConceptCteStep.EVENT_FILTER);
+	}
+
 }

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SingleSelectFilter.java

@@ -1,10 +1,16 @@
 package com.bakdata.conquery.models.datasets.concepts.filters.specific;
 
+import java.util.Set;
+
 import com.bakdata.conquery.apiv1.frontend.FrontendFilterType;
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.models.datasets.concepts.filters.Filter;
 import com.bakdata.conquery.models.query.filter.event.SelectFilterNode;
 import com.bakdata.conquery.models.query.queryplan.filter.FilterNode;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectFilterUtil;
+import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters;
 import net.minidev.json.annotate.JsonIgnore;
 
 /**
@@ -25,4 +31,15 @@ public FilterNode<?> createFilterNode(String value) {
 	public String getFilterType() {
 		return FrontendFilterType.Fields.SELECT;
 	}
+
+	@Override
+	public SqlFilters convertToSqlFilter(FilterContext<String> filterContext) {
+		return SelectFilterUtil.convert(this, filterContext, new String[]{filterContext.getValue()});
+	}
+
+	@Override
+	public Set<ConceptCteStep> getRequiredSqlSteps() {
+		return ConceptCteStep.withOptionalSteps(ConceptCteStep.EVENT_FILTER);
+	}
+
 }

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java

@@ -3,6 +3,7 @@
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 import javax.annotation.Nullable;
 import javax.validation.constraints.NotNull;
@@ -31,6 +32,10 @@
 import com.bakdata.conquery.models.query.queryplan.aggregators.specific.sum.MoneySumAggregator;
 import com.bakdata.conquery.models.query.queryplan.aggregators.specific.sum.RealSumAggregator;
 import com.bakdata.conquery.models.query.queryplan.filter.FilterNode;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext;
+import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters;
+import com.bakdata.conquery.sql.conversion.model.select.SumSqlAggregator;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import lombok.Data;
 import lombok.NoArgsConstructor;
@@ -102,6 +107,16 @@ public FilterNode createFilterNode(RANGE value) {
 		return new RangeFilterNode(range, getAggregator());
 	}
 
+	@Override
+	public SqlFilters convertToSqlFilter(FilterContext<RANGE> filterContext) {
+		return SumSqlAggregator.create(this, filterContext).getSqlFilters();
+	}
+
+	@Override
+	public Set<ConceptCteStep> getRequiredSqlSteps() {
+		return ConceptCteStep.withOptionalSteps(ConceptCteStep.AGGREGATION_FILTER);
+	}
+
 	@JsonIgnore
 	private ColumnAggregator<?> getAggregator() {
 		if (getSubtractColumn() == null) {

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java

@@ -1,6 +1,7 @@
 package com.bakdata.conquery.models.datasets.concepts.select;
 
 import java.util.List;
+import java.util.Set;
 
 import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept;
 import com.bakdata.conquery.io.cps.CPSBase;
@@ -17,6 +18,9 @@
 import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator;
 import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo;
 import com.bakdata.conquery.models.types.ResultType;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext;
+import com.bakdata.conquery.sql.conversion.model.select.SqlSelects;
 import com.fasterxml.jackson.annotation.JsonBackReference;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
@@ -117,4 +121,15 @@ public boolean isForConnectorsTable() {
 
 		return valid;
 	}
+
+	@JsonIgnore
+	public SqlSelects convertToSqlSelects(SelectContext selectContext) {
+		throw new UnsupportedOperationException("SQL conversion of select %s not implemented yet.".formatted(getClass()));
+	}
+
+	@JsonIgnore
+	public Set<ConceptCteStep> getRequiredSqlSteps() {
+		return ConceptCteStep.MANDATORY_STEPS;
+	}
+
 }

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java

@@ -4,11 +4,14 @@
 import java.util.stream.Collectors;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.models.datasets.Table;
 import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.select.Select;
 import com.bakdata.conquery.models.datasets.concepts.select.concept.UniversalSelect;
-import com.bakdata.conquery.models.datasets.Table;
 import com.bakdata.conquery.models.query.queryplan.aggregators.specific.ExistsAggregator;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext;
+import com.bakdata.conquery.sql.conversion.model.select.ExistsSqlAggregator;
+import com.bakdata.conquery.sql.conversion.model.select.SqlSelects;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import lombok.Getter;
 
@@ -23,6 +26,11 @@ public ExistsAggregator createAggregator() {
 		return new ExistsAggregator(getRequiredTables());
 	}
 
+	@Override
+	public SqlSelects convertToSqlSelects(SelectContext selectContext) {
+		return ExistsSqlAggregator.create(this, selectContext).getSqlSelects();
+	}
+
 	private Set<Table> collectRequiredTables() {
 		return this.getHolder().findConcept().getConnectors().stream().map(Connector::getTable).collect(Collectors.toSet());
 	}

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java

@@ -8,6 +8,9 @@
 import com.bakdata.conquery.models.index.InternToExternMapper;
 import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator;
 import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.FirstValueAggregator;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext;
+import com.bakdata.conquery.sql.conversion.model.select.FirstValueSqlAggregator;
+import com.bakdata.conquery.sql.conversion.model.select.SqlSelects;
 import com.fasterxml.jackson.annotation.JsonCreator;
 
 @CPSType(id = "FIRST", base = Select.class)
@@ -24,4 +27,9 @@ public Aggregator<?> createAggregator() {
 		return new FirstValueAggregator<>(getColumn());
 	}
 
+	@Override
+	public SqlSelects convertToSqlSelects(SelectContext selectContext) {
+		return FirstValueSqlAggregator.create(this, selectContext).getSqlSelects();
+	}
+
 }