diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java index 45e15d870e..5685ba240c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java @@ -1,13 +1,10 @@ package com.bakdata.conquery.models.datasets.concepts.filters.specific; -import java.util.Set; - import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.query.filter.event.MultiSelectFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectFilterUtil; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; @@ -42,9 +39,4 @@ public SqlFilters convertToSqlFilter(FilterContext filterContext) { return SelectFilterUtil.convert(this, filterContext, filterContext.getValue()); } - @Override - public Set getRequiredSqlSteps() { - return ConnectorCteStep.withOptionalSteps(ConnectorCteStep.EVENT_FILTER); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java index 505729894a..8b4ec67998 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java @@ -3,7 +3,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Set; import javax.validation.constraints.NotNull; @@ -20,7 +19,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.DistinctValuesWrapperAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.CountAggregator; import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.model.aggregator.CountSqlAggregator; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; @@ -79,9 +77,4 @@ public SqlFilters convertToSqlFilter(FilterContext filterContex return CountSqlAggregator.create(this, filterContext).getSqlFilters(); } - @Override - public Set getRequiredSqlSteps() { - return ConnectorCteStep.withOptionalSteps(ConnectorCteStep.AGGREGATION_FILTER); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountQuartersFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountQuartersFilter.java index 9d4fc693fc..4c54a403cf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountQuartersFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountQuartersFilter.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.models.datasets.concepts.filters.specific; import java.util.EnumSet; -import java.util.Set; import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration; import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; @@ -15,10 +14,9 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.specific.CountQuartersOfDateRangeAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.CountQuartersOfDatesAggregator; import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; -import com.bakdata.conquery.sql.conversion.model.aggregator.CountQuartersSqlAggregator; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; +import com.bakdata.conquery.sql.conversion.model.select.CountQuartersSqlAggregator; import lombok.Getter; import lombok.Setter; @@ -53,8 +51,4 @@ public SqlFilters convertToSqlFilter(FilterContext filterContex return CountQuartersSqlAggregator.create(this, filterContext).getSqlFilters(); } - @Override - public Set getRequiredSqlSteps() { - return ConnectorCteStep.withOptionalSteps(ConnectorCteStep.AGGREGATION_FILTER); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java index fdca832758..088342f596 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java @@ -2,7 +2,6 @@ import java.time.temporal.ChronoUnit; import java.util.EnumSet; -import java.util.Set; import javax.validation.constraints.NotNull; @@ -17,7 +16,6 @@ import com.bakdata.conquery.models.exceptions.ConceptConfigurationException; import com.bakdata.conquery.models.query.filter.event.DateDistanceFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.model.aggregator.DateDistanceSqlAggregator; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; @@ -59,9 +57,4 @@ public SqlFilters convertToSqlFilter(FilterContext filterContex return DateDistanceSqlAggregator.create(this, filterContext).getSqlFilters(); } - @Override - public Set getRequiredSqlSteps() { - return ConnectorCteStep.withOptionalSteps(ConnectorCteStep.EVENT_FILTER); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java index 4f9ad7f1b7..2886f0d4fc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java @@ -19,7 +19,6 @@ import com.bakdata.conquery.models.exceptions.ConceptConfigurationException; import com.bakdata.conquery.models.query.filter.event.FlagColumnsFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.model.aggregator.FlagSqlAggregator; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; @@ -99,8 +98,4 @@ public SqlFilters convertToSqlFilter(FilterContext filterContext) { return FlagSqlAggregator.create(this, filterContext).getSqlFilters(); } - @Override - public Set getRequiredSqlSteps() { - return ConnectorCteStep.withOptionalSteps(ConnectorCteStep.EVENT_FILTER); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/MultiSelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/MultiSelectFilter.java index 0c098c13b5..338ddf7ab1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/MultiSelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/MultiSelectFilter.java @@ -1,13 +1,10 @@ package com.bakdata.conquery.models.datasets.concepts.filters.specific; -import java.util.Set; - import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.query.filter.event.MultiSelectFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectFilterUtil; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; @@ -41,9 +38,4 @@ public SqlFilters convertToSqlFilter(FilterContext filterContext) { return SelectFilterUtil.convert(this, filterContext, filterContext.getValue()); } - @Override - public Set getRequiredSqlSteps() { - return ConnectorCteStep.withOptionalSteps(ConnectorCteStep.EVENT_FILTER); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java index e6884bc392..6e56e83d7d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.models.datasets.concepts.filters.specific; import java.math.BigDecimal; -import java.util.Set; import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration; import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; @@ -17,7 +16,6 @@ import com.bakdata.conquery.models.query.filter.event.number.MoneyFilterNode; import com.bakdata.conquery.models.query.filter.event.number.RealFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.model.aggregator.NumberSqlAggregator; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; @@ -64,9 +62,4 @@ public SqlFilters convertToSqlFilter(FilterContext filterContext) { return NumberSqlAggregator.create(this, filterContext).getSqlFilters(); } - @Override - public Set getRequiredSqlSteps() { - return ConnectorCteStep.withOptionalSteps(ConnectorCteStep.EVENT_FILTER); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SingleSelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SingleSelectFilter.java index 2e4e325862..0e0210626a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SingleSelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SingleSelectFilter.java @@ -1,13 +1,10 @@ package com.bakdata.conquery.models.datasets.concepts.filters.specific; -import java.util.Set; - import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.query.filter.event.SelectFilterNode; import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectFilterUtil; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; @@ -37,9 +34,4 @@ public SqlFilters convertToSqlFilter(FilterContext filterContext) { return SelectFilterUtil.convert(this, filterContext, new String[]{filterContext.getValue()}); } - @Override - public Set getRequiredSqlSteps() { - return ConnectorCteStep.withOptionalSteps(ConnectorCteStep.EVENT_FILTER); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java index 8e66e32f2d..a1b8105d2d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java @@ -3,7 +3,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Set; import javax.annotation.Nullable; import javax.validation.constraints.NotNull; @@ -32,7 +31,6 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.specific.sum.MoneySumAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.sum.RealSumAggregator; import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.model.aggregator.SumDistinctSqlAggregator; import com.bakdata.conquery.sql.conversion.model.aggregator.SumSqlAggregator; @@ -116,14 +114,6 @@ public SqlFilters convertToSqlFilter(FilterContext filterContext) { return SumSqlAggregator.create(this, filterContext).getSqlFilters(); } - @Override - public Set getRequiredSqlSteps() { - if (distinctByColumn != null && !distinctByColumn.isEmpty()) { - return ConnectorCteStep.withOptionalSteps(ConnectorCteStep.JOIN_PREDECESSORS, ConnectorCteStep.AGGREGATION_FILTER); - } - return ConnectorCteStep.withOptionalSteps(ConnectorCteStep.AGGREGATION_FILTER); - } - @JsonIgnore private ColumnAggregator getAggregator() { if (getSubtractColumn() == null) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java index 6ad158df87..1077eac482 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java @@ -19,7 +19,7 @@ import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; -import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; import com.fasterxml.jackson.annotation.JsonBackReference; import com.fasterxml.jackson.annotation.JsonIgnore; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java index 368e756c8e..28be95b528 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.datasets.concepts.select.concept.UniversalSelect; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.ExistsAggregator; -import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.aggregator.ExistsSqlAggregator; import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; import com.fasterxml.jackson.annotation.JsonIgnore; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java index 8a0c4341bf..4729637d80 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/FirstValueSelect.java @@ -8,7 +8,7 @@ import com.bakdata.conquery.models.index.InternToExternMapper; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.FirstValueAggregator; -import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.aggregator.FirstValueSqlAggregator; import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/LastValueSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/LastValueSelect.java index 0e4b5b32bb..aa48276681 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/LastValueSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/LastValueSelect.java @@ -8,7 +8,7 @@ import com.bakdata.conquery.models.index.InternToExternMapper; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.LastValueAggregator; -import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.aggregator.LastValueSqlAggregator; import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/RandomValueSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/RandomValueSelect.java index 94c0f1c2af..79f9cab147 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/RandomValueSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/RandomValueSelect.java @@ -8,7 +8,7 @@ import com.bakdata.conquery.models.index.InternToExternMapper; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.RandomValueAggregator; -import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.aggregator.RandomValueSqlAggregator; import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountQuartersSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountQuartersSelect.java index f883ac1dc0..cb24bf8e45 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountQuartersSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountQuartersSelect.java @@ -11,8 +11,8 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.CountQuartersOfDateRangeAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.CountQuartersOfDatesAggregator; -import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; -import com.bakdata.conquery.sql.conversion.model.aggregator.CountQuartersSqlAggregator; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; +import com.bakdata.conquery.sql.conversion.model.select.CountQuartersSqlAggregator; import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountSelect.java index 46d104ed4b..e6dac93589 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/CountSelect.java @@ -14,7 +14,7 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.DistinctValuesWrapperAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.CountAggregator; -import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.aggregator.CountSqlAggregator; import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; import lombok.Data; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateDistanceSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateDistanceSelect.java index 0711d2d7a1..6b3aa7ebc6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateDistanceSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateDistanceSelect.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.DateDistanceAggregator; -import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.aggregator.DateDistanceSqlAggregator; import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java index 49532d731a..5393c6cac1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/FlagSelect.java @@ -12,7 +12,7 @@ import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.FlagsAggregator; -import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.aggregator.FlagSqlAggregator; import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; import com.fasterxml.jackson.annotation.JsonCreator; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java index d09e6283d9..680d50cc9e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java @@ -4,7 +4,6 @@ import java.util.Collections; import java.util.EnumSet; import java.util.List; -import java.util.Set; import javax.validation.constraints.NotNull; @@ -25,8 +24,7 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.specific.sum.IntegerSumAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.sum.MoneySumAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.sum.RealSumAggregator; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; -import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.aggregator.SumDistinctSqlAggregator; import com.bakdata.conquery.sql.conversion.model.aggregator.SumSqlAggregator; import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; @@ -136,12 +134,4 @@ public SqlSelects convertToSqlSelects(SelectContext selectContext) { return SumSqlAggregator.create(this, selectContext).getSqlSelects(); } - @Override - public Set getRequiredSqlSteps() { - if (distinctByColumn != null && !distinctByColumn.isEmpty()) { - return ConnectorCteStep.withOptionalSteps(ConnectorCteStep.JOIN_PREDECESSORS); - } - return ConnectorCteStep.MANDATORY_STEPS; - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SharedAliases.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SharedAliases.java new file mode 100644 index 0000000000..e365dd84aa --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SharedAliases.java @@ -0,0 +1,13 @@ +package com.bakdata.conquery.sql.conversion; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@Getter +@RequiredArgsConstructor +public enum SharedAliases { + + PRIMARY_COLUMN("pid"); + + private final String alias; +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java index 862489550e..88781c941b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java @@ -3,6 +3,7 @@ import java.sql.Date; import java.util.Optional; +import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QualifyingUtil; @@ -24,7 +25,6 @@ class InvertCte extends DateAggregationCte { public static final String ROWS_LEFT_TABLE_NAME = "rows_left"; public static final String ROWS_RIGHT_TABLE_NAME = "rows_right"; - public static final String PRIMARY_COLUMN_FIELD_NAME = "primary_column"; private final DateAggregationCteStep cteStep; @@ -41,7 +41,7 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context) Field leftPrimaryColumn = QualifyingUtil.qualify(primaryColumn, ROWS_LEFT_TABLE_NAME); Field rightPrimaryColumn = QualifyingUtil.qualify(primaryColumn, ROWS_RIGHT_TABLE_NAME); Field coalescedPrimaryColumn = DSL.coalesce(leftPrimaryColumn, rightPrimaryColumn) - .as(PRIMARY_COLUMN_FIELD_NAME); + .as(SharedAliases.PRIMARY_COLUMN.getAlias()); Selects invertSelects = getInvertSelects(rowNumberStep, coalescedPrimaryColumn, context); TableOnConditionStep fromTable = selfJoinWithShiftedRows(leftPrimaryColumn, rightPrimaryColumn, rowNumberStep); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java index 902e5e8254..c3be63a466 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java @@ -1,27 +1,26 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; import java.util.List; +import java.util.function.Predicate; +import java.util.stream.Collectors; -import com.bakdata.conquery.sql.conversion.model.QualifyingUtil; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; import org.jooq.Condition; -import org.jooq.Field; class AggregationFilterCte extends ConnectorCte { + @Override + public ConnectorCteStep cteStep() { + return ConnectorCteStep.AGGREGATION_FILTER; + } + @Override public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { - String predecessorTableName = tableContext.getConnectorTables().getPredecessor(cteStep()); - Field primaryColumn = QualifyingUtil.qualify(tableContext.getPrimaryColumn(), predecessorTableName); - Selects aggregationFilterSelects = Selects.builder() - .primaryColumn(primaryColumn) - .sqlSelects(getForAggregationFilterSelects(tableContext)) - .build() - .qualify(predecessorTableName); + Selects aggregationFilterSelects = getAggregationFilterSelects(tableContext); List aggregationFilterConditions = tableContext.getSqlFilters().stream() .flatMap(conceptFilter -> conceptFilter.getWhereClauses().getGroupFilters().stream()) @@ -33,17 +32,23 @@ public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { .conditions(aggregationFilterConditions); } - private List getForAggregationFilterSelects(CQTableContext tableContext) { - return tableContext.allSqlSelects().stream() - .flatMap(sqlSelects -> sqlSelects.getFinalSelects().stream()) - .filter(sqlSelect -> !sqlSelect.isUniversal()) - .distinct() - .toList(); - } + private Selects getAggregationFilterSelects(CQTableContext tableContext) { + + QueryStep previous = tableContext.getPrevious(); + Selects previousSelects = previous.getQualifiedSelects(); + List forAggregationFilterStep = + tableContext.allSqlSelects().stream() + .flatMap(sqlSelects -> sqlSelects.getFinalSelects().stream()) + .filter(Predicate.not(SqlSelect::isUniversal)) + .map(sqlSelect -> sqlSelect.qualify(previous.getCteName())) + .collect(Collectors.toList()); + + return Selects.builder() + .primaryColumn(previousSelects.getPrimaryColumn()) + .validityDate(previousSelects.getValidityDate()) + .sqlSelects(forAggregationFilterStep) + .build(); - @Override - public ConnectorCteStep cteStep() { - return ConnectorCteStep.AGGREGATION_FILTER; } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java index 65d4f80b86..33facee40c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java @@ -18,7 +18,6 @@ public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { List requiredInAggregationFilterStep = tableContext.allSqlSelects().stream() .flatMap(sqlSelects -> sqlSelects.getAggregationSelects().stream()) - .distinct() .toList(); Selects aggregationSelectSelects = Selects.builder() diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index 1e6b61387a..73ec110454 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -1,11 +1,9 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.Set; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -25,6 +23,7 @@ import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.sql.conversion.model.filter.WhereConditionWrapper; import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; import org.jooq.Condition; @@ -40,7 +39,7 @@ public CQConceptConverter(SqlFunctionProvider functionProvider) { new PreprocessingCte(), new EventFilterCte(), new AggregationSelectCte(), - new JoinPredecessorsCte(), + new JoinBranchesCte(), new AggregationFilterCte(), new FinalConnectorCte() ); @@ -80,8 +79,7 @@ private CQTableContext createConceptCteContext(CQConcept cqConcept, ConversionCo String conceptLabel = context.getNameGenerator().conceptName(cqConcept); Optional validityDateSelect = convertValidityDate(cqTable, tableName, conceptLabel); - Set requiredSteps = getRequiredSteps(cqTable, context.dateRestrictionActive(), validityDateSelect); - ConnectorTables connectorTables = new ConnectorTables(conceptLabel, requiredSteps, tableName, context.getNameGenerator()); + ConnectorTables connectorTables = new ConnectorTables(conceptLabel, tableName, context.getNameGenerator()); // convert filters List allFiltersForTable = new ArrayList<>(); @@ -108,30 +106,6 @@ private CQTableContext createConceptCteContext(CQConcept cqConcept, ConversionCo .build(); } - /** - * Determines if event/aggregation filter steps are required. - * - *

- * {@link ConnectorCteStep#MANDATORY_STEPS} are allways part of any concept conversion. - */ - private Set getRequiredSteps(CQTable table, boolean dateRestrictionRequired, Optional validityDateSelect) { - Set requiredSteps = new HashSet<>(ConnectorCteStep.MANDATORY_STEPS); - - if (dateRestrictionApplicable(dateRestrictionRequired, validityDateSelect)) { - requiredSteps.add(ConnectorCteStep.EVENT_FILTER); - } - - table.getFilters().stream() - .flatMap(filterValue -> filterValue.getFilter().getRequiredSqlSteps().stream()) - .forEach(requiredSteps::add); - - Stream.concat(table.getConcept().getSelects().stream(), table.getSelects().stream()) - .flatMap(select -> select.getRequiredSqlSteps().stream()) - .forEach(requiredSteps::add); - - return requiredSteps; - } - private Optional convertValidityDate( CQTable table, String tableName, diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCteStep.java index 374a881b71..b250f84129 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCteStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCteStep.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; -import java.util.HashSet; import java.util.Set; import com.bakdata.conquery.sql.conversion.model.CteStep; @@ -14,22 +13,18 @@ public enum ConnectorCteStep implements CteStep { PREPROCESSING("preprocessing", null), EVENT_FILTER("event_filter", PREPROCESSING), AGGREGATION_SELECT("group_select", EVENT_FILTER), - JOIN_PREDECESSORS("join_predecessors", AGGREGATION_SELECT), - AGGREGATION_FILTER("group_filter", JOIN_PREDECESSORS), + JOIN_BRANCHES("join_branches", AGGREGATION_SELECT), + AGGREGATION_FILTER("group_filter", JOIN_BRANCHES), FINAL("", AGGREGATION_FILTER); - public static final Set MANDATORY_STEPS = Set.of( - ConnectorCteStep.PREPROCESSING, - ConnectorCteStep.AGGREGATION_SELECT, - ConnectorCteStep.FINAL - ); + public static final Set MANDATORY_STEPS = Set.of(values()); private final String suffix; private final ConnectorCteStep predecessor; - public static Set withOptionalSteps(ConnectorCteStep... connectorCteStep) { - Set steps = new HashSet<>(MANDATORY_STEPS); - steps.addAll(Set.of(connectorCteStep)); - return steps; + @Override + public String cteName(String conceptLabel) { + return "%s-%s".formatted(conceptLabel, this.suffix); } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorTables.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorTables.java index 89d9af5cb1..4c0c0467b8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorTables.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorTables.java @@ -1,14 +1,12 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; -import java.util.Set; - import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.conversion.model.SqlTables; class ConnectorTables extends SqlTables { - public ConnectorTables(String conceptLabel, Set requiredSteps, String rootTableName, NameGenerator nameGenerator) { - super(conceptLabel, requiredSteps, rootTableName, nameGenerator); + public ConnectorTables(String conceptLabel, String rootTableName, NameGenerator nameGenerator) { + super(conceptLabel, ConnectorCteStep.MANDATORY_STEPS, rootTableName, nameGenerator); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterCte.java index 68f2760796..703933d196 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterCte.java @@ -2,13 +2,18 @@ import java.util.List; import java.util.Optional; +import java.util.stream.Stream; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QualifyingUtil; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.aggregator.SumDistinctSqlAggregator; import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; +import com.bakdata.conquery.sql.conversion.model.select.ExtractingSqlSelect; +import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; import org.jooq.Condition; import org.jooq.Field; @@ -41,16 +46,48 @@ private Selects getEventFilterSelects(CQTableContext tableContext) { validityDate = Optional.of(validityDate.get().qualify(predecessorTableName)); } - List sqlSelects = tableContext.allSqlSelects().stream() - .flatMap(selects -> selects.getAggregationSelects().stream()) - .map(sqlSelect -> sqlSelect.createColumnReference(predecessorTableName)) - .toList(); + List eventFilterSelects = + tableContext.allSqlSelects().stream() + .flatMap(sqlSelects -> collectForEventFilterStep(sqlSelects).stream()) + .flatMap(sqlSelect -> referenceRequiredColumns(sqlSelect, predecessorTableName)) + .toList(); return Selects.builder() .primaryColumn(primaryColumn) .validityDate(validityDate) - .sqlSelects(sqlSelects) + .sqlSelects(eventFilterSelects) .build(); } + /** + * Collects the columns required in {@link ConnectorCteStep#AGGREGATION_SELECT}, but also columns additional tables require (like the ones created by the + * {@link SumDistinctSqlAggregator}). An additional predecessor can contain an N-ary tree of predecessors itself (like all {@link QueryStep}s), so we want to + * look for the deepest predeceasing QueryStep leafs and collect their {@link SqlSelects}, because they expect this CTE to contain all their + * {@link SqlSelect#requiredColumns()}. + */ + private static List collectForEventFilterStep(SqlSelects sqlSelects) { + return Stream.concat( + sqlSelects.getAggregationSelects().stream(), + sqlSelects.getAdditionalPredecessor().map(EventFilterCte::collectDeepestPredecessorsColumns).orElse(Stream.empty()) + ) + .toList(); + } + + /** + * Recursively looks for the deepest predecessors of the given additional predecessor and collects their {@link SqlSelect}. + */ + private static Stream collectDeepestPredecessorsColumns(QueryStep additionalPredecessor) { + if (additionalPredecessor.getPredecessors().isEmpty()) { + return Stream.concat( + additionalPredecessor.getSelects().getSqlSelects().stream(), + additionalPredecessor.getGroupBy().stream().map(FieldWrapper::new) // some required columns may just be referenced in a GROUP BY statement + ); + } + return additionalPredecessor.getPredecessors().stream().flatMap(EventFilterCte::collectDeepestPredecessorsColumns); + } + + private static Stream> referenceRequiredColumns(SqlSelect sqlSelect, String predecessorTableName) { + return sqlSelect.requiredColumns().stream().map(column -> new ExtractingSqlSelect<>(predecessorTableName, column, Object.class)); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FinalConnectorCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FinalConnectorCte.java index 2521452687..df5298ab89 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FinalConnectorCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FinalConnectorCte.java @@ -1,19 +1,10 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; import java.util.List; -import java.util.Optional; -import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingContext; -import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingTables; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.LogicalOperation; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.TableLike; class FinalConnectorCte extends ConnectorCte { @@ -22,19 +13,17 @@ protected QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { List forFinalStep = tableContext.allSqlSelects().stream() .flatMap(sqlSelects -> sqlSelects.getFinalSelects().stream()) - .distinct() .toList(); - if (tableContext.getValidityDate().isEmpty() || tableContext.isExcludedFromDateAggregation()) { - Selects finalConceptSelects = Selects.builder() - .primaryColumn(tableContext.getPrimaryColumn()) - .sqlSelects(forFinalStep) - .build(); - return QueryStep.builder() - .selects(finalConceptSelects); - } + QueryStep previous = tableContext.getPrevious(); + Selects previousSelects = previous.getQualifiedSelects(); + Selects finalConceptSelects = Selects.builder() + .primaryColumn(previousSelects.getPrimaryColumn()) + .validityDate(previousSelects.getValidityDate()) + .sqlSelects(forFinalStep) + .build(); - return applyIntervalPacking(forFinalStep, tableContext); + return QueryStep.builder().selects(finalConceptSelects); } @Override @@ -42,53 +31,4 @@ protected ConnectorCteStep cteStep() { return ConnectorCteStep.FINAL; } - private QueryStep.QueryStepBuilder applyIntervalPacking(List forFinalStep, CQTableContext tableContext) { - - String conceptLabel = tableContext.getConceptLabel(); - IntervalPackingTables intervalPackingTables = - IntervalPackingTables.forConcept(conceptLabel, tableContext.getConnectorTables(), tableContext.getNameGenerator()); - - IntervalPackingContext intervalPackingContext = - IntervalPackingContext.builder() - .nodeLabel(conceptLabel) - .primaryColumn(tableContext.getPrimaryColumn()) - .validityDate(tableContext.getValidityDate().get()) - .intervalPackingTables(intervalPackingTables) - .build(); - - QueryStep finalIntervalPackingStep = tableContext.getConversionContext() - .getSqlDialect() - .getIntervalPacker() - .createIntervalPackingSteps(intervalPackingContext); - - return joinSelectsAndFiltersWithIntervalPackingStep(forFinalStep, finalIntervalPackingStep, tableContext); - } - - private QueryStep.QueryStepBuilder joinSelectsAndFiltersWithIntervalPackingStep( - List forFinalStep, - QueryStep finalIntervalPackingStep, - CQTableContext tableContext - ) { - QueryStep finalSelectsAndFilterStep = tableContext.getPrevious(); - Field primaryColumn = finalSelectsAndFilterStep.getQualifiedSelects().getPrimaryColumn(); - Optional validityDate = Optional.of(finalIntervalPackingStep.getQualifiedSelects().getValidityDate().get()); - - TableLike joinedTable = QueryStepJoiner.constructJoinedTable( - List.of(finalSelectsAndFilterStep, finalIntervalPackingStep), - LogicalOperation.AND, - tableContext.getConversionContext() - ); - - Selects finalConceptSelects = Selects.builder() - .primaryColumn(primaryColumn) - .validityDate(validityDate) - .sqlSelects(forFinalStep) - .build(); - - return QueryStep.builder() - .selects(finalConceptSelects) - .fromTable(joinedTable) - .predecessors(List.of(finalSelectsAndFilterStep, finalIntervalPackingStep)); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java new file mode 100644 index 0000000000..0190e19994 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java @@ -0,0 +1,105 @@ +package com.bakdata.conquery.sql.conversion.cqelement.concept; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingContext; +import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingTables; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.conversion.model.LogicalOperation; +import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; +import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.aggregator.SumDistinctSqlAggregator; +import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.TableLike; + +/** + * Joins the {@link ConnectorCteStep#AGGREGATION_SELECT} with the interval packing branch for the aggregated validity date and optional additional predecessors. + *

+ * Joining is optional - if a validity date is not present, the node is excluded from time aggregation or if there is no additional predecessor, no join will + * take place. See {@link SumDistinctSqlAggregator} for an example of additional predecessors. + * + * 

+ *     {@code
+ *     "join_branches" as (
+ *  	  select
+ *  	    coalesce("group_select"."pid", "interval_complete"."pid", "row_number_filtered"."pid") as "pid",
+ *  	    "interval_complete"."concept_concept-1_validity_date",
+ *  	    "row_number_filtered"."sum_distinct-1"
+ *  	  from "group_select"
+ *  	    join "interval_complete"
+ *  	      on "group_select"."pid" = "interval_complete"."pid"
+ *  	    join "row_number_filtered"
+ *  	      on "interval_complete"."pid" = "row_number_filtered"."pid"
+ *  	)
+ *     }
+ *
+ */ +class JoinBranchesCte extends ConnectorCte { + + @Override + protected ConnectorCteStep cteStep() { + return ConnectorCteStep.JOIN_BRANCHES; + } + + @Override + protected QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { + + List queriesToJoin = new ArrayList<>(); + queriesToJoin.add(tableContext.getPrevious()); + + Optional validityDate; + if (tableContext.getValidityDate().isEmpty() || tableContext.isExcludedFromDateAggregation()) { + validityDate = Optional.empty(); + } + else { + QueryStep finalIntervalPackingStep = applyIntervalPacking(tableContext); + queriesToJoin.add(finalIntervalPackingStep); + validityDate = finalIntervalPackingStep.getQualifiedSelects().getValidityDate(); + } + + tableContext.allSqlSelects().stream() + .flatMap(sqlSelects -> sqlSelects.getAdditionalPredecessor().stream()) + .forEach(queriesToJoin::add); + + Field primaryColumn = QueryStepJoiner.coalescePrimaryColumns(queriesToJoin); + List mergedSqlSelects = QueryStepJoiner.mergeSelects(queriesToJoin); + Selects selects = Selects.builder() + .primaryColumn(primaryColumn) + .validityDate(validityDate) + .sqlSelects(mergedSqlSelects) + .build(); + + TableLike fromTable = QueryStepJoiner.constructJoinedTable(queriesToJoin, LogicalOperation.AND, tableContext.getConversionContext()); + + return QueryStep.builder() + .selects(selects) + .fromTable(fromTable) + .predecessors(queriesToJoin); + } + + private static QueryStep applyIntervalPacking(CQTableContext tableContext) { + + String conceptLabel = tableContext.getConceptLabel(); + IntervalPackingTables intervalPackingTables = + IntervalPackingTables.forConcept(conceptLabel, tableContext.getConnectorTables(), tableContext.getNameGenerator()); + + IntervalPackingContext intervalPackingContext = + IntervalPackingContext.builder() + .nodeLabel(conceptLabel) + .primaryColumn(tableContext.getPrimaryColumn()) + .validityDate(tableContext.getValidityDate().get()) + .intervalPackingTables(intervalPackingTables) + .build(); + + return tableContext.getConversionContext() + .getSqlDialect() + .getIntervalPacker() + .createIntervalPackingSteps(intervalPackingContext); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinPredecessorsCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinPredecessorsCte.java deleted file mode 100644 index 9086c215f1..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinPredecessorsCte.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; - -import java.util.ArrayList; -import java.util.List; - -import com.bakdata.conquery.sql.conversion.model.LogicalOperation; -import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; -import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.TableLike; - -class JoinPredecessorsCte extends ConnectorCte { - - @Override - protected QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { - - List queriesToJoin = new ArrayList<>(); - queriesToJoin.add(tableContext.getPrevious()); - tableContext.allSqlSelects().stream() - .flatMap(sqlSelects -> sqlSelects.getAdditionalPredecessors().stream()) - .forEach(queriesToJoin::add); - - Field primaryColumn = QueryStepJoiner.coalescePrimaryColumns(queriesToJoin); - List mergedSelects = QueryStepJoiner.mergeSelects(queriesToJoin); - Selects selects = Selects.builder() - .primaryColumn(primaryColumn) - .sqlSelects(mergedSelects) - .build(); - - TableLike fromTable = QueryStepJoiner.constructJoinedTable(queriesToJoin, LogicalOperation.AND, tableContext.getConversionContext()); - - return QueryStep.builder() - .selects(selects) - .fromTable(fromTable) - .predecessors(queriesToJoin); - } - - @Override - protected ConnectorCteStep cteStep() { - return ConnectorCteStep.JOIN_PREDECESSORS; - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java index 06e51a6cb3..36e1b35dfc 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java @@ -2,11 +2,13 @@ import java.util.List; +import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; import org.jooq.Condition; +import org.jooq.Field; class PreprocessingCte extends ConnectorCte { @@ -14,11 +16,13 @@ public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { List forPreprocessing = tableContext.allSqlSelects().stream() .flatMap(sqlSelects -> sqlSelects.getPreprocessingSelects().stream()) - .distinct() .toList(); + // we alias the primary column, so we can rely upon in other places that it has a specific name + Field aliasesPrimaryColumn = tableContext.getPrimaryColumn().as(SharedAliases.PRIMARY_COLUMN.getAlias()); + Selects preprocessingSelects = Selects.builder() - .primaryColumn(tableContext.getPrimaryColumn()) + .primaryColumn(aliasesPrimaryColumn) .validityDate(tableContext.getValidityDate()) .sqlSelects(forPreprocessing) .build(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QualifyingUtil.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QualifyingUtil.java index cfd1fe0ef6..a4665c4957 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QualifyingUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QualifyingUtil.java @@ -15,7 +15,7 @@ public static Field qualify(Field field, String qualifier) { public static List qualify(List sqlSelects, String qualifier) { return sqlSelects.stream() - .map(sqlSelect -> sqlSelect.createAliasReference(qualifier)) + .map(sqlSelect -> sqlSelect.qualify(qualifier)) .collect(Collectors.toList()); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java index a7a5ed1874..301376406e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java @@ -7,6 +7,7 @@ import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.cqelement.aggregation.DateAggregationDates; import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; @@ -21,8 +22,6 @@ public class QueryStepJoiner { - private static final String PRIMARY_COLUMN_NAME = "primary_column"; - public static ConversionContext joinChildren( Iterable children, ConversionContext context, @@ -103,7 +102,7 @@ public static Field coalescePrimaryColumns(List querySteps) { .map(queryStep -> queryStep.getQualifiedSelects().getPrimaryColumn()) .collect(Collectors.toList()); return DSL.coalesce(primaryColumns.get(0), primaryColumns.subList(1, primaryColumns.size()).toArray()) - .as(PRIMARY_COLUMN_NAME); + .as(SharedAliases.PRIMARY_COLUMN.getAlias()); } public static List mergeSelects(List querySteps) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java index 9a2d4740f8..02d7149535 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java @@ -38,8 +38,7 @@ public Selects blockValidityDate() { public Selects qualify(String qualifier) { Field qualifiedPrimaryColumn = DSL.field(DSL.name(qualifier, this.primaryColumn.getName())); List sqlSelects = this.sqlSelects.stream() - .map(sqlSelect -> sqlSelect.createAliasReference(qualifier)) - .distinct() + .map(sqlSelect -> sqlSelect.qualify(qualifier)) .collect(Collectors.toList()); SelectsBuilder builder = Selects.builder() @@ -61,12 +60,14 @@ public List> all() { ) .flatMap(Function.identity()) .map(select -> (Field) select) + .distinct() .collect(Collectors.toList()); } public List> explicitSelects() { return this.sqlSelects.stream() .map(SqlSelect::select) + .distinct() .collect(Collectors.toList()); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java index 2ebae196cb..f01086c69a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountQuartersSqlAggregator.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.model.aggregator; +package com.bakdata.conquery.sql.conversion.model.select; import java.sql.Date; @@ -9,14 +9,11 @@ import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.CountQuartersSelect; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; -import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.conversion.model.aggregator.SqlAggregator; import com.bakdata.conquery.sql.conversion.model.filter.CountCondition; import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; -import com.bakdata.conquery.sql.conversion.model.select.ExtractingSqlSelect; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; -import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; import lombok.Value; import org.jooq.Field; import org.jooq.impl.DSL; @@ -34,13 +31,9 @@ private CountQuartersSqlAggregator( SqlFunctionProvider functionProvider, IRange filterValue ) { - ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>( - connectorTables.getPredecessor(ConnectorCteStep.PREPROCESSING), - column.getName(), - Date.class - ); + ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(connectorTables.getRootTable(), column.getName(), Date.class); - Field qualifiedRootSelect = rootSelect.createAliasReference(connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_SELECT)).select(); + Field qualifiedRootSelect = rootSelect.qualify(connectorTables.cteName(ConnectorCteStep.EVENT_FILTER)).select(); FieldWrapper countQuartersField = new FieldWrapper<>( DSL.countDistinct(functionProvider.yearQuarter(qualifiedRootSelect)).as(alias), column.getName() @@ -51,13 +44,13 @@ private CountQuartersSqlAggregator( .aggregationSelect(countQuartersField); if (filterValue == null) { - ExtractingSqlSelect finalSelect = countQuartersField.createAliasReference(connectorTables.getPredecessor(ConnectorCteStep.FINAL)); + ExtractingSqlSelect finalSelect = countQuartersField.qualify(connectorTables.getPredecessor(ConnectorCteStep.FINAL)); this.sqlSelects = builder.finalSelect(finalSelect).build(); - this.whereClauses = WhereClauses.builder().build(); + this.whereClauses = WhereClauses.empty(); } else { this.sqlSelects = builder.build(); - Field qualified = countQuartersField.createAliasReference(connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_FILTER)).select(); + Field qualified = countQuartersField.qualify(connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_FILTER)).select(); CountCondition countCondition = new CountCondition(qualified, filterValue); this.whereClauses = WhereClauses.builder() .groupFilter(countCondition) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java index a43528dc66..afcbb50f17 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/CountSqlAggregator.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.CountSelect; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; -import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.CountCondition; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; @@ -33,13 +33,9 @@ private CountSqlAggregator( SqlTables connectorTables, IRange filterValue ) { - ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>( - connectorTables.getPredecessor(ConnectorCteStep.PREPROCESSING), - countColumn.getName(), - Object.class - ); + ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(connectorTables.getRootTable(), countColumn.getName(), Object.class); - Field qualifiedRootSelect = rootSelect.createAliasReference(connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_SELECT)).select(); + Field qualifiedRootSelect = rootSelect.qualify(connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_SELECT)).select(); Field countField = countType == CountType.DISTINCT ? DSL.countDistinct(qualifiedRootSelect) : DSL.count(qualifiedRootSelect); @@ -50,14 +46,14 @@ private CountSqlAggregator( .aggregationSelect(countGroupBy); if (filterValue == null) { - ExtractingSqlSelect finalSelect = countGroupBy.createAliasReference(connectorTables.getPredecessor(ConnectorCteStep.FINAL)); + ExtractingSqlSelect finalSelect = countGroupBy.qualify(connectorTables.getPredecessor(ConnectorCteStep.FINAL)); this.sqlSelects = builder.finalSelect(finalSelect).build(); - this.whereClauses = null; + this.whereClauses = WhereClauses.empty(); } else { this.sqlSelects = builder.build(); - Field qualifiedCountSelect = - countGroupBy.createAliasReference(connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_FILTER)).select(); + String predecessor = connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_FILTER); + Field qualifiedCountSelect = countGroupBy.qualify(predecessor).select(); CountCondition countCondition = new CountCondition(qualifiedCountSelect, filterValue); this.whereClauses = WhereClauses.builder() .groupFilter(countCondition) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java index eb9e5ff8c6..fc25932318 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/DateDistanceSqlAggregator.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; -import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.DateDistanceCondition; @@ -55,22 +55,21 @@ private DateDistanceSqlAggregator( if (filterValue == null) { - Field qualifiedDateDistance = dateDistanceSelect.createAliasReference(connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_SELECT)) + Field qualifiedDateDistance = dateDistanceSelect.qualify(connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_SELECT)) .select(); FieldWrapper minDateDistance = new FieldWrapper<>(DSL.min(qualifiedDateDistance).as(alias)); - ExtractingSqlSelect finalSelect = minDateDistance.createAliasReference(connectorTables.getPredecessor(ConnectorCteStep.FINAL)); + ExtractingSqlSelect finalSelect = minDateDistance.qualify(connectorTables.getPredecessor(ConnectorCteStep.FINAL)); this.sqlSelects = builder.aggregationSelect(minDateDistance) .finalSelect(finalSelect) .build(); - this.whereClauses = null; + this.whereClauses = WhereClauses.empty(); } else { this.sqlSelects = builder.build(); - Field - qualifiedDateDistanceSelect = - dateDistanceSelect.createAliasReference(connectorTables.getPredecessor(ConnectorCteStep.EVENT_FILTER)).select(); + String predecessorCte = connectorTables.getPredecessor(ConnectorCteStep.EVENT_FILTER); + Field qualifiedDateDistanceSelect = dateDistanceSelect.qualify(predecessorCte).select(); WhereCondition dateDistanceCondition = new DateDistanceCondition(qualifiedDateDistanceSelect, filterValue); this.whereClauses = WhereClauses.builder() .eventFilter(dateDistanceCondition) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/ExistsSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/ExistsSqlAggregator.java index f245ecdbfa..b6a9e88087 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/ExistsSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/ExistsSqlAggregator.java @@ -1,7 +1,7 @@ package com.bakdata.conquery.sql.conversion.model.aggregator; import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.ExistsSelect; -import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; @@ -20,7 +20,7 @@ private ExistsSqlAggregator(String alias) { this.sqlSelects = SqlSelects.builder() .finalSelect(existsSelect) .build(); - this.whereClauses = WhereClauses.builder().build(); + this.whereClauses = WhereClauses.empty(); } public static ExistsSqlAggregator create(ExistsSelect existsSelect, SelectContext selectContext) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FirstValueSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FirstValueSqlAggregator.java index 329a4101c8..21cd524818 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FirstValueSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FirstValueSqlAggregator.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.select.connector.FirstValueSelect; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; -import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.SqlTables; @@ -31,7 +31,7 @@ private FirstValueSqlAggregator( SqlTables connectorTables, SqlFunctionProvider functionProvider ) { - String rootTableName = connectorTables.getPredecessor(ConnectorCteStep.PREPROCESSING); + String rootTableName = connectorTables.getRootTable(); String columnName = column.getName(); ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(rootTableName, columnName, Object.class); @@ -39,10 +39,10 @@ private FirstValueSqlAggregator( validityDate.map(_validityDate -> _validityDate.qualify(connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_SELECT))) .map(ColumnDateRange::toFields) .orElse(Collections.emptyList()); - Field qualifiedRootSelect = rootSelect.createAliasReference(connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_SELECT)).select(); + Field qualifiedRootSelect = rootSelect.qualify(connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_SELECT)).select(); FieldWrapper firstGroupBy = new FieldWrapper<>(functionProvider.first(qualifiedRootSelect, validityDateFields).as(alias), columnName); - ExtractingSqlSelect finalSelect = firstGroupBy.createAliasReference(connectorTables.getPredecessor(ConnectorCteStep.FINAL)); + ExtractingSqlSelect finalSelect = firstGroupBy.qualify(connectorTables.getPredecessor(ConnectorCteStep.FINAL)); this.sqlSelects = SqlSelects.builder() .preprocessingSelect(rootSelect) @@ -50,7 +50,7 @@ private FirstValueSqlAggregator( .finalSelect(finalSelect) .build(); - this.whereClauses = WhereClauses.builder().build(); + this.whereClauses = WhereClauses.empty(); } public static FirstValueSqlAggregator create(FirstValueSelect firstValueSelect, SelectContext selectContext) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java index 2f59256bb0..a6b81c8be3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java @@ -11,7 +11,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.FlagSelect; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; -import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.FlagCondition; @@ -78,13 +78,12 @@ public static FlagSqlAggregator create(FlagSelect flagSelect, SelectContext sele SqlFunctionProvider functionProvider = selectContext.getParentContext().getSqlDialect().getFunctionProvider(); SqlTables connectorTables = selectContext.getConnectorTables(); - String rootTable = connectorTables.getPredecessor(ConnectorCteStep.PREPROCESSING); - Map rootSelects = createFlagRootSelectMap(flagSelect, rootTable); + Map rootSelects = createFlagRootSelectMap(flagSelect, connectorTables.getRootTable()); String alias = selectContext.getNameGenerator().selectName(flagSelect); FieldWrapper flagAggregation = createFlagSelect(alias, connectorTables, functionProvider, rootSelects); - ExtractingSqlSelect finalSelect = flagAggregation.createAliasReference(connectorTables.getPredecessor(ConnectorCteStep.FINAL)); + ExtractingSqlSelect finalSelect = flagAggregation.qualify(connectorTables.getPredecessor(ConnectorCteStep.FINAL)); SqlSelects sqlSelects = SqlSelects.builder().preprocessingSelects(rootSelects.values()) .aggregationSelect(flagAggregation) @@ -151,7 +150,9 @@ private static FieldWrapper createFlagSelect( // and stuff them into 1 array field Field flagsArray = functionProvider.asArray(flagAggregations).as(alias); - return new FieldWrapper<>(flagsArray); + // we also need the references for all flag columns for the flag aggregation of multiple columns + String[] requiredColumns = flagFieldsMap.values().stream().map(Field::getName).toArray(String[]::new); + return new FieldWrapper<>(flagsArray, requiredColumns); } private static Map> createRootSelectReferences(SqlTables connectorTables, Map flagRootSelectMap) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/LastValueSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/LastValueSqlAggregator.java index f094f97aba..465ad37989 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/LastValueSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/LastValueSqlAggregator.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.select.connector.LastValueSelect; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; -import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.SqlTables; @@ -31,18 +31,17 @@ private LastValueSqlAggregator( SqlTables connectorTables, SqlFunctionProvider functionProvider ) { - String rootTableName = connectorTables.getPredecessor(ConnectorCteStep.PREPROCESSING); String columnName = column.getName(); - ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(rootTableName, columnName, Object.class); + ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(connectorTables.getRootTable(), columnName, Object.class); List> validityDateFields = validityDate.map(_validityDate -> _validityDate.qualify(connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_SELECT))) .map(ColumnDateRange::toFields) .orElse(Collections.emptyList()); - Field qualifiedRootSelect = rootSelect.createAliasReference(connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_SELECT)).select(); + Field qualifiedRootSelect = rootSelect.qualify(connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_SELECT)).select(); FieldWrapper lastGroupBy = new FieldWrapper<>(functionProvider.last(qualifiedRootSelect, validityDateFields).as(alias), columnName); - ExtractingSqlSelect finalSelect = lastGroupBy.createAliasReference(connectorTables.getPredecessor(ConnectorCteStep.FINAL)); + ExtractingSqlSelect finalSelect = lastGroupBy.qualify(connectorTables.getPredecessor(ConnectorCteStep.FINAL)); this.sqlSelects = SqlSelects.builder() .preprocessingSelect(rootSelect) @@ -50,7 +49,7 @@ private LastValueSqlAggregator( .finalSelect(finalSelect) .build(); - this.whereClauses = WhereClauses.builder().build(); + this.whereClauses = WhereClauses.empty(); } public static LastValueSqlAggregator create(LastValueSelect lastValueSelect, SelectContext selectContext) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/NumberSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/NumberSqlAggregator.java index 08a84356f1..c465af1ad4 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/NumberSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/NumberSqlAggregator.java @@ -30,12 +30,7 @@ public NumberSqlAggregator( IRange filterValue ) { Class numberClass = NumberMapUtil.NUMBER_MAP.get(column.getType()); - - ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>( - connectorTables.getPredecessor(ConnectorCteStep.PREPROCESSING), - column.getName(), - numberClass - ); + ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(connectorTables.getRootTable(), column.getName(), numberClass); Field eventFilterCtePredecessor = connectorTables.qualifyOnPredecessor(ConnectorCteStep.EVENT_FILTER, rootSelect.aliased()); NumberCondition condition = new NumberCondition(eventFilterCtePredecessor, filterValue); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/RandomValueSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/RandomValueSqlAggregator.java index 29763678a1..b441ffeae0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/RandomValueSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/RandomValueSqlAggregator.java @@ -3,7 +3,7 @@ import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.select.connector.RandomValueSelect; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; -import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; @@ -25,14 +25,12 @@ private RandomValueSqlAggregator( SqlTables connectorTables, SqlFunctionProvider functionProvider ) { - String rootTableName = connectorTables.getPredecessor(ConnectorCteStep.PREPROCESSING); - String columnName = column.getName(); - ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(rootTableName, columnName, Object.class); + ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(connectorTables.getRootTable(), column.getName(), Object.class); - Field qualifiedRootSelect = rootSelect.createAliasReference(connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_SELECT)).select(); - FieldWrapper randomGroupBy = new FieldWrapper<>(functionProvider.random(qualifiedRootSelect).as(alias), columnName); + Field qualifiedRootSelect = rootSelect.qualify(connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_SELECT)).select(); + FieldWrapper randomGroupBy = new FieldWrapper<>(functionProvider.random(qualifiedRootSelect).as(alias), column.getName()); - ExtractingSqlSelect finalSelect = randomGroupBy.createAliasReference(connectorTables.getPredecessor(ConnectorCteStep.FINAL)); + ExtractingSqlSelect finalSelect = randomGroupBy.qualify(connectorTables.getPredecessor(ConnectorCteStep.FINAL)); this.sqlSelects = SqlSelects.builder() .preprocessingSelect(rootSelect) @@ -40,7 +38,7 @@ private RandomValueSqlAggregator( .finalSelect(finalSelect) .build(); - this.whereClauses = WhereClauses.builder().build(); + this.whereClauses = WhereClauses.empty(); } public static RandomValueSqlAggregator create(RandomValueSelect randomValueSelect, SelectContext selectContext) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumDistinctSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumDistinctSqlAggregator.java index f8938fca59..2a4f65b218 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumDistinctSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumDistinctSqlAggregator.java @@ -2,6 +2,7 @@ import java.math.BigDecimal; import java.util.List; +import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -11,8 +12,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.SumSelect; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; -import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.CteStep; import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.conversion.model.QualifyingUtil; @@ -27,9 +27,47 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.Value; +import org.jooq.Condition; import org.jooq.Field; import org.jooq.impl.DSL; +/** + * Conversion of a {@link SumSelect} with {@link SumSelect#getDistinctByColumn()}. Sum's the values of a column for each row which is distinct by the + * distinct-by columns by creating 2 additional CTEs. We can't use our usual {@link ConnectorCteStep#PREPROCESSING} CTE for achieving distinctness, because + * it's used for the conversion of other selects where distinctness by distinct-by columns is not required and would cause wrong results. + *

+ * + * 

+ *  The two additional CTEs this aggregator creates
+ * 	
    
+ * 	    
  1. 
+ * 	        Assign a row number to each row partitioned by the distinct by columns to ensure distinctness.
+ *            {@code
+ * 	        	"row_number_assigned" as (
+ *   			  select
+ *   			    "pid",
+ *   			    "value",
+ *   			    row_number() over (partition by "pid", "k1", "k2") "row_number"
+ *   			  from "event_filter"
+ *   			)
+ *            }
+ * 	    
    2. 
+ * 	    
  2. 
+ * 	        Sum all entries of a subject where the row number = 1, thus only summing distinct entries.
+ *            {@code
+ * 	        "sum_distinct_select-1-row_number_filtered" as (
+ *   		  select
+ *   		    "pid",
+ *   		    sum("value") "sum_distinct_select-1"
+ *   		  from "row_number_assigned"
+ *   		  where "row_number" = 1
+ *   		  group by "pid"
+ *   		),
+ *            }
+ * 	    
    3. 
+ * 	

+ *
+ */ @Value public class SumDistinctSqlAggregator implements SqlAggregator { @@ -37,8 +75,8 @@ public class SumDistinctSqlAggregator implements SqlAggregator { @RequiredArgsConstructor private enum SumDistinctCteStep implements CteStep { - GROUP_BY_DISTINCT_COLUMNS("grouped_by_distinct_columns", null), - SUM_DISTINCT("sum_distinct", GROUP_BY_DISTINCT_COLUMNS); + ROW_NUMBER_ASSIGNED("row_number_assigned", null), + ROW_NUMBER_FILTERED("row_number_filtered", ROW_NUMBER_ASSIGNED); private final String suffix; private final SumDistinctCteStep predecessor; @@ -57,51 +95,41 @@ public SumDistinctSqlAggregator( IRange filterValue, Field primaryColumn, SqlTables connectorTables, - SqlFunctionProvider functionProvider, NameGenerator nameGenerator ) { - // preprocesssing - String rootTable = connectorTables.getRootTable(); - Class numberClass1 = NumberMapUtil.NUMBER_MAP.get(sumColumn.getType()); - ExtractingSqlSelect sumColumnRootSelect = new ExtractingSqlSelect<>(rootTable, sumColumn.getName(), numberClass1); + // preprocessing + Class numberClass = NumberMapUtil.NUMBER_MAP.get(sumColumn.getType()); + ExtractingSqlSelect sumColumnRootSelect = new ExtractingSqlSelect<>(connectorTables.getRootTable(), sumColumn.getName(), numberClass); List> distinctByRootSelects = distinctByColumns.stream() .map(column -> new ExtractingSqlSelect<>( - rootTable, column.getName(), Object.class) + connectorTables.getRootTable(), column.getName(), Object.class) ) .toList(); - // sum column grouped by distinct columns - String predecessor = connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_SELECT); - ExtractingSqlSelect qualifiedRootSelect = sumColumnRootSelect.createAliasReference(predecessor); - FieldWrapper firstSelect = new FieldWrapper<>(functionProvider.first(qualifiedRootSelect.select(), List.of()).as(alias)); - QueryStep distinctColumnsStep = getGroupByDistinctColumnsStep(alias, primaryColumn, nameGenerator, predecessor, firstSelect, distinctByRootSelects); - - // sum select - Field firstSumColumn = firstSelect.createAliasReference(distinctColumnsStep.getCteName()).select(); - FieldWrapper distinctSum = new FieldWrapper<>(DSL.sum(firstSumColumn).as(alias)); - - // sum aggregation - QueryStep sumDistinctCte = getSumDistinctStep(alias, primaryColumn, nameGenerator, distinctSum, distinctColumnsStep); + // additional predecessors + QueryStep rowNumberCte = createRowNumberCte(primaryColumn, sumColumnRootSelect, distinctByRootSelects, alias, connectorTables, nameGenerator); + Field rootSelectQualified = sumColumnRootSelect.qualify(rowNumberCte.getCteName()).select(); + FieldWrapper distinctSum = new FieldWrapper<>(DSL.sum(rootSelectQualified).as(alias)); + QueryStep rowNumberFilteredCte = createRowNumberFilteredCte(rowNumberCte, primaryColumn, distinctSum, alias, nameGenerator); SqlSelects.SqlSelectsBuilder builder = SqlSelects.builder() .preprocessingSelect(sumColumnRootSelect) .preprocessingSelects(distinctByRootSelects) - .additionalPredecessor(sumDistinctCte); + .additionalPredecessor(Optional.of(rowNumberFilteredCte)); if (filterValue != null) { this.sqlSelects = builder.build(); - Field - qualifiedSumSelect = - distinctSum.createAliasReference(connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_FILTER)).select(); + String groupFilterPredecessor = connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_FILTER); + Field qualifiedSumSelect = distinctSum.qualify(groupFilterPredecessor).select(); SumCondition sumCondition = new SumCondition(qualifiedSumSelect, filterValue); this.whereClauses = WhereClauses.builder() .groupFilter(sumCondition) .build(); } else { - ExtractingSqlSelect finalSelect = distinctSum.createAliasReference(connectorTables.getPredecessor(ConnectorCteStep.FINAL)); + ExtractingSqlSelect finalSelect = distinctSum.qualify(connectorTables.getPredecessor(ConnectorCteStep.FINAL)); this.sqlSelects = builder.finalSelect(finalSelect).build(); - this.whereClauses = WhereClauses.builder().build(); + this.whereClauses = WhereClauses.empty(); } } @@ -113,7 +141,6 @@ public static SumDistinctSqlAggregator create(SumSelect sumSelect, SelectContext null, selectContext.getParentContext().getPrimaryColumn(), selectContext.getConnectorTables(), - selectContext.getParentContext().getSqlDialect().getFunctionProvider(), selectContext.getNameGenerator() ); } @@ -126,58 +153,74 @@ public static SumDistinctSqlAggregator create(SumSelect sumSelect, SelectContext filterContext.getValue(), filterContext.getParentContext().getPrimaryColumn(), filterContext.getConnectorTables(), - filterContext.getParentContext().getSqlDialect().getFunctionProvider(), filterContext.getNameGenerator() ); } - private static QueryStep getSumDistinctStep( - String alias, + /** + * Assigns row numbers for each partition over the pid and the distinct by columns. If the values per pid in the distinct by columns are duplicated, + * the row number will be incremented for each duplicated entry. + */ + private static QueryStep createRowNumberCte( Field primaryColumn, - NameGenerator nameGenerator, - FieldWrapper distinctSum, - QueryStep distinctColumnsStep + ExtractingSqlSelect sumColumnRootSelect, + List> distinctByRootSelects, + String alias, + SqlTables conceptTables, + NameGenerator nameGenerator ) { - Field qualifiedPrimaryColumn = QualifyingUtil.qualify(primaryColumn, distinctColumnsStep.getCteName()); + String predecessor = conceptTables.getPredecessor(ConnectorCteStep.AGGREGATION_SELECT); - Selects sumDistinctSelects = Selects.builder() - .primaryColumn(qualifiedPrimaryColumn) - .sqlSelect(distinctSum) - .build(); + Field qualifiedPrimaryColumn = QualifyingUtil.qualify(primaryColumn, predecessor); + ExtractingSqlSelect qualifiedSumRootSelect = sumColumnRootSelect.qualify(predecessor); + + List> partitioningFields = Stream.concat( + Stream.of(qualifiedPrimaryColumn), + distinctByRootSelects.stream().map(sqlSelect -> sqlSelect.qualify(predecessor).select()) + ) + .collect(Collectors.toList()); + FieldWrapper rowNumber = new FieldWrapper<>( + DSL.rowNumber().over(DSL.partitionBy(partitioningFields)).as(ROW_NUMBER_ALIAS), + partitioningFields.stream().map(Field::getName).toArray(String[]::new) + ); + + Selects rowNumberAssignedSelects = Selects.builder() + .primaryColumn(qualifiedPrimaryColumn) + .sqlSelects(List.of(qualifiedSumRootSelect, rowNumber)) + .build(); return QueryStep.builder() - .cteName(nameGenerator.cteStepName(SumDistinctCteStep.SUM_DISTINCT, alias)) - .selects(sumDistinctSelects) - .fromTable(QueryStep.toTableLike(distinctColumnsStep.getCteName())) - .predecessors(List.of(distinctColumnsStep)) - .groupBy(List.of(qualifiedPrimaryColumn)) + .cteName(nameGenerator.cteStepName(SumDistinctCteStep.ROW_NUMBER_ASSIGNED, alias)) + .selects(rowNumberAssignedSelects) + .fromTable(QueryStep.toTableLike(predecessor)) .build(); } - private static QueryStep getGroupByDistinctColumnsStep( - String alias, + /** + * Sums up the sum column values but only those whose row number is 1. Thus, only unique entries will be summed up. + */ + private static QueryStep createRowNumberFilteredCte( + QueryStep rowNumberCte, Field primaryColumn, - NameGenerator nameGenerator, - String predecessor, - FieldWrapper firstSelect, - List> distinctByRootSelects + FieldWrapper sumSelect, + String alias, + NameGenerator nameGenerator ) { - Field qualifiedPrimaryColumn = QualifyingUtil.qualify(primaryColumn, predecessor); - Selects selects = Selects.builder() - .primaryColumn(qualifiedPrimaryColumn) - .sqlSelect(firstSelect) - .build(); + Selects rowNumberFilteredSelects = Selects.builder() + .primaryColumn(primaryColumn) + .sqlSelects(List.of(sumSelect)) + .build(); - List> groupByFields = Stream.concat( - Stream.of(qualifiedPrimaryColumn), - distinctByRootSelects.stream().map(sqlSelect -> sqlSelect.createAliasReference(predecessor)).map(ExtractingSqlSelect::select) - ).collect(Collectors.toList()); + Condition firstOccurrence = DSL.field(DSL.name(rowNumberCte.getCteName(), ROW_NUMBER_ALIAS)) + .eq(DSL.val(1)); return QueryStep.builder() - .cteName(nameGenerator.cteStepName(SumDistinctCteStep.GROUP_BY_DISTINCT_COLUMNS, alias)) - .selects(selects) - .fromTable(QueryStep.toTableLike(predecessor)) - .groupBy(groupByFields) + .cteName(nameGenerator.cteStepName(SumDistinctCteStep.ROW_NUMBER_FILTERED, alias)) + .selects(rowNumberFilteredSelects) + .fromTable(QueryStep.toTableLike(rowNumberCte.getCteName())) + .conditions(List.of(firstOccurrence)) + .predecessors(List.of(rowNumberCte)) + .groupBy(List.of(primaryColumn)) .build(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java index 2e6563e068..a07e8de542 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumSqlAggregator.java @@ -10,7 +10,7 @@ import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.SumSelect; import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; -import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; +import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.SumCondition; import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; @@ -37,45 +37,40 @@ private SumSqlAggregator( Class numberClass = NumberMapUtil.NUMBER_MAP.get(sumColumn.getType()); List> preprocessingSelects = new ArrayList<>(); - ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>( - connectorTables.getPredecessor(ConnectorCteStep.PREPROCESSING), - sumColumn.getName(), - numberClass - ); + ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>(connectorTables.getRootTable(), sumColumn.getName(), numberClass); preprocessingSelects.add(rootSelect); - String aggregationSelectPredecessor = connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_SELECT); - Field sumField; + String eventFilterCte = connectorTables.cteName(ConnectorCteStep.EVENT_FILTER); + Field sumField = rootSelect.qualify(eventFilterCte).select(); + FieldWrapper sumGroupBy; if (subtractColumn != null) { ExtractingSqlSelect subtractColumnRootSelect = new ExtractingSqlSelect<>( - connectorTables.getPredecessor(ConnectorCteStep.PREPROCESSING), + connectorTables.getRootTable(), subtractColumn.getName(), numberClass ); preprocessingSelects.add(subtractColumnRootSelect); - Field qualifiedRootSelect = rootSelect.createAliasReference(aggregationSelectPredecessor).select(); - Field qualifiedSubtractRootSelect = subtractColumnRootSelect.createAliasReference(aggregationSelectPredecessor).select(); - sumField = qualifiedRootSelect.minus(qualifiedSubtractRootSelect); + Field subtractField = subtractColumnRootSelect.qualify(eventFilterCte).select(); + sumGroupBy = new FieldWrapper<>(DSL.sum(sumField.minus(subtractField)).as(alias), sumColumn.getName(), subtractColumn.getName()); } else { - sumField = rootSelect.createAliasReference(aggregationSelectPredecessor).select(); + sumGroupBy = new FieldWrapper<>(DSL.sum(sumField).as(alias), sumColumn.getName()); } - FieldWrapper sumGroupBy = new FieldWrapper<>(DSL.sum(sumField).as(alias), sumColumn.getName()); SqlSelects.SqlSelectsBuilder builder = SqlSelects.builder() .preprocessingSelects(preprocessingSelects) .aggregationSelect(sumGroupBy); if (filterValue == null) { - ExtractingSqlSelect finalSelect = sumGroupBy.createAliasReference(connectorTables.getPredecessor(ConnectorCteStep.FINAL)); + ExtractingSqlSelect finalSelect = sumGroupBy.qualify(connectorTables.getPredecessor(ConnectorCteStep.FINAL)); this.sqlSelects = builder.finalSelect(finalSelect).build(); - this.whereClauses = null; + this.whereClauses = WhereClauses.empty(); } else { this.sqlSelects = builder.build(); - Field qualifiedSumGroupBy = - sumGroupBy.createAliasReference(connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_FILTER)).select(); + String predecessor = connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_FILTER); + Field qualifiedSumGroupBy = sumGroupBy.qualify(predecessor).select(); SumCondition sumCondition = new SumCondition(qualifiedSumGroupBy, filterValue); this.whereClauses = WhereClauses.builder() .groupFilter(sumCondition) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/WhereClauses.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/WhereClauses.java index 052994a391..fa3f1b9901 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/WhereClauses.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/WhereClauses.java @@ -25,4 +25,8 @@ public WhereClauses negated() { ); } + public static WhereClauses empty() { + return WhereClauses.builder().build(); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExtractingSqlSelect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExtractingSqlSelect.java index 860125b3d5..53d0e62137 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExtractingSqlSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExtractingSqlSelect.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.sql.conversion.model.select; -import lombok.EqualsAndHashCode; +import java.util.List; + import lombok.Value; import org.jooq.Field; import org.jooq.impl.DSL; @@ -13,12 +14,10 @@ * @param type of column */ @Value -@EqualsAndHashCode public class ExtractingSqlSelect implements SqlSelect { String table; String column; - @EqualsAndHashCode.Exclude Class columnClass; @Override @@ -32,12 +31,12 @@ public Field aliased() { } @Override - public String columnName() { - return column; + public List requiredColumns() { + return List.of(column); } @Override - public ExtractingSqlSelect createAliasReference(String qualifier) { + public ExtractingSqlSelect qualify(String qualifier) { Field aliased = aliased(); return new ExtractingSqlSelect<>( qualifier, diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FieldWrapper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FieldWrapper.java index 6f3dba73d6..92f84cbe1e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FieldWrapper.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/FieldWrapper.java @@ -1,38 +1,49 @@ package com.bakdata.conquery.sql.conversion.model.select; +import java.util.List; + import lombok.EqualsAndHashCode; import org.jooq.Field; import org.jooq.impl.DSL; +/** + * Wrapper for a {@link Field}. + */ @EqualsAndHashCode public class FieldWrapper implements SqlSelect { private final Field field; - private final String columnName; + @EqualsAndHashCode.Exclude + private final List requiredColumns; /** - * Wrapper for a {@link Field}. - *

- * {@link FieldWrapper#columnName()} will return the given column name of the given field. - * - * @param field The field to wrap, e.g. {@code DSL.sum(DSL.field(DSL.name("foo", "bar"))).as("foo_bar")}; - * @param columnName The "root" column name of the wrapped field, e.g. "bar" + * @param field The field to wrap, e.g. {@code DSL.sum(DSL.field(DSL.name("foo", "bar"))).as("foo_bar")}; + * @param requiredColumns All columns this {@link FieldWrapper} requires in the previous CTE/table to be present. + */ + public FieldWrapper(Field field, String... requiredColumns) { + this.field = field; + this.requiredColumns = List.of(requiredColumns); + } + + /** + * @param field The field to wrap, e.g. {@code DSL.sum(DSL.field(DSL.name("foo", "bar"))).as("foo_bar")}; + * @param predecessor The {@link ExtractingSqlSelect} containing the required column this {@link FieldWrapper} uses. */ - public FieldWrapper(Field field, String columnName) { + public FieldWrapper(Field field, ExtractingSqlSelect predecessor) { this.field = field; - this.columnName = columnName; + this.requiredColumns = predecessor.requiredColumns(); } + /** * Wrapper for a {@link Field}. *

- * {@link FieldWrapper#columnName()} will return the alias of the given field. * * @param field @param field The field to wrap, e.g. {@code DSL.field(DSL.name("fizz", "buzz"))).as("fizz_buzz")}; */ public FieldWrapper(Field field) { this.field = field; - this.columnName = field.getName(); + this.requiredColumns = List.of(field.getName()); } @Override @@ -46,12 +57,12 @@ public Field aliased() { } @Override - public String columnName() { - return this.columnName; + public List requiredColumns() { + return this.requiredColumns; } @Override - public ExtractingSqlSelect createAliasReference(String qualifier) { + public ExtractingSqlSelect qualify(String qualifier) { Field aliased = aliased(); return new ExtractingSqlSelect<>( qualifier, diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/SelectContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java similarity index 84% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/SelectContext.java rename to backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java index a3d8270334..1eecb68a62 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/SelectContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java @@ -1,10 +1,11 @@ -package com.bakdata.conquery.sql.conversion.cqelement.concept; +package com.bakdata.conquery.sql.conversion.model.select; import java.util.Optional; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.conversion.model.SqlTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java index b2a795b0bf..d2f9ea7bdb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java @@ -1,6 +1,8 @@ package com.bakdata.conquery.sql.conversion.model.select; +import java.util.List; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; import org.jooq.Field; public interface SqlSelect { @@ -18,29 +20,17 @@ public interface SqlSelect { Field aliased(); /** - * @return Plain column name of this {@link SqlSelect}. - * For example, {@code "bar"}. + * All column names this {@link SqlSelect} requires to build its {@link SqlSelect#select()}. */ - String columnName(); + List requiredColumns(); /** * @return Creates a reference to the alias of this SqlSelect qualified onto the given qualifier. */ - ExtractingSqlSelect createAliasReference(String qualifier); + ExtractingSqlSelect qualify(String qualifier); /** - * @return Creates a reference to the column of this SqlSelect qualified onto the given qualifier. - */ - default ExtractingSqlSelect createColumnReference(String qualifier) { - return new ExtractingSqlSelect<>( - qualifier, - columnName(), - Object.class - ); - } - - /** - * @return Determines if this SqlSelect is only part of the final concept conversion CTE and has no predeceasing selects. + * @return Determines if this is only part of the {@link ConnectorCteStep#FINAL} CTE and has no predeceasing selects. */ default boolean isUniversal() { return false; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelects.java index a41300dd3e..de9cd2eeb7 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelects.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.sql.conversion.model.select; import java.util.List; +import java.util.Optional; import com.bakdata.conquery.sql.conversion.model.QueryStep; import lombok.Builder; @@ -18,7 +19,7 @@ public class SqlSelects { // Empty if only used in aggregation select @Singular List finalSelects; - // Additional predecessors these SqlSelects require - @Singular - List additionalPredecessors; + // An additional predecessor these SqlSelects require + @Builder.Default + Optional additionalPredecessor = Optional.empty(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java index cf73d6ff07..311a59cfc6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java @@ -80,7 +80,7 @@ private SqlExecutionResult createStatementAndExecute(SqlManagedQuery sqlQuery, C String sqlString = sqlQuery.getSqlQuery().getSql(); List> resultTypes = sqlQuery.getSqlQuery().getResultInfos().stream().map(ResultInfo::getType).collect(Collectors.toList()); - log.debug("Executing query: \n{}", sqlString); + log.info("Executing query: \n{}", sqlString); try (Statement statement = connection.createStatement(); ResultSet resultSet = statement.executeQuery(sqlString)) { diff --git a/backend/src/test/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorTablesTest.java b/backend/src/test/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorTablesTest.java index 630a2d0f6b..860c06d8af 100644 --- a/backend/src/test/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorTablesTest.java +++ b/backend/src/test/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorTablesTest.java @@ -5,6 +5,7 @@ import java.util.stream.Stream; import com.bakdata.conquery.sql.conversion.model.NameGenerator; +import com.bakdata.conquery.sql.conversion.model.SqlTables; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -17,10 +18,23 @@ class ConnectorTablesTest { public static final int NAME_MAX_LENGTH = 127; private static final NameGenerator NAME_GENERATOR = new NameGenerator(NAME_MAX_LENGTH); + private static class TestSqlTables extends SqlTables { + + public static Set MANDATORY_STEPS = Set.of( + ConnectorCteStep.PREPROCESSING, + ConnectorCteStep.AGGREGATION_SELECT, + ConnectorCteStep.FINAL + ); + + public TestSqlTables(String nodeLabel, Set requiredSteps, String rootTableName, NameGenerator nameGenerator) { + super(nodeLabel, requiredSteps, rootTableName, nameGenerator); + } + } + @ParameterizedTest @MethodSource("requiredStepsProvider") public void getPredecessorTableName(Set requiredSteps, ConnectorCteStep step, String expectedPredecessorTableName) { - ConnectorTables connectorTables = new ConnectorTables(CONCEPT_LABEL, requiredSteps, ROOT_TABLE, NAME_GENERATOR); + TestSqlTables connectorTables = new TestSqlTables(CONCEPT_LABEL, requiredSteps, ROOT_TABLE, NAME_GENERATOR); Assertions.assertEquals( expectedPredecessorTableName, connectorTables.getPredecessor(step) @@ -31,11 +45,11 @@ public static Stream requiredStepsProvider() { return Stream.of( // AGGREGATION_SELECT and FINAL direct predecessors missing - Arguments.of(ConnectorCteStep.MANDATORY_STEPS, ConnectorCteStep.PREPROCESSING, ROOT_TABLE), - Arguments.of(ConnectorCteStep.MANDATORY_STEPS, ConnectorCteStep.EVENT_FILTER, ConnectorCteStep.PREPROCESSING.cteName(CONCEPT_LABEL)), - Arguments.of(ConnectorCteStep.MANDATORY_STEPS, ConnectorCteStep.AGGREGATION_SELECT, ConnectorCteStep.PREPROCESSING.cteName(CONCEPT_LABEL)), - Arguments.of(ConnectorCteStep.MANDATORY_STEPS, ConnectorCteStep.AGGREGATION_FILTER, ConnectorCteStep.AGGREGATION_SELECT.cteName(CONCEPT_LABEL)), - Arguments.of(ConnectorCteStep.MANDATORY_STEPS, ConnectorCteStep.FINAL, ConnectorCteStep.AGGREGATION_SELECT.cteName(CONCEPT_LABEL)), + Arguments.of(TestSqlTables.MANDATORY_STEPS, ConnectorCteStep.PREPROCESSING, ROOT_TABLE), + Arguments.of(TestSqlTables.MANDATORY_STEPS, ConnectorCteStep.EVENT_FILTER, ConnectorCteStep.PREPROCESSING.cteName(CONCEPT_LABEL)), + Arguments.of(TestSqlTables.MANDATORY_STEPS, ConnectorCteStep.AGGREGATION_SELECT, ConnectorCteStep.PREPROCESSING.cteName(CONCEPT_LABEL)), + Arguments.of(TestSqlTables.MANDATORY_STEPS, ConnectorCteStep.AGGREGATION_FILTER, ConnectorCteStep.AGGREGATION_SELECT.cteName(CONCEPT_LABEL)), + Arguments.of(TestSqlTables.MANDATORY_STEPS, ConnectorCteStep.FINAL, ConnectorCteStep.AGGREGATION_SELECT.cteName(CONCEPT_LABEL)), // only FINAL direct predecessor missing Arguments.of( @@ -68,7 +82,7 @@ public static Stream requiredStepsProvider() { } private static Set withAdditionalSteps(Set additionalSteps) { - return Stream.concat(ConnectorCteStep.MANDATORY_STEPS.stream(), additionalSteps.stream()).collect(Collectors.toSet()); + return Stream.concat(TestSqlTables.MANDATORY_STEPS.stream(), additionalSteps.stream()).collect(Collectors.toSet()); } } diff --git a/backend/src/test/resources/tests/sql/combined/combined.json b/backend/src/test/resources/tests/sql/combined/combined.json new file mode 100644 index 0000000000..89a5ea927f --- /dev/null +++ b/backend/src/test/resources/tests/sql/combined/combined.json @@ -0,0 +1,134 @@ +{ + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, + "label": "Query that utilizes all possible select conversion steps in 1 query", + "expectedCsv": "tests/sql/combined/expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "ids": [ + "concept" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "concept.connector", + "selects": [ + "concept.connector.first_value", + "concept.connector.sum_distinct" + ], + "filters": [ + { + "filter": "concept.connector.value", + "type": "REAL_RANGE", + "value": { + "min": 0.0, + "max": 1.5 + } + }, + { + "filter": "concept.connector.count_value", + "type": "REAL_RANGE", + "value": { + "min": 1 + } + } + ] + } + ], + "selects": [ + "concept.exists" + ] + } + }, + "concepts": [ + { + "name": "concept", + "type": "TREE", + "connectors": [ + { + "label": "connector", + "table": "table", + "validityDates": [ + { + "label": "datum", + "startColumn": "table.datum_start", + "endColumn": "table.datum_end" + } + ], + "filters": [ + { + "label": "count_value", + "column": "table.value", + "type": "COUNT" + }, + { + "label": "value", + "description": "xy", + "column": "table.value", + "type": "NUMBER" + } + ], + "selects": [ + { + "label": "first_value", + "column": "table.value", + "type": "FIRST" + }, + { + "name": "sum_distinct", + "type": "SUM", + "column": "table.value", + "distinctByColumn": [ + "table.k1", + "table.k2" + ] + } + ] + } + ], + "selects": [ + { + "type": "EXISTS", + "name": "exists" + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/combined/content.csv", + "name": "table", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum_start", + "type": "DATE" + }, + { + "name": "datum_end", + "type": "DATE" + }, + { + "name": "value", + "type": "REAL" + }, + { + "name": "k1", + "type": "STRING" + }, + { + "name": "k2", + "type": "STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/combined/content.csv b/backend/src/test/resources/tests/sql/combined/content.csv new file mode 100644 index 0000000000..b1445f5c4f --- /dev/null +++ b/backend/src/test/resources/tests/sql/combined/content.csv @@ -0,0 +1,10 @@ +pid,datum_start,datum_end,value,k1,k2 +1,2012-01-01,2012-01-01,1,a,b +1,2013-01-01,2013-12-31,1,a,a +2,2012-01-01,2012-01-01,1.01,a,a +2,2012-01-02,2012-01-02,1,a,a +2,2012-01-05,2012-01-10,1,a,b +3,2012-01-01,2012-01-03,0.5,a,b +4,2012-01-01,2012-01-04,0.5,a,b +5,2012-01-01,2012-01-03,1,a,b +5,2012-01-04,2012-01-05,0.5,a,a diff --git a/backend/src/test/resources/tests/sql/combined/expected.csv b/backend/src/test/resources/tests/sql/combined/expected.csv new file mode 100644 index 0000000000..1673bf3b9f --- /dev/null +++ b/backend/src/test/resources/tests/sql/combined/expected.csv @@ -0,0 +1,6 @@ +result,dates,concept exists,concept first_value,concept sum_distinct +1,"{2012-01-01/2012-01-01,2013-01-01/2013-12-31}",1,1.0,2.0 +2,"{2012-01-01/2012-01-02,2012-01-05/2012-01-10}",1,1.01,2.01 +3,{2012-01-01/2012-01-03},1,0.5,0.5 +4,{2012-01-01/2012-01-04},1,0.5,0.5 +5,{2012-01-01/2012-01-05},1,1.0,1.5