From 939b537a0234656d4895d78b5db7ab2552624b43 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 4 Jan 2024 15:18:49 +0100 Subject: [PATCH 1/4] fixes accessing externalId after wrapping as object --- .../com/bakdata/conquery/resources/admin/ui/idmapping.html.ftl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/idmapping.html.ftl b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/idmapping.html.ftl index b9a97a53cd..79cc206cca 100644 --- a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/idmapping.html.ftl +++ b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/idmapping.html.ftl @@ -4,7 +4,7 @@ <#list c as key, val>
${key}
- <#list val as externalIdPart> + <#list val.externalId as externalIdPart>
${externalIdPart}
From 03c3052d3d92e75c94f2ea97ca1ad0964a75e36a Mon Sep 17 00:00:00 2001 From: Jonas Arnhold Date: Tue, 9 Jan 2024 12:58:20 +0100 Subject: [PATCH 2/4] Implement diffsum/distinct SumSelect/SumFilter conversion (#3251) --- .../concepts/filters/specific/SumFilter.java | 7 + .../select/connector/specific/SumSelect.java | 14 ++ .../sql/conversion/NodeConversions.java | 2 + .../cqelement/ConversionContext.java | 2 + .../cqelement/concept/CQConceptConverter.java | 7 +- .../cqelement/concept/ConceptCteContext.java | 8 +- .../cqelement/concept/ConceptCteStep.java | 3 +- .../concept/JoinPredecessorsCte.java | 46 +++++ .../dialect/HanaSqlFunctionProvider.java | 12 +- .../dialect/PostgreSqlFunctionProvider.java | 12 +- .../dialect/SqlFunctionProvider.java | 6 +- .../sql/conversion/model/QueryStepJoiner.java | 4 +- .../conversion/model/select/SqlSelects.java | 4 + .../select/SumDistinctSqlAggregator.java | 191 ++++++++++++++++++ .../model/select/SumSqlAggregator.java | 55 +++-- .../tests/sql/filter/sum/diffsum/content.csv | 22 ++ .../sql/filter/sum/diffsum/diffsum.spec.json | 75 +++++++ .../tests/sql/filter/sum/diffsum/expected.csv | 8 + .../tests/sql/filter/sum/distinct/content.csv | 9 + .../sum/distinct/distinct-sum.spec.json | 76 +++++++ .../sql/filter/sum/distinct/expected.csv | 3 + .../tests/sql/selects/sum/diffsum/content.csv | 22 ++ .../sql/selects/sum/diffsum/diffsum.spec.json | 68 +++++++ .../sql/selects/sum/diffsum/expected.csv | 13 ++ .../sql/selects/sum/distinct/content.csv | 8 + .../sum/distinct/distinct-sum.spec.json | 69 +++++++ .../sql/selects/sum/distinct/expected.csv | 5 + 27 files changed, 715 insertions(+), 36 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinPredecessorsCte.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SumDistinctSqlAggregator.java create mode 100644 backend/src/test/resources/tests/sql/filter/sum/diffsum/content.csv create mode 100644 backend/src/test/resources/tests/sql/filter/sum/diffsum/diffsum.spec.json create mode 100644 backend/src/test/resources/tests/sql/filter/sum/diffsum/expected.csv create mode 100644 backend/src/test/resources/tests/sql/filter/sum/distinct/content.csv create mode 100644 backend/src/test/resources/tests/sql/filter/sum/distinct/distinct-sum.spec.json create mode 100644 backend/src/test/resources/tests/sql/filter/sum/distinct/expected.csv create mode 100644 backend/src/test/resources/tests/sql/selects/sum/diffsum/content.csv create mode 100644 backend/src/test/resources/tests/sql/selects/sum/diffsum/diffsum.spec.json create mode 100644 backend/src/test/resources/tests/sql/selects/sum/diffsum/expected.csv create mode 100644 backend/src/test/resources/tests/sql/selects/sum/distinct/content.csv create mode 100644 backend/src/test/resources/tests/sql/selects/sum/distinct/distinct-sum.spec.json create mode 100644 backend/src/test/resources/tests/sql/selects/sum/distinct/expected.csv diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java index 54f5395e62..a4edf084a6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java @@ -35,6 +35,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; +import com.bakdata.conquery.sql.conversion.model.select.SumDistinctSqlAggregator; import com.bakdata.conquery.sql.conversion.model.select.SumSqlAggregator; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; @@ -109,11 +110,17 @@ public FilterNode createFilterNode(RANGE value) { @Override public SqlFilters convertToSqlFilter(FilterContext filterContext) { + if (distinctByColumn != null && !distinctByColumn.isEmpty()) { + return SumDistinctSqlAggregator.create(this, filterContext).getSqlFilters(); + } return SumSqlAggregator.create(this, filterContext).getSqlFilters(); } @Override public Set getRequiredSqlSteps() { + if (distinctByColumn != null && !distinctByColumn.isEmpty()) { + return ConceptCteStep.withOptionalSteps(ConceptCteStep.JOIN_PREDECESSORS, ConceptCteStep.AGGREGATION_FILTER); + } return ConceptCteStep.withOptionalSteps(ConceptCteStep.AGGREGATION_FILTER); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java index 77903d5312..91c1708446 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/SumSelect.java @@ -4,6 +4,7 @@ import java.util.Collections; import java.util.EnumSet; import java.util.List; +import java.util.Set; import javax.validation.constraints.NotNull; @@ -24,8 +25,10 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.specific.sum.IntegerSumAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.sum.MoneySumAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.sum.RealSumAggregator; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; +import com.bakdata.conquery.sql.conversion.model.select.SumDistinctSqlAggregator; import com.bakdata.conquery.sql.conversion.model.select.SumSqlAggregator; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -127,7 +130,18 @@ public boolean isColumnsOfSameType() { @Override public SqlSelects convertToSqlSelects(SelectContext selectContext) { + if (distinctByColumn != null && !distinctByColumn.isEmpty()) { + return SumDistinctSqlAggregator.create(this, selectContext).getSqlSelects(); + } return SumSqlAggregator.create(this, selectContext).getSqlSelects(); } + @Override + public Set getRequiredSqlSteps() { + if (distinctByColumn != null && !distinctByColumn.isEmpty()) { + return ConceptCteStep.withOptionalSteps(ConceptCteStep.JOIN_PREDECESSORS); + } + return ConceptCteStep.MANDATORY_STEPS; + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java index 81b936e558..03f8e22906 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java @@ -6,6 +6,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; import com.bakdata.conquery.sql.conversion.model.NameGenerator; +import org.jooq.impl.DSL; /** * Entry point for converting {@link QueryDescription} to an SQL query. @@ -27,6 +28,7 @@ public ConversionContext convert(QueryDescription queryDescription) { .nameGenerator(new NameGenerator(config.getDialect().getNameMaxLength())) .nodeConversions(this) .sqlDialect(this.dialect) + .primaryColumn(DSL.field(DSL.name(config.getPrimaryColumn()))) .build(); return convert(queryDescription, initialCtx); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java index b2f991a3c8..39aa1e89c2 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java @@ -14,6 +14,7 @@ import lombok.Singular; import lombok.Value; import lombok.With; +import org.jooq.Field; @Value @With @@ -27,6 +28,7 @@ public class ConversionContext implements Context { @Singular List querySteps; SqlQuery finalQuery; + Field primaryColumn; CDateRange dateRestrictionRange; boolean negation; boolean isGroupBy; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index d140b7a3ca..8bb57203fd 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -28,7 +28,6 @@ import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; import org.jooq.Condition; -import org.jooq.impl.DSL; public class CQConceptConverter implements NodeConverter { @@ -41,6 +40,7 @@ public CQConceptConverter(SqlFunctionProvider functionProvider) { new PreprocessingCte(), new EventFilterCte(), new AggregationSelectCte(), + new JoinPredecessorsCte(), new AggregationFilterCte(), new FinalConceptCte() ); @@ -103,7 +103,6 @@ private ConceptCteContext createConceptCteContext(CQConcept cqConcept, Conversio .conversionContext(context) .filters(allFiltersForTable) .selects(conceptSelects) - .primaryColumn(DSL.field(DSL.name(context.getConfig().getPrimaryColumn()))) .validityDate(validityDateSelect) .isExcludedFromDateAggregation(cqConcept.isExcludeFromTimeAggregation()) .conceptTables(conceptTables) @@ -128,6 +127,10 @@ private Set getRequiredSteps(CQTable table, boolean dateRestrict .flatMap(filterValue -> filterValue.getFilter().getRequiredSqlSteps().stream()) .forEach(requiredSteps::add); + Stream.concat(table.getConcept().getSelects().stream(), table.getSelects().stream()) + .flatMap(select -> select.getRequiredSqlSteps().stream()) + .forEach(requiredSteps::add); + return requiredSteps; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteContext.java index 6ad120e9ca..871b1566b1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteContext.java @@ -24,7 +24,6 @@ class ConceptCteContext implements Context { ConversionContext conversionContext; String conceptLabel; - Field primaryColumn; Optional validityDate; boolean isExcludedFromDateAggregation; List selects; @@ -43,6 +42,13 @@ public Stream allConceptSelects() { ); } + public Field getPrimaryColumn() { + if (previous == null) { + return conversionContext.getPrimaryColumn(); + } + return previous.getQualifiedSelects().getPrimaryColumn(); + } + @Override public NameGenerator getNameGenerator() { return conversionContext.getNameGenerator(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteStep.java index 464ece302e..0b231a89d8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptCteStep.java @@ -12,7 +12,8 @@ public enum ConceptCteStep implements CteStep { PREPROCESSING("preprocessing", null), EVENT_FILTER("event_filter", PREPROCESSING), AGGREGATION_SELECT("group_select", EVENT_FILTER), - AGGREGATION_FILTER("group_filter", AGGREGATION_SELECT), + JOIN_PREDECESSORS("join_predecessors", AGGREGATION_SELECT), + AGGREGATION_FILTER("group_filter", JOIN_PREDECESSORS), FINAL("", AGGREGATION_FILTER); public static final Set MANDATORY_STEPS = Set.of(ConceptCteStep.PREPROCESSING, ConceptCteStep.AGGREGATION_SELECT, ConceptCteStep.FINAL); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinPredecessorsCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinPredecessorsCte.java new file mode 100644 index 0000000000..ab5df6a751 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinPredecessorsCte.java @@ -0,0 +1,46 @@ +package com.bakdata.conquery.sql.conversion.cqelement.concept; + +import java.util.ArrayList; +import java.util.List; + +import com.bakdata.conquery.sql.conversion.model.LogicalOperation; +import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; +import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.TableLike; + +class JoinPredecessorsCte extends ConceptCte { + + @Override + protected QueryStep.QueryStepBuilder convertStep(ConceptCteContext conceptCteContext) { + + List queriesToJoin = new ArrayList<>(); + queriesToJoin.add(conceptCteContext.getPrevious()); + conceptCteContext.allConceptSelects() + .flatMap(sqlSelects -> sqlSelects.getAdditionalPredecessors().stream()) + .forEach(queriesToJoin::add); + + Field primaryColumn = QueryStepJoiner.coalescePrimaryColumns(queriesToJoin); + List mergedSelects = QueryStepJoiner.mergeSelects(queriesToJoin); + Selects selects = Selects.builder() + .primaryColumn(primaryColumn) + .sqlSelects(mergedSelects) + .build(); + + TableLike fromTable = QueryStepJoiner.constructJoinedTable(queriesToJoin, LogicalOperation.AND, conceptCteContext.getConversionContext()); + + return QueryStep.builder() + .selects(selects) + .fromTable(fromTable) + .predecessors(queriesToJoin); + } + + @Override + protected ConceptCteStep cteStep() { + return ConceptCteStep.JOIN_PREDECESSORS; + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java index 9adef30865..c6c3131043 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java @@ -171,24 +171,24 @@ public Field toDateField(String dateExpression) { } @Override - public Field first(Field column, List> orderByColumns) { + public Field first(Field column, List> orderByColumns) { if (orderByColumns.isEmpty()) { orderByColumns = List.of(column); } - return DSL.field(DSL.sql("FIRST_VALUE({0} {1})", column, DSL.orderBy(orderByColumns))); + return DSL.field(DSL.sql("FIRST_VALUE({0} {1})", column, DSL.orderBy(orderByColumns)), column.getType()); } @Override - public Field last(Field column, List> orderByColumns) { + public Field last(Field column, List> orderByColumns) { if (orderByColumns.isEmpty()) { orderByColumns = List.of(column); } - return DSL.field(DSL.sql("LAST_VALUE({0} {1} DESC)", column, DSL.orderBy(orderByColumns))); + return DSL.field(DSL.sql("LAST_VALUE({0} {1} DESC)", column, DSL.orderBy(orderByColumns)), column.getType()); } @Override - public Field random(Field column) { - return DSL.field(DSL.sql("FIRST_VALUE({0} {1})", column, DSL.orderBy(DSL.function("RAND", Object.class)))); + public Field random(Field column) { + return DSL.field(DSL.sql("FIRST_VALUE({0} {1})", column, DSL.orderBy(DSL.function("RAND", Object.class))), column.getType()); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java index ebc2ebd4cb..16d14de28d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java @@ -158,22 +158,22 @@ public Field addDays(Field dateColumn, int amountOfDays) { } @Override - public Field first(Field column, List> orderByColumn) { - return DSL.field(DSL.sql("({0})[1]", DSL.arrayAgg(column))); + public Field first(Field column, List> orderByColumn) { + return DSL.field(DSL.sql("({0})[1]", DSL.arrayAgg(column)), column.getType()); } @Override - public Field last(Field column, List> orderByColumns) { + public Field last(Field column, List> orderByColumns) { String orderByClause = orderByColumns.stream() .map(Field::toString) .collect(Collectors.joining(", ", "ORDER BY ", " DESC")); - return DSL.field(DSL.sql("({0})[1]", DSL.arrayAgg(DSL.field("%s %s".formatted(column, orderByClause))))); + return DSL.field(DSL.sql("({0})[1]", DSL.arrayAgg(DSL.field("%s %s".formatted(column, orderByClause)))), column.getType()); } @Override - public Field random(Field column) { + public Field random(Field column) { WindowSpecificationRowsStep orderByRandomClause = DSL.orderBy(DSL.function("random", Object.class)); - return DSL.field(DSL.sql("({0})[1]", DSL.arrayAgg(DSL.field("%s %s".formatted(column, orderByRandomClause))))); + return DSL.field(DSL.sql("({0})[1]", DSL.arrayAgg(DSL.field("%s %s".formatted(column, orderByRandomClause)))), column.getType()); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index 59b6fd69f6..b93623eb3f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -57,11 +57,11 @@ public interface SqlFunctionProvider { Field addDays(Field dateColumn, int amountOfDays); - Field first(Field field, List> orderByColumn); + Field first(Field field, List> orderByColumn); - Field last(Field column, List> orderByColumns); + Field last(Field column, List> orderByColumns); - Field random(Field column); + Field random(Field column); Condition likeRegex(Field field, String pattern); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java index a7e24ea2fe..432dc9f1e3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java @@ -111,7 +111,7 @@ public static TableLike constructJoinedTable(List queriesToJo return joinedQuery; } - private static Field coalescePrimaryColumns(List querySteps) { + public static Field coalescePrimaryColumns(List querySteps) { List> primaryColumns = querySteps.stream() .map(queryStep -> queryStep.getQualifiedSelects().getPrimaryColumn()) .collect(Collectors.toList()); @@ -119,7 +119,7 @@ private static Field coalescePrimaryColumns(List querySteps) .as(PRIMARY_COLUMN_NAME); } - private static List mergeSelects(List querySteps) { + public static List mergeSelects(List querySteps) { return querySteps.stream() .flatMap(queryStep -> queryStep.getQualifiedSelects().getSqlSelects().stream()) .collect(Collectors.toList()); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelects.java index be42762148..a41300dd3e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelects.java @@ -2,6 +2,7 @@ import java.util.List; +import com.bakdata.conquery.sql.conversion.model.QueryStep; import lombok.Builder; import lombok.Singular; import lombok.Value; @@ -17,4 +18,7 @@ public class SqlSelects { // Empty if only used in aggregation select @Singular List finalSelects; + // Additional predecessors these SqlSelects require + @Singular + List additionalPredecessors; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SumDistinctSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SumDistinctSqlAggregator.java new file mode 100644 index 0000000000..897a6af54a --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SumDistinctSqlAggregator.java @@ -0,0 +1,191 @@ +package com.bakdata.conquery.sql.conversion.model.select; + +import java.math.BigDecimal; +import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.bakdata.conquery.models.common.IRange; +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.datasets.concepts.filters.specific.SumFilter; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.SumSelect; +import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; +import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; +import com.bakdata.conquery.sql.conversion.cqelement.concept.NumberMapUtil; +import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectContext; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.conversion.model.CteStep; +import com.bakdata.conquery.sql.conversion.model.NameGenerator; +import com.bakdata.conquery.sql.conversion.model.QualifyingUtil; +import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.conversion.model.filter.SumCondition; +import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; +import lombok.Value; +import org.jooq.Field; +import org.jooq.impl.DSL; + +@Value +public class SumDistinctSqlAggregator implements SqlAggregator { + + private enum SumDistinctCteStep implements CteStep { + + GROUP_BY_DISTINCT_COLUMNS("grouped_by_distinct_columns", null), + SUM_DISTINCT("sum_distinct", GROUP_BY_DISTINCT_COLUMNS); + + private final String suffix; + private final SumDistinctCteStep predecessor; + + SumDistinctCteStep(String suffix, SumDistinctCteStep predecessor) { + this.suffix = suffix; + this.predecessor = predecessor; + } + + @Override + public String cteName(String nodeLabel) { + return "%s-%s".formatted(nodeLabel, this.suffix); + } + + @Override + public CteStep predecessor() { + return this.predecessor; + } + } + + private static final String ROW_NUMBER_ALIAS = "row_number"; + private static final String SUM_DISTINCT_SUFFIX = "sum_distinct"; + + SqlSelects sqlSelects; + WhereClauses whereClauses; + + public SumDistinctSqlAggregator( + Column sumColumn, + List distinctByColumns, + String alias, + IRange filterValue, + Field primaryColumn, + SqlTables conceptTables, + SqlFunctionProvider functionProvider, + NameGenerator nameGenerator + ) { + // preprocesssing + String rootTable = conceptTables.getRootTable(); + Class numberClass1 = NumberMapUtil.NUMBER_MAP.get(sumColumn.getType()); + ExtractingSqlSelect sumColumnRootSelect = new ExtractingSqlSelect<>(rootTable, sumColumn.getName(), numberClass1); + List> distinctByRootSelects = distinctByColumns.stream() + .map(column -> new ExtractingSqlSelect<>( + rootTable, column.getName(), Object.class) + ) + .toList(); + + // sum column grouped by distinct columns + String predecessor = conceptTables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT); + ExtractingSqlSelect qualifiedRootSelect = sumColumnRootSelect.createAliasedReference(predecessor); + FieldWrapper firstSelect = new FieldWrapper<>(functionProvider.first(qualifiedRootSelect.select(), List.of()).as(alias)); + QueryStep distinctColumnsStep = getGroupByDistinctColumnsStep(alias, primaryColumn, nameGenerator, predecessor, firstSelect, distinctByRootSelects); + + // sum select + Field firstSumColumn = firstSelect.createAliasedReference(distinctColumnsStep.getCteName()).select(); + FieldWrapper distinctSum = new FieldWrapper<>(DSL.sum(firstSumColumn).as(alias)); + + // sum aggregation + QueryStep sumDistinctCte = getSumDistinctStep(alias, primaryColumn, nameGenerator, distinctSum, distinctColumnsStep); + + SqlSelects.SqlSelectsBuilder builder = SqlSelects.builder() + .preprocessingSelect(sumColumnRootSelect) + .preprocessingSelects(distinctByRootSelects) + .additionalPredecessor(sumDistinctCte); + + if (filterValue != null) { + this.sqlSelects = builder.build(); + Field qualifiedSumSelect = distinctSum.createAliasedReference(conceptTables.getPredecessor(ConceptCteStep.AGGREGATION_FILTER)).select(); + SumCondition sumCondition = new SumCondition(qualifiedSumSelect, filterValue); + this.whereClauses = WhereClauses.builder() + .groupFilter(sumCondition) + .build(); + } + else { + ExtractingSqlSelect finalSelect = distinctSum.createAliasedReference(conceptTables.getPredecessor(ConceptCteStep.FINAL)); + this.sqlSelects = builder.finalSelect(finalSelect).build(); + this.whereClauses = WhereClauses.builder().build(); + } + } + + public static SumDistinctSqlAggregator create(SumSelect sumSelect, SelectContext selectContext) { + return new SumDistinctSqlAggregator( + sumSelect.getColumn(), + sumSelect.getDistinctByColumn(), + selectContext.getNameGenerator().selectName(sumSelect), + null, + selectContext.getParentContext().getPrimaryColumn(), + selectContext.getConceptTables(), + selectContext.getParentContext().getSqlDialect().getFunctionProvider(), + selectContext.getNameGenerator() + ); + } + + public static > SumDistinctSqlAggregator create(SumFilter sumFilter, FilterContext filterContext) { + return new SumDistinctSqlAggregator( + sumFilter.getColumn(), + sumFilter.getDistinctByColumn(), + filterContext.getNameGenerator().selectName(sumFilter), + filterContext.getValue(), + filterContext.getParentContext().getPrimaryColumn(), + filterContext.getConceptTables(), + filterContext.getParentContext().getSqlDialect().getFunctionProvider(), + filterContext.getNameGenerator() + ); + } + + private static QueryStep getSumDistinctStep( + String alias, + Field primaryColumn, + NameGenerator nameGenerator, + FieldWrapper distinctSum, + QueryStep distinctColumnsStep + ) { + Field qualifiedPrimaryColumn = QualifyingUtil.qualify(primaryColumn, distinctColumnsStep.getCteName()); + + Selects sumDistinctSelects = Selects.builder() + .primaryColumn(qualifiedPrimaryColumn) + .sqlSelect(distinctSum) + .build(); + + return QueryStep.builder() + .cteName(nameGenerator.cteStepName(SumDistinctCteStep.SUM_DISTINCT, alias)) + .selects(sumDistinctSelects) + .fromTable(QueryStep.toTableLike(distinctColumnsStep.getCteName())) + .predecessors(List.of(distinctColumnsStep)) + .groupBy(List.of(qualifiedPrimaryColumn)) + .build(); + } + + private static QueryStep getGroupByDistinctColumnsStep( + String alias, + Field primaryColumn, + NameGenerator nameGenerator, + String predecessor, + FieldWrapper firstSelect, + List> distinctByRootSelects + ) { + Field qualifiedPrimaryColumn = QualifyingUtil.qualify(primaryColumn, predecessor); + Selects selects = Selects.builder() + .primaryColumn(qualifiedPrimaryColumn) + .sqlSelect(firstSelect) + .build(); + + List> groupByFields = Stream.concat( + Stream.of(qualifiedPrimaryColumn), + distinctByRootSelects.stream().map(sqlSelect -> sqlSelect.createAliasedReference(predecessor)).map(ExtractingSqlSelect::select) + ).collect(Collectors.toList()); + + return QueryStep.builder() + .cteName(nameGenerator.cteStepName(SumDistinctCteStep.GROUP_BY_DISTINCT_COLUMNS, alias)) + .selects(selects) + .fromTable(QueryStep.toTableLike(predecessor)) + .groupBy(groupByFields) + .build(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SumSqlAggregator.java index fb5d5822e3..7037bfb096 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SumSqlAggregator.java @@ -1,6 +1,8 @@ package com.bakdata.conquery.sql.conversion.model.select; import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; import com.bakdata.conquery.models.common.IRange; import com.bakdata.conquery.models.datasets.Column; @@ -25,24 +27,42 @@ public class SumSqlAggregator implements SqlAggregator { private SumSqlAggregator( Column sumColumn, + Column subtractColumn, String alias, SqlTables conceptTables, IRange filterValue ) { Class numberClass = NumberMapUtil.NUMBER_MAP.get(sumColumn.getType()); + List> preprocessingSelects = new ArrayList<>(); + ExtractingSqlSelect rootSelect = new ExtractingSqlSelect<>( conceptTables.getPredecessor(ConceptCteStep.PREPROCESSING), sumColumn.getName(), numberClass ); + preprocessingSelects.add(rootSelect); + + String aggregationSelectPredecessor = conceptTables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT); + Field sumField; + if (subtractColumn != null) { + ExtractingSqlSelect subtractColumnRootSelect = new ExtractingSqlSelect<>( + conceptTables.getPredecessor(ConceptCteStep.PREPROCESSING), + subtractColumn.getName(), + numberClass + ); + preprocessingSelects.add(subtractColumnRootSelect); - Field - qualifiedRootSelect = - rootSelect.createAliasedReference(conceptTables.getPredecessor(ConceptCteStep.AGGREGATION_SELECT)).select(); - FieldWrapper sumGroupBy = new FieldWrapper<>(DSL.sum(qualifiedRootSelect).as(alias), sumColumn.getName()); + Field qualifiedRootSelect = rootSelect.createAliasedReference(aggregationSelectPredecessor).select(); + Field qualifiedSubtractRootSelect = subtractColumnRootSelect.createAliasedReference(aggregationSelectPredecessor).select(); + sumField = qualifiedRootSelect.minus(qualifiedSubtractRootSelect); + } + else { + sumField = rootSelect.createAliasedReference(aggregationSelectPredecessor).select(); + } + FieldWrapper sumGroupBy = new FieldWrapper<>(DSL.sum(sumField).as(alias), sumColumn.getName()); SqlSelects.SqlSelectsBuilder builder = SqlSelects.builder() - .preprocessingSelect(rootSelect) + .preprocessingSelects(preprocessingSelects) .aggregationSelect(sumGroupBy); if (filterValue == null) { @@ -61,18 +81,23 @@ private SumSqlAggregator( } public static SumSqlAggregator create(SumSelect sumSelect, SelectContext selectContext) { - Column sumColumn = sumSelect.getColumn(); - String alias = selectContext.getNameGenerator().selectName(sumSelect); - return new SumSqlAggregator(sumColumn, alias, selectContext.getConceptTables(), null); + return new SumSqlAggregator( + sumSelect.getColumn(), + sumSelect.getSubtractColumn(), + selectContext.getNameGenerator().selectName(sumSelect), + selectContext.getConceptTables(), + null + ); } - public static SumSqlAggregator create( - SumFilter> sumFilter, - FilterContext> filterContext - ) { - Column sumColumn = sumFilter.getColumn(); - String alias = filterContext.getNameGenerator().selectName(sumFilter); - return new SumSqlAggregator(sumColumn, alias, filterContext.getConceptTables(), filterContext.getValue()); + public static > SqlAggregator create(SumFilter sumFilter, FilterContext filterContext) { + return new SumSqlAggregator( + sumFilter.getColumn(), + sumFilter.getSubtractColumn(), + filterContext.getNameGenerator().selectName(sumFilter), + filterContext.getConceptTables(), + filterContext.getValue() + ); } } diff --git a/backend/src/test/resources/tests/sql/filter/sum/diffsum/content.csv b/backend/src/test/resources/tests/sql/filter/sum/diffsum/content.csv new file mode 100644 index 0000000000..f092c3cb60 --- /dev/null +++ b/backend/src/test/resources/tests/sql/filter/sum/diffsum/content.csv @@ -0,0 +1,22 @@ +pid,plus,minus +1,100,50 +2,100,0 +3,200,0 +4,250,50 +5,250,0 +6,150,51 +7,150,0 +8,100,0 +8,100,0 +8,100,0 +9,150,100 +9,150,100 +9,150,100 +10,200,200 +10,200,0 +11,50,0 +11,50,0 +11,50,0 +12,100,99 +12,100,99 +12,100,99 diff --git a/backend/src/test/resources/tests/sql/filter/sum/diffsum/diffsum.spec.json b/backend/src/test/resources/tests/sql/filter/sum/diffsum/diffsum.spec.json new file mode 100644 index 0000000000..954df631ec --- /dev/null +++ b/backend/src/test/resources/tests/sql/filter/sum/diffsum/diffsum.spec.json @@ -0,0 +1,75 @@ +{ + "label": "SUM with subtract column filter", + "type": "SQL_QUERY_TEST", + "expectedCsv": "tests/sql/filter/sum/diffsum/expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "ids": [ + "sum" + ], + "type": "CONCEPT", + "label": "sum", + "tables": [ + { + "id": "sum.sum_connector", + "filters": [ + { + "filter": "sum.sum_connector.diffsum_filter", + "type": "INTEGER_RANGE", + "value": { + "min": 100, + "max": 200 + } + } + ] + } + ] + } + ] + } + }, + "concepts": [ + { + "label": "sum", + "type": "TREE", + "connectors": [ + { + "label": "sum_connector", + "table": "table1", + "filters": { + "type": "SUM", + "name": "diffsum_filter", + "subtractColumn": "table1.minus", + "column": "table1.plus" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/filter/sum/diffsum/content.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "plus", + "type": "INTEGER" + }, + { + "name": "minus", + "type": "INTEGER" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/filter/sum/diffsum/expected.csv b/backend/src/test/resources/tests/sql/filter/sum/diffsum/expected.csv new file mode 100644 index 0000000000..acc00916b5 --- /dev/null +++ b/backend/src/test/resources/tests/sql/filter/sum/diffsum/expected.csv @@ -0,0 +1,8 @@ +result,dates +2,{} +3,{} +4,{} +7,{} +9,{} +10,{} +11,{} diff --git a/backend/src/test/resources/tests/sql/filter/sum/distinct/content.csv b/backend/src/test/resources/tests/sql/filter/sum/distinct/content.csv new file mode 100644 index 0000000000..12626d3707 --- /dev/null +++ b/backend/src/test/resources/tests/sql/filter/sum/distinct/content.csv @@ -0,0 +1,9 @@ +pid,k1,k2,value +1,a,b,1 +1,a,b,1 +2,a,,0 +3,a,b,1 +3,a,c,1 +4,a,b,1 +4,b,a,1 +4,c,a,1 diff --git a/backend/src/test/resources/tests/sql/filter/sum/distinct/distinct-sum.spec.json b/backend/src/test/resources/tests/sql/filter/sum/distinct/distinct-sum.spec.json new file mode 100644 index 0000000000..fdb030fb8d --- /dev/null +++ b/backend/src/test/resources/tests/sql/filter/sum/distinct/distinct-sum.spec.json @@ -0,0 +1,76 @@ +{ + "label": "SUM filter with DISTINCT BY columns", + "type": "SQL_QUERY_TEST", + "expectedCsv": "tests/sql/filter/sum/distinct/expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "ids": [ + "sum" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "sum.sum_connector", + "filters": [ + { + "filter": "sum.sum_connector.sum_distinct_filter", + "type": "INTEGER_RANGE", + "value": { + "min": 1, + "max": 2 + } + } + ] + } + ] + } + }, + "concepts": [ + { + "name": "sum", + "type": "TREE", + "connectors": [ + { + "name": "sum_connector", + "table": "table", + "filters": { + "name": "sum_distinct_filter", + "type": "SUM", + "column": "table.value", + "distinctByColumn": [ + "table.k1", + "table.k2" + ] + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/filter/sum/distinct/content.csv", + "name": "table", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "k1", + "type": "STRING" + }, + { + "name": "k2", + "type": "STRING" + }, + { + "name": "value", + "type": "INTEGER" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/filter/sum/distinct/expected.csv b/backend/src/test/resources/tests/sql/filter/sum/distinct/expected.csv new file mode 100644 index 0000000000..f4a54b0956 --- /dev/null +++ b/backend/src/test/resources/tests/sql/filter/sum/distinct/expected.csv @@ -0,0 +1,3 @@ +result,dates +1,{} +3,{} diff --git a/backend/src/test/resources/tests/sql/selects/sum/diffsum/content.csv b/backend/src/test/resources/tests/sql/selects/sum/diffsum/content.csv new file mode 100644 index 0000000000..f092c3cb60 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/sum/diffsum/content.csv @@ -0,0 +1,22 @@ +pid,plus,minus +1,100,50 +2,100,0 +3,200,0 +4,250,50 +5,250,0 +6,150,51 +7,150,0 +8,100,0 +8,100,0 +8,100,0 +9,150,100 +9,150,100 +9,150,100 +10,200,200 +10,200,0 +11,50,0 +11,50,0 +11,50,0 +12,100,99 +12,100,99 +12,100,99 diff --git a/backend/src/test/resources/tests/sql/selects/sum/diffsum/diffsum.spec.json b/backend/src/test/resources/tests/sql/selects/sum/diffsum/diffsum.spec.json new file mode 100644 index 0000000000..8bf6e1bcc1 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/sum/diffsum/diffsum.spec.json @@ -0,0 +1,68 @@ +{ + "label": "SUM with subtract column select", + "type": "SQL_QUERY_TEST", + "expectedCsv": "tests/sql/selects/sum/diffsum/expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "ids": [ + "sum" + ], + "type": "CONCEPT", + "label": "sum", + "tables": [ + { + "id": "sum.sum_connector", + "selects": [ + "sum.sum_connector.diffsum_select" + ] + } + ] + } + ] + } + }, + "concepts": [ + { + "label": "sum", + "type": "TREE", + "connectors": [ + { + "label": "sum_connector", + "table": "table1", + "selects": { + "type": "SUM", + "name": "diffsum_select", + "subtractColumn": "table1.minus", + "column": "table1.plus" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/selects/sum/diffsum/content.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "plus", + "type": "INTEGER" + }, + { + "name": "minus", + "type": "INTEGER" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/selects/sum/diffsum/expected.csv b/backend/src/test/resources/tests/sql/selects/sum/diffsum/expected.csv new file mode 100644 index 0000000000..7ddc949d45 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/sum/diffsum/expected.csv @@ -0,0 +1,13 @@ +result,dates,sum diffsum_select +2,{},100 +9,{},150 +4,{},200 +11,{},150 +12,{},3 +10,{},200 +7,{},150 +3,{},200 +6,{},99 +5,{},250 +1,{},50 +8,{},300 diff --git a/backend/src/test/resources/tests/sql/selects/sum/distinct/content.csv b/backend/src/test/resources/tests/sql/selects/sum/distinct/content.csv new file mode 100644 index 0000000000..3b47786fac --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/sum/distinct/content.csv @@ -0,0 +1,8 @@ +pid,k1,k2,value +1,a,b,1 +1,a,b,1 +2,a,,1 +3,a,b,1 +3,a,c,1 +4,a,b,1 +4,b,a,1 diff --git a/backend/src/test/resources/tests/sql/selects/sum/distinct/distinct-sum.spec.json b/backend/src/test/resources/tests/sql/selects/sum/distinct/distinct-sum.spec.json new file mode 100644 index 0000000000..4ee1faff0a --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/sum/distinct/distinct-sum.spec.json @@ -0,0 +1,69 @@ +{ + "label": "SUM select with DISTINCT BY columns", + "type": "SQL_QUERY_TEST", + "expectedCsv": "tests/sql/selects/sum/distinct/expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "ids": [ + "concept" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "concept.connector", + "selects": [ + "concept.connector.sum_distinct_select" + ] + } + ] + } + }, + "concepts": [ + { + "name": "concept", + "type": "TREE", + "connectors": [ + { + "name": "connector", + "table": "table", + "selects": { + "name": "sum_distinct_select", + "type": "SUM", + "column": "table.value", + "distinctByColumn": [ + "table.k1", + "table.k2" + ] + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/selects/sum/distinct/content.csv", + "name": "table", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "k1", + "type": "STRING" + }, + { + "name": "k2", + "type": "STRING" + }, + { + "name": "value", + "type": "INTEGER" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/selects/sum/distinct/expected.csv b/backend/src/test/resources/tests/sql/selects/sum/distinct/expected.csv new file mode 100644 index 0000000000..8bfff145b6 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/sum/distinct/expected.csv @@ -0,0 +1,5 @@ +result,dates,concept sum_distinct_select +1,{},1 +2,{},1 +3,{},2 +4,{},2 From 811656ca93d21af19c13079817214591210dfafa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 16 Jan 2024 20:35:30 +0000 Subject: [PATCH 3/4] Bump org.apache.shiro:shiro-core from 1.10.1 to 1.13.0 in /backend Bumps [org.apache.shiro:shiro-core](https://github.com/apache/shiro) from 1.10.1 to 1.13.0. - [Release notes](https://github.com/apache/shiro/releases) - [Changelog](https://github.com/apache/shiro/blob/main/RELEASE-NOTES) - [Commits](https://github.com/apache/shiro/compare/shiro-root-1.10.1...shiro-root-1.13.0) --- updated-dependencies: - dependency-name: org.apache.shiro:shiro-core dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- backend/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pom.xml b/backend/pom.xml index 5f4b400d24..4cdceb0364 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -207,7 +207,7 @@ org.apache.shiro shiro-core - 1.10.1 + 1.13.0 From 572a44243e005657f687ca1f6bde5a08e4a1d3a4 Mon Sep 17 00:00:00 2001 From: Jonas Arnhold Date: Thu, 18 Jan 2024 11:06:38 +0100 Subject: [PATCH 4/4] Rework tests to easily switch between Worker and SQL mode (#3266) --- .../integration/ConqueryIntegrationTests.java | 9 +- .../conquery/integration/IntegrationTest.java | 4 +- .../integration/IntegrationTests.java | 104 ++++++++---------- .../integration/json/ConqueryTestSpec.java | 14 +++ .../conquery/integration/json/FormTest.java | 26 +---- .../integration/json/JsonIntegrationTest.java | 93 +++------------- .../conquery/integration/json/QueryTest.java | 30 +---- .../integration/json/SqlQueryTest.java | 93 ---------------- .../conquery/integration/json/SqlSpec.java | 28 +++++ .../integration/json/SqlTestDataImporter.java | 56 ++++++++++ .../integration/json/TestDataImporter.java | 31 ++++++ .../json/WorkerTestDataImporter.java | 82 ++++++++++++++ .../integration/json/filter/FilterTest.java | 36 +----- .../sql/dialect/HanaSqlIntegrationTests.java | 12 +- .../dialect/PostgreSqlIntegrationTests.java | 13 ++- .../tests/ProgrammaticIntegrationTest.java | 14 ++- .../util/support/StandaloneSupport.java | 7 ++ .../conquery/util/support/TestConquery.java | 12 +- .../tests/sql/and/different_concept/and.json | 5 +- .../and/same_concept/and_same_concept.json | 5 +- .../date_restriction_date_column.json | 5 +- .../date_restriction_no_validity_date.json | 5 +- .../only_max/only_max.spec.json | 5 +- .../only_min/only_min.spec.json | 5 +- .../daterange_column.spec.json | 11 +- .../date_restriction_simple_date.json | 5 +- .../big_multi_select.spec.json | 5 +- .../tests/sql/filter/count/count.json | 5 +- .../filter/count_distinct/count_distinct.json | 5 +- .../centuries/centuries.spec.json | 5 +- .../days_with_date_restriction.spec.json | 5 +- .../days_without_date_restriction.json | 5 +- .../date_distance/decades/decades.spec.json | 5 +- .../date_distance/months/months.spec.json | 5 +- .../date_distance/years/years.spec.json | 5 +- .../multi_select/multi_select.spec.json | 5 +- .../tests/sql/filter/number/number.spec.json | 5 +- .../integer_range/number_money.spec.json | 5 +- .../real_range/number_money.spec.json | 5 +- .../number_only_max/number_only_max.spec.json | 5 +- .../number_only_min/number_only_min.spec.json | 5 +- .../single_select/single_select.spec.json | 5 +- .../sql/filter/sum/diffsum/diffsum.spec.json | 2 +- .../sum/distinct/distinct-sum.spec.json | 2 +- .../resources/tests/sql/filter/sum/sum.json | 5 +- .../resources/tests/sql/not/not.spec.json | 5 +- .../sql/or/different_concept/or.spec.json | 5 +- .../or/same_concept/or_same_concept.spec.json | 5 +- .../tests/sql/selects/count/count.json | 5 +- .../count_distinct/count_distinct.json | 5 +- .../centuries/centuries.spec.json | 5 +- .../days_with_date_restriction.spec.json | 5 +- .../days_without_date_restriction.json | 5 +- .../date_distance/decades/decades.spec.json | 5 +- .../date_distance/months/months.spec.json | 5 +- .../date_distance/years/years.spec.json | 5 +- .../exists_with_other_selects.spec.json | 5 +- .../exists/single_exists/exists.spec.json | 5 +- .../selects/first/date/number_date.spec.json | 5 +- .../first/money/number_money.spec.json | 5 +- .../sql/selects/first/number/first.spec.json | 5 +- .../tests/sql/selects/last/last.spec.json | 5 +- .../tests/sql/selects/random/random.spec.json | 5 +- .../same_select_2_times/same_select.json | 5 +- .../sql/selects/sum/diffsum/diffsum.spec.json | 2 +- .../sum/distinct/distinct-sum.spec.json | 2 +- .../resources/tests/sql/selects/sum/sum.json | 5 +- .../aggregation/block/and/none.spec.json | 5 +- .../aggregation/block/not/negate.spec.json | 5 +- .../aggregation/intersect/intersect.spec.json | 5 +- .../aggregation/merge/merge.spec.json | 5 +- .../aggregation/negate/negate.spec.json | 5 +- .../no_validity_date.spec.json | 5 +- .../only_1_date/only_1_date.spec.json | 5 +- .../default/validity_date_default.json | 5 +- .../validity_date_excluded.json | 5 +- .../multirange/multi_range.spec.json | 5 +- .../tests/sql/tree/nested/nested.spec.json | 5 +- .../tree/prefix_range/prefix_range.spec.json | 5 +- .../tests/sql/tree/simple/simple.spec.json | 5 +- 80 files changed, 572 insertions(+), 396 deletions(-) delete mode 100644 backend/src/test/java/com/bakdata/conquery/integration/json/SqlQueryTest.java create mode 100644 backend/src/test/java/com/bakdata/conquery/integration/json/SqlSpec.java create mode 100644 backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java create mode 100644 backend/src/test/java/com/bakdata/conquery/integration/json/TestDataImporter.java create mode 100644 backend/src/test/java/com/bakdata/conquery/integration/json/WorkerTestDataImporter.java diff --git a/backend/src/test/java/com/bakdata/conquery/integration/ConqueryIntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/ConqueryIntegrationTests.java index 51de9a0557..7c643bb4cd 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/ConqueryIntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/ConqueryIntegrationTests.java @@ -4,6 +4,9 @@ import java.util.stream.Stream; import com.bakdata.conquery.TestTags; +import com.bakdata.conquery.integration.json.TestDataImporter; +import com.bakdata.conquery.integration.json.WorkerTestDataImporter; +import com.bakdata.conquery.util.support.StandaloneSupport; import org.junit.jupiter.api.DynamicNode; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.TestFactory; @@ -11,6 +14,7 @@ public class ConqueryIntegrationTests extends IntegrationTests { public static final String DEFAULT_SQL_TEST_ROOT = "tests/sql/"; + public static final TestDataImporter WORKER_TEST_DATA_IMPORTER = new WorkerTestDataImporter(); public ConqueryIntegrationTests() { super("tests/", "com.bakdata.conquery.integration"); @@ -23,11 +27,10 @@ public List jsonTests() { return super.jsonTests(); } - @Override @TestFactory @Tag(TestTags.INTEGRATION_PROGRAMMATIC) - public Stream programmaticTests() { - return super.programmaticTests(); + public Stream runProgrammaticTests() { + return super.programmaticTests(WORKER_TEST_DATA_IMPORTER, StandaloneSupport.Mode.WORKER); } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTest.java b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTest.java index 3517cb2a6b..286c247cea 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTest.java @@ -3,6 +3,7 @@ import java.io.File; import java.util.Map; +import com.bakdata.conquery.integration.json.TestDataImporter; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.util.io.Cloner; import com.bakdata.conquery.util.io.ConqueryMDC; @@ -62,6 +63,7 @@ final class Wrapper implements Executable { private final String name; private final IntegrationTests integrationTests; private final IntegrationTest test; + private final TestDataImporter testImporter; @Override public void execute() throws Throwable { @@ -75,7 +77,7 @@ public void execute() throws Throwable { final ConqueryConfig clonedConfig = Cloner.clone(integrationTests.getConfig(), Map.of(), IntegrationTests.MAPPER); final ConqueryConfig overridenConfig = test.overrideConfig(clonedConfig, integrationTests.getWorkDir()); - final TestConquery testConquery = integrationTests.getCachedConqueryInstance(integrationTests.getWorkDir(), overridenConfig); + final TestConquery testConquery = integrationTests.getCachedConqueryInstance(integrationTests.getWorkDir(), overridenConfig, testImporter); try { testConquery.beforeEach(); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java index 8bed58f836..077722c747 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java @@ -21,14 +21,17 @@ import com.bakdata.conquery.TestTags; import com.bakdata.conquery.integration.json.JsonIntegrationTest; -import com.bakdata.conquery.integration.sql.dialect.TestSqlDialect; +import com.bakdata.conquery.integration.json.TestDataImporter; +import com.bakdata.conquery.integration.json.WorkerTestDataImporter; import com.bakdata.conquery.integration.tests.ProgrammaticIntegrationTest; import com.bakdata.conquery.io.cps.CPSTypeIdResolver; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.config.Dialect; import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.util.support.ConfigOverride; +import com.bakdata.conquery.util.support.StandaloneSupport; import com.bakdata.conquery.util.support.TestConquery; import com.codahale.metrics.SharedMetricRegistries; import com.fasterxml.jackson.databind.ObjectMapper; @@ -88,25 +91,15 @@ public IntegrationTests(String defaultTestRoot, String defaultTestRootPackage) { } public List jsonTests() { + TestDataImporter testImporter = new WorkerTestDataImporter(); final String testRoot = Objects.requireNonNullElse(System.getenv(TestTags.TEST_DIRECTORY_ENVIRONMENT_VARIABLE), defaultTestRoot); ResourceTree tree = scanForResources(testRoot, JSON_TEST_PATTERN); - - // collect tests from directory - if (tree.getChildren().isEmpty()) { - throw new RuntimeException("Could not find tests in " + testRoot); - } - final ResourceTree reduced = tree.reduce(); - - if (reduced.getChildren().isEmpty()) { - return Collections.singletonList(collectTests(reduced)); - } - return reduced.getChildren().values().stream() - .map(this::collectTests) - .collect(Collectors.toList()); + Dialect dialect = null; + return collectTestTree(tree, testRoot, testImporter, dialect); } @SneakyThrows - public Stream programmaticTests() { + public Stream programmaticTests(TestDataImporter testImporter, StandaloneSupport.Mode mode) { String regexFilter = System.getenv(TestTags.TEST_PROGRAMMATIC_REGEX_FILTER); List> programmatic = CPSTypeIdResolver.SCAN_RESULT.getClassesImplementing(ProgrammaticIntegrationTest.class.getName()) @@ -137,61 +130,62 @@ public Stream programmaticTests() { throw new RuntimeException(e); } }) - .map(this::createDynamicProgrammaticTestNode); + .filter(test -> test.isEnabled(mode)) + .map(programmaticIntegrationTest -> createDynamicProgrammaticTestNode(programmaticIntegrationTest, testImporter)); + } + + @SneakyThrows + public Stream sqlProgrammaticTests(SqlConnectorConfig sqlConfig, TestDataImporter testDataImporter) { + this.config.setSqlConnectorConfig(sqlConfig); + return programmaticTests(testDataImporter, StandaloneSupport.Mode.SQL); } @SneakyThrows - public List sqlTests(TestSqlDialect sqlDialect, SqlConnectorConfig sqlConfig) { + public List sqlQueryTests(SqlConnectorConfig sqlConfig, TestDataImporter testDataImporter) { this.config.setSqlConnectorConfig(sqlConfig); final String testRoot = Objects.requireNonNullElse(System.getenv(TestTags.SQL_BACKEND_TEST_DIRECTORY_ENVIRONMENT_VARIABLE), defaultTestRoot); ResourceTree tree = scanForResources(testRoot, SQL_TEST_PATTERN); - // collect tests from directory + return collectTestTree(tree, testRoot, testDataImporter, sqlConfig.getDialect()); + } + + private List collectTestTree(ResourceTree tree, String testRoot, TestDataImporter testImporter, Dialect sqlDialect) { if (tree.getChildren().isEmpty()) { - log.warn("Could not find tests in {}", testRoot); - return Collections.emptyList(); + throw new RuntimeException("Could not find tests in " + testRoot); } final ResourceTree reduced = tree.reduce(); + if (reduced.getChildren().isEmpty()) { - return collectSqlTests(reduced, sqlDialect, sqlConfig).stream().toList(); + return Collections.singletonList(collectTests(reduced, testImporter, sqlDialect)); } return reduced.getChildren().values().stream() - .flatMap(child -> collectSqlTests(child, sqlDialect, sqlConfig).stream()) + .map(currentDir -> collectTests(currentDir, testImporter, sqlDialect)) .collect(Collectors.toList()); } - private DynamicTest createDynamicProgrammaticTestNode(ProgrammaticIntegrationTest test) { + private DynamicTest createDynamicProgrammaticTestNode(ProgrammaticIntegrationTest test, TestDataImporter testImporter) { return DynamicTest.dynamicTest( test.getClass().getSimpleName(), //classpath URI URI.create("classpath:/" + test.getClass().getName().replace('.', '/') + ".java"), - new IntegrationTest.Wrapper(test.getClass().getSimpleName(), this, test) + new IntegrationTest.Wrapper(test.getClass().getSimpleName(), this, test, testImporter) ); } - private DynamicNode collectTests(ResourceTree currentDir) { + private DynamicNode collectTests(ResourceTree currentDir, TestDataImporter testImporter, Dialect sqlDialect) { if (currentDir.getValue() != null) { - return readTest(currentDir.getValue(), currentDir.getName()); + Optional dynamicTest = readTest(currentDir.getValue(), currentDir.getName(), testImporter, sqlDialect); + if (dynamicTest.isPresent()) { + return dynamicTest.get(); + } } List list = new ArrayList<>(); for (ResourceTree child : currentDir.getChildren().values()) { - list.add(collectTests(child)); + list.add(collectTests(child, testImporter, sqlDialect)); } return toDynamicContainer(currentDir, list); } - private Optional collectSqlTests(ResourceTree currentDir, TestSqlDialect sqlDialect, SqlConnectorConfig sqlConfig) { - if (currentDir.getValue() != null) { - return readSqlTest(currentDir.getValue(), currentDir.getName(), sqlDialect, sqlConfig); - } - List list = new ArrayList<>(); - for (ResourceTree child : currentDir.getChildren().values()) { - Optional sqlTest = collectSqlTests(child, sqlDialect, sqlConfig); - sqlTest.ifPresent(list::add); - } - return Optional.of(toDynamicContainer(currentDir, list)); - } - private static DynamicContainer toDynamicContainer(ResourceTree currentDir, List list) { list.sort(Comparator.comparing(DynamicNode::getDisplayName)); return dynamicContainer( @@ -201,26 +195,13 @@ private static DynamicContainer toDynamicContainer(ResourceTree currentDir, List ); } - private DynamicTest readTest(Resource resource, String name) { - try (InputStream in = resource.open()) { - JsonIntegrationTest test = JsonIntegrationTest.Distributed.create(in); - return wrapTest(resource, name, test); - } - catch (Exception e) { - return wrapError(resource, name, e); - } - finally { - resource.close(); - } - } - - private Optional readSqlTest(Resource resource, String name, TestSqlDialect sqlDialect, SqlConnectorConfig sqlConnectorConfig) { + private Optional readTest(Resource resource, String name, TestDataImporter testImporter, Dialect sqlDialect) { try (InputStream in = resource.open()) { - JsonIntegrationTest.Sql test = JsonIntegrationTest.Sql.create(in, sqlDialect, sqlConnectorConfig); - if (!test.isAllowedTest(sqlConnectorConfig.getDialect())) { - return Optional.empty(); + JsonIntegrationTest test = new JsonIntegrationTest(in); + if (test.getTestSpec().isEnabled(sqlDialect)) { + return Optional.of(wrapTest(resource, name, test, testImporter)); } - return Optional.of(wrapTest(resource, name, test)); + return Optional.empty(); } catch (Exception e) { return Optional.of(wrapError(resource, name, e)); @@ -240,7 +221,7 @@ private static DynamicTest wrapError(Resource resource, String name, Exception e ); } - private DynamicTest wrapTest(Resource resource, String name, JsonIntegrationTest test) { + private DynamicTest wrapTest(Resource resource, String name, JsonIntegrationTest test, TestDataImporter testImporter) { String testLabel = Optional.ofNullable(test.getTestSpec().getLabel()) // If no label was defined use the filename part before the first dot .orElse(name.split("\\.", 1)[0]); @@ -257,13 +238,14 @@ private DynamicTest wrapTest(Resource resource, String name, JsonIntegrationTest new IntegrationTest.Wrapper( testLabel, this, - test + test, + testImporter ) ); } @SneakyThrows - public synchronized TestConquery getCachedConqueryInstance(File workDir, ConqueryConfig conf) { + public synchronized TestConquery getCachedConqueryInstance(File workDir, ConqueryConfig conf, TestDataImporter testDataImporter) { // This should be fast enough and a stable comparison String confString = CONFIG_WRITER.writeValueAsString(conf); if (!reusedInstances.containsKey(confString)) { @@ -271,7 +253,7 @@ public synchronized TestConquery getCachedConqueryInstance(File workDir, Conquer // We do it here so the config "hash" is not influenced by the port settings ConfigOverride.configureRandomPorts(conf); log.trace("Creating a new test conquery instance for test {}", conf); - TestConquery conquery = new TestConquery(workDir, conf); + TestConquery conquery = new TestConquery(workDir, conf, testDataImporter); reusedInstances.put(confString, conquery); conquery.beforeAll(); } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java index fc93697703..353fa2db06 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java @@ -14,6 +14,7 @@ import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.config.Dialect; import com.bakdata.conquery.models.exceptions.JSONException; import com.bakdata.conquery.models.exceptions.ValidatorHelper; import com.bakdata.conquery.models.identifiable.Identifiable; @@ -43,11 +44,21 @@ public abstract class ConqueryTestSpec { @Setter private String label; + @Setter + @Getter + @Nullable + private String description; + @Setter @Getter @Nullable private ConqueryConfig config; + @Setter + @Getter + @Nullable + SqlSpec sqlSpec; + public ConqueryConfig overrideConfig(ConqueryConfig config) { if (getConfig() != null) { @@ -144,6 +155,9 @@ public static List parseSubTreeList(TestSupport support, ArrayNode node, return result; } + public boolean isEnabled(Dialect sqlDialect) { + return sqlSpec == null || sqlSpec.isEnabled() && sqlSpec.isAllowedTest(sqlDialect); + } /** * Replaces occurrences of the string "${dataset}" with the id of the current dataset of the {@link StandaloneSupport}. diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java index 12fd1c29ea..1d16d4486b 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.integration.json; -import static com.bakdata.conquery.integration.common.LoadingUtil.importIdMapping; -import static com.bakdata.conquery.integration.common.LoadingUtil.importSecondaryIds; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.fail; @@ -85,31 +83,9 @@ public boolean isWithConcepts() { @Override public void importRequiredData(StandaloneSupport support) throws Exception { - importSecondaryIds(support, content.getSecondaryIds()); - support.waitUntilWorkDone(); - - LoadingUtil.importTables(support, content.getTables(), content.isAutoConcept()); - support.waitUntilWorkDone(); - log.info("{} IMPORT TABLES", getLabel()); - - importConcepts(support, rawConcepts); - support.waitUntilWorkDone(); - log.info("{} IMPORT CONCEPTS", getLabel()); - - LoadingUtil.importTableContents(support, content.getTables()); - support.waitUntilWorkDone(); - - importIdMapping(support, content); - support.waitUntilWorkDone(); - - log.info("{} IMPORT TABLE CONTENTS", getLabel()); - LoadingUtil.importPreviousQueries(support, content, support.getTestUser()); - - support.waitUntilWorkDone(); - + support.getTestImporter().importFormTestData(support, this); log.info("{} PARSE JSON FORM DESCRIPTION", getLabel()); form = parseForm(support); - } @Override diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/JsonIntegrationTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/JsonIntegrationTest.java index a59541e877..29fbc9c444 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/JsonIntegrationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/JsonIntegrationTest.java @@ -7,17 +7,11 @@ import javax.validation.Validator; import com.bakdata.conquery.integration.IntegrationTest; -import com.bakdata.conquery.integration.sql.CsvTableImporter; -import com.bakdata.conquery.integration.sql.dialect.TestSqlDialect; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.config.Dialect; -import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.exceptions.ValidatorHelper; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.bakdata.conquery.models.messages.namespaces.specific.UpdateMatchingStatsMessage; -import com.bakdata.conquery.models.worker.DistributedNamespace; import com.bakdata.conquery.util.support.StandaloneSupport; import com.fasterxml.jackson.databind.ObjectReader; import io.dropwizard.jersey.validation.Validators; @@ -29,14 +23,28 @@ @Getter @Slf4j @RequiredArgsConstructor -public abstract class JsonIntegrationTest extends IntegrationTest.Simple { +public class JsonIntegrationTest extends IntegrationTest.Simple { + private final ConqueryTestSpec testSpec; public static final ObjectReader TEST_SPEC_READER = Jackson.MAPPER.readerFor(ConqueryTestSpec.class); public static final Validator VALIDATOR = Validators.newValidator(); - public abstract void execute(StandaloneSupport conquery) throws Exception; + public JsonIntegrationTest(InputStream in) throws IOException { + this.testSpec = TEST_SPEC_READER.readValue(in.readAllBytes()); + in.close(); + } + + @Override + public void execute(StandaloneSupport conquery) throws Exception { + ValidatorHelper.failOnError(log, VALIDATOR.validate(testSpec)); + testSpec.importRequiredData(conquery); + testSpec.executeTest(conquery); + } - public abstract ConqueryTestSpec getTestSpec(); + @Override + public ConqueryConfig overrideConfig(final ConqueryConfig conf, final File workDir) { + return getTestSpec().overrideConfig(conf); + } public static T readJson(DatasetId dataset, String json) throws IOException { return readJson(dataset, json, TEST_SPEC_READER); @@ -56,71 +64,4 @@ private static T readJson(DatasetId dataset, String return jsonReader.readValue(json); } - @Override - public ConqueryConfig overrideConfig(final ConqueryConfig conf, final File workDir) { - return getTestSpec().overrideConfig(conf); - } - - @Getter - public static class Distributed extends JsonIntegrationTest { - - private final ConqueryTestSpec testSpec; - - private Distributed(ConqueryTestSpec testSpec) { - this.testSpec = testSpec; - } - - public static Distributed create(InputStream in) throws IOException { - ConqueryTestSpec testSpec = TEST_SPEC_READER.readValue(in.readAllBytes()); - in.close(); - return new Distributed(testSpec); - } - - @Override - public void execute(StandaloneSupport conquery) throws Exception { - ValidatorHelper.failOnError(log, VALIDATOR.validate(testSpec)); - - testSpec.importRequiredData(conquery); - - //ensure the metadata is collected - DistributedNamespace namespace = (DistributedNamespace) conquery.getNamespace(); - namespace.getWorkerHandler().sendToAll(new UpdateMatchingStatsMessage(conquery.getNamespace().getStorage().getAllConcepts())); - conquery.waitUntilWorkDone(); - - testSpec.executeTest(conquery); - } - - } - - @Getter - public static class Sql extends JsonIntegrationTest { - - private final SqlQueryTest testSpec; - - private Sql(SqlQueryTest testSpec, TestSqlDialect sqlDialect, SqlConnectorConfig sqlConnectorConfig) { - this.testSpec = testSpec; - this.testSpec.setTableImporter(new CsvTableImporter(sqlDialect.getDSLContext(), sqlDialect, sqlConnectorConfig)); - } - - public static Sql create(InputStream in, TestSqlDialect sqlDialect, SqlConnectorConfig sqlConnectorConfig) throws IOException { - SqlQueryTest testSpec = TEST_SPEC_READER.readValue(in.readAllBytes()); - in.close(); - return new Sql(testSpec, sqlDialect, sqlConnectorConfig); - } - - @Override - public void execute(StandaloneSupport conquery) throws Exception { - ConqueryTestSpec testSpec = getTestSpec(); - ValidatorHelper.failOnError(log, VALIDATOR.validate(testSpec)); - testSpec.importRequiredData(conquery); - testSpec.executeTest(conquery); - } - - public boolean isAllowedTest(Dialect dialect) { - SqlQueryTest testSpec = getTestSpec(); - return testSpec.getSupportedDialects() == null || testSpec.getSupportedDialects().contains(dialect); - } - - } - } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/QueryTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/QueryTest.java index 2da08ed5ed..069908624b 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/QueryTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/QueryTest.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.integration.json; -import static com.bakdata.conquery.integration.common.LoadingUtil.*; - import java.util.List; import javax.validation.Valid; @@ -48,7 +46,6 @@ public class QueryTest extends AbstractQueryEngineTest { @JsonIgnore private Query query; - @Override public Query getQuery() { return query; @@ -56,32 +53,7 @@ public Query getQuery() { @Override public void importRequiredData(StandaloneSupport support) throws Exception { - - importSecondaryIds(support, content.getSecondaryIds()); - - importInternToExternMappers(support, internToExternMappings); - - importSearchIndexes(support, searchIndexes); - support.waitUntilWorkDone(); - - importTables(support, content.getTables(), content.isAutoConcept()); - support.waitUntilWorkDone(); - - importConcepts(support, rawConcepts); - support.waitUntilWorkDone(); - - importTableContents(support, content.getTables()); - support.waitUntilWorkDone(); - - importIdMapping(support, content); - support.waitUntilWorkDone(); - - importPreviousQueries(support, content, support.getTestUser()); - support.waitUntilWorkDone(); - - updateMatchingStats(support); - support.waitUntilWorkDone(); - + support.getTestImporter().importQueryTestData(support, this); query = IntegrationUtils.parseQuery(support, rawQuery); } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/SqlQueryTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/SqlQueryTest.java deleted file mode 100644 index 1f4cba6ceb..0000000000 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/SqlQueryTest.java +++ /dev/null @@ -1,93 +0,0 @@ -package com.bakdata.conquery.integration.json; - -import java.io.IOException; -import java.util.List; - -import javax.validation.Valid; -import javax.validation.constraints.NotNull; - -import com.bakdata.conquery.apiv1.query.Query; -import com.bakdata.conquery.integration.common.RequiredData; -import com.bakdata.conquery.integration.common.RequiredTable; -import com.bakdata.conquery.integration.common.ResourceFile; -import com.bakdata.conquery.integration.sql.CsvTableImporter; -import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.models.config.Dialect; -import com.bakdata.conquery.models.datasets.Table; -import com.bakdata.conquery.models.datasets.concepts.Concept; -import com.bakdata.conquery.models.exceptions.JSONException; -import com.bakdata.conquery.util.support.StandaloneSupport; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ArrayNode; -import lombok.Getter; -import lombok.Setter; -import lombok.extern.slf4j.Slf4j; - -@Slf4j -@Setter -@Getter -@CPSType(id = "SQL_QUERY_TEST", base = ConqueryTestSpec.class) -public class SqlQueryTest extends AbstractQueryEngineTest { - - List supportedDialects; - - String description; - - @NotNull - private ResourceFile expectedCsv; - - @NotNull - @JsonProperty("query") - private JsonNode rawQuery; - - @Valid - @NotNull - private RequiredData content; - - @NotNull - @JsonProperty("concepts") - private ArrayNode rawConcepts; - - @JsonIgnore - private Query query; - - @JsonIgnore - private CsvTableImporter csvTableImporter; - - @Override - public void importRequiredData(StandaloneSupport support) throws IOException, JSONException { - - importTables(support); - importConcepts(support); - - Query parsedQuery = ConqueryTestSpec.parseSubTree(support, getRawQuery(), Query.class); - setQuery(parsedQuery); - - for (RequiredTable table : content.getTables()) { - this.csvTableImporter.importTableIntoDatabase(table); - } - } - - public void setTableImporter(CsvTableImporter csvTableImporter) { - this.csvTableImporter = csvTableImporter; - } - - private void importTables(StandaloneSupport support) { - for (RequiredTable rTable : getContent().getTables()) { - final Table table = rTable.toTable(support.getDataset(), support.getNamespaceStorage().getCentralRegistry()); - support.getNamespaceStorage().addTable(table); - } - } - - private void importConcepts(StandaloneSupport support) throws IOException, JSONException { - List> - concepts = - ConqueryTestSpec.parseSubTreeList(support, getRawConcepts(), Concept.class, concept -> concept.setDataset(support.getDataset())); - for (Concept concept : concepts) { - support.getNamespaceStorage().updateConcept(concept); - } - } - -} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/SqlSpec.java b/backend/src/test/java/com/bakdata/conquery/integration/json/SqlSpec.java new file mode 100644 index 0000000000..f5333f19bb --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/SqlSpec.java @@ -0,0 +1,28 @@ +package com.bakdata.conquery.integration.json; + +import java.util.List; + +import javax.annotation.Nullable; + +import com.bakdata.conquery.models.config.Dialect; +import lombok.Value; + +@Value +public class SqlSpec { + + boolean isEnabled; + + @Nullable + List supportedDialects; + + /** + * @return True if a test spec is allowed for a specific dialect. + */ + public boolean isAllowedTest(Dialect dialect) { + if (dialect == null) { + return true; + } + return supportedDialects == null || supportedDialects.contains(dialect); + } + +} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java new file mode 100644 index 0000000000..6cdd7f4728 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java @@ -0,0 +1,56 @@ +package com.bakdata.conquery.integration.json; + +import java.io.IOException; +import java.util.List; + +import com.bakdata.conquery.integration.common.RequiredData; +import com.bakdata.conquery.integration.common.RequiredTable; +import com.bakdata.conquery.integration.json.filter.FilterTest; +import com.bakdata.conquery.integration.sql.CsvTableImporter; +import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.Concept; +import com.bakdata.conquery.models.exceptions.JSONException; +import com.bakdata.conquery.util.support.StandaloneSupport; +import com.fasterxml.jackson.databind.node.ArrayNode; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class SqlTestDataImporter implements TestDataImporter { + + private final CsvTableImporter csvTableImporter; + + @Override + public void importQueryTestData(StandaloneSupport support, QueryTest test) throws Exception { + RequiredData content = test.getContent(); + importTables(support, content); + importConcepts(support, test.getRawConcepts()); + for (RequiredTable table : content.getTables()) { + csvTableImporter.importTableIntoDatabase(table); + } + } + + @Override + public void importFormTestData(StandaloneSupport support, FormTest formTest) throws Exception { + throw new UnsupportedOperationException("Not implemented yet."); + } + + @Override + public void importFilterTestData(StandaloneSupport support, FilterTest filterTest) throws Exception { + throw new UnsupportedOperationException("Not implemented yet."); + } + + private void importTables(StandaloneSupport support, RequiredData content) { + for (RequiredTable rTable : content.getTables()) { + final Table table = rTable.toTable(support.getDataset(), support.getNamespaceStorage().getCentralRegistry()); + support.getNamespaceStorage().addTable(table); + } + } + + private void importConcepts(StandaloneSupport support, ArrayNode rawConcepts) throws IOException, JSONException { + List> concepts = + ConqueryTestSpec.parseSubTreeList(support, rawConcepts, Concept.class, concept -> concept.setDataset(support.getDataset())); + for (Concept concept : concepts) { + support.getNamespaceStorage().updateConcept(concept); + } + } +} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/TestDataImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/json/TestDataImporter.java new file mode 100644 index 0000000000..a292f4e08c --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/TestDataImporter.java @@ -0,0 +1,31 @@ +package com.bakdata.conquery.integration.json; + +import com.bakdata.conquery.integration.json.filter.FilterTest; +import com.bakdata.conquery.util.support.StandaloneSupport; + +public interface TestDataImporter { + + void importQueryTestData(StandaloneSupport support, QueryTest queryTest) throws Exception; + + void importFormTestData(StandaloneSupport support, FormTest formTest) throws Exception; + + void importFilterTestData(StandaloneSupport support, FilterTest filterTest) throws Exception; + + @FunctionalInterface + interface CheckedRunnable extends Runnable { + + @Override + default void run() throws RuntimeException { + try { + runThrows(); + } + catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + void runThrows() throws E; + + } + +} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/WorkerTestDataImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/json/WorkerTestDataImporter.java new file mode 100644 index 0000000000..2dc7c40803 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/WorkerTestDataImporter.java @@ -0,0 +1,82 @@ +package com.bakdata.conquery.integration.json; + +import static com.bakdata.conquery.integration.common.LoadingUtil.*; + +import java.util.Collections; + +import com.bakdata.conquery.integration.common.RequiredData; +import com.bakdata.conquery.integration.json.filter.FilterTest; +import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeConnector; +import com.bakdata.conquery.models.messages.namespaces.specific.UpdateMatchingStatsMessage; +import com.bakdata.conquery.models.worker.DistributedNamespace; +import com.bakdata.conquery.util.support.StandaloneSupport; + +public class WorkerTestDataImporter implements TestDataImporter { + + @Override + public void importQueryTestData(StandaloneSupport support, QueryTest test) throws Exception { + + RequiredData content = test.getContent(); + + importSecondaryIds(support, content.getSecondaryIds()); + importInternToExternMappers(support, test.getInternToExternMappings()); + + waitUntilDone(support, () -> importSearchIndexes(support, test.getSearchIndexes())); + waitUntilDone(support, () -> importTables(support, content.getTables(), content.isAutoConcept())); + waitUntilDone(support, () -> importConcepts(support, test.getRawConcepts())); + waitUntilDone(support, () -> importTableContents(support, content.getTables())); + waitUntilDone(support, () -> importIdMapping(support, content)); + waitUntilDone(support, () -> importPreviousQueries(support, content, support.getTestUser())); + waitUntilDone(support, () -> updateMatchingStats(support)); + + sendUpdateMatchingStatsMessage(support); + } + + @Override + public void importFormTestData(StandaloneSupport support, FormTest test) throws Exception { + + RequiredData content = test.getContent(); + + waitUntilDone(support, () -> importSecondaryIds(support, content.getSecondaryIds())); + waitUntilDone(support, () -> importTables(support, content.getTables(), content.isAutoConcept())); + waitUntilDone(support, () -> importConcepts(support, test.getRawConcepts())); + waitUntilDone(support, () -> importTableContents(support, content.getTables())); + waitUntilDone(support, () -> importIdMapping(support, content)); + waitUntilDone(support, () -> importPreviousQueries(support, content, support.getTestUser())); + } + + @Override + public void importFilterTestData(StandaloneSupport support, FilterTest test) throws Exception { + + RequiredData content = test.getContent(); + + importInternToExternMappers(support, test.getInternToExternMappings()); + importSearchIndexes(support, test.getSearchIndices()); + waitUntilDone(support, () -> importTables(support, content.getTables(), content.isAutoConcept())); + + test.setConnector(ConqueryTestSpec.parseSubTree( + support, + test.getRawConnector(), + ConceptTreeConnector.class, + conn -> conn.setConcept(test.getConcept()) + ) + ); + test.getConcept().setConnectors(Collections.singletonList((ConceptTreeConnector) test.getConnector())); + + waitUntilDone(support, () -> uploadConcept(support, support.getDataset(), test.getConcept())); + waitUntilDone(support, () -> importTableContents(support, content.getTables())); + waitUntilDone(support, () -> updateMatchingStats(support)); + } + + private void waitUntilDone(StandaloneSupport support, CheckedRunnable runnable) { + runnable.run(); + support.waitUntilWorkDone(); + } + + private static void sendUpdateMatchingStatsMessage(StandaloneSupport support) { + DistributedNamespace namespace = (DistributedNamespace) support.getNamespace(); + namespace.getWorkerHandler().sendToAll(new UpdateMatchingStatsMessage(support.getNamespace().getStorage().getAllConcepts())); + support.waitUntilWorkDone(); + } + +} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/filter/FilterTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/filter/FilterTest.java index 2bdfbb7708..58547e6c8c 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/filter/FilterTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/filter/FilterTest.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.integration.json.filter; -import static com.bakdata.conquery.integration.common.LoadingUtil.*; import static org.assertj.core.api.Assertions.assertThat; import java.io.IOException; @@ -17,7 +16,6 @@ import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.apiv1.query.concept.specific.CQDateRestriction; -import com.bakdata.conquery.integration.common.LoadingUtil; import com.bakdata.conquery.integration.common.RequiredData; import com.bakdata.conquery.integration.common.ResourceFile; import com.bakdata.conquery.integration.json.AbstractQueryEngineTest; @@ -26,9 +24,7 @@ import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.serializer.SerializationTestUtil; import com.bakdata.conquery.models.common.Range; -import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.concepts.Connector; -import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeConnector; import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.models.exceptions.ConceptConfigurationException; import com.bakdata.conquery.models.exceptions.JSONException; @@ -92,27 +88,6 @@ public void importRequiredData(StandaloneSupport support) throws Exception { content = parseSubTree(support, rawContent, RequiredData.class); - importInternToExternMappers(support, internToExternMappings); - importSearchIndexes(support, searchIndices); - LoadingUtil.importTables(support, content.getTables(), content.isAutoConcept()); - support.waitUntilWorkDone(); - - importConcepts(support); - support.waitUntilWorkDone(); - - query = parseQuery(support); - - importTableContents(support, content.getTables()); - support.waitUntilWorkDone(); - - updateMatchingStats(support); - support.waitUntilWorkDone(); - } - - - private void importConcepts(StandaloneSupport support) throws JSONException, IOException { - Dataset dataset = support.getDataset(); - concept = new TreeConcept(); concept.setLabel("concept"); @@ -123,17 +98,12 @@ private void importConcepts(StandaloneSupport support) throws JSONException, IOE ((ObjectNode) rawConnector.get("filters")).put("name", "filter"); - connector = parseSubTree( - support, - rawConnector, - ConceptTreeConnector.class, - conn -> conn.setConcept(concept) - ); + support.getTestImporter().importFilterTestData(support, this); - concept.setConnectors(Collections.singletonList((ConceptTreeConnector) connector)); - LoadingUtil.uploadConcept(support, dataset, concept); + query = parseQuery(support); } + private Query parseQuery(StandaloneSupport support) throws JSONException, IOException { final String filterId = support.getDataset().getName() + ".concept.connector.filter"; rawFilterValue.put("filter", filterId); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java index bd8ef1547c..5ca44ad7d6 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java @@ -17,6 +17,9 @@ import com.bakdata.conquery.TestTags; import com.bakdata.conquery.integration.ConqueryIntegrationTests; import com.bakdata.conquery.integration.IntegrationTests; +import com.bakdata.conquery.integration.json.SqlTestDataImporter; +import com.bakdata.conquery.integration.json.TestDataImporter; +import com.bakdata.conquery.integration.sql.CsvTableImporter; import com.bakdata.conquery.integration.sql.testcontainer.hana.HanaContainer; import com.bakdata.conquery.models.config.Dialect; import com.bakdata.conquery.models.config.SqlConnectorConfig; @@ -61,7 +64,7 @@ public HanaSqlIntegrationTests() { @TestFactory @Tag(TestTags.INTEGRATION_SQL_BACKEND) - public List sqlBackendTests() { + public Stream sqlBackendTests() { TestContextProvider provider = useLocalHanaDb ? new HanaTestcontainerContextProvider() @@ -71,8 +74,13 @@ public List sqlBackendTests() { DSLContext dslContext = provider.getDslContext(); SqlConnectorConfig config = provider.getSqlConnectorConfig(); + TestHanaDialect testHanaDialect = new TestHanaDialect(dslContext); + TestDataImporter testDataImporter = new SqlTestDataImporter(new CsvTableImporter(dslContext, testHanaDialect, config)); - return super.sqlTests(new TestHanaDialect(dslContext), config); + return Stream.concat( + super.sqlProgrammaticTests(config, testDataImporter), + super.sqlQueryTests(config, testDataImporter).stream() + ); } @SneakyThrows diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java index 47fed09765..3053d6f9e1 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java @@ -2,11 +2,14 @@ import java.util.Collections; import java.util.List; +import java.util.stream.Stream; import com.bakdata.conquery.TestTags; import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.integration.ConqueryIntegrationTests; import com.bakdata.conquery.integration.IntegrationTests; +import com.bakdata.conquery.integration.json.SqlTestDataImporter; +import com.bakdata.conquery.integration.sql.CsvTableImporter; import com.bakdata.conquery.models.config.Dialect; import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.models.error.ConqueryError; @@ -43,6 +46,7 @@ public class PostgreSqlIntegrationTests extends IntegrationTests { private static DSLContext dslContext; private static SqlConnectorConfig sqlConfig; private static TestSqlDialect testSqlDialect; + private static SqlTestDataImporter testDataImporter; public PostgreSqlIntegrationTests() { super(ConqueryIntegrationTests.DEFAULT_SQL_TEST_ROOT, "com.bakdata.conquery.integration"); @@ -69,6 +73,7 @@ static void before() { .build(); dslContext = DslContextFactory.create(sqlConfig); testSqlDialect = new TestPostgreSqlDialect(dslContext); + testDataImporter = new SqlTestDataImporter(new CsvTableImporter(dslContext, testSqlDialect, sqlConfig)); } @Test @@ -87,11 +92,13 @@ public void shouldThrowException() { .hasMessageContaining("$org.postgresql.util.PSQLException"); } - @TestFactory @Tag(TestTags.INTEGRATION_SQL_BACKEND) - public List sqlBackendTests() { - return super.sqlTests(testSqlDialect, sqlConfig); + public Stream sqlBackendTests() { + return Stream.concat( + super.sqlProgrammaticTests(sqlConfig, testDataImporter), + super.sqlQueryTests(sqlConfig, testDataImporter).stream() + ); } public static class TestPostgreSqlDialect extends PostgreSqlDialect implements TestSqlDialect { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ProgrammaticIntegrationTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ProgrammaticIntegrationTest.java index 07288c21cb..03f7a049c1 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ProgrammaticIntegrationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ProgrammaticIntegrationTest.java @@ -1,7 +1,19 @@ package com.bakdata.conquery.integration.tests; +import java.util.Set; + import com.bakdata.conquery.integration.IntegrationTest; +import com.bakdata.conquery.util.support.StandaloneSupport; public interface ProgrammaticIntegrationTest extends IntegrationTest { - + + default boolean isEnabled(StandaloneSupport.Mode mode) { + return forModes().contains(mode); + } + + default Set forModes() { + // Worker mode is standard until SQL connector has full feature support + return Set.of(StandaloneSupport.Mode.WORKER); + } + } diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java b/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java index 3a3bb8d1a6..7e246c5818 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java @@ -13,6 +13,7 @@ import com.bakdata.conquery.commands.PreprocessorCommand; import com.bakdata.conquery.commands.ShardNode; +import com.bakdata.conquery.integration.json.TestDataImporter; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.models.auth.AuthorizationController; @@ -37,6 +38,10 @@ @RequiredArgsConstructor public class StandaloneSupport implements TestSupport { + public enum Mode {WORKER, SQL} + + @Getter + private final Mode mode; private final TestConquery testConquery; @Getter private final Namespace namespace; @@ -52,6 +57,8 @@ public class StandaloneSupport implements TestSupport { private final AdminDatasetProcessor datasetsProcessor; @Getter private final User testUser; + @Getter + private final TestDataImporter testImporter; public AuthorizationController getAuthorizationController() { return testConquery.getStandaloneCommand().getManagerNode().getAuthController(); diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java index 012aee41f2..1832db13ec 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java @@ -20,6 +20,7 @@ import com.bakdata.conquery.commands.ShardNode; import com.bakdata.conquery.commands.StandaloneCommand; import com.bakdata.conquery.integration.IntegrationTests; +import com.bakdata.conquery.integration.json.TestDataImporter; import com.bakdata.conquery.integration.sql.SqlStandaloneCommand; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.mode.cluster.ClusterManager; @@ -56,6 +57,7 @@ public class TestConquery { private static final ConcurrentHashMap NAME_COUNTS = new ConcurrentHashMap<>(); private final File tmpDir; private final ConqueryConfig config; + private final TestDataImporter testDataImporter; @Getter private StandaloneCommand standaloneCommand; @Getter @@ -196,7 +198,7 @@ public void beforeEach() { private synchronized StandaloneSupport createSupport(DatasetId datasetId, String name) { if (config.getSqlConnectorConfig().isEnabled()) { - return buildSupport(datasetId, name); + return buildSupport(datasetId, name, StandaloneSupport.Mode.SQL); } return buildDistributedSupport(datasetId, name); } @@ -213,10 +215,10 @@ private synchronized StandaloneSupport buildDistributedSupport(DatasetId dataset .build() .until(() -> clusterState.getWorkerHandlers().get(datasetId).getWorkers().size() == clusterState.getShardNodes().size()); - return buildSupport(datasetId, name); + return buildSupport(datasetId, name, StandaloneSupport.Mode.WORKER); } - private StandaloneSupport buildSupport(DatasetId datasetId, String name) { + private StandaloneSupport buildSupport(DatasetId datasetId, String name, StandaloneSupport.Mode mode) { DatasetRegistry datasets = standaloneCommand.getManager().getDatasetRegistry(); Namespace ns = datasets.get(datasetId); @@ -238,6 +240,7 @@ private StandaloneSupport buildSupport(DatasetId datasetId, String name) { Cloner.clone(config, Map.of(Validator.class, standaloneCommand.getManagerNode().getEnvironment().getValidator()), IntegrationTests.MAPPER); StandaloneSupport support = new StandaloneSupport( + mode, this, ns, ns.getStorage().getDataset(), @@ -246,7 +249,8 @@ private StandaloneSupport buildSupport(DatasetId datasetId, String name) { standaloneCommand.getManagerNode().getAdmin().getAdminProcessor(), standaloneCommand.getManagerNode().getAdmin().getAdminDatasetProcessor(), // Getting the User from AuthorizationConfig - testUser + testUser, + testDataImporter ); support.waitUntilWorkDone(); diff --git a/backend/src/test/resources/tests/sql/and/different_concept/and.json b/backend/src/test/resources/tests/sql/and/different_concept/and.json index 589e3716f0..3430dbf361 100644 --- a/backend/src/test/resources/tests/sql/and/different_concept/and.json +++ b/backend/src/test/resources/tests/sql/and/different_concept/and.json @@ -1,7 +1,10 @@ { "label": "Simple AND query for 3 different concepts", "expectedCsv": "tests/sql/and/different_concept/expected.csv", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "query": { "type": "CONCEPT_QUERY", "root": { diff --git a/backend/src/test/resources/tests/sql/and/same_concept/and_same_concept.json b/backend/src/test/resources/tests/sql/and/same_concept/and_same_concept.json index be1c5256a0..837bdfdad7 100644 --- a/backend/src/test/resources/tests/sql/and/same_concept/and_same_concept.json +++ b/backend/src/test/resources/tests/sql/and/same_concept/and_same_concept.json @@ -1,7 +1,10 @@ { "label": "Simple AND query for same concept", "expectedCsv": "tests/sql/and/same_concept/expected.csv", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "query": { "type": "CONCEPT_QUERY", "root": { diff --git a/backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/date_restriction_date_column.json b/backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/date_restriction_date_column.json index 8a0eb93977..6314927ffa 100644 --- a/backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/date_restriction_date_column.json +++ b/backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/date_restriction_date_column.json @@ -1,6 +1,9 @@ { "label": "Date restriction with multiple validity dates and dateColumn", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/date_restriction/date_restriction_date_column/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/date_restriction/date_restriction_no_validity_date/date_restriction_no_validity_date.json b/backend/src/test/resources/tests/sql/date_restriction/date_restriction_no_validity_date/date_restriction_no_validity_date.json index bdabef0fa6..36f6351d7a 100644 --- a/backend/src/test/resources/tests/sql/date_restriction/date_restriction_no_validity_date/date_restriction_no_validity_date.json +++ b/backend/src/test/resources/tests/sql/date_restriction/date_restriction_no_validity_date/date_restriction_no_validity_date.json @@ -1,7 +1,10 @@ { "label": "Date restriction query without validity date", "description": "If a date restriction is active, but there is no validity date defined to apply the date restriction on, the date restriction filter should not be applied.", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/date_restriction/date_restriction_no_validity_date/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/date_restriction/only_max/only_max.spec.json b/backend/src/test/resources/tests/sql/date_restriction/only_max/only_max.spec.json index 9220e7c67d..bcb014510a 100644 --- a/backend/src/test/resources/tests/sql/date_restriction/only_max/only_max.spec.json +++ b/backend/src/test/resources/tests/sql/date_restriction/only_max/only_max.spec.json @@ -1,6 +1,9 @@ { "label": "Date restriction query with simple date validity date and only max date restriction set", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/date_restriction/only_max/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/date_restriction/only_min/only_min.spec.json b/backend/src/test/resources/tests/sql/date_restriction/only_min/only_min.spec.json index d18d037355..78fb514ca9 100644 --- a/backend/src/test/resources/tests/sql/date_restriction/only_min/only_min.spec.json +++ b/backend/src/test/resources/tests/sql/date_restriction/only_min/only_min.spec.json @@ -1,6 +1,9 @@ { "label": "Date restriction query with simple date validity date and only min date restriction set", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/date_restriction/only_min/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/date_restriction/postgres_daterange/daterange_column.spec.json b/backend/src/test/resources/tests/sql/date_restriction/postgres_daterange/daterange_column.spec.json index ced6bcd7c3..8e4f3ba260 100644 --- a/backend/src/test/resources/tests/sql/date_restriction/postgres_daterange/daterange_column.spec.json +++ b/backend/src/test/resources/tests/sql/date_restriction/postgres_daterange/daterange_column.spec.json @@ -1,8 +1,11 @@ { - "type": "SQL_QUERY_TEST", - "supportedDialects": [ - "POSTGRESQL" - ], + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true, + "supportedDialects": [ + "POSTGRESQL" + ] + }, "label": "Date restriction with a daterange column validity date (PostgreSQL only) ", "expectedCsv": "tests/sql/date_restriction/postgres_daterange/expected.csv", "query": { diff --git a/backend/src/test/resources/tests/sql/date_restriction/simple_date/date_restriction_simple_date.json b/backend/src/test/resources/tests/sql/date_restriction/simple_date/date_restriction_simple_date.json index bf3d1b02e2..d01bd8c240 100644 --- a/backend/src/test/resources/tests/sql/date_restriction/simple_date/date_restriction_simple_date.json +++ b/backend/src/test/resources/tests/sql/date_restriction/simple_date/date_restriction_simple_date.json @@ -1,6 +1,9 @@ { "label": "Date restriction query with simple date validity date", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/date_restriction/simple_date/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/filter/big_multi_select/big_multi_select.spec.json b/backend/src/test/resources/tests/sql/filter/big_multi_select/big_multi_select.spec.json index d76b438b99..1f8ad5ab9b 100644 --- a/backend/src/test/resources/tests/sql/filter/big_multi_select/big_multi_select.spec.json +++ b/backend/src/test/resources/tests/sql/filter/big_multi_select/big_multi_select.spec.json @@ -1,6 +1,9 @@ { "label": "BIG_MULTI_SELECT filter", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/filter/big_multi_select/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/filter/count/count.json b/backend/src/test/resources/tests/sql/filter/count/count.json index 97cfc31977..832b1ae0f2 100644 --- a/backend/src/test/resources/tests/sql/filter/count/count.json +++ b/backend/src/test/resources/tests/sql/filter/count/count.json @@ -1,6 +1,9 @@ { "label": "COUNT filter", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/filter/count/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/filter/count_distinct/count_distinct.json b/backend/src/test/resources/tests/sql/filter/count_distinct/count_distinct.json index 808ba77a89..f1850ae927 100644 --- a/backend/src/test/resources/tests/sql/filter/count_distinct/count_distinct.json +++ b/backend/src/test/resources/tests/sql/filter/count_distinct/count_distinct.json @@ -1,6 +1,9 @@ { "label": "COUNT DISTINCT filter", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/filter/count_distinct/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/filter/date_distance/centuries/centuries.spec.json b/backend/src/test/resources/tests/sql/filter/date_distance/centuries/centuries.spec.json index 4f1eb7c5e8..1636cb130a 100644 --- a/backend/src/test/resources/tests/sql/filter/date_distance/centuries/centuries.spec.json +++ b/backend/src/test/resources/tests/sql/filter/date_distance/centuries/centuries.spec.json @@ -1,6 +1,9 @@ { "label": "DATE_DISTANCE filter query with timeUnit CENTURIES", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/filter/date_distance/centuries/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/filter/date_distance/days_with_date_restriction/days_with_date_restriction.spec.json b/backend/src/test/resources/tests/sql/filter/date_distance/days_with_date_restriction/days_with_date_restriction.spec.json index 8ca88bff09..4d6e824de6 100644 --- a/backend/src/test/resources/tests/sql/filter/date_distance/days_with_date_restriction/days_with_date_restriction.spec.json +++ b/backend/src/test/resources/tests/sql/filter/date_distance/days_with_date_restriction/days_with_date_restriction.spec.json @@ -1,5 +1,8 @@ { - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "label": "DATE_DISTANCE filter query with timeUnit DAYS and date restriction set", "expectedCsv": "tests/sql/filter/date_distance/days_with_date_restriction/expected.csv", "query": { diff --git a/backend/src/test/resources/tests/sql/filter/date_distance/days_without_date_restriction/days_without_date_restriction.json b/backend/src/test/resources/tests/sql/filter/date_distance/days_without_date_restriction/days_without_date_restriction.json index 7abee9d64e..4cd696025d 100644 --- a/backend/src/test/resources/tests/sql/filter/date_distance/days_without_date_restriction/days_without_date_restriction.json +++ b/backend/src/test/resources/tests/sql/filter/date_distance/days_without_date_restriction/days_without_date_restriction.json @@ -1,5 +1,8 @@ { - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "label": "DATE_DISTANCE filter query with timeUnit DAYS and without date restriction set", "expectedCsv": "tests/sql/filter/date_distance/days_without_date_restriction/expected.csv", "query": { diff --git a/backend/src/test/resources/tests/sql/filter/date_distance/decades/decades.spec.json b/backend/src/test/resources/tests/sql/filter/date_distance/decades/decades.spec.json index a37ef37aa9..6298a36b55 100644 --- a/backend/src/test/resources/tests/sql/filter/date_distance/decades/decades.spec.json +++ b/backend/src/test/resources/tests/sql/filter/date_distance/decades/decades.spec.json @@ -1,5 +1,8 @@ { - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "label": "DATE_DISTANCE filter query with timeUnit DECADES", "expectedCsv": "tests/sql/filter/date_distance/decades/expected.csv", "query": { diff --git a/backend/src/test/resources/tests/sql/filter/date_distance/months/months.spec.json b/backend/src/test/resources/tests/sql/filter/date_distance/months/months.spec.json index 976063c0e1..fcfa8852a5 100644 --- a/backend/src/test/resources/tests/sql/filter/date_distance/months/months.spec.json +++ b/backend/src/test/resources/tests/sql/filter/date_distance/months/months.spec.json @@ -1,5 +1,8 @@ { - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "label": "DATE_DISTANCE filter query with timeUnit MONTHS", "expectedCsv": "tests/sql/filter/date_distance/months/expected.csv", "query": { diff --git a/backend/src/test/resources/tests/sql/filter/date_distance/years/years.spec.json b/backend/src/test/resources/tests/sql/filter/date_distance/years/years.spec.json index 47dd2ccb64..e68ab6140f 100644 --- a/backend/src/test/resources/tests/sql/filter/date_distance/years/years.spec.json +++ b/backend/src/test/resources/tests/sql/filter/date_distance/years/years.spec.json @@ -1,5 +1,8 @@ { - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "label": "DATE_DISTANCE filter query with timeUnit YEARS", "expectedCsv": "tests/sql/filter/date_distance/years/expected.csv", "query": { diff --git a/backend/src/test/resources/tests/sql/filter/multi_select/multi_select.spec.json b/backend/src/test/resources/tests/sql/filter/multi_select/multi_select.spec.json index abed219f08..817e2c270e 100644 --- a/backend/src/test/resources/tests/sql/filter/multi_select/multi_select.spec.json +++ b/backend/src/test/resources/tests/sql/filter/multi_select/multi_select.spec.json @@ -1,6 +1,9 @@ { "label": "SELECT filter", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/filter/multi_select/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/filter/number/number.spec.json b/backend/src/test/resources/tests/sql/filter/number/number.spec.json index 3a95affd20..d99a9b22a6 100644 --- a/backend/src/test/resources/tests/sql/filter/number/number.spec.json +++ b/backend/src/test/resources/tests/sql/filter/number/number.spec.json @@ -1,6 +1,9 @@ { "label": "Single Number-Real-Range Filter Query", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/filter/number/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/filter/number_money/integer_range/number_money.spec.json b/backend/src/test/resources/tests/sql/filter/number_money/integer_range/number_money.spec.json index 3cd96d2e77..f65241485a 100644 --- a/backend/src/test/resources/tests/sql/filter/number_money/integer_range/number_money.spec.json +++ b/backend/src/test/resources/tests/sql/filter/number_money/integer_range/number_money.spec.json @@ -1,6 +1,9 @@ { "label": "Single Number-Real-Range Filter Query with INTEGER RANGE on column with MONEY type", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/filter/number_money/integer_range/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/filter/number_money/real_range/number_money.spec.json b/backend/src/test/resources/tests/sql/filter/number_money/real_range/number_money.spec.json index 316bb21c83..9c869eaf1b 100644 --- a/backend/src/test/resources/tests/sql/filter/number_money/real_range/number_money.spec.json +++ b/backend/src/test/resources/tests/sql/filter/number_money/real_range/number_money.spec.json @@ -1,6 +1,9 @@ { "label": "Single Number-Real-Range Filter Query with REAL RANGE on column with MONEY type", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/filter/number_money/real_range/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/filter/number_only_max/number_only_max.spec.json b/backend/src/test/resources/tests/sql/filter/number_only_max/number_only_max.spec.json index 21c5d216bc..30fb26fb40 100644 --- a/backend/src/test/resources/tests/sql/filter/number_only_max/number_only_max.spec.json +++ b/backend/src/test/resources/tests/sql/filter/number_only_max/number_only_max.spec.json @@ -1,6 +1,9 @@ { "label": "Single Number-Real-Range Filter Query (only max val)", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/filter/number_only_max/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/filter/number_only_min/number_only_min.spec.json b/backend/src/test/resources/tests/sql/filter/number_only_min/number_only_min.spec.json index 533f25f06c..dc2e91b21d 100644 --- a/backend/src/test/resources/tests/sql/filter/number_only_min/number_only_min.spec.json +++ b/backend/src/test/resources/tests/sql/filter/number_only_min/number_only_min.spec.json @@ -1,6 +1,9 @@ { "label": "Single Number-Real-Range Filter Query (only min val)", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/filter/number_only_min/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/filter/single_select/single_select.spec.json b/backend/src/test/resources/tests/sql/filter/single_select/single_select.spec.json index 1e41b46eb3..78b79d1fe9 100644 --- a/backend/src/test/resources/tests/sql/filter/single_select/single_select.spec.json +++ b/backend/src/test/resources/tests/sql/filter/single_select/single_select.spec.json @@ -1,6 +1,9 @@ { "label": "SINGLE_SELECT filter", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/filter/single_select/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/filter/sum/diffsum/diffsum.spec.json b/backend/src/test/resources/tests/sql/filter/sum/diffsum/diffsum.spec.json index 954df631ec..0ce06bb054 100644 --- a/backend/src/test/resources/tests/sql/filter/sum/diffsum/diffsum.spec.json +++ b/backend/src/test/resources/tests/sql/filter/sum/diffsum/diffsum.spec.json @@ -1,6 +1,6 @@ { "label": "SUM with subtract column filter", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", "expectedCsv": "tests/sql/filter/sum/diffsum/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/filter/sum/distinct/distinct-sum.spec.json b/backend/src/test/resources/tests/sql/filter/sum/distinct/distinct-sum.spec.json index fdb030fb8d..15e6d32008 100644 --- a/backend/src/test/resources/tests/sql/filter/sum/distinct/distinct-sum.spec.json +++ b/backend/src/test/resources/tests/sql/filter/sum/distinct/distinct-sum.spec.json @@ -1,6 +1,6 @@ { "label": "SUM filter with DISTINCT BY columns", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", "expectedCsv": "tests/sql/filter/sum/distinct/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/filter/sum/sum.json b/backend/src/test/resources/tests/sql/filter/sum/sum.json index baa717730e..46b973960b 100644 --- a/backend/src/test/resources/tests/sql/filter/sum/sum.json +++ b/backend/src/test/resources/tests/sql/filter/sum/sum.json @@ -1,6 +1,9 @@ { "label": "SUM filter", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/filter/sum/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/not/not.spec.json b/backend/src/test/resources/tests/sql/not/not.spec.json index d25ca38e41..6c4dd49ba1 100644 --- a/backend/src/test/resources/tests/sql/not/not.spec.json +++ b/backend/src/test/resources/tests/sql/not/not.spec.json @@ -1,6 +1,9 @@ { "label": "Simple Negation Query", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/not/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/or/different_concept/or.spec.json b/backend/src/test/resources/tests/sql/or/different_concept/or.spec.json index 9d2a1182f5..f0e2a3dc36 100644 --- a/backend/src/test/resources/tests/sql/or/different_concept/or.spec.json +++ b/backend/src/test/resources/tests/sql/or/different_concept/or.spec.json @@ -1,7 +1,10 @@ { "label": "Simple OR query for 3 different concepts", "expectedCsv": "tests/sql/or/different_concept/expected.csv", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "query": { "type": "CONCEPT_QUERY", "root": { diff --git a/backend/src/test/resources/tests/sql/or/same_concept/or_same_concept.spec.json b/backend/src/test/resources/tests/sql/or/same_concept/or_same_concept.spec.json index 09d532cbc6..b711ca5862 100644 --- a/backend/src/test/resources/tests/sql/or/same_concept/or_same_concept.spec.json +++ b/backend/src/test/resources/tests/sql/or/same_concept/or_same_concept.spec.json @@ -1,7 +1,10 @@ { "label": "Simple OR query for same concept", "expectedCsv": "tests/sql/or/same_concept/expected.csv", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "query": { "type": "CONCEPT_QUERY", "root": { diff --git a/backend/src/test/resources/tests/sql/selects/count/count.json b/backend/src/test/resources/tests/sql/selects/count/count.json index 59a198385c..9ad2ed5532 100644 --- a/backend/src/test/resources/tests/sql/selects/count/count.json +++ b/backend/src/test/resources/tests/sql/selects/count/count.json @@ -1,6 +1,9 @@ { "label": "COUNT select", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/selects/count/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/selects/count_distinct/count_distinct.json b/backend/src/test/resources/tests/sql/selects/count_distinct/count_distinct.json index 6fae17ff62..9c03c95c3c 100644 --- a/backend/src/test/resources/tests/sql/selects/count_distinct/count_distinct.json +++ b/backend/src/test/resources/tests/sql/selects/count_distinct/count_distinct.json @@ -1,6 +1,9 @@ { "label": "COUNT DISTINCT select", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/selects/count_distinct/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/centuries/centuries.spec.json b/backend/src/test/resources/tests/sql/selects/date_distance/centuries/centuries.spec.json index 1b396a93cb..c84acd8f7f 100644 --- a/backend/src/test/resources/tests/sql/selects/date_distance/centuries/centuries.spec.json +++ b/backend/src/test/resources/tests/sql/selects/date_distance/centuries/centuries.spec.json @@ -1,6 +1,9 @@ { "label": "DATE_DISTANCE select with timeUnit CENTURIES", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/selects/date_distance/centuries/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/days_with_date_restriction/days_with_date_restriction.spec.json b/backend/src/test/resources/tests/sql/selects/date_distance/days_with_date_restriction/days_with_date_restriction.spec.json index 64610cf0fc..fe822b59ea 100644 --- a/backend/src/test/resources/tests/sql/selects/date_distance/days_with_date_restriction/days_with_date_restriction.spec.json +++ b/backend/src/test/resources/tests/sql/selects/date_distance/days_with_date_restriction/days_with_date_restriction.spec.json @@ -1,5 +1,8 @@ { - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "label": "DATE_DISTANCE select with timeUnit DAYS and date restriction set", "expectedCsv": "tests/sql/selects/date_distance/days_with_date_restriction/expected.csv", "query": { diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/days_without_date_restriction/days_without_date_restriction.json b/backend/src/test/resources/tests/sql/selects/date_distance/days_without_date_restriction/days_without_date_restriction.json index b043ab276a..31974a82d7 100644 --- a/backend/src/test/resources/tests/sql/selects/date_distance/days_without_date_restriction/days_without_date_restriction.json +++ b/backend/src/test/resources/tests/sql/selects/date_distance/days_without_date_restriction/days_without_date_restriction.json @@ -1,5 +1,8 @@ { - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "label": "DATE_DISTANCE select with timeUnit DAYS and without date restriction set", "expectedCsv": "tests/sql/selects/date_distance/days_without_date_restriction/expected.csv", "query": { diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/decades/decades.spec.json b/backend/src/test/resources/tests/sql/selects/date_distance/decades/decades.spec.json index 4ba867331d..f95246c58b 100644 --- a/backend/src/test/resources/tests/sql/selects/date_distance/decades/decades.spec.json +++ b/backend/src/test/resources/tests/sql/selects/date_distance/decades/decades.spec.json @@ -1,5 +1,8 @@ { - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "label": "DATE_DISTANCE select with timeUnit DECADES", "expectedCsv": "tests/sql/selects/date_distance/decades/expected.csv", "query": { diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/months/months.spec.json b/backend/src/test/resources/tests/sql/selects/date_distance/months/months.spec.json index fa7f598daf..467c4577b7 100644 --- a/backend/src/test/resources/tests/sql/selects/date_distance/months/months.spec.json +++ b/backend/src/test/resources/tests/sql/selects/date_distance/months/months.spec.json @@ -1,5 +1,8 @@ { - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "label": "DATE_DISTANCE select with timeUnit MONTHS", "expectedCsv": "tests/sql/selects/date_distance/months/expected.csv", "query": { diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/years/years.spec.json b/backend/src/test/resources/tests/sql/selects/date_distance/years/years.spec.json index 18d36c167c..3f22a5ad04 100644 --- a/backend/src/test/resources/tests/sql/selects/date_distance/years/years.spec.json +++ b/backend/src/test/resources/tests/sql/selects/date_distance/years/years.spec.json @@ -1,5 +1,8 @@ { - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "label": "DATE_DISTANCE select with timeUnit YEARS", "expectedCsv": "tests/sql/selects/date_distance/years/expected.csv", "query": { diff --git a/backend/src/test/resources/tests/sql/selects/exists/exists_with_other_selects/exists_with_other_selects.spec.json b/backend/src/test/resources/tests/sql/selects/exists/exists_with_other_selects/exists_with_other_selects.spec.json index 8e6fdf0a59..782447789f 100644 --- a/backend/src/test/resources/tests/sql/selects/exists/exists_with_other_selects/exists_with_other_selects.spec.json +++ b/backend/src/test/resources/tests/sql/selects/exists/exists_with_other_selects/exists_with_other_selects.spec.json @@ -1,5 +1,8 @@ { - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "label": "EXISTS select with other selects", "description": "EXISTS as concept level select should be in before table-level selects like first in final result set", "expectedCsv": "tests/sql/selects/exists/exists_with_other_selects/expected.csv", diff --git a/backend/src/test/resources/tests/sql/selects/exists/single_exists/exists.spec.json b/backend/src/test/resources/tests/sql/selects/exists/single_exists/exists.spec.json index 0ac7b238ad..1e8eb0f01b 100644 --- a/backend/src/test/resources/tests/sql/selects/exists/single_exists/exists.spec.json +++ b/backend/src/test/resources/tests/sql/selects/exists/single_exists/exists.spec.json @@ -1,5 +1,8 @@ { - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "label": "Single Concept EXISTS", "expectedCsv": "tests/sql/selects/exists/single_exists/expected.csv", "query": { diff --git a/backend/src/test/resources/tests/sql/selects/first/date/number_date.spec.json b/backend/src/test/resources/tests/sql/selects/first/date/number_date.spec.json index fb5064165b..6a32bbeddd 100644 --- a/backend/src/test/resources/tests/sql/selects/first/date/number_date.spec.json +++ b/backend/src/test/resources/tests/sql/selects/first/date/number_date.spec.json @@ -1,6 +1,9 @@ { "label": "FIRST on column of date type", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/selects/first/date/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/selects/first/money/number_money.spec.json b/backend/src/test/resources/tests/sql/selects/first/money/number_money.spec.json index 6387f16ae3..b439d91c6c 100644 --- a/backend/src/test/resources/tests/sql/selects/first/money/number_money.spec.json +++ b/backend/src/test/resources/tests/sql/selects/first/money/number_money.spec.json @@ -1,6 +1,9 @@ { "label": "FIRST on column of MONEY type", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/selects/first/money/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/selects/first/number/first.spec.json b/backend/src/test/resources/tests/sql/selects/first/number/first.spec.json index b67fc762a4..6cd94a1f59 100644 --- a/backend/src/test/resources/tests/sql/selects/first/number/first.spec.json +++ b/backend/src/test/resources/tests/sql/selects/first/number/first.spec.json @@ -1,6 +1,9 @@ { "label": "FIRST select with validity date", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/selects/first/number/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/selects/last/last.spec.json b/backend/src/test/resources/tests/sql/selects/last/last.spec.json index e746100281..40f0e5d3ea 100644 --- a/backend/src/test/resources/tests/sql/selects/last/last.spec.json +++ b/backend/src/test/resources/tests/sql/selects/last/last.spec.json @@ -1,6 +1,9 @@ { "label": "LAST select with validity date", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/selects/last/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/selects/random/random.spec.json b/backend/src/test/resources/tests/sql/selects/random/random.spec.json index 0f5f58f3ed..fa72e7f57d 100644 --- a/backend/src/test/resources/tests/sql/selects/random/random.spec.json +++ b/backend/src/test/resources/tests/sql/selects/random/random.spec.json @@ -1,6 +1,9 @@ { "label": "RANDOM select with validity date", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/selects/random/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/selects/same_select_2_times/same_select.json b/backend/src/test/resources/tests/sql/selects/same_select_2_times/same_select.json index 989690b3e1..6af6400c69 100644 --- a/backend/src/test/resources/tests/sql/selects/same_select_2_times/same_select.json +++ b/backend/src/test/resources/tests/sql/selects/same_select_2_times/same_select.json @@ -1,7 +1,10 @@ { "label": "Same select 2 times", "description": "Selecting the same select 2 times should not cause a name collision", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/selects/same_select_2_times/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/selects/sum/diffsum/diffsum.spec.json b/backend/src/test/resources/tests/sql/selects/sum/diffsum/diffsum.spec.json index 8bf6e1bcc1..e260817455 100644 --- a/backend/src/test/resources/tests/sql/selects/sum/diffsum/diffsum.spec.json +++ b/backend/src/test/resources/tests/sql/selects/sum/diffsum/diffsum.spec.json @@ -1,6 +1,6 @@ { "label": "SUM with subtract column select", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", "expectedCsv": "tests/sql/selects/sum/diffsum/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/selects/sum/distinct/distinct-sum.spec.json b/backend/src/test/resources/tests/sql/selects/sum/distinct/distinct-sum.spec.json index 4ee1faff0a..767614085e 100644 --- a/backend/src/test/resources/tests/sql/selects/sum/distinct/distinct-sum.spec.json +++ b/backend/src/test/resources/tests/sql/selects/sum/distinct/distinct-sum.spec.json @@ -1,6 +1,6 @@ { "label": "SUM select with DISTINCT BY columns", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", "expectedCsv": "tests/sql/selects/sum/distinct/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/selects/sum/sum.json b/backend/src/test/resources/tests/sql/selects/sum/sum.json index c74f2672c0..4ab0d65458 100644 --- a/backend/src/test/resources/tests/sql/selects/sum/sum.json +++ b/backend/src/test/resources/tests/sql/selects/sum/sum.json @@ -1,6 +1,9 @@ { "label": "SUM select", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "expectedCsv": "tests/sql/selects/sum/expected.csv", "query": { "type": "CONCEPT_QUERY", diff --git a/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/block/and/none.spec.json b/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/block/and/none.spec.json index 08df2df906..d5f00804a3 100644 --- a/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/block/and/none.spec.json +++ b/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/block/and/none.spec.json @@ -1,7 +1,10 @@ { "label": "NONE date aggregation of 2 concepts", "expectedCsv": "tests/sql/selects/validity_date/aggregation/block/and/expected.csv", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "query": { "type": "CONCEPT_QUERY", "dateAggregationMode": "NONE", diff --git a/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/block/not/negate.spec.json b/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/block/not/negate.spec.json index 4353e5e05f..7c52280e6c 100644 --- a/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/block/not/negate.spec.json +++ b/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/block/not/negate.spec.json @@ -1,7 +1,10 @@ { "label": "BLOCK as default date aggregation of a NEGATION CONCEPT node", "expectedCsv": "tests/sql/selects/validity_date/aggregation/block/not/expected.csv", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "query": { "type": "CONCEPT_QUERY", "root": { diff --git a/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/intersect/intersect.spec.json b/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/intersect/intersect.spec.json index 8fbe89af22..c413e3143f 100644 --- a/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/intersect/intersect.spec.json +++ b/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/intersect/intersect.spec.json @@ -1,7 +1,10 @@ { "label": "INTERSECT date aggregation of 2 concepts", "expectedCsv": "tests/sql/selects/validity_date/aggregation/intersect/expected.csv", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "query": { "type": "CONCEPT_QUERY", "dateAggregationMode": "INTERSECT", diff --git a/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/merge/merge.spec.json b/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/merge/merge.spec.json index 001f95e744..170696cec2 100644 --- a/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/merge/merge.spec.json +++ b/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/merge/merge.spec.json @@ -1,7 +1,10 @@ { "label": "MERGE date aggregation of 2 concepts", "expectedCsv": "tests/sql/selects/validity_date/aggregation/merge/expected.csv", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "query": { "type": "CONCEPT_QUERY", "dateAggregationMode": "MERGE", diff --git a/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/negate/negate.spec.json b/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/negate/negate.spec.json index 91f3a5a2a5..a9eb91baf6 100644 --- a/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/negate/negate.spec.json +++ b/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/negate/negate.spec.json @@ -1,7 +1,10 @@ { "label": "LOGICAL date aggregation of a NEGATION CONCEPT node", "expectedCsv": "tests/sql/selects/validity_date/aggregation/negate/expected.csv", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "query": { "type": "CONCEPT_QUERY", "dateAggregationMode": "LOGICAL", diff --git a/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/no_validity_date/no_validity_date.spec.json b/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/no_validity_date/no_validity_date.spec.json index 90c2552301..8bfa69fdc4 100644 --- a/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/no_validity_date/no_validity_date.spec.json +++ b/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/no_validity_date/no_validity_date.spec.json @@ -1,7 +1,10 @@ { "label": "MERGE date aggregation of 2 concepts but no one has a validity date", "expectedCsv": "tests/sql/selects/validity_date/aggregation/no_validity_date/expected.csv", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "query": { "type": "CONCEPT_QUERY", "dateAggregationMode": "MERGE", diff --git a/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/only_1_date/only_1_date.spec.json b/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/only_1_date/only_1_date.spec.json index dc3af6f560..6d66047fe1 100644 --- a/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/only_1_date/only_1_date.spec.json +++ b/backend/src/test/resources/tests/sql/selects/validity_date/aggregation/only_1_date/only_1_date.spec.json @@ -1,7 +1,10 @@ { "label": "MERGE date aggregation of 2 concepts but only 1 has a validity date", "expectedCsv": "tests/sql/selects/validity_date/aggregation/only_1_date/expected.csv", - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "query": { "type": "CONCEPT_QUERY", "dateAggregationMode": "MERGE", diff --git a/backend/src/test/resources/tests/sql/selects/validity_date/default/validity_date_default.json b/backend/src/test/resources/tests/sql/selects/validity_date/default/validity_date_default.json index 5d86de2e64..d976c28a94 100644 --- a/backend/src/test/resources/tests/sql/selects/validity_date/default/validity_date_default.json +++ b/backend/src/test/resources/tests/sql/selects/validity_date/default/validity_date_default.json @@ -1,5 +1,8 @@ { - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "label": "Validity date as default select without date restriction set", "description": "If validity dates exist and the concept is not excluded from time aggregation, validity dates should be part of the final selects - regardless if a date restriction is set.", "expectedCsv": "tests/sql/selects/validity_date/default/expected.csv", diff --git a/backend/src/test/resources/tests/sql/selects/validity_date/excluded_from_time_aggregation/validity_date_excluded.json b/backend/src/test/resources/tests/sql/selects/validity_date/excluded_from_time_aggregation/validity_date_excluded.json index b2a928865b..5a45613b4a 100644 --- a/backend/src/test/resources/tests/sql/selects/validity_date/excluded_from_time_aggregation/validity_date_excluded.json +++ b/backend/src/test/resources/tests/sql/selects/validity_date/excluded_from_time_aggregation/validity_date_excluded.json @@ -1,5 +1,8 @@ { - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "label": "Validity date excluded from time aggregation", "description": "If a concept is excluded from time aggregation, validity dates should not be part of the final select.", "expectedCsv": "tests/sql/selects/validity_date/excluded_from_time_aggregation/expected.csv", diff --git a/backend/src/test/resources/tests/sql/selects/validity_date/multirange/multi_range.spec.json b/backend/src/test/resources/tests/sql/selects/validity_date/multirange/multi_range.spec.json index 92f9a37549..f7e8f5b92e 100644 --- a/backend/src/test/resources/tests/sql/selects/validity_date/multirange/multi_range.spec.json +++ b/backend/src/test/resources/tests/sql/selects/validity_date/multirange/multi_range.spec.json @@ -1,5 +1,8 @@ { - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "label": "Validity date with multiple ranges for subjects", "description": "If multiple ranges of a concepts' validity date exist, that they are aggregated properly.", "expectedCsv": "tests/sql/selects/validity_date/multirange/expected.csv", diff --git a/backend/src/test/resources/tests/sql/tree/nested/nested.spec.json b/backend/src/test/resources/tests/sql/tree/nested/nested.spec.json index e7270db1be..466673114a 100644 --- a/backend/src/test/resources/tests/sql/tree/nested/nested.spec.json +++ b/backend/src/test/resources/tests/sql/tree/nested/nested.spec.json @@ -1,5 +1,8 @@ { - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "label": "Simple TREE concept with 1 child with EQUAL CTCondition selected", "expectedCsv": "tests/sql/tree/nested/expected.csv", "query": { diff --git a/backend/src/test/resources/tests/sql/tree/prefix_range/prefix_range.spec.json b/backend/src/test/resources/tests/sql/tree/prefix_range/prefix_range.spec.json index d1a314febf..b6161700d1 100644 --- a/backend/src/test/resources/tests/sql/tree/prefix_range/prefix_range.spec.json +++ b/backend/src/test/resources/tests/sql/tree/prefix_range/prefix_range.spec.json @@ -1,5 +1,8 @@ { - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "label": "PREFIX_RANGE condition test", "expectedCsv": "tests/query/MULTIPLE_CONNECTORS_QUERY/expected.csv", "query": { diff --git a/backend/src/test/resources/tests/sql/tree/simple/simple.spec.json b/backend/src/test/resources/tests/sql/tree/simple/simple.spec.json index c09a3534ad..b36915b328 100644 --- a/backend/src/test/resources/tests/sql/tree/simple/simple.spec.json +++ b/backend/src/test/resources/tests/sql/tree/simple/simple.spec.json @@ -1,5 +1,8 @@ { - "type": "SQL_QUERY_TEST", + "type": "QUERY_TEST", + "sqlSpec": { + "isEnabled": true + }, "label": "Simple TREE concept with 1 child with EQUAL CTCondition selected", "expectedCsv": "tests/sql/tree/simple/expected.csv", "query": {