diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java index 8a9138bf05..4c413a3678 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java @@ -6,7 +6,6 @@ import java.util.Set; import java.util.function.Consumer; -import javax.annotation.CheckForNull; import javax.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; @@ -110,7 +109,7 @@ public void resolve(final QueryResolveContext context) { for (CQTable connector : concept.getTables()) { final Table table = connector.getConnector().getTable(); - final Column secondaryIdColumn = findSecondaryIdColumn(table); + final Column secondaryIdColumn = table.findSecondaryIdColumn(secondaryId); if (secondaryIdColumn != null && !concept.isExcludeFromSecondaryId()) { withSecondaryId.add(secondaryIdColumn); @@ -127,23 +126,6 @@ public void resolve(final QueryResolveContext context) { } } - /** - * selects the right column for the given secondaryId from a table - */ - @CheckForNull - private Column findSecondaryIdColumn(Table table) { - - for (Column col : table.getColumns()) { - if (col.getSecondaryId() == null || !secondaryId.equals(col.getSecondaryId())) { - continue; - } - - return col; - } - - return null; - } - @Override public List getResultInfos() { final List resultInfos = new ArrayList<>(); @@ -170,4 +152,4 @@ public CQElement getReusableComponents() { public RequiredEntities collectRequiredEntities(QueryExecutionContext context) { return query.collectRequiredEntities(context); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/CQTable.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/CQTable.java index b858c3741d..b165cfbd59 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/CQTable.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/CQTable.java @@ -1,7 +1,9 @@ package com.bakdata.conquery.apiv1.query.concept.filter; +import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Objects; import javax.annotation.CheckForNull; import javax.validation.Valid; @@ -10,6 +12,8 @@ import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.io.jackson.serializer.NsIdRef; import com.bakdata.conquery.io.jackson.serializer.NsIdRefCollection; +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.datasets.SecondaryIdDescription; import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.models.datasets.concepts.select.Select; @@ -88,4 +92,11 @@ public ValidityDate findValidityDate() { return null; } + public boolean hasSelectedSecondaryId(SecondaryIdDescription secondaryId) { + return Arrays.stream(connector.getTable().getColumns()) + .map(Column::getSecondaryId) + .filter(Objects::nonNull) + .anyMatch(o -> Objects.equals(secondaryId, o)); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java index 0cd55977d1..0601b943ca 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java @@ -21,6 +21,7 @@ import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -70,8 +71,8 @@ public FilterNode createNode() { return getFilter().createFilterNode(getValue()); } - public SqlFilters convertToSqlFilter(ConversionContext context, SqlTables connectorTables) { - FilterContext filterContext = new FilterContext<>(value, context, connectorTables); + public SqlFilters convertToSqlFilter(SqlIdColumns ids, ConversionContext context, SqlTables connectorTables) { + FilterContext filterContext = new FilterContext<>(ids, value, context, connectorTables); SqlFilters sqlFilters = filter.convertToSqlFilter(filterContext); if (context.isNegation()) { return new SqlFilters(sqlFilters.getSelects(), sqlFilters.getWhereClauses().negated()); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java index bed4ace016..e2f988f019 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java @@ -1,11 +1,9 @@ package com.bakdata.conquery.apiv1.query.concept.specific; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Locale; -import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -21,7 +19,6 @@ import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.jackson.serializer.NsIdRefCollection; import com.bakdata.conquery.models.common.CDateSet; -import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.models.datasets.concepts.Connector; @@ -234,12 +231,7 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { existsAggregators.forEach(agg -> agg.setReference(conceptSpecificNode)); // Select if matching secondaryId available - final boolean hasSelectedSecondaryId = - Arrays.stream(table.getConnector().getTable().getColumns()) - .map(Column::getSecondaryId) - .filter(Objects::nonNull) - .anyMatch(o -> Objects.equals(context.getSelectedSecondaryId(), o)); - + final boolean hasSelectedSecondaryId = table.hasSelectedSecondaryId(context.getSelectedSecondaryId()); final ConceptNode node = new ConceptNode( conceptSpecificNode, diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java index 856fdd54b7..4d59ac4910 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java @@ -5,6 +5,7 @@ import java.util.Set; import java.util.stream.Stream; +import javax.annotation.CheckForNull; import javax.validation.Valid; import javax.validation.constraints.NotNull; @@ -78,4 +79,22 @@ public Column getColumnByName(@NotNull String columnName) { .findFirst() .orElseThrow(() -> new IllegalStateException(String.format("Column %s not found", columnName))); } + + /** + * selects the right column for the given secondaryId from this table + */ + @CheckForNull + public Column findSecondaryIdColumn(SecondaryIdDescription secondaryId) { + + for (Column col : columns) { + if (col.getSecondaryId() == null || !secondaryId.equals(col.getSecondaryId())) { + continue; + } + + return col; + } + + return null; + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/ConceptSqlQuery.java b/backend/src/main/java/com/bakdata/conquery/sql/ConceptSqlQuery.java deleted file mode 100644 index 9891a70047..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/ConceptSqlQuery.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.bakdata.conquery.sql; - -import java.util.List; - -import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.sql.conversion.model.SqlQuery; -import lombok.Value; -import org.jooq.Record; -import org.jooq.Select; -import org.jooq.conf.ParamType; - -@Value -public class ConceptSqlQuery implements SqlQuery { - - String sqlString; - List resultInfos; - - public ConceptSqlQuery(Select finalQuery, List resultInfos) { - this.sqlString = finalQuery.getSQL(ParamType.INLINED); - this.resultInfos = resultInfos; - } - - @Override - public String getSql() { - return this.sqlString; - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java index 03f8e22906..81b936e558 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java @@ -6,7 +6,6 @@ import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; import com.bakdata.conquery.sql.conversion.model.NameGenerator; -import org.jooq.impl.DSL; /** * Entry point for converting {@link QueryDescription} to an SQL query. @@ -28,7 +27,6 @@ public ConversionContext convert(QueryDescription queryDescription) { .nameGenerator(new NameGenerator(config.getDialect().getNameMaxLength())) .nodeConversions(this) .sqlDialect(this.dialect) - .primaryColumn(DSL.field(DSL.name(config.getPrimaryColumn()))) .build(); return convert(queryDescription, initialCtx); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SharedAliases.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SharedAliases.java index e365dd84aa..16374562d4 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SharedAliases.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SharedAliases.java @@ -7,7 +7,9 @@ @RequiredArgsConstructor public enum SharedAliases { - PRIMARY_COLUMN("pid"); + PRIMARY_COLUMN("primary_id"), + SECONDARY_ID("secondary_id"), + DATES_COLUMN("dates"); private final String alias; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java index 1e8d32f988..51421ef813 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java @@ -2,8 +2,13 @@ import java.util.List; +import javax.annotation.Nullable; + +import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; +import com.bakdata.conquery.apiv1.query.concept.specific.CQDateRestriction; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.models.datasets.SecondaryIdDescription; import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.NodeConversions; import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; @@ -14,7 +19,6 @@ import lombok.Singular; import lombok.Value; import lombok.With; -import org.jooq.Field; @Value @With @@ -22,15 +26,33 @@ public class ConversionContext implements Context { SqlConnectorConfig config; + NodeConversions nodeConversions; + SqlDialect sqlDialect; + NameGenerator nameGenerator; + @Singular List querySteps; + + @Nullable SqlQuery finalQuery; - Field primaryColumn; + + /** + * An optional date restriction range. Is set when converting a {@link CQDateRestriction}. + */ + @Nullable CDateRange dateRestrictionRange; + + /** + * An optional secondary id to group results by in addition to the primary id. Only set when converting {@link SecondaryIdQuery}s. + */ + @Nullable + SecondaryIdDescription secondaryIdDescription; + boolean negation; + boolean isGroupBy; public boolean dateRestrictionActive() { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java index 1bbd58cf75..e253655c98 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java @@ -44,7 +44,7 @@ public QueryStep apply( .carryThroughSelects(carryThroughSelects) .dateAggregationDates(dateAggregationDates) .dateAggregationTables(aggregationAction.tableNames(nameGenerator)) - .primaryColumn(joinedStep.getQualifiedSelects().getPrimaryColumn()) + .ids(joinedStep.getQualifiedSelects().getIds()) .functionProvider(this.functionProvider) .intervalPacker(this.intervalPacker) .nameGenerator(nameGenerator) @@ -62,7 +62,7 @@ public QueryStep apply( IntervalPackingContext intervalPackingContext = IntervalPackingContext.builder() .nodeLabel(joinedCteLabel) - .primaryColumn(predecessorSelects.getPrimaryColumn()) + .ids(predecessorSelects.getIds()) .validityDate(predecessorSelects.getValidityDate().get()) .predecessor(finalDateAggregationStep) .carryThroughSelects(carryThroughSelects) @@ -89,7 +89,7 @@ public QueryStep invertAggregatedIntervals(QueryStep baseStep, NameGenerator nam .carryThroughSelects(baseStepQualifiedSelects.getSqlSelects()) .dateAggregationDates(dateAggregationDates) .dateAggregationTables(dateAggregationTables) - .primaryColumn(baseStepQualifiedSelects.getPrimaryColumn()) + .ids(baseStepQualifiedSelects.getIds()) .functionProvider(this.functionProvider) .intervalPacker(this.intervalPacker) .nameGenerator(nameGenerator) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java index 9430b046a4..d393463f98 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java @@ -9,19 +9,20 @@ import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.NameGenerator; +import com.bakdata.conquery.sql.conversion.model.Qualifiable; import com.bakdata.conquery.sql.conversion.model.QualifyingUtil; import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; import lombok.Builder; import lombok.Value; -import org.jooq.Field; @Value @Builder(toBuilder = true) -class DateAggregationContext implements Context { +class DateAggregationContext implements Context, Qualifiable { - Field primaryColumn; + SqlIdColumns ids; List carryThroughSelects; SqlTables dateAggregationTables; DateAggregationDates dateAggregationDates; @@ -55,9 +56,10 @@ public List getSteps(DateAggregationCteStep dateAggregationCteStep) { return this.intervalMergeSteps.get(dateAggregationCteStep); } + @Override public DateAggregationContext qualify(String qualifier) { return this.toBuilder() - .primaryColumn(QualifyingUtil.qualify(this.primaryColumn, qualifier)) + .ids(this.ids.qualify(qualifier)) .carryThroughSelects(QualifyingUtil.qualify(this.carryThroughSelects, qualifier)) .dateAggregationDates(this.dateAggregationDates.qualify(qualifier)) .build(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java index a061bc6117..bbd8a93070 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java @@ -29,7 +29,7 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context) context.getCarryThroughSelects() ); Selects selects = Selects.builder() - .primaryColumn(context.getPrimaryColumn()) + .ids(context.getIds()) .sqlSelects(intermediateTableSelects) .build(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java index 88781c941b..e4bc58d590 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java @@ -1,14 +1,16 @@ package com.bakdata.conquery.sql.conversion.cqelement.aggregation; import java.sql.Date; +import java.util.List; import java.util.Optional; +import java.util.stream.Stream; -import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QualifyingUtil; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import lombok.Getter; import org.jooq.Condition; import org.jooq.Field; @@ -37,29 +39,24 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context) QueryStep rowNumberStep = context.getStep(InvertCteStep.ROW_NUMBER); - Field primaryColumn = context.getPrimaryColumn(); - Field leftPrimaryColumn = QualifyingUtil.qualify(primaryColumn, ROWS_LEFT_TABLE_NAME); - Field rightPrimaryColumn = QualifyingUtil.qualify(primaryColumn, ROWS_RIGHT_TABLE_NAME); - Field coalescedPrimaryColumn = DSL.coalesce(leftPrimaryColumn, rightPrimaryColumn) - .as(SharedAliases.PRIMARY_COLUMN.getAlias()); + SqlIdColumns ids = context.getIds(); + SqlIdColumns leftIds = ids.qualify(ROWS_LEFT_TABLE_NAME); + SqlIdColumns rightIds = ids.qualify(ROWS_RIGHT_TABLE_NAME); + SqlIdColumns coalescedIds = SqlIdColumns.coalesce(List.of(leftIds, rightIds)); - Selects invertSelects = getInvertSelects(rowNumberStep, coalescedPrimaryColumn, context); - TableOnConditionStep fromTable = selfJoinWithShiftedRows(leftPrimaryColumn, rightPrimaryColumn, rowNumberStep); + Selects invertSelects = getInvertSelects(rowNumberStep, coalescedIds, context); + TableOnConditionStep fromTable = selfJoinWithShiftedRows(leftIds, rightIds, rowNumberStep); return QueryStep.builder() .selects(invertSelects) .fromTable(fromTable); } - private Selects getInvertSelects( - QueryStep rowNumberStep, - Field coalescedPrimaryColumn, - DateAggregationContext context - ) { + private Selects getInvertSelects(QueryStep rowNumberStep, SqlIdColumns coalescedIds, DateAggregationContext context) { SqlFunctionProvider functionProvider = context.getFunctionProvider(); ColumnDateRange validityDate = rowNumberStep.getSelects().getValidityDate().get(); - + Field rangeStart = DSL.coalesce( QualifyingUtil.qualify(validityDate.getEnd(), ROWS_LEFT_TABLE_NAME), functionProvider.toDateField(functionProvider.getMinDateExpression()) @@ -71,26 +68,28 @@ private Selects getInvertSelects( ).as(DateAggregationCte.RANGE_END); return Selects.builder() - .primaryColumn(coalescedPrimaryColumn) + .ids(coalescedIds) .validityDate(Optional.of(ColumnDateRange.of(rangeStart, rangeEnd))) .sqlSelects(context.getCarryThroughSelects()) .build(); } - private TableOnConditionStep selfJoinWithShiftedRows(Field leftPrimaryColumn, Field rightPrimaryColumn, QueryStep rowNumberStep) { + private TableOnConditionStep selfJoinWithShiftedRows(SqlIdColumns leftPrimaryColumn, SqlIdColumns rightPrimaryColumn, QueryStep rowNumberStep) { Field leftRowNumber = DSL.field(DSL.name(ROWS_LEFT_TABLE_NAME, RowNumberCte.ROW_NUMBER_FIELD_NAME), Integer.class) .plus(1); Field rightRowNumber = DSL.field(DSL.name(ROWS_RIGHT_TABLE_NAME, RowNumberCte.ROW_NUMBER_FIELD_NAME), Integer.class); - Condition joinCondition = leftPrimaryColumn.eq(rightPrimaryColumn) - .and(leftRowNumber.eq(rightRowNumber)); + Condition[] joinConditions = Stream.concat( + Stream.of(leftRowNumber.eq(rightRowNumber)), + SqlIdColumns.join(leftPrimaryColumn, rightPrimaryColumn).stream() + ) + .toArray(Condition[]::new); TableLike rowNumberTable = QueryStep.toTableLike(rowNumberStep.getCteName()); return rowNumberTable.asTable(ROWS_LEFT_TABLE_NAME) .fullJoin(rowNumberTable.asTable(ROWS_RIGHT_TABLE_NAME)) - .on(joinCondition); + .on(joinConditions); } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java index cd290e4fcd..369e1ed057 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/NodeNoOverlapCte.java @@ -60,7 +60,7 @@ private QueryStep.QueryStepBuilder createNoOverlapStep( Field asRangeStart = start.as(DateAggregationCte.RANGE_START); String intermediateTableCteName = dateAggregationTables.getPredecessor(getCteStep()); Selects nodeNoOverlapSelects = Selects.builder() - .primaryColumn(context.getPrimaryColumn()) + .ids(context.getIds()) .validityDate(Optional.of(ColumnDateRange.of(asRangeStart, asRangeEnd))) .sqlSelects(context.getCarryThroughSelects()) .build(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java index 35761f0ef1..82f9d443b5 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/OverlapCte.java @@ -33,7 +33,7 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context) context.getFunctionProvider() ); Selects overlapSelects = Selects.builder() - .primaryColumn(context.getPrimaryColumn()) + .ids(context.getIds()) .validityDate(Optional.of(overlapValidityDate)) .sqlSelects(context.getCarryThroughSelects()) .build(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/PostgreSqlDateAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/PostgreSqlDateAggregator.java index d11e9eeb02..1ebec7b57e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/PostgreSqlDateAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/PostgreSqlDateAggregator.java @@ -49,7 +49,7 @@ public QueryStep apply( ColumnDateRange aggregatedValidityDate = getAggregatedValidityDate(dateAggregationDates, dateAggregationAction, joinedStepCteName); Selects dateAggregationSelects = Selects.builder() - .primaryColumn(joinedStep.getQualifiedSelects().getPrimaryColumn()) + .ids(joinedStep.getQualifiedSelects().getIds()) .validityDate(Optional.ofNullable(aggregatedValidityDate)) .sqlSelects(QualifyingUtil.qualify(carryThroughSelects, joinedStepCteName)) .build(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java index b3387fdae6..53492f0982 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/RowNumberCte.java @@ -6,6 +6,7 @@ import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; import lombok.Getter; @@ -28,17 +29,17 @@ public RowNumberCte(DateAggregationCteStep cteStep) { @Override protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context) { - Field primaryColumn = context.getPrimaryColumn(); + SqlIdColumns ids = context.getIds(); ColumnDateRange aggregatedValidityDate = context.getDateAggregationDates().getValidityDates().get(0); - Field rowNumber = DSL.rowNumber().over(DSL.partitionBy(primaryColumn).orderBy(aggregatedValidityDate.getStart())) + Field rowNumber = DSL.rowNumber().over(DSL.partitionBy(ids.toFields()).orderBy(aggregatedValidityDate.getStart())) .as(ROW_NUMBER_FIELD_NAME); ArrayList selects = new ArrayList<>(context.getCarryThroughSelects()); selects.add(new FieldWrapper<>(rowNumber)); Selects rowNumberSelects = Selects.builder() - .primaryColumn(primaryColumn) + .ids(ids) .validityDate(Optional.of(aggregatedValidityDate)) .sqlSelects(selects) .build(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java index c3be63a466..bdc886bdb0 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java @@ -44,7 +44,7 @@ private Selects getAggregationFilterSelects(CQTableContext tableContext) { .collect(Collectors.toList()); return Selects.builder() - .primaryColumn(previousSelects.getPrimaryColumn()) + .ids(previousSelects.getIds()) .validityDate(previousSelects.getValidityDate()) .sqlSelects(forAggregationFilterStep) .build(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java index 33facee40c..c9c1881239 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java @@ -2,11 +2,10 @@ import java.util.List; -import com.bakdata.conquery.sql.conversion.model.QualifyingUtil; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; -import org.jooq.Field; class AggregationSelectCte extends ConnectorCte { @@ -14,20 +13,20 @@ class AggregationSelectCte extends ConnectorCte { public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { String predecessor = tableContext.getConnectorTables().getPredecessor(ConnectorCteStep.AGGREGATION_SELECT); - Field primaryColumn = QualifyingUtil.qualify(tableContext.getPrimaryColumn(), predecessor); + SqlIdColumns ids = tableContext.getIds().qualify(predecessor); List requiredInAggregationFilterStep = tableContext.allSqlSelects().stream() .flatMap(sqlSelects -> sqlSelects.getAggregationSelects().stream()) .toList(); Selects aggregationSelectSelects = Selects.builder() - .primaryColumn(primaryColumn) + .ids(ids) .sqlSelects(requiredInAggregationFilterStep) .build(); return QueryStep.builder() .selects(aggregationSelectSelects) - .groupBy(List.of(primaryColumn)); + .groupBy(ids.toFields()); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index db58dc0bfd..843fb04b25 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -9,12 +9,14 @@ import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeChild; import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeNode; import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingContext; import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep; @@ -26,6 +28,7 @@ import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.ConditionType; import com.bakdata.conquery.sql.conversion.model.filter.ConditionUtil; @@ -36,6 +39,7 @@ import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; +import com.google.common.base.Preconditions; import lombok.Getter; import lombok.RequiredArgsConstructor; import org.jooq.Condition; @@ -104,7 +108,7 @@ private Optional convertCqTable(String conceptLabel, CQConcept cqConc private static QueryStep finishConceptConversion(String conceptLabel, QueryStep predecessor, CQConcept cqConcept, ConversionContext context) { Selects predecessorSelects = predecessor.getQualifiedSelects(); - SelectContext selectContext = SelectContext.forUniversalSelects(predecessorSelects.getPrimaryColumn(), predecessorSelects.getValidityDate(), context); + SelectContext selectContext = SelectContext.forUniversalSelects(predecessorSelects.getIds(), predecessorSelects.getValidityDate(), context); List universalSelects = cqConcept.getSelects().stream() .map(select -> select.convertToSqlSelects(selectContext)) .flatMap(sqlSelects -> sqlSelects.getFinalSelects().stream()) @@ -136,7 +140,7 @@ private CQTableContext createTableContext(String conceptLabel, CQConcept cqConce String conceptConnectorLabel = nameGenerator.conceptConnectorName(cqConcept, connector); String tableName = connector.getTable().getName(); - Field primaryColumn = DSL.field(DSL.name(conversionContext.getConfig().getPrimaryColumn())); + SqlIdColumns ids = convertIds(cqConcept, cqTable, conversionContext); Optional tablesValidityDate = convertValidityDate(cqTable, tableName, conversionContext); SqlTables connectorTables = ConnectorCteStep.createTables(conceptConnectorLabel, tableName, nameGenerator); @@ -147,7 +151,7 @@ private CQTableContext createTableContext(String conceptLabel, CQConcept cqConce SqlTables intervalPackingTables = IntervalPackingCteStep.getTables(conceptConnectorLabel, preprocessingCteName, nameGenerator); intervalPackingContext = IntervalPackingContext.builder() .nodeLabel(conceptConnectorLabel) - .primaryColumn(primaryColumn) + .ids(ids) .validityDate(tablesValidityDate.get()) .intervalPackingTables(intervalPackingTables) .build(); @@ -156,13 +160,13 @@ private CQTableContext createTableContext(String conceptLabel, CQConcept cqConce // convert filters List allSqlFiltersForTable = new ArrayList<>(); cqTable.getFilters().stream() - .map(filterValue -> filterValue.convertToSqlFilter(conversionContext, connectorTables)) + .map(filterValue -> filterValue.convertToSqlFilter(ids, conversionContext, connectorTables)) .forEach(allSqlFiltersForTable::add); collectConditionFilters(cqConcept.getElements(), cqTable, functionProvider).ifPresent(allSqlFiltersForTable::add); getDateRestriction(conversionContext, tablesValidityDate).ifPresent(allSqlFiltersForTable::add); // convert selects - SelectContext selectContext = SelectContext.forConnectorSelects(primaryColumn, tablesValidityDate, connectorTables, conversionContext); + SelectContext selectContext = SelectContext.forConnectorSelects(ids, tablesValidityDate, connectorTables, conversionContext); List allSelectsForTable = cqTable.getSelects().stream() .map(select -> select.convertToSqlSelects(selectContext)) .toList(); @@ -170,7 +174,7 @@ private CQTableContext createTableContext(String conceptLabel, CQConcept cqConce return CQTableContext.builder() .conceptLabel(conceptLabel) .conceptConnectorLabel(conceptConnectorLabel) - .primaryColumn(primaryColumn) + .ids(ids) .validityDate(tablesValidityDate) .sqlSelects(allSelectsForTable) .sqlFilters(allSqlFiltersForTable) @@ -180,6 +184,31 @@ private CQTableContext createTableContext(String conceptLabel, CQConcept cqConce .build(); } + private static SqlIdColumns convertIds(CQConcept cqConcept, CQTable cqTable, ConversionContext conversionContext) { + + Field primaryColumn = DSL.field(DSL.name(conversionContext.getConfig().getPrimaryColumn())).as(SharedAliases.PRIMARY_COLUMN.getAlias()); + + if (cqConcept.isExcludeFromSecondaryId() + || conversionContext.getSecondaryIdDescription() == null + || !cqTable.hasSelectedSecondaryId(conversionContext.getSecondaryIdDescription()) + ) { + return new SqlIdColumns(primaryColumn); + } + + Column secondaryIdColumn = cqTable.getConnector().getTable().findSecondaryIdColumn(conversionContext.getSecondaryIdDescription()); + + Preconditions.checkArgument( + secondaryIdColumn != null, + "Expecting Table %s to have a matching secondary id for %s".formatted( + cqTable.getConnector().getTable(), + conversionContext.getSecondaryIdDescription() + ) + ); + + Field secondaryId = DSL.field(DSL.name(secondaryIdColumn.getName())).as(SharedAliases.SECONDARY_ID.getAlias()); + return new SqlIdColumns(primaryColumn, secondaryId); + } + private static Optional convertValidityDate(CQTable cqTable, String connectorLabel, ConversionContext context) { if (Objects.isNull(cqTable.findValidityDate())) { return Optional.empty(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java index 9dfc5a164b..548d66ccfb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java @@ -13,12 +13,12 @@ import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; import lombok.Builder; import lombok.Value; import lombok.With; -import org.jooq.Field; @Value @Builder @@ -26,7 +26,7 @@ class CQTableContext implements Context { String conceptLabel; String conceptConnectorLabel; - Field primaryColumn; + SqlIdColumns ids; Optional validityDate; List sqlSelects; List sqlFilters; @@ -43,11 +43,11 @@ public List allSqlSelects() { return Stream.concat(sqlSelects.stream(), sqlFilters.stream().map(SqlFilters::getSelects)).toList(); } - public Field getPrimaryColumn() { + public SqlIdColumns getIds() { if (previous == null) { - return this.primaryColumn; + return ids; } - return previous.getSelects().getPrimaryColumn(); + return previous.getQualifiedSelects().getIds(); } public Optional getIntervalPackingContext() { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterCte.java index 703933d196..eba3200c82 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterCte.java @@ -5,9 +5,9 @@ import java.util.stream.Stream; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QualifyingUtil; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.aggregator.SumDistinctSqlAggregator; import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.sql.conversion.model.select.ExtractingSqlSelect; @@ -15,7 +15,6 @@ import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; import org.jooq.Condition; -import org.jooq.Field; class EventFilterCte extends ConnectorCte { @@ -38,8 +37,7 @@ public ConnectorCteStep cteStep() { private Selects getEventFilterSelects(CQTableContext tableContext) { String predecessorTableName = tableContext.getConnectorTables().getPredecessor(cteStep()); - - Field primaryColumn = QualifyingUtil.qualify(tableContext.getPrimaryColumn(), predecessorTableName); + SqlIdColumns ids = tableContext.getIds().qualify(predecessorTableName); Optional validityDate = tableContext.getValidityDate(); if (validityDate.isPresent()) { @@ -53,7 +51,7 @@ private Selects getEventFilterSelects(CQTableContext tableContext) { .toList(); return Selects.builder() - .primaryColumn(primaryColumn) + .ids(ids) .validityDate(validityDate) .sqlSelects(eventFilterSelects) .build(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java index ca41d70864..22d67d50ea 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java @@ -4,6 +4,7 @@ import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.model.NameGenerator; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; import lombok.Value; @@ -13,6 +14,7 @@ public class FilterContext implements Context { /** * A filter value ({@link FilterValue#getValue()}) */ + SqlIdColumns ids; V value; ConversionContext parentContext; SqlTables connectorTables; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java index df8b175157..131c3f7cc8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java @@ -10,9 +10,9 @@ import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.aggregator.SumDistinctSqlAggregator; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; -import org.jooq.Field; import org.jooq.Record; import org.jooq.TableLike; @@ -66,10 +66,10 @@ protected QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { .flatMap(sqlSelects -> sqlSelects.getAdditionalPredecessor().stream()) .forEach(queriesToJoin::add); - Field primaryColumn = QueryStepJoiner.coalescePrimaryColumns(queriesToJoin); + SqlIdColumns ids = QueryStepJoiner.coalesceIds(queriesToJoin); List mergedSqlSelects = QueryStepJoiner.mergeSelects(queriesToJoin); Selects selects = Selects.builder() - .primaryColumn(primaryColumn) + .ids(ids) .validityDate(validityDate) .sqlSelects(mergedSqlSelects) .build(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java index 36e1b35dfc..60094e2c8e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java @@ -2,13 +2,11 @@ import java.util.List; -import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; import org.jooq.Condition; -import org.jooq.Field; class PreprocessingCte extends ConnectorCte { @@ -18,11 +16,8 @@ public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { .flatMap(sqlSelects -> sqlSelects.getPreprocessingSelects().stream()) .toList(); - // we alias the primary column, so we can rely upon in other places that it has a specific name - Field aliasesPrimaryColumn = tableContext.getPrimaryColumn().as(SharedAliases.PRIMARY_COLUMN.getAlias()); - Selects preprocessingSelects = Selects.builder() - .primaryColumn(aliasesPrimaryColumn) + .ids(tableContext.getIds()) .validityDate(tableContext.getValidityDate()) .sqlSelects(forPreprocessing) .build(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java index 2db795a310..cb60c686c5 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/AnsiSqlIntervalPacker.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.sql.conversion.model.QualifyingUtil; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; import lombok.RequiredArgsConstructor; @@ -30,11 +31,11 @@ public QueryStep createIntervalPackingSteps(IntervalPackingContext context) { private QueryStep createPreviousEndStep(IntervalPackingContext context) { String sourceTableName = context.getIntervalPackingTables().getRootTable(); - Field primaryColumn = QualifyingUtil.qualify(context.getPrimaryColumn(), sourceTableName); + SqlIdColumns ids = context.getIds().qualify(sourceTableName); ColumnDateRange validityDate = context.getValidityDate().qualify(sourceTableName); Field previousEnd = DSL.max(validityDate.getEnd()) - .over(DSL.partitionBy(primaryColumn) + .over(DSL.partitionBy(ids.toFields()) .orderBy(validityDate.getStart(), validityDate.getEnd()) .rowsBetweenUnboundedPreceding() .andPreceding(1)) @@ -44,7 +45,7 @@ private QueryStep createPreviousEndStep(IntervalPackingContext context) { qualifiedSelects.add(new FieldWrapper<>(previousEnd)); Selects previousEndSelects = Selects.builder() - .primaryColumn(primaryColumn) + .ids(ids) .validityDate(Optional.of(validityDate)) .sqlSelects(qualifiedSelects) .build(); @@ -61,7 +62,7 @@ private QueryStep createRangeIndexStep(QueryStep previousEndStep, IntervalPackin String previousEndCteName = previousEndStep.getCteName(); Selects previousEndSelects = previousEndStep.getQualifiedSelects(); - Field primaryColumn = previousEndSelects.getPrimaryColumn(); + SqlIdColumns ids = previousEndSelects.getIds(); ColumnDateRange validityDate = previousEndSelects.getValidityDate().get(); Field previousEnd = DSL.field(DSL.name(previousEndCteName, IntervalPacker.PREVIOUS_END_FIELD_NAME), Date.class); @@ -69,7 +70,7 @@ private QueryStep createRangeIndexStep(QueryStep previousEndStep, IntervalPackin DSL.sum( DSL.when(validityDate.getStart().greaterThan(previousEnd), DSL.val(1)) .otherwise(DSL.inline(null, Integer.class))) - .over(DSL.partitionBy(primaryColumn) + .over(DSL.partitionBy(ids.toFields()) .orderBy(validityDate.getStart(), validityDate.getEnd()) .rowsUnboundedPreceding()) .as(IntervalPacker.RANGE_INDEX_FIELD_NAME); @@ -78,7 +79,7 @@ private QueryStep createRangeIndexStep(QueryStep previousEndStep, IntervalPackin qualifiedSelects.add(new FieldWrapper<>(rangeIndex)); Selects rangeIndexSelects = Selects.builder() - .primaryColumn(primaryColumn) + .ids(ids) .validityDate(Optional.of(validityDate)) .sqlSelects(qualifiedSelects) .build(); @@ -95,7 +96,7 @@ private QueryStep createIntervalCompleteStep(QueryStep rangeIndexStep, IntervalP String rangeIndexCteName = rangeIndexStep.getCteName(); Selects rangeIndexSelects = rangeIndexStep.getQualifiedSelects(); - Field primaryColumn = rangeIndexSelects.getPrimaryColumn(); + SqlIdColumns ids = rangeIndexSelects.getIds(); ColumnDateRange validityDate = rangeIndexSelects.getValidityDate().get(); Field rangeStart = DSL.min(validityDate.getStart()).as(IntervalPacker.RANGE_START_MIN_FIELD_NAME); @@ -104,14 +105,14 @@ private QueryStep createIntervalCompleteStep(QueryStep rangeIndexStep, IntervalP List qualifiedSelects = QualifyingUtil.qualify(context.getCarryThroughSelects(), rangeIndexCteName); Selects intervalCompleteSelects = Selects.builder() - .primaryColumn(primaryColumn) + .ids(ids) .validityDate(Optional.of(ColumnDateRange.of(rangeStart, rangeEnd))) .sqlSelects(qualifiedSelects) .build(); // we group range start and end by range index List> groupBySelects = new ArrayList<>(); - groupBySelects.add(primaryColumn); + groupBySelects.addAll(ids.toFields()); groupBySelects.add(rangeIndex); qualifiedSelects.stream().map(SqlSelect::select).forEach(groupBySelects::add); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java index 22ef19e1f7..046574f64e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/IntervalPackingContext.java @@ -9,11 +9,11 @@ import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.NameGenerator; import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; import lombok.Builder; import lombok.Value; -import org.jooq.Field; @Value @Builder @@ -24,7 +24,7 @@ public class IntervalPackingContext implements Context { */ String nodeLabel; - Field primaryColumn; + SqlIdColumns ids; ColumnDateRange validityDate; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java index 0643315e3b..9e0e773cfc 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/intervalpacking/PostgreSqlIntervalPacker.java @@ -1,16 +1,14 @@ package com.bakdata.conquery.sql.conversion.cqelement.intervalpacking; -import java.util.List; import java.util.Optional; import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.QualifyingUtil; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import lombok.RequiredArgsConstructor; -import org.jooq.Field; /** * PostgreSql supports interval packing with a native range function. @@ -26,13 +24,13 @@ public class PostgreSqlIntervalPacker implements IntervalPacker { public QueryStep createIntervalPackingSteps(IntervalPackingContext context) { String sourceTableName = context.getIntervalPackingTables().getRootTable(); - Field primaryColumn = QualifyingUtil.qualify(context.getPrimaryColumn(), sourceTableName); + SqlIdColumns ids = context.getIds().qualify(sourceTableName); ColumnDateRange qualifiedValidityDate = context.getValidityDate().qualify(sourceTableName); ColumnDateRange aggregatedValidityDate = this.functionProvider.aggregated(qualifiedValidityDate) .asValidityDateRange(context.getNodeLabel()); Selects selectsWithAggregatedValidityDate = Selects.builder() - .primaryColumn(primaryColumn) + .ids(ids) .validityDate(Optional.of(aggregatedValidityDate)) .sqlSelects(context.getCarryThroughSelects()) .build(); @@ -41,7 +39,7 @@ public QueryStep createIntervalPackingSteps(IntervalPackingContext context) { .cteName(context.getIntervalPackingTables().cteName(IntervalPackingCteStep.INTERVAL_COMPLETE)) .selects(selectsWithAggregatedValidityDate) .fromTable(QueryStep.toTableLike(sourceTableName)) - .groupBy(List.of(primaryColumn)) + .groupBy(ids.toFields()) .predecessors(Optional.ofNullable(context.getPredecessor()).stream().toList()) .build(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java index 7ff8be5ffe..04a75e5b74 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java @@ -15,6 +15,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.CQConceptConverter; import com.bakdata.conquery.sql.conversion.model.QueryStepTransformer; import com.bakdata.conquery.sql.conversion.query.ConceptQueryConverter; +import com.bakdata.conquery.sql.conversion.query.SecondaryIdQueryConverter; import com.bakdata.conquery.sql.conversion.supplier.DateNowSupplier; import com.bakdata.conquery.sql.conversion.supplier.SystemDateNowSupplier; import com.bakdata.conquery.sql.execution.SqlCDateSetParser; @@ -51,7 +52,8 @@ default List> getDefaultNodeConverters() { new CQOrConverter(), new CQNegationConverter(), new CQConceptConverter(), - new ConceptQueryConverter(new QueryStepTransformer(getDSLContext())) + new ConceptQueryConverter(new QueryStepTransformer(getDSLContext())), + new SecondaryIdQueryConverter() ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index 536a168a7c..0aad646a93 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -118,23 +118,21 @@ default Condition in(Field column, String[] values) { default TableOnConditionStep innerJoin( Table leftPartQueryBase, QueryStep rightPartQS, - Field leftPartPrimaryColumn, - Field rightPartPrimaryColumn + List joinConditions ) { return leftPartQueryBase .innerJoin(DSL.name(rightPartQS.getCteName())) - .on(leftPartPrimaryColumn.eq(rightPartPrimaryColumn)); + .on(joinConditions.toArray(Condition[]::new)); } default TableOnConditionStep fullOuterJoin( Table leftPartQueryBase, QueryStep rightPartQS, - Field leftPartPrimaryColumn, - Field rightPartPrimaryColumn + List joinConditions ) { return leftPartQueryBase .fullOuterJoin(DSL.name(rightPartQS.getCteName())) - .on(leftPartPrimaryColumn.eq(rightPartPrimaryColumn)); + .on(joinConditions.toArray(Condition[]::new)); } default Field toDateField(String dateExpression) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java index 1acaa39f68..1d4fb6a46a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java @@ -9,7 +9,7 @@ import org.jooq.Field; @Getter -public class ColumnDateRange { +public class ColumnDateRange implements Qualifiable { private static final String DATE_RESTRICTION_COLUMN_NAME = "date_restriction"; private static final String VALIDITY_DATE_COLUMN_NAME_SUFFIX = "_validity_date"; @@ -75,6 +75,7 @@ public List> toFields() { .collect(Collectors.toList()); } + @Override public ColumnDateRange qualify(String qualifier) { if (isSingleColumnRange()) { return new ColumnDateRange(QualifyingUtil.qualify(getRange(), qualifier), getAlias()); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Qualifiable.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Qualifiable.java new file mode 100644 index 0000000000..a5c54fa5e2 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Qualifiable.java @@ -0,0 +1,7 @@ +package com.bakdata.conquery.sql.conversion.model; + +public interface Qualifiable { + + T qualify(String qualifier); + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java index 7b5389729e..323cdc6b06 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java @@ -7,13 +7,12 @@ import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; -import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.cqelement.aggregation.DateAggregationDates; import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; -import org.jooq.Field; +import org.jooq.Condition; import org.jooq.Record; import org.jooq.Table; import org.jooq.TableLike; @@ -45,7 +44,7 @@ public static QueryStep joinSteps( ConversionContext context ) { String joinedCteName = context.getNameGenerator().joinedNodeName(logicalOperation); - Field primaryColumn = coalescePrimaryColumns(queriesToJoin); + SqlIdColumns ids = coalesceIds(queriesToJoin); List mergedSelects = mergeSelects(queriesToJoin); TableLike joinedTable = constructJoinedTable(queriesToJoin, logicalOperation, context); @@ -57,15 +56,15 @@ public static QueryStep joinSteps( DateAggregationDates dateAggregationDates = DateAggregationDates.forSteps(queriesToJoin); if (dateAggregationAction == DateAggregationAction.BLOCK || dateAggregationDates.dateAggregationImpossible()) { - joinedStep = buildJoinedStep(primaryColumn, mergedSelects, Optional.empty(), joinedStepBuilder); + joinedStep = buildJoinedStep(ids, mergedSelects, Optional.empty(), joinedStepBuilder); } // if there is only 1 child node containing a validity date, we just keep it as overall validity date for the joined node else if (dateAggregationDates.getValidityDates().size() == 1) { ColumnDateRange validityDate = dateAggregationDates.getValidityDates().get(0); - joinedStep = buildJoinedStep(primaryColumn, mergedSelects, Optional.of(validityDate), joinedStepBuilder); + joinedStep = buildJoinedStep(ids, mergedSelects, Optional.of(validityDate), joinedStepBuilder); } else { - joinedStep = buildStepAndAggregateDates(primaryColumn, mergedSelects, joinedStepBuilder, dateAggregationDates, dateAggregationAction, context); + joinedStep = buildStepAndAggregateDates(ids, mergedSelects, joinedStepBuilder, dateAggregationDates, dateAggregationAction, context); } return joinedStep; } @@ -85,51 +84,40 @@ public static TableLike constructJoinedTable(List queriesToJo QueryStep leftPartQS = queriesToJoin.get(i); QueryStep rightPartQS = queriesToJoin.get(i + 1); - Field leftPartPrimaryColumn = leftPartQS.getQualifiedSelects().getPrimaryColumn(); - Field rightPartPrimaryColumn = rightPartQS.getQualifiedSelects().getPrimaryColumn(); + SqlIdColumns leftIds = leftPartQS.getQualifiedSelects().getIds(); + SqlIdColumns rightIds = rightPartQS.getQualifiedSelects().getIds(); - joinedQuery = joinType.join(joinedQuery, rightPartQS, leftPartPrimaryColumn, rightPartPrimaryColumn); + List joinConditions = SqlIdColumns.join(leftIds, rightIds); + + joinedQuery = joinType.join(joinedQuery, rightPartQS, joinConditions); } return joinedQuery; } - @FunctionalInterface - private interface JoinType { - TableOnConditionStep join( - Table leftPartQueryBase, - QueryStep rightPartQS, - Field leftPartPrimaryColumn, - Field rightPartPrimaryColumn - ); - } - - public static Field coalescePrimaryColumns(List querySteps) { - List> primaryColumns = querySteps.stream() - .map(queryStep -> queryStep.getQualifiedSelects().getPrimaryColumn()) - .collect(Collectors.toList()); - return DSL.coalesce(primaryColumns.get(0), primaryColumns.subList(1, primaryColumns.size()).toArray()) - .as(SharedAliases.PRIMARY_COLUMN.getAlias()); - } - public static List mergeSelects(List querySteps) { return querySteps.stream() .flatMap(queryStep -> queryStep.getQualifiedSelects().getSqlSelects().stream()) .collect(Collectors.toList()); } + public static SqlIdColumns coalesceIds(List querySteps) { + List ids = querySteps.stream().map(QueryStep::getQualifiedSelects).map(Selects::getIds).toList(); + return SqlIdColumns.coalesce(ids); + } + private static Table getIntitialJoinTable(List queriesToJoin) { return DSL.table(DSL.name(queriesToJoin.get(0).getCteName())); } private static QueryStep buildJoinedStep( - Field primaryColumn, + SqlIdColumns ids, List mergedSelects, Optional validityDate, QueryStep.QueryStepBuilder builder ) { Selects selects = Selects.builder() - .primaryColumn(primaryColumn) + .ids(ids) .sqlSelects(mergedSelects) .validityDate(validityDate) .build(); @@ -137,7 +125,7 @@ private static QueryStep buildJoinedStep( } private static QueryStep buildStepAndAggregateDates( - Field primaryColumn, + SqlIdColumns ids, List mergedSelects, QueryStep.QueryStepBuilder builder, DateAggregationDates dateAggregationDates, @@ -146,7 +134,7 @@ private static QueryStep buildStepAndAggregateDates( ) { List withAllValidityDates = new ArrayList<>(mergedSelects); withAllValidityDates.addAll(dateAggregationDates.allStartsAndEnds()); - QueryStep joinedStep = buildJoinedStep(primaryColumn, withAllValidityDates, Optional.empty(), builder); + QueryStep joinedStep = buildJoinedStep(ids, withAllValidityDates, Optional.empty(), builder); SqlDateAggregator sqlDateAggregator = context.getSqlDialect().getDateAggregator(); return sqlDateAggregator.apply( @@ -158,4 +146,13 @@ private static QueryStep buildStepAndAggregateDates( ); } + @FunctionalInterface + private interface JoinType { + TableOnConditionStep join( + Table leftPartQueryBase, + QueryStep rightPartQS, + List joinConditions + ); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java index 02d7149535..5b0129d133 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java @@ -11,13 +11,12 @@ import lombok.Singular; import lombok.Value; import org.jooq.Field; -import org.jooq.impl.DSL; @Value @Builder(toBuilder = true) public class Selects { - Field primaryColumn; + SqlIdColumns ids; @Builder.Default Optional validityDate = Optional.empty(); @Singular @@ -36,13 +35,11 @@ public Selects blockValidityDate() { } public Selects qualify(String qualifier) { - Field qualifiedPrimaryColumn = DSL.field(DSL.name(qualifier, this.primaryColumn.getName())); - List sqlSelects = this.sqlSelects.stream() - .map(sqlSelect -> sqlSelect.qualify(qualifier)) - .collect(Collectors.toList()); + SqlIdColumns ids = this.ids.qualify(qualifier); + List sqlSelects = this.sqlSelects.stream().map(sqlSelect -> sqlSelect.qualify(qualifier)).collect(Collectors.toList()); SelectsBuilder builder = Selects.builder() - .primaryColumn(qualifiedPrimaryColumn) + .ids(ids) .sqlSelects(sqlSelects); if (this.validityDate.isPresent()) { @@ -54,7 +51,7 @@ public Selects qualify(String qualifier) { public List> all() { return Stream.of( - Stream.of(this.primaryColumn), + this.ids.toFields().stream(), this.validityDate.stream().flatMap(range -> range.toFields().stream()), this.sqlSelects.stream().map(SqlSelect::select) ) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java new file mode 100644 index 0000000000..029c970b24 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java @@ -0,0 +1,94 @@ +package com.bakdata.conquery.sql.conversion.model; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.annotation.Nullable; + +import com.bakdata.conquery.sql.conversion.SharedAliases; +import lombok.Getter; +import lombok.NonNull; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.impl.DSL; + +public class SqlIdColumns implements Qualifiable { + + @Getter + private final Field primaryColumn; + + @Nullable + private final Field secondaryId; + + private final List> ids; + + public SqlIdColumns(Field primaryColumn, @NonNull Field secondaryId) { + this.primaryColumn = primaryColumn; + this.secondaryId = secondaryId; + this.ids = Stream.concat(Stream.of(this.primaryColumn), Stream.ofNullable(this.secondaryId)).collect(Collectors.toList()); + } + + public SqlIdColumns(Field primaryColumn) { + this.primaryColumn = primaryColumn; + this.secondaryId = null; + this.ids = List.of(this.primaryColumn); + } + + @Override + public SqlIdColumns qualify(String qualifier) { + Field primaryColumn = QualifyingUtil.qualify(this.primaryColumn, qualifier); + if (this.secondaryId == null) { + return new SqlIdColumns(primaryColumn); + } + Field secondaryId = QualifyingUtil.qualify(this.secondaryId, qualifier); + return new SqlIdColumns(primaryColumn, secondaryId); + } + + public Optional> getSecondaryId() { + return Optional.ofNullable(this.secondaryId); + } + + public List> toFields() { + return this.ids; + } + + public static List join(SqlIdColumns leftIds, SqlIdColumns rightIds) { + Condition joinPrimariesCondition = leftIds.getPrimaryColumn().eq(rightIds.getPrimaryColumn()); + Condition joinSecondariesCondition; + if (leftIds.getSecondaryId().isPresent() && rightIds.getSecondaryId().isPresent()) { + joinSecondariesCondition = leftIds.getSecondaryId().get().eq(rightIds.getSecondaryId().get()); + } + else { + joinSecondariesCondition = DSL.noCondition(); + } + return List.of(joinPrimariesCondition, joinSecondariesCondition); + } + + public static SqlIdColumns coalesce(List selectsIds) { + + List> primaryColumns = new ArrayList<>(selectsIds.size()); + List> secondaryIds = new ArrayList<>(selectsIds.size()); + selectsIds.forEach(ids -> { + primaryColumns.add(ids.getPrimaryColumn()); + ids.getSecondaryId().ifPresent(secondaryIds::add); + }); + + Field coalescedPrimaryColumn = coalesceFields(primaryColumns).as(SharedAliases.PRIMARY_COLUMN.getAlias()); + if (secondaryIds.isEmpty()) { + return new SqlIdColumns(coalescedPrimaryColumn); + } + Field coalescedSecondaryIds = coalesceFields(secondaryIds).as(SharedAliases.SECONDARY_ID.getAlias()); + return new SqlIdColumns(coalescedPrimaryColumn, coalescedSecondaryIds); + } + + private static Field coalesceFields(List> fields) { + if (fields.size() == 1) { + return fields.get(0).coerce(Object.class); + } + return DSL.coalesce(fields.get(0), fields.subList(1, fields.size()).toArray()); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumDistinctSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumDistinctSqlAggregator.java index e2ec302ade..834dd2c28e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumDistinctSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SumDistinctSqlAggregator.java @@ -14,9 +14,9 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.model.CteStep; import com.bakdata.conquery.sql.conversion.model.NameGenerator; -import com.bakdata.conquery.sql.conversion.model.QualifyingUtil; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.SumCondition; import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; @@ -93,7 +93,7 @@ public SumDistinctSqlAggregator( List distinctByColumns, String alias, IRange filterValue, - Field primaryColumn, + SqlIdColumns ids, SqlTables connectorTables, NameGenerator nameGenerator ) { @@ -107,10 +107,10 @@ public SumDistinctSqlAggregator( .toList(); // additional predecessors - QueryStep rowNumberCte = createRowNumberCte(primaryColumn, sumColumnRootSelect, distinctByRootSelects, alias, connectorTables, nameGenerator); + QueryStep rowNumberCte = createRowNumberCte(ids, sumColumnRootSelect, distinctByRootSelects, alias, connectorTables, nameGenerator); Field rootSelectQualified = sumColumnRootSelect.qualify(rowNumberCte.getCteName()).select(); FieldWrapper distinctSum = new FieldWrapper<>(DSL.sum(rootSelectQualified).as(alias)); - QueryStep rowNumberFilteredCte = createRowNumberFilteredCte(rowNumberCte, primaryColumn, distinctSum, alias, nameGenerator); + QueryStep rowNumberFilteredCte = createRowNumberFilteredCte(rowNumberCte, distinctSum, alias, nameGenerator); SqlSelects.SqlSelectsBuilder builder = SqlSelects.builder() .preprocessingSelect(sumColumnRootSelect) @@ -139,7 +139,7 @@ public static SumDistinctSqlAggregator create(SumSelect sumSelect, SelectContext sumSelect.getDistinctByColumn(), selectContext.getNameGenerator().selectName(sumSelect), null, - selectContext.getParentContext().getPrimaryColumn(), + selectContext.getIds(), selectContext.getConnectorTables(), selectContext.getNameGenerator() ); @@ -151,7 +151,7 @@ public static SumDistinctSqlAggregator create(SumSelect sumSelect, SelectContext sumFilter.getDistinctByColumn(), filterContext.getNameGenerator().selectName(sumFilter), filterContext.getValue(), - filterContext.getParentContext().getPrimaryColumn(), + filterContext.getIds(), filterContext.getConnectorTables(), filterContext.getNameGenerator() ); @@ -162,7 +162,7 @@ public static SumDistinctSqlAggregator create(SumSelect sumSelect, SelectContext * the row number will be incremented for each duplicated entry. */ private static QueryStep createRowNumberCte( - Field primaryColumn, + SqlIdColumns ids, ExtractingSqlSelect sumColumnRootSelect, List> distinctByRootSelects, String alias, @@ -170,12 +170,11 @@ private static QueryStep createRowNumberCte( NameGenerator nameGenerator ) { String predecessor = connectorTables.getPredecessor(ConnectorCteStep.AGGREGATION_SELECT); - - Field qualifiedPrimaryColumn = QualifyingUtil.qualify(primaryColumn, predecessor); + SqlIdColumns qualifiedIds = ids.qualify(predecessor); ExtractingSqlSelect qualifiedSumRootSelect = sumColumnRootSelect.qualify(predecessor); List> partitioningFields = Stream.concat( - Stream.of(qualifiedPrimaryColumn), + qualifiedIds.toFields().stream(), distinctByRootSelects.stream().map(sqlSelect -> sqlSelect.qualify(predecessor).select()) ) .collect(Collectors.toList()); @@ -185,7 +184,7 @@ private static QueryStep createRowNumberCte( ); Selects rowNumberAssignedSelects = Selects.builder() - .primaryColumn(qualifiedPrimaryColumn) + .ids(qualifiedIds) .sqlSelects(List.of(qualifiedSumRootSelect, rowNumber)) .build(); @@ -201,13 +200,14 @@ private static QueryStep createRowNumberCte( */ private static QueryStep createRowNumberFilteredCte( QueryStep rowNumberCte, - Field primaryColumn, FieldWrapper sumSelect, String alias, NameGenerator nameGenerator ) { + SqlIdColumns ids = rowNumberCte.getQualifiedSelects().getIds(); + Selects rowNumberFilteredSelects = Selects.builder() - .primaryColumn(primaryColumn) + .ids(ids) .sqlSelects(List.of(sumSelect)) .build(); @@ -220,7 +220,7 @@ private static QueryStep createRowNumberFilteredCte( .fromTable(QueryStep.toTableLike(rowNumberCte.getCteName())) .conditions(List.of(firstOccurrence)) .predecessors(List.of(rowNumberCte)) - .groupBy(List.of(primaryColumn)) + .groupBy(ids.toFields()) .build(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java index 82144768c0..ab41df1f05 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SelectContext.java @@ -6,29 +6,29 @@ import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.NameGenerator; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlTables; import lombok.Value; -import org.jooq.Field; @Value public class SelectContext implements Context { - Field primaryColumn; + SqlIdColumns ids; Optional validityDate; SqlTables connectorTables; ConversionContext parentContext; - public static SelectContext forUniversalSelects(Field primaryColumn, Optional validityDate, ConversionContext conversionContext) { - return new SelectContext(primaryColumn, validityDate, null, conversionContext); + public static SelectContext forUniversalSelects(SqlIdColumns ids, Optional validityDate, ConversionContext conversionContext) { + return new SelectContext(ids, validityDate, null, conversionContext); } public static SelectContext forConnectorSelects( - Field primaryColumn, + SqlIdColumns ids, Optional validityDate, SqlTables connectorTables, ConversionContext conversionContext ) { - return new SelectContext(primaryColumn, validityDate, connectorTables, conversionContext); + return new SelectContext(ids, validityDate, connectorTables, conversionContext); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java index d2f9ea7bdb..53df5356a3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/SqlSelect.java @@ -1,11 +1,12 @@ package com.bakdata.conquery.sql.conversion.model.select; + import java.util.List; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConnectorCteStep; +import com.bakdata.conquery.sql.conversion.model.Qualifiable; import org.jooq.Field; -public interface SqlSelect { +public interface SqlSelect extends Qualifiable> { /** * @return The whole (aliased) SQL expression of this {@link SqlSelect}. @@ -25,12 +26,7 @@ public interface SqlSelect { List requiredColumns(); /** - * @return Creates a reference to the alias of this SqlSelect qualified onto the given qualifier. - */ - ExtractingSqlSelect qualify(String qualifier); - - /** - * @return Determines if this is only part of the {@link ConnectorCteStep#FINAL} CTE and has no predeceasing selects. + * @return Determines if this SqlSelect is only part of the final concept conversion CTE and has no predeceasing selects. */ default boolean isUniversal() { return false; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java index 2cc84c0201..526d97383c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java @@ -6,8 +6,8 @@ import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.models.query.DateAggregationMode; -import com.bakdata.conquery.sql.ConceptSqlQuery; import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; @@ -22,7 +22,6 @@ public class ConceptQueryConverter implements NodeConverter { - public static final String FINAL_VALIDITY_DATE_COLUMN_NAME = "dates"; private final QueryStepTransformer queryStepTransformer; public ConceptQueryConverter(QueryStepTransformer queryStepTransformer) { @@ -64,7 +63,7 @@ else if (preFinalSelects.getValidityDate().isEmpty()) { return preFinalSelects.withValidityDate(ColumnDateRange.of(emptyRange)); } Field validityDateStringAggregation = functionProvider.validityDateStringAggregation(preFinalSelects.getValidityDate().get()) - .as(FINAL_VALIDITY_DATE_COLUMN_NAME); + .as(SharedAliases.DATES_COLUMN.getAlias()); return preFinalSelects.withValidityDate(ColumnDateRange.of(validityDateStringAggregation)); } @@ -73,7 +72,7 @@ private List> getFinalGroupBySelects(Selects preFinalSelects, SqlDialec return Collections.emptyList(); } List> groupBySelects = new ArrayList<>(); - groupBySelects.add(preFinalSelects.getPrimaryColumn()); + groupBySelects.addAll(preFinalSelects.getIds().toFields()); groupBySelects.addAll(preFinalSelects.explicitSelects()); return groupBySelects; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptSqlQuery.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptSqlQuery.java index a5a7e67517..b7e7c2aca3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptSqlQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptSqlQuery.java @@ -4,25 +4,30 @@ import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.sql.conversion.model.SqlQuery; -import lombok.Value; +import lombok.Getter; import org.jooq.Record; import org.jooq.Select; import org.jooq.conf.ParamType; -@Value +@Getter class ConceptSqlQuery implements SqlQuery { - String query; + String sqlString; List resultInfos; - public ConceptSqlQuery(Select query, List resultInfos) { - this.query = query.getSQL(ParamType.INLINED); + public ConceptSqlQuery(Select finalQuery, List resultInfos) { + this.sqlString = finalQuery.getSQL(ParamType.INLINED); + this.resultInfos = resultInfos; + } + + protected ConceptSqlQuery(String sqlString, List resultInfos) { + this.sqlString = sqlString; this.resultInfos = resultInfos; } @Override public String getSql() { - return this.query; + return this.sqlString; } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdQueryConverter.java new file mode 100644 index 0000000000..2b10605fc4 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdQueryConverter.java @@ -0,0 +1,29 @@ +package com.bakdata.conquery.sql.conversion.query; + +import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; +import com.google.common.base.Preconditions; + +public class SecondaryIdQueryConverter implements NodeConverter { + + @Override + public Class getConversionClass() { + return SecondaryIdQuery.class; + } + + @Override + public ConversionContext convert(SecondaryIdQuery query, ConversionContext context) { + + ConversionContext withConvertedQuery = context.getNodeConversions().convert( + query.getQuery(), + context.withSecondaryIdDescription(query.getSecondaryId()) + ); + + Preconditions.checkArgument(withConvertedQuery.getFinalQuery() != null, "The SecondaryIdQuery's query should be converted by now."); + SecondaryIdSqlQuery secondaryIdSqlQuery = SecondaryIdSqlQuery.overwriteResultInfos(withConvertedQuery.getFinalQuery(), query.getResultInfos()); + + return withConvertedQuery.withFinalQuery(secondaryIdSqlQuery); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdSqlQuery.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdSqlQuery.java new file mode 100644 index 0000000000..aaa1e17399 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdSqlQuery.java @@ -0,0 +1,18 @@ +package com.bakdata.conquery.sql.conversion.query; + +import java.util.List; + +import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.sql.conversion.model.SqlQuery; + +public class SecondaryIdSqlQuery extends ConceptSqlQuery { + + private SecondaryIdSqlQuery(String sqlString, List resultInfos) { + super(sqlString, resultInfos); + } + + public static SecondaryIdSqlQuery overwriteResultInfos(SqlQuery query, List resultInfos) { + return new SecondaryIdSqlQuery(query.getSql(), resultInfos); + } + +} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java index a7e945f33d..375fa9d140 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java @@ -27,6 +27,7 @@ public class SqlTestDataImporter implements TestDataImporter { @Override public void importQueryTestData(StandaloneSupport support, QueryTest test) throws Exception { RequiredData content = test.getContent(); + importSecondaryIds(support, content.getSecondaryIds()); importTables(support, content.getTables(), true); importConcepts(support, test.getRawConcepts()); importTableContents(support, content.getTables()); diff --git a/backend/src/test/resources/tests/sql/secondary_id/SECONDARY_IDS.test.json b/backend/src/test/resources/tests/sql/secondary_id/SECONDARY_IDS.test.json new file mode 100644 index 0000000000..bbdc441bd2 --- /dev/null +++ b/backend/src/test/resources/tests/sql/secondary_id/SECONDARY_IDS.test.json @@ -0,0 +1,145 @@ +{ + "type": "QUERY_TEST", + "label": "SECONDARY_ID Test", + "expectedCsv": "tests/sql/secondary_id/expected.csv", + "query": { + "type": "SECONDARY_ID_QUERY", + "secondaryId": "secondary", + "root": { + "ids": [ + "number" + ], + "type": "CONCEPT", + "excludeFromSecondaryId": false, + "tables": [ + { + "id": "number.number_connector", + "filters": [ + { + "filter": "number.number_connector.value", + "type": "REAL_RANGE", + "value": { + "min": 1, + "max": 1 + } + } + ] + }, + { + "id": "number.number_connector2", + "filters": [ + { + "filter": "number.number_connector2.value", + "type": "REAL_RANGE", + "value": { + "min": 1, + "max": 1 + } + } + ] + } + ] + } + }, + "concepts": [ + { + "name": "number", + "type": "TREE", + "connectors": [ + { + "name": "number_connector", + "table": "table1", + "validityDates": { + "label": "datum", + "startColumn": "table1.datum_start", + "endColumn": "table1.datum_end" + }, + "filters": { + "label": "value", + "description": "xy", + "column": "table1.value", + "type": "NUMBER" + } + }, + { + "name": "number_connector2", + "table": "table2", + "validityDates": { + "label": "datum", + "startColumn": "table2.datum_start", + "endColumn": "table2.datum_end" + }, + "filters": { + "label": "value", + "description": "xy", + "column": "table2.value", + "type": "NUMBER" + } + } + ] + } + ], + "content": { + "secondaryIds": [ + { + "name": "secondary" + } + ], + "tables": [ + { + "csv": "tests/sql/secondary_id/content.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "sid", + "type": "STRING", + "secondaryId": "secondary" + }, + { + "name": "value", + "type": "REAL" + }, + { + "name": "datum_start", + "type": "DATE" + }, + { + "name": "datum_end", + "type": "DATE" + } + ] + }, + { + "csv": "tests/sql/secondary_id/content2.csv", + "name": "table2", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "sid", + "type": "STRING", + "secondaryId": "secondary" + }, + { + "name": "value", + "type": "REAL" + }, + { + "name": "datum_start", + "type": "DATE" + }, + { + "name": "datum_end", + "type": "DATE" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/secondary_id/content.csv b/backend/src/test/resources/tests/sql/secondary_id/content.csv new file mode 100644 index 0000000000..2b33b32208 --- /dev/null +++ b/backend/src/test/resources/tests/sql/secondary_id/content.csv @@ -0,0 +1,7 @@ +pid,sid,value,datum_start,datum_end +a,f_a1,1,2014-06-30,2015-06-30 +a,f_a1,1,2016-06-30,2016-06-30 +a,f_a2,1,2014-06-30,2015-06-30 +a,,1,2010-06-30,2010-06-30 +a,f_a3,1.01,2014-06-30,2015-06-30 +b,f_b1,1,2015-02-03,2015-06-30 diff --git a/backend/src/test/resources/tests/sql/secondary_id/content2.csv b/backend/src/test/resources/tests/sql/secondary_id/content2.csv new file mode 100644 index 0000000000..fb989445e9 --- /dev/null +++ b/backend/src/test/resources/tests/sql/secondary_id/content2.csv @@ -0,0 +1,7 @@ +pid,sid,value,datum_start,datum_end +a,f_a4,1,2024-06-30,2025-06-30 +a,f_a4,1,2026-06-30,2026-06-30 +a,f_a4,1,2024-06-30,2025-06-30 +a,,13,2020-06-30,2020-06-30 +a,f_a5,1.01,2024-06-30,2025-06-30 +b,f_b6,1,2025-02-03,2025-06-30 diff --git a/backend/src/test/resources/tests/sql/secondary_id/date_mode_logical/SECONDARY_IDS.test.json b/backend/src/test/resources/tests/sql/secondary_id/date_mode_logical/SECONDARY_IDS.test.json new file mode 100644 index 0000000000..87ccc5a412 --- /dev/null +++ b/backend/src/test/resources/tests/sql/secondary_id/date_mode_logical/SECONDARY_IDS.test.json @@ -0,0 +1,165 @@ +{ + "type": "QUERY_TEST", + "label": "SECONDARY_ID Test", + "expectedCsv": "tests/sql/secondary_id/date_mode_logical/expected.csv", + "query": { + "type": "SECONDARY_ID_QUERY", + "secondaryId": "secondary", + "dateAggregationMode": "LOGICAL", + "root": { + "type": "AND", + "children": [ + { + "ids": [ + "number" + ], + "type": "CONCEPT", + "excludeFromSecondaryId": false, + "label": "vs", + "tables": [ + { + "id": "number.number_connector", + "filters": [ + { + "filter": "number.number_connector.value", + "type": "REAL_RANGE", + "value": { + "min": 1, + "max": 1 + } + } + ] + } + ] + }, + { + "ids": [ + "number" + ], + "type": "CONCEPT", + "excludeFromSecondaryId": false, + "label": "vs", + "tables": [ + { + "id": "number.number_connector2", + "filters": [ + { + "filter": "number.number_connector2.value", + "type": "REAL_RANGE", + "value": { + "min": 1, + "max": 1 + } + } + ] + } + ] + } + ] + } + }, + "concepts": [ + { + "label": "number", + "type": "TREE", + "connectors": [ + { + "label": "number_connector", + "table": "table1", + "validityDates": { + "label": "datum", + "startColumn": "table1.datum_start", + "endColumn": "table1.datum_end" + }, + "filters": { + "label": "value", + "description": "xy", + "column": "table1.value", + "type": "NUMBER" + } + }, + { + "label": "number_connector2", + "table": "table12", + "validityDates": { + "label": "datum", + "startColumn": "table12.datum_start", + "endColumn": "table12.datum_end" + }, + "filters": { + "label": "value", + "description": "xy", + "column": "table12.value", + "type": "NUMBER" + } + } + ] + } + ], + "content": { + "secondaryIds": [ + { + "name": "secondary" + }, + { + "name": "ignored" + } + ], + "tables": [ + { + "csv": "tests/sql/secondary_id/date_mode_logical/content.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "sid", + "type": "STRING", + "secondaryId": "secondary" + }, + { + "name": "value", + "type": "REAL" + }, + { + "name": "datum_start", + "type": "DATE" + }, + { + "name": "datum_end", + "type": "DATE" + } + ] + }, + { + "csv": "tests/sql/secondary_id/date_mode_logical/content2.csv", + "name": "table12", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "sid", + "type": "STRING", + "secondaryId": "secondary" + }, + { + "name": "value", + "type": "REAL" + }, + { + "name": "datum_start", + "type": "DATE" + }, + { + "name": "datum_end", + "type": "DATE" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/secondary_id/date_mode_logical/content.csv b/backend/src/test/resources/tests/sql/secondary_id/date_mode_logical/content.csv new file mode 100644 index 0000000000..3081ae37de --- /dev/null +++ b/backend/src/test/resources/tests/sql/secondary_id/date_mode_logical/content.csv @@ -0,0 +1,5 @@ +pid,sid,value,datum_start,datum_end +a,f_a1,1,2014-06-30,2015-06-30 +a,f_a1,1,2016-06-30,2027-06-30 +a,f_a2,1,2014-06-30,2015-06-30 +a,,1,2010-06-30,2010-06-30 diff --git a/backend/src/test/resources/tests/sql/secondary_id/date_mode_logical/content2.csv b/backend/src/test/resources/tests/sql/secondary_id/date_mode_logical/content2.csv new file mode 100644 index 0000000000..670e9fc815 --- /dev/null +++ b/backend/src/test/resources/tests/sql/secondary_id/date_mode_logical/content2.csv @@ -0,0 +1,5 @@ +pid,sid,value,datum_start,datum_end +a,f_a1,1,2024-06-30,2025-06-30 +a,f_a1,1,2026-06-30,2026-06-30 +a,f_a2,1,2024-06-30,2025-06-30 +a,,13,2020-06-30,2020-06-30 diff --git a/backend/src/test/resources/tests/sql/secondary_id/date_mode_logical/expected.csv b/backend/src/test/resources/tests/sql/secondary_id/date_mode_logical/expected.csv new file mode 100644 index 0000000000..2834b221b9 --- /dev/null +++ b/backend/src/test/resources/tests/sql/secondary_id/date_mode_logical/expected.csv @@ -0,0 +1,3 @@ +result,secondary,dates +a,f_a1,"{2024-06-30/2025-06-30,2026-06-30/2026-06-30}" +a,f_a2,{} diff --git a/backend/src/test/resources/tests/sql/secondary_id/excluded/SECONDARY_IDS_EXCLUDED.test.json b/backend/src/test/resources/tests/sql/secondary_id/excluded/SECONDARY_IDS_EXCLUDED.test.json new file mode 100644 index 0000000000..f9880afcae --- /dev/null +++ b/backend/src/test/resources/tests/sql/secondary_id/excluded/SECONDARY_IDS_EXCLUDED.test.json @@ -0,0 +1,101 @@ +{ + "type": "QUERY_TEST", + "label": "SECONDARY_ID_EXCLUDED Test", + "expectedCsv": "tests/query/SECONDARY_ID_EXCLUDED/expected.csv", + "query": { + "type": "SECONDARY_ID_QUERY", + "secondaryId": "secondary", + "root": { + "type": "AND", + "children": [ + { + "ids": [ + "concept" + ], + "type": "CONCEPT", + "label": "vs", + "excludeFromSecondaryId": false, + "tables": [ + { + "id": "concept.connector1" + } + ] + }, + { + "ids": [ + "concept" + ], + "excludeFromSecondaryId": true, + "type": "CONCEPT", + "tables": [ + { + "id": "concept.connector1", + "filters": [ + { + "filter": "concept.connector1.value", + "type": "INTEGER_RANGE", + "value": { + "min": 2 + } + } + ] + } + ] + } + ] + } + }, + "concepts": [ + { + "name": "concept", + "type": "TREE", + "connectors": [ + { + "name": "connector1", + "table": "table1", + "validityDates": { + "name": "datum", + "column": "table1.datum" + }, + "filters": { + "name": "value", + "column": "table1.value", + "type": "COUNT" + } + } + ] + } + ], + "content": { + "secondaryIds": [ + { + "name": "secondary" + } + ], + "tables": [ + { + "csv": "tests/query/SECONDARY_ID_EXCLUDED/content.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "sid", + "type": "STRING", + "secondaryId": "secondary" + }, + { + "name": "value", + "type": "STRING" + }, + { + "name": "datum", + "type": "DATE" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/secondary_id/expected.csv b/backend/src/test/resources/tests/sql/secondary_id/expected.csv new file mode 100644 index 0000000000..f325999afe --- /dev/null +++ b/backend/src/test/resources/tests/sql/secondary_id/expected.csv @@ -0,0 +1,6 @@ +result,secondary,dates +a,f_a1,"{2014-06-30/2015-06-30,2016-06-30/2016-06-30}" +a,f_a2,{2014-06-30/2015-06-30} +b,f_b1,{2015-02-03/2015-06-30} +a,f_a4,"{2024-06-30/2025-06-30,2026-06-30/2026-06-30}" +b,f_b6,{2025-02-03/2025-06-30} \ No newline at end of file diff --git a/backend/src/test/resources/tests/sql/secondary_id/mixed/SECONDARY_IDS_MIXED.test.json b/backend/src/test/resources/tests/sql/secondary_id/mixed/SECONDARY_IDS_MIXED.test.json new file mode 100644 index 0000000000..28f013fb3a --- /dev/null +++ b/backend/src/test/resources/tests/sql/secondary_id/mixed/SECONDARY_IDS_MIXED.test.json @@ -0,0 +1,152 @@ +{ + "type": "QUERY_TEST", + "label": "SECONDARY_ID_MIXED Test", + "expectedCsv": "tests/sql/secondary_id/mixed/expected.csv", + "query": { + "type": "SECONDARY_ID_QUERY", + "secondaryId": "secondary", + "root": { + "ids": [ + "concept" + ], + "type": "CONCEPT", + "excludeFromSecondaryId": false, + "tables": [ + { + "id": "concept.connector1", + "filters": [ + { + "filter": "concept.connector1.filter", + "type": "REAL_RANGE", + "value": { + "min": 1, + "max": 1 + } + } + ] + }, + { + "id": "concept.connector2", + "filters": [ + { + "filter": "concept.connector2.filter", + "type": "REAL_RANGE", + "value": { + "min": 1, + "max": 1 + } + } + ] + } + ] + } + }, + "concepts": [ + { + "name": "concept", + "type": "TREE", + "connectors": [ + { + "label": "connector1", + "table": "table", + "validityDates": { + "label": "datum", + "startColumn": "table.datum_start", + "endColumn": "table.datum_end" + }, + "filters": { + "label": "filter", + "description": "xy", + "column": "table.value", + "type": "NUMBER" + } + }, + { + "label": "connector2", + "table": "table2", + "validityDates": { + "label": "datum", + "startColumn": "table2.datum_start", + "endColumn": "table2.datum_end" + }, + "filters": { + "label": "filter", + "description": "xy", + "column": "table2.value", + "type": "NUMBER" + } + } + ] + } + ], + "content": { + "secondaryIds": [ + { + "name": "secondary" + }, + { + "name": "ignored" + } + ], + "tables": [ + { + "csv": "tests/sql/secondary_id/mixed/content.csv", + "name": "table", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "sid", + "type": "STRING", + "secondaryId": "secondary" + }, + { + "name": "value", + "type": "REAL" + }, + { + "name": "datum_start", + "type": "DATE" + }, + { + "name": "datum_end", + "type": "DATE" + }, + { + "name": "ignored", + "type": "STRING", + "secondaryId": "ignored" + } + ] + }, + { + "csv": "tests/sql/secondary_id/mixed/content2.csv", + "name": "table2", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "sid", + "type": "STRING" + }, + { + "name": "value", + "type": "REAL" + }, + { + "name": "datum_start", + "type": "DATE" + }, + { + "name": "datum_end", + "type": "DATE" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/secondary_id/mixed/content.csv b/backend/src/test/resources/tests/sql/secondary_id/mixed/content.csv new file mode 100644 index 0000000000..62918f9a9b --- /dev/null +++ b/backend/src/test/resources/tests/sql/secondary_id/mixed/content.csv @@ -0,0 +1,7 @@ +pid,sid,value,datum_start,datum_end,ignored +a,f_a1,1,2014-06-30,2015-06-30,"a" +a,f_a1,1,2016-06-30,2016-06-30,"a" +a,f_a2,1,2014-06-30,2015-06-30,"a" +a,,1,2010-06-30,2010-06-30,"a" +a,f_a3,1.01,2014-06-30,2015-06-30,"a" +b,f_b1,1,2015-02-03,2015-06-30,"a" diff --git a/backend/src/test/resources/tests/sql/secondary_id/mixed/content2.csv b/backend/src/test/resources/tests/sql/secondary_id/mixed/content2.csv new file mode 100644 index 0000000000..fb989445e9 --- /dev/null +++ b/backend/src/test/resources/tests/sql/secondary_id/mixed/content2.csv @@ -0,0 +1,7 @@ +pid,sid,value,datum_start,datum_end +a,f_a4,1,2024-06-30,2025-06-30 +a,f_a4,1,2026-06-30,2026-06-30 +a,f_a4,1,2024-06-30,2025-06-30 +a,,13,2020-06-30,2020-06-30 +a,f_a5,1.01,2024-06-30,2025-06-30 +b,f_b6,1,2025-02-03,2025-06-30 diff --git a/backend/src/test/resources/tests/sql/secondary_id/mixed/expected.csv b/backend/src/test/resources/tests/sql/secondary_id/mixed/expected.csv new file mode 100644 index 0000000000..c0ecc10b2c --- /dev/null +++ b/backend/src/test/resources/tests/sql/secondary_id/mixed/expected.csv @@ -0,0 +1,4 @@ +result,secondary,dates +a,f_a2,"{2014-06-30/2015-06-30,2024-06-30/2025-06-30,2026-06-30/2026-06-30}" +a,f_a1,"{2014-06-30/2015-06-30,2016-06-30/2016-06-30,2024-06-30/2025-06-30,2026-06-30/2026-06-30}" +b,f_b1,"{2015-02-03/2015-06-30,2025-02-03/2025-06-30}"