Implement SecondaryId conversion (#3317)

backend/src/main/java/com/bakdata/conquery/apiv1/query/SecondaryIdQuery.java

@@ -6,7 +6,6 @@
 import java.util.Set;
 import java.util.function.Consumer;
 
-import javax.annotation.CheckForNull;
 import javax.validation.constraints.NotNull;
 
 import com.bakdata.conquery.apiv1.query.concept.filter.CQTable;
@@ -110,7 +109,7 @@ public void resolve(final QueryResolveContext context) {
 
 			for (CQTable connector : concept.getTables()) {
 				final Table table = connector.getConnector().getTable();
-				final Column secondaryIdColumn = findSecondaryIdColumn(table);
+				final Column secondaryIdColumn = table.findSecondaryIdColumn(secondaryId);
 
 				if (secondaryIdColumn != null && !concept.isExcludeFromSecondaryId()) {
 					withSecondaryId.add(secondaryIdColumn);
@@ -127,23 +126,6 @@ public void resolve(final QueryResolveContext context) {
 		}
 	}
 
-	/**
-	 * selects the right column for the given secondaryId from a table
-	 */
-	@CheckForNull
-	private Column findSecondaryIdColumn(Table table) {
-
-		for (Column col : table.getColumns()) {
-			if (col.getSecondaryId() == null || !secondaryId.equals(col.getSecondaryId())) {
-				continue;
-			}
-
-			return col;
-		}
-
-		return null;
-	}
-
 	@Override
 	public List<ResultInfo> getResultInfos() {
 		final List<ResultInfo> resultInfos = new ArrayList<>();
@@ -170,4 +152,4 @@ public CQElement getReusableComponents() {
 	public RequiredEntities collectRequiredEntities(QueryExecutionContext context) {
 		return query.collectRequiredEntities(context);
 	}
-}
+}

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/CQTable.java

@@ -1,7 +1,9 @@
 package com.bakdata.conquery.apiv1.query.concept.filter;
 
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Objects;
 
 import javax.annotation.CheckForNull;
 import javax.validation.Valid;
@@ -10,6 +12,8 @@
 import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept;
 import com.bakdata.conquery.io.jackson.serializer.NsIdRef;
 import com.bakdata.conquery.io.jackson.serializer.NsIdRefCollection;
+import com.bakdata.conquery.models.datasets.Column;
+import com.bakdata.conquery.models.datasets.SecondaryIdDescription;
 import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.ValidityDate;
 import com.bakdata.conquery.models.datasets.concepts.select.Select;
@@ -88,4 +92,11 @@ public ValidityDate findValidityDate() {
 		return null;
 	}
 
+	public boolean hasSelectedSecondaryId(SecondaryIdDescription secondaryId) {
+		return Arrays.stream(connector.getTable().getColumns())
+					 .map(Column::getSecondaryId)
+					 .filter(Objects::nonNull)
+					 .anyMatch(o -> Objects.equals(secondaryId, o));
+	}
+
 }

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java

@@ -21,6 +21,7 @@
 import com.bakdata.conquery.models.query.queryplan.filter.FilterNode;
 import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext;
 import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext;
+import com.bakdata.conquery.sql.conversion.model.SqlIdColumns;
 import com.bakdata.conquery.sql.conversion.model.SqlTables;
 import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters;
 import com.fasterxml.jackson.annotation.JsonIgnore;
@@ -70,8 +71,8 @@ public FilterNode<?> createNode() {
 		return getFilter().createFilterNode(getValue());
 	}
 
-	public SqlFilters convertToSqlFilter(ConversionContext context, SqlTables connectorTables) {
-		FilterContext<VALUE> filterContext = new FilterContext<>(value, context, connectorTables);
+	public SqlFilters convertToSqlFilter(SqlIdColumns ids, ConversionContext context, SqlTables connectorTables) {
+		FilterContext<VALUE> filterContext = new FilterContext<>(ids, value, context, connectorTables);
 		SqlFilters sqlFilters = filter.convertToSqlFilter(filterContext);
 		if (context.isNegation()) {
 			return new SqlFilters(sqlFilters.getSelects(), sqlFilters.getWhereClauses().negated());

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java

@@ -1,11 +1,9 @@
 package com.bakdata.conquery.apiv1.query.concept.specific;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Locale;
-import java.util.Objects;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -21,7 +19,6 @@
 import com.bakdata.conquery.io.jackson.View;
 import com.bakdata.conquery.io.jackson.serializer.NsIdRefCollection;
 import com.bakdata.conquery.models.common.CDateSet;
-import com.bakdata.conquery.models.datasets.Column;
 import com.bakdata.conquery.models.datasets.concepts.Concept;
 import com.bakdata.conquery.models.datasets.concepts.ConceptElement;
 import com.bakdata.conquery.models.datasets.concepts.Connector;
@@ -234,12 +231,7 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) {
 			existsAggregators.forEach(agg -> agg.setReference(conceptSpecificNode));
 
 			// Select if matching secondaryId available
-			final boolean hasSelectedSecondaryId =
-					Arrays.stream(table.getConnector().getTable().getColumns())
-						  .map(Column::getSecondaryId)
-						  .filter(Objects::nonNull)
-						  .anyMatch(o -> Objects.equals(context.getSelectedSecondaryId(), o));
-
+			final boolean hasSelectedSecondaryId = table.hasSelectedSecondaryId(context.getSelectedSecondaryId());
 
 			final ConceptNode node = new ConceptNode(
 					conceptSpecificNode,

backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java

@@ -5,6 +5,7 @@
 import java.util.Set;
 import java.util.stream.Stream;
 
+import javax.annotation.CheckForNull;
 import javax.validation.Valid;
 import javax.validation.constraints.NotNull;
 
@@ -78,4 +79,22 @@ public Column getColumnByName(@NotNull String columnName) {
 					 .findFirst()
 					 .orElseThrow(() -> new IllegalStateException(String.format("Column %s not found", columnName)));
 	}
+
+	/**
+	 * selects the right column for the given secondaryId from this table
+	 */
+	@CheckForNull
+	public Column findSecondaryIdColumn(SecondaryIdDescription secondaryId) {
+
+		for (Column col : columns) {
+			if (col.getSecondaryId() == null || !secondaryId.equals(col.getSecondaryId())) {
+				continue;
+			}
+
+			return col;
+		}
+
+		return null;
+	}
+
 }

backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java

@@ -6,7 +6,6 @@
 import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext;
 import com.bakdata.conquery.sql.conversion.dialect.SqlDialect;
 import com.bakdata.conquery.sql.conversion.model.NameGenerator;
-import org.jooq.impl.DSL;
 
 /**
  * Entry point for converting {@link QueryDescription} to an SQL query.
@@ -28,7 +27,6 @@ public ConversionContext convert(QueryDescription queryDescription) {
 														.nameGenerator(new NameGenerator(config.getDialect().getNameMaxLength()))
 														.nodeConversions(this)
 														.sqlDialect(this.dialect)
-														.primaryColumn(DSL.field(DSL.name(config.getPrimaryColumn())))
 														.build();
 		return convert(queryDescription, initialCtx);
 	}

backend/src/main/java/com/bakdata/conquery/sql/conversion/SharedAliases.java

@@ -7,7 +7,9 @@
 @RequiredArgsConstructor
 public enum SharedAliases {
 
-	PRIMARY_COLUMN("pid");
+	PRIMARY_COLUMN("primary_id"),
+	SECONDARY_ID("secondary_id"),
+	DATES_COLUMN("dates");
 
 	private final String alias;
 }

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java

@@ -2,8 +2,13 @@
 
 import java.util.List;
 
+import javax.annotation.Nullable;
+
+import com.bakdata.conquery.apiv1.query.SecondaryIdQuery;
+import com.bakdata.conquery.apiv1.query.concept.specific.CQDateRestriction;
 import com.bakdata.conquery.models.common.daterange.CDateRange;
 import com.bakdata.conquery.models.config.SqlConnectorConfig;
+import com.bakdata.conquery.models.datasets.SecondaryIdDescription;
 import com.bakdata.conquery.sql.conversion.Context;
 import com.bakdata.conquery.sql.conversion.NodeConversions;
 import com.bakdata.conquery.sql.conversion.dialect.SqlDialect;
@@ -14,23 +19,40 @@
 import lombok.Singular;
 import lombok.Value;
 import lombok.With;
-import org.jooq.Field;
 
 @Value
 @With
 @Builder(toBuilder = true)
 public class ConversionContext implements Context {
 
 	SqlConnectorConfig config;
+
 	NodeConversions nodeConversions;
+
 	SqlDialect sqlDialect;
+
 	NameGenerator nameGenerator;
+
 	@Singular
 	List<QueryStep> querySteps;
+
+	@Nullable
 	SqlQuery finalQuery;
-	Field<Object> primaryColumn;
+
+	/**
+	 * An optional date restriction range. Is set when converting a {@link CQDateRestriction}.
+	 */
+	@Nullable
 	CDateRange dateRestrictionRange;
+
+	/**
+	 * An optional secondary id to group results by in addition to the primary id. Only set when converting {@link SecondaryIdQuery}s.
+	 */
+	@Nullable
+	SecondaryIdDescription secondaryIdDescription;
+
 	boolean negation;
+
 	boolean isGroupBy;
 
 	public boolean dateRestrictionActive() {

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/AnsiSqlDateAggregator.java

@@ -44,7 +44,7 @@ public QueryStep apply(
 									  .carryThroughSelects(carryThroughSelects)
 									  .dateAggregationDates(dateAggregationDates)
 									  .dateAggregationTables(aggregationAction.tableNames(nameGenerator))
-									  .primaryColumn(joinedStep.getQualifiedSelects().getPrimaryColumn())
+									  .ids(joinedStep.getQualifiedSelects().getIds())
 									  .functionProvider(this.functionProvider)
 									  .intervalPacker(this.intervalPacker)
 									  .nameGenerator(nameGenerator)
@@ -62,7 +62,7 @@ public QueryStep apply(
 		IntervalPackingContext intervalPackingContext =
 				IntervalPackingContext.builder()
 									  .nodeLabel(joinedCteLabel)
-									  .primaryColumn(predecessorSelects.getPrimaryColumn())
+									  .ids(predecessorSelects.getIds())
 									  .validityDate(predecessorSelects.getValidityDate().get())
 									  .predecessor(finalDateAggregationStep)
 									  .carryThroughSelects(carryThroughSelects)
@@ -89,7 +89,7 @@ public QueryStep invertAggregatedIntervals(QueryStep baseStep, NameGenerator nam
 															   .carryThroughSelects(baseStepQualifiedSelects.getSqlSelects())
 															   .dateAggregationDates(dateAggregationDates)
 															   .dateAggregationTables(dateAggregationTables)
-															   .primaryColumn(baseStepQualifiedSelects.getPrimaryColumn())
+															   .ids(baseStepQualifiedSelects.getIds())
 															   .functionProvider(this.functionProvider)
 															   .intervalPacker(this.intervalPacker)
 															   .nameGenerator(nameGenerator)

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationContext.java

@@ -9,19 +9,20 @@
 import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker;
 import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider;
 import com.bakdata.conquery.sql.conversion.model.NameGenerator;
+import com.bakdata.conquery.sql.conversion.model.Qualifiable;
 import com.bakdata.conquery.sql.conversion.model.QualifyingUtil;
 import com.bakdata.conquery.sql.conversion.model.QueryStep;
+import com.bakdata.conquery.sql.conversion.model.SqlIdColumns;
 import com.bakdata.conquery.sql.conversion.model.SqlTables;
 import com.bakdata.conquery.sql.conversion.model.select.SqlSelect;
 import lombok.Builder;
 import lombok.Value;
-import org.jooq.Field;
 
 @Value
 @Builder(toBuilder = true)
-class DateAggregationContext implements Context {
+class DateAggregationContext implements Context, Qualifiable<DateAggregationContext> {
 
-	Field<Object> primaryColumn;
+	SqlIdColumns ids;
 	List<SqlSelect> carryThroughSelects;
 	SqlTables dateAggregationTables;
 	DateAggregationDates dateAggregationDates;
@@ -55,9 +56,10 @@ public List<QueryStep> getSteps(DateAggregationCteStep dateAggregationCteStep) {
 		return this.intervalMergeSteps.get(dateAggregationCteStep);
 	}
 
+	@Override
 	public DateAggregationContext qualify(String qualifier) {
 		return this.toBuilder()
-				   .primaryColumn(QualifyingUtil.qualify(this.primaryColumn, qualifier))
+				   .ids(this.ids.qualify(qualifier))
 				   .carryThroughSelects(QualifyingUtil.qualify(this.carryThroughSelects, qualifier))
 				   .dateAggregationDates(this.dateAggregationDates.qualify(qualifier))
 				   .build();

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/IntermediateTableCte.java

@@ -29,7 +29,7 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context)
 				context.getCarryThroughSelects()
 		);
 		Selects selects = Selects.builder()
-								 .primaryColumn(context.getPrimaryColumn())
+								 .ids(context.getIds())
 								 .sqlSelects(intermediateTableSelects)
 								 .build();