Absolute form conversion

backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java

@@ -28,7 +28,6 @@
 import com.bakdata.conquery.apiv1.execution.ResultAsset;
 import com.bakdata.conquery.apiv1.query.CQElement;
 import com.bakdata.conquery.apiv1.query.ConceptQuery;
-import com.bakdata.conquery.apiv1.query.EditorQuery;
 import com.bakdata.conquery.apiv1.query.ExternalUpload;
 import com.bakdata.conquery.apiv1.query.ExternalUploadResult;
 import com.bakdata.conquery.apiv1.query.Query;
@@ -137,11 +136,11 @@ public Stream<ExecutionStatus> getQueriesFiltered(Dataset datasetId, UriBuilder
 	 */
 	private static boolean canFrontendRender(ManagedExecution q) {
 		//TODO FK: should this be used to fill into canExpand instead of hiding the Executions?
-		if (!(q instanceof EditorQuery)) {
+		if (!(q instanceof ManagedQuery)) {
 			return false;
 		}
 
-		final Query query = ((EditorQuery) q).getQuery();
+		final Query query = ((ManagedQuery) q).getQuery();
 
 		if (query instanceof ConceptQuery) {
 			return isFrontendStructure(((ConceptQuery) query).getRoot());
@@ -291,14 +290,15 @@ public FullExecutionStatus getQueryFullStatus(ManagedExecution query, Subject su
 	public ExternalUploadResult uploadEntities(Subject subject, Dataset dataset, ExternalUpload upload) {
 
 		final Namespace namespace = datasetRegistry.get(dataset.getId());
-		final CQExternal.ResolveStatistic
-				statistic =
-				CQExternal.resolveEntities(upload.getValues(), upload.getFormat(), namespace
-						.getStorage()
-						.getIdMapping(), config.getIdColumns(), config.getLocale()
-																	  .getDateReader(), upload.isOneRowPerEntity()
-
-				);
+		final CQExternal.ResolveStatistic statistic = CQExternal.resolveEntities(
+				upload.getValues(),
+				upload.getFormat(),
+				namespace.getStorage().getIdMapping(),
+				config.getIdColumns(),
+				config.getLocale().getDateReader(),
+				upload.isOneRowPerEntity(),
+				true
+		);
 
 		// Resolving nothing is a problem thus we fail.
 		if (statistic.getResolved().isEmpty()) {

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQReusedQuery.java

@@ -7,12 +7,12 @@
 import javax.annotation.Nullable;
 
 import com.bakdata.conquery.apiv1.query.CQElement;
-import com.bakdata.conquery.apiv1.query.EditorQuery;
 import com.bakdata.conquery.apiv1.query.Query;
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.io.jackson.View;
 import com.bakdata.conquery.models.error.ConqueryError;
 import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId;
+import com.bakdata.conquery.models.query.ManagedQuery;
 import com.bakdata.conquery.models.query.QueryExecutionContext;
 import com.bakdata.conquery.models.query.QueryPlanContext;
 import com.bakdata.conquery.models.query.QueryResolveContext;
@@ -48,7 +48,7 @@ public CQReusedQuery(ManagedExecutionId executionId){
 	private ManagedExecutionId queryId;
 
 	@JsonIgnore
-	private EditorQuery query;
+	private ManagedQuery query;
 
 	@JsonView(View.InternalCommunication.class)
 	private Query resolvedQuery;
@@ -74,7 +74,7 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) {
 
 	@Override
 	public void resolve(QueryResolveContext context) {
-		query = (EditorQuery) context.getStorage().getExecution(queryId);
+		query = (ManagedQuery) context.getStorage().getExecution(queryId);
 		if (query == null) {
 			throw new ConqueryError.ExecutionCreationResolveError(queryId);
 		}

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java

@@ -80,7 +80,7 @@ public class CQExternal extends CQElement {
 	/**
 	 * Maps from Entity to the computed time-frame.
 	 */
-	@Getter(AccessLevel.PRIVATE)
+	@Getter
 	@JsonView(View.InternalCommunication.class)
 	private Map<String, CDateSet> valuesResolved;
 
@@ -249,7 +249,8 @@ public void resolve(QueryResolveContext context) {
 								context.getNamespace().getStorage().getIdMapping(),
 								context.getConfig().getIdColumns(),
 								context.getConfig().getLocale().getDateReader(),
-								onlySingles
+								onlySingles,
+								context.getConfig().getSqlConnectorConfig().isEnabled()
 				);
 
 		if (resolved.getResolved().isEmpty()) {
@@ -296,7 +297,7 @@ public static class ResolveStatistic {
 	/**
 	 * Helper method to try and resolve entities in values using the specified format.
 	 */
-	public static ResolveStatistic resolveEntities(@NotEmpty String[][] values, @NotEmpty List<String> format, EntityIdMap mapping, IdColumnConfig idColumnConfig, @NotNull DateReader dateReader, boolean onlySingles) {
+	public static ResolveStatistic resolveEntities(@NotEmpty String[][] values, @NotEmpty List<String> format, EntityIdMap mapping, IdColumnConfig idColumnConfig, @NotNull DateReader dateReader, boolean onlySingles, boolean isInSqlMode) {
 		final Map<String, CDateSet> resolved = new HashMap<>();
 
 		final List<String[]> unresolvedDate = new ArrayList<>();
@@ -329,7 +330,10 @@ public static ResolveStatistic resolveEntities(@NotEmpty String[][] values, @Not
 				continue;
 			}
 
-			String resolvedId = tryResolveId(row, readers, mapping);
+			// TODO proper implementation of EntityIdMap#resolve for SQL mode
+			String resolvedId = isInSqlMode
+								? String.valueOf(row[0])
+								: tryResolveId(row, readers, mapping);
 
 			if (resolvedId == null) {
 				unresolvedId.add(row);

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java

@@ -126,7 +126,7 @@ public SqlSelects convertToSqlSelects(SelectContext selectContext) {
 	}
 
 	@JsonIgnore
-	public boolean requiresIntervalPacking() {
+	public boolean isEventDateSelect() {
 		return false;
 	}
 

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java

@@ -35,7 +35,8 @@ public SqlSelects convertToSqlSelects(SelectContext selectContext) {
 	}
 
 	@Override
-	public boolean requiresIntervalPacking() {
+	public boolean isEventDateSelect() {
 		return true;
 	}
+
 }

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java

@@ -36,7 +36,7 @@ public SqlSelects convertToSqlSelects(SelectContext selectContext) {
 	}
 
 	@Override
-	public boolean requiresIntervalPacking() {
+	public boolean isEventDateSelect() {
 		return true;
 	}
 }

backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java

@@ -136,7 +136,7 @@ protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject
 	@Override
 	public void cancel() {
 		log.debug("Sending cancel message to all workers.");
-		getNamespace().getWorkerHandler().sendToAll(new CancelQuery(getId()));
+		((DistributedNamespace) getNamespace()).getWorkerHandler().sendToAll(new CancelQuery(getId()));
 	}
 
 	@Override
@@ -179,7 +179,4 @@ public boolean allSubQueriesDone() {
 		}
 	}
 
-	public DistributedNamespace getNamespace() {
-		return (DistributedNamespace) super.getNamespace();
-	}
 }

backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java

@@ -8,9 +8,9 @@
 
 import com.bakdata.conquery.apiv1.execution.ExecutionStatus;
 import com.bakdata.conquery.apiv1.execution.FullExecutionStatus;
-import com.bakdata.conquery.apiv1.query.EditorQuery;
 import com.bakdata.conquery.apiv1.query.Query;
 import com.bakdata.conquery.apiv1.query.QueryDescription;
+import com.bakdata.conquery.apiv1.query.SecondaryIdQuery;
 import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept;
 import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery;
 import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal;
@@ -42,7 +42,7 @@
 @ToString(callSuper = true)
 @Slf4j
 @CPSType(base = ManagedExecution.class, id = "MANAGED_QUERY")
-public class ManagedQuery extends ManagedExecution implements EditorQuery, SingleTableResult, InternalExecution<ShardResult> {
+public class ManagedQuery extends ManagedExecution implements SingleTableResult, InternalExecution<ShardResult> {
 
 	// Needs to be resolved externally before being executed
 	private Query query;
@@ -100,12 +100,18 @@ public long resultRowCount() {
 		return lastResultCount;
 	}
 
-
-
 	@Override
 	public void setStatusBase(@NonNull Subject subject, @NonNull ExecutionStatus status) {
+
 		super.setStatusBase(subject, status);
-		enrichStatusBase(status);
+		status.setNumberOfResults(getLastResultCount());
+
+		Query query = getQuery();
+		status.setQueryType(query.getClass().getAnnotation(CPSType.class).id());
+
+		if (query instanceof SecondaryIdQuery secondaryIdQuery) {
+			status.setSecondaryId((secondaryIdQuery).getSecondaryId().getId());
+		}
 	}
 
 	protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject, FullExecutionStatus status) {

backend/src/main/java/com/bakdata/conquery/sql/conversion/SharedAliases.java

@@ -9,7 +9,29 @@ public enum SharedAliases {
 
 	PRIMARY_COLUMN("primary_id"),
 	SECONDARY_ID("secondary_id"),
-	DATES_COLUMN("dates");
+	DATES_COLUMN("dates"),
+	DATE_RESTRICTION("date_restriction"),
+
+	NOP_TABLE("nop_table"),
+
+	// form related
+	INDEX_START("index_start"),
+	QUARTER_START("quarter_start"),
+	QUARTER_END("quarter_end"),
+	YEAR_START("year_start"),
+	YEAR_END("year_end"),
+	YEAR_END_QUARTER_ALIGNED("year_end_quarter_aligned"),
+	YEAR_ALIGNED_COUNT("year_aligned_count"),
+	QUARTER_ALIGNED_COUNT("quarter_aligned_count"),
+	DAY_ALIGNED_COUNT("day_aligned_count"),
+	SERIES_INDEX("index"),
+	RESOLUTION("resolution"),
+	INDEX("index"),
+	INDEX_SELECTOR("index_selector"),
+	INDEX_START_POSITIVE("index_start_positive"),
+	INDEX_START_NEGATIVE("index_start_negative"),
+	STRATIFICATION_BOUNDS("stratification_bounds"),
+	OBSERVATION_SCOPE("scope");
 
 	private final String alias;
 }

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java

@@ -2,7 +2,7 @@
 
 import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd;
 import com.bakdata.conquery.sql.conversion.NodeConverter;
-import com.bakdata.conquery.sql.conversion.model.LogicalOperation;
+import com.bakdata.conquery.sql.conversion.model.ConqueryJoinType;
 import com.bakdata.conquery.sql.conversion.model.QueryStep;
 import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner;
 
@@ -21,7 +21,7 @@ public ConversionContext convert(CQAnd andNode, ConversionContext context) {
 		QueryStep joined = QueryStepJoiner.joinChildren(
 				andNode.getChildren(),
 				context,
-				LogicalOperation.AND,
+				ConqueryJoinType.INNER_JOIN,
 				andNode.getDateAction()
 		);
 		return context.withQueryStep(joined);

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java

@@ -0,0 +1,84 @@
+package com.bakdata.conquery.sql.conversion.cqelement;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal;
+import com.bakdata.conquery.models.common.CDateSet;
+import com.bakdata.conquery.sql.conversion.NodeConverter;
+import com.bakdata.conquery.sql.conversion.SharedAliases;
+import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider;
+import com.bakdata.conquery.sql.conversion.model.ColumnDateRange;
+import com.bakdata.conquery.sql.conversion.model.QueryStep;
+import com.bakdata.conquery.sql.conversion.model.Selects;
+import com.bakdata.conquery.sql.conversion.model.SqlIdColumns;
+import com.google.common.base.Preconditions;
+import org.jooq.Field;
+import org.jooq.Record;
+import org.jooq.Table;
+import org.jooq.impl.DSL;
+
+/**
+ * TODO this is just a minimal implementation to make form conversion testcases work
+ */
+public class CQExternalConverter implements NodeConverter<CQExternal> {
+
+	private static final String CQ_EXTERNAL_CTE_NAME = "external";
+
+	@Override
+	public Class<? extends CQExternal> getConversionClass() {
+		return CQExternal.class;
+	}
+
+	@Override
+	public ConversionContext convert(CQExternal external, ConversionContext context) {
+
+		SqlFunctionProvider functionProvider = context.getSqlDialect().getFunctionProvider();
+		List<QueryStep> unions = external.getValuesResolved()
+										 .entrySet().stream()
+										 .flatMap(entry -> createRowSelects(entry, functionProvider).stream())
+										 .toList();
+
+		Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 converted resolved row when converting a CQExternal");
+		QueryStep externalStep = QueryStep.createUnionStep(unions, CQ_EXTERNAL_CTE_NAME, Collections.emptyList());
+		return context.withQueryStep(externalStep);
+	}
+
+	/**
+	 * For each entry, we need to create a SELECT statement of static values for each pid -> date set. For dialects that support date multiranges, 1 row per ID
+	 * is sufficient. For other dialects there can be multiple rows with the same pid -> date range from the date set.
+	 */
+	private static List<QueryStep> createRowSelects(Map.Entry<String, CDateSet> entry, SqlFunctionProvider functionProvider) {
+
+		Field<Object> primaryColumn = DSL.field(DSL.val(entry.getKey())).coerce(Object.class).as(SharedAliases.PRIMARY_COLUMN.getAlias());
+		SqlIdColumns ids = new SqlIdColumns(primaryColumn);
+
+		List<ColumnDateRange> validityDateEntries = functionProvider.forCDateSet(entry.getValue(), SharedAliases.DATES_COLUMN);
+		return validityDateEntries.stream()
+								  .map(validityDateEntry -> createRowSelect(ids, validityDateEntry, functionProvider))
+								  .toList();
+	}
+
+	/**
+	 * Creates a SELECT statement of static values for each pid -> date entry, like
+	 * <pre>{@code select 1 as "pid", '[2021-01-01,2022-01-01)'::daterange as "date_range"}</pre>
+	 */
+	private static QueryStep createRowSelect(SqlIdColumns ids, ColumnDateRange validityDate, SqlFunctionProvider functionProvider) {
+
+		Selects selects = Selects.builder()
+								 .ids(ids)
+								 .validityDate(Optional.ofNullable(validityDate))
+								 .build();
+
+		// not all SQL dialects can create a SELECT statement without a FROM clause,
+		// so we ensure there is some no-op table to select the static values from
+		Table<? extends Record> table = functionProvider.getNoOpTable();
+
+		return QueryStep.builder()
+						.selects(selects)
+						.fromTable(table)
+						.build();
+	}
+}

backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java

@@ -2,7 +2,7 @@
 
 import com.bakdata.conquery.apiv1.query.concept.specific.CQOr;
 import com.bakdata.conquery.sql.conversion.NodeConverter;
-import com.bakdata.conquery.sql.conversion.model.LogicalOperation;
+import com.bakdata.conquery.sql.conversion.model.ConqueryJoinType;
 import com.bakdata.conquery.sql.conversion.model.QueryStep;
 import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner;
 
@@ -21,7 +21,7 @@ public ConversionContext convert(CQOr orNode, ConversionContext context) {
 		QueryStep joined = QueryStepJoiner.joinChildren(
 				orNode.getChildren(),
 				context,
-				LogicalOperation.OR,
+				ConqueryJoinType.OUTER_JOIN,
 				orNode.getDateAction()
 		);
 		return context.withQueryStep(joined);