From c8d883156cef590dfdb7477ae706df0ffbf06eb2 Mon Sep 17 00:00:00 2001 From: Jonas Arnhold Date: Tue, 26 Mar 2024 20:43:18 +0100 Subject: [PATCH] Implement absolute form conversion --- .../conquery/apiv1/QueryProcessor.java | 22 +- .../conquery/apiv1/query/EditorQuery.java | 27 -- .../query/concept/specific/CQReusedQuery.java | 6 +- .../concept/specific/external/CQExternal.java | 11 +- .../datasets/concepts/select/Select.java | 2 +- .../specific/EventDateUnionSelect.java | 3 +- .../specific/EventDurationSumSelect.java | 2 +- .../forms/managed/ManagedInternalForm.java | 5 +- .../conquery/models/query/ManagedQuery.java | 16 +- .../sql/conversion/SharedAliases.java | 16 +- .../cqelement/CQExternalConverter.java | 82 +++++++ .../cqelement/ConversionContext.java | 7 + .../aggregation/DateAggregationCte.java | 2 +- .../cqelement/aggregation/InvertCte.java | 6 +- .../concept/AggregationFilterCte.java | 5 +- .../concept/AggregationSelectCte.java | 19 +- .../cqelement/concept/CQTableContext.java | 16 +- .../cqelement/concept/ConnectorCte.java | 4 +- .../cqelement/concept/EventFilterCte.java | 59 +++-- .../cqelement/concept/JoinBranchesCte.java | 40 ++- .../cqelement/concept/PreprocessingCte.java | 56 ++++- .../cqelement/concept/TablePathGenerator.java | 17 +- .../conversion/dialect/HanaSqlDialect.java | 9 + .../conversion/dialect/PostgreSqlDialect.java | 9 + .../sql/conversion/dialect/SqlDialect.java | 14 +- .../dialect/SqlFunctionProvider.java | 40 ++- .../forms/AbsoluteStratification.java | 42 ++++ .../sql/conversion/forms/FormCteStep.java | 42 ++++ .../forms/HanaStratificationTableFactory.java | 101 ++++++++ .../PostgresStratificationTableFactory.java | 162 ++++++++++++ .../conquery/sql/conversion/forms/README.md | 203 +++++++++++++++ .../forms/StratificationTableFactory.java | 232 ++++++++++++++++++ .../sql/conversion/model/ColumnDateRange.java | 23 +- .../model/ConceptConversionTables.java | 33 --- .../conversion/model/LogicalOperation.java | 3 +- .../sql/conversion/model/NameGenerator.java | 1 + .../sql/conversion/model/QueryStep.java | 17 +- .../sql/conversion/model/QueryStepJoiner.java | 31 ++- .../model/QueryStepTransformer.java | 16 +- .../sql/conversion/model/Selects.java | 7 + .../sql/conversion/model/SqlIdColumns.java | 80 +++--- .../sql/conversion/model/SqlQuery.java | 31 ++- .../model/StratificationSqlIdColumns.java | 155 ++++++++++++ .../query/AbsoluteFormQueryConverter.java | 140 +++++++++++ .../sql/conversion/query/ConceptSqlQuery.java | 33 --- .../query/SecondaryIdQueryConverter.java | 3 +- .../conversion/query/SecondaryIdSqlQuery.java | 18 -- .../integration/common/LoadingUtil.java | 7 +- .../json/AbstractQueryEngineTest.java | 4 +- .../integration/json/SqlTestDataImporter.java | 11 +- .../integration/json/TestDataImporter.java | 5 + .../json/WorkerTestDataImporter.java | 5 - .../dialect/PostgreSqlIntegrationTests.java | 26 +- .../test/resources/shared/alter.concept.json | 54 ++++ .../resources/shared/geschlecht.concept.json | 49 ++++ .../shared/two_connector.concept.json | 59 +++++ .../src/test/resources/shared/vers_stamm.csv | 23 ++ .../resources/shared/vers_stamm.table.json | 26 ++ .../form/ABSOLUT/ALIGNMENT/QUARTER YEAR.json | 46 ++++ .../form/ABSOLUT/ALIGNMENT/YEAR QUARTER.json | 46 ++++ .../sql/form/ABSOLUT/ALIGNMENT/YEAR YEAR.json | 46 ++++ .../form/ABSOLUT/ALIGNMENT/quarter year.csv | 8 + .../ALIGNMENT/year quarter expected.csv | 3 + .../MULTIPLE_FEATURES/MULTIPLE_FEATURES.json | 58 +++++ .../ABSOLUT/MULTIPLE_FEATURES/expected.csv | 7 + .../ABS_EXPORT_FORM_SECONDARY_ID.json | 79 ++++++ .../form/ABSOLUT/SIMPLE/ABS_EXPORT_FORM.json | 45 ++++ .../SIMPLE/ABS_EXPORT_FORM_WITH_SELECT.json | 50 ++++ 68 files changed, 2228 insertions(+), 297 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/EditorQuery.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/HanaStratificationTableFactory.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/PostgresStratificationTableFactory.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/README.md create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ConceptConversionTables.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptSqlQuery.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdSqlQuery.java create mode 100644 backend/src/test/resources/shared/alter.concept.json create mode 100644 backend/src/test/resources/shared/geschlecht.concept.json create mode 100644 backend/src/test/resources/shared/two_connector.concept.json create mode 100644 backend/src/test/resources/shared/vers_stamm.csv create mode 100644 backend/src/test/resources/shared/vers_stamm.table.json create mode 100644 backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/QUARTER YEAR.json create mode 100644 backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/YEAR QUARTER.json create mode 100644 backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/YEAR YEAR.json create mode 100644 backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/quarter year.csv create mode 100644 backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/year quarter expected.csv create mode 100644 backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/MULTIPLE_FEATURES.json create mode 100644 backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/expected.csv create mode 100644 backend/src/test/resources/tests/sql/form/ABSOLUT/SECONDARY_ID/ABS_EXPORT_FORM_SECONDARY_ID.json create mode 100644 backend/src/test/resources/tests/sql/form/ABSOLUT/SIMPLE/ABS_EXPORT_FORM.json create mode 100644 backend/src/test/resources/tests/sql/form/ABSOLUT/SIMPLE/ABS_EXPORT_FORM_WITH_SELECT.json diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index 163c5c9bc5a..a2a5d38fc2f 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -28,7 +28,6 @@ import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.apiv1.query.ConceptQuery; -import com.bakdata.conquery.apiv1.query.EditorQuery; import com.bakdata.conquery.apiv1.query.ExternalUpload; import com.bakdata.conquery.apiv1.query.ExternalUploadResult; import com.bakdata.conquery.apiv1.query.Query; @@ -137,11 +136,11 @@ public Stream getQueriesFiltered(Dataset datasetId, UriBuilder */ private static boolean canFrontendRender(ManagedExecution q) { //TODO FK: should this be used to fill into canExpand instead of hiding the Executions? - if (!(q instanceof EditorQuery)) { + if (!(q instanceof ManagedQuery)) { return false; } - final Query query = ((EditorQuery) q).getQuery(); + final Query query = ((ManagedQuery) q).getQuery(); if (query instanceof ConceptQuery) { return isFrontendStructure(((ConceptQuery) query).getRoot()); @@ -291,14 +290,15 @@ public FullExecutionStatus getQueryFullStatus(ManagedExecution query, Subject su public ExternalUploadResult uploadEntities(Subject subject, Dataset dataset, ExternalUpload upload) { final Namespace namespace = datasetRegistry.get(dataset.getId()); - final CQExternal.ResolveStatistic - statistic = - CQExternal.resolveEntities(upload.getValues(), upload.getFormat(), namespace - .getStorage() - .getIdMapping(), config.getIdColumns(), config.getLocale() - .getDateReader(), upload.isOneRowPerEntity() - - ); + final CQExternal.ResolveStatistic statistic = CQExternal.resolveEntities( + upload.getValues(), + upload.getFormat(), + namespace.getStorage().getIdMapping(), + config.getIdColumns(), + config.getLocale().getDateReader(), + upload.isOneRowPerEntity(), + true + ); // Resolving nothing is a problem thus we fail. if (statistic.getResolved().isEmpty()) { diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/EditorQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/EditorQuery.java deleted file mode 100644 index 21c2096e1f2..00000000000 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/EditorQuery.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.bakdata.conquery.apiv1.query; - -import com.bakdata.conquery.apiv1.execution.ExecutionStatus; -import com.bakdata.conquery.io.cps.CPSType; -import com.bakdata.conquery.models.query.ManagedQuery; - -/** - * Common abstraction for intersecting parts of {@link ManagedQuery}. - */ -public interface EditorQuery { - - Query getQuery(); - - Long getLastResultCount(); - - default void enrichStatusBase(ExecutionStatus status) { - status.setNumberOfResults(getLastResultCount()); - - Query query = getQuery(); - status.setQueryType(query.getClass().getAnnotation(CPSType.class).id()); - - if (query instanceof SecondaryIdQuery) { - status.setSecondaryId(((SecondaryIdQuery) query).getSecondaryId().getId()); - } - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQReusedQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQReusedQuery.java index 199e8d5088f..496fa511205 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQReusedQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQReusedQuery.java @@ -7,12 +7,12 @@ import javax.annotation.Nullable; import com.bakdata.conquery.apiv1.query.CQElement; -import com.bakdata.conquery.apiv1.query.EditorQuery; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; @@ -48,7 +48,7 @@ public CQReusedQuery(ManagedExecutionId executionId){ private ManagedExecutionId queryId; @JsonIgnore - private EditorQuery query; + private ManagedQuery query; @JsonView(View.InternalCommunication.class) private Query resolvedQuery; @@ -74,7 +74,7 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { @Override public void resolve(QueryResolveContext context) { - query = (EditorQuery) context.getStorage().getExecution(queryId); + query = (ManagedQuery) context.getStorage().getExecution(queryId); if (query == null) { throw new ConqueryError.ExecutionCreationResolveError(queryId); } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java index 23f6a774f13..e53c923b38d 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/external/CQExternal.java @@ -80,7 +80,7 @@ public class CQExternal extends CQElement { /** * Maps from Entity to the computed time-frame. */ - @Getter(AccessLevel.PRIVATE) + @Getter @JsonView(View.InternalCommunication.class) private Map valuesResolved; @@ -249,7 +249,8 @@ public void resolve(QueryResolveContext context) { context.getNamespace().getStorage().getIdMapping(), context.getConfig().getIdColumns(), context.getConfig().getLocale().getDateReader(), - onlySingles + onlySingles, + context.getConfig().getSqlConnectorConfig().isEnabled() ); if (resolved.getResolved().isEmpty()) { @@ -296,7 +297,7 @@ public static class ResolveStatistic { /** * Helper method to try and resolve entities in values using the specified format. */ - public static ResolveStatistic resolveEntities(@NotEmpty String[][] values, @NotEmpty List format, EntityIdMap mapping, IdColumnConfig idColumnConfig, @NotNull DateReader dateReader, boolean onlySingles) { + public static ResolveStatistic resolveEntities(@NotEmpty String[][] values, @NotEmpty List format, EntityIdMap mapping, IdColumnConfig idColumnConfig, @NotNull DateReader dateReader, boolean onlySingles, boolean isInSqlMode) { final Map resolved = new HashMap<>(); final List unresolvedDate = new ArrayList<>(); @@ -329,7 +330,9 @@ public static ResolveStatistic resolveEntities(@NotEmpty String[][] values, @Not continue; } - String resolvedId = tryResolveId(row, readers, mapping); + String resolvedId = isInSqlMode + ? String.valueOf(row[0]) + : tryResolveId(row, readers, mapping); if (resolvedId == null) { unresolvedId.add(row); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java index 1b86d613406..5f8b842376d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java @@ -126,7 +126,7 @@ public SqlSelects convertToSqlSelects(SelectContext selectContext) { } @JsonIgnore - public boolean requiresIntervalPacking() { + public boolean isEventDateSelect() { return false; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java index 56acc039fab..a740d28c5cf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java @@ -35,7 +35,8 @@ public SqlSelects convertToSqlSelects(SelectContext selectContext) { } @Override - public boolean requiresIntervalPacking() { + public boolean isEventDateSelect() { return true; } + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java index ea94d047d46..5aeea6ea256 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java @@ -36,7 +36,7 @@ public SqlSelects convertToSqlSelects(SelectContext selectContext) { } @Override - public boolean requiresIntervalPacking() { + public boolean isEventDateSelect() { return true; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java index 5e4170944a3..61ff7940900 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java @@ -137,7 +137,7 @@ protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject @Override public void cancel() { log.debug("Sending cancel message to all workers."); - getNamespace().getWorkerHandler().sendToAll(new CancelQuery(getId())); + ((DistributedNamespace) getNamespace()).getWorkerHandler().sendToAll(new CancelQuery(getId())); } @Override @@ -180,7 +180,4 @@ public boolean allSubQueriesDone() { } } - public DistributedNamespace getNamespace() { - return (DistributedNamespace) super.getNamespace(); - } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java index 460fefa906b..0251783e3df 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java @@ -8,9 +8,9 @@ import com.bakdata.conquery.apiv1.execution.ExecutionStatus; import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; -import com.bakdata.conquery.apiv1.query.EditorQuery; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.apiv1.query.QueryDescription; +import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; @@ -42,7 +42,7 @@ @ToString(callSuper = true) @Slf4j @CPSType(base = ManagedExecution.class, id = "MANAGED_QUERY") -public class ManagedQuery extends ManagedExecution implements EditorQuery, SingleTableResult, InternalExecution { +public class ManagedQuery extends ManagedExecution implements SingleTableResult, InternalExecution { // Needs to be resolved externally before being executed private Query query; @@ -100,12 +100,18 @@ public long resultRowCount() { return lastResultCount; } - - @Override public void setStatusBase(@NonNull Subject subject, @NonNull ExecutionStatus status) { + super.setStatusBase(subject, status); - enrichStatusBase(status); + status.setNumberOfResults(getLastResultCount()); + + Query query1 = getQuery(); + status.setQueryType(query1.getClass().getAnnotation(CPSType.class).id()); + + if (query1 instanceof SecondaryIdQuery) { + status.setSecondaryId(((SecondaryIdQuery) query1).getSecondaryId().getId()); + } } protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject, FullExecutionStatus status) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SharedAliases.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SharedAliases.java index 16374562d46..586913f6425 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SharedAliases.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SharedAliases.java @@ -9,7 +9,21 @@ public enum SharedAliases { PRIMARY_COLUMN("primary_id"), SECONDARY_ID("secondary_id"), - DATES_COLUMN("dates"); + DATES_COLUMN("dates"), + DATE_RESTRICTION("date_restriction"), + + NOP_TABLE("nop_table"), + + // form related + RESOLUTION("resolution"), + INDEX("index"), + STRATIFICATION_RANGE("stratification_range"), + DATE_START("date_start"), + DATE_END("date_end"), + DATE_SERIES("date_series"), + INDEX_DATE("index_date"), + INDEX_START_POSITIVE("index_start_positive"), + INDEX_START_NEGATIVE("index_start_negative"); private final String alias; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java new file mode 100644 index 00000000000..60cb5226f17 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java @@ -0,0 +1,82 @@ +package com.bakdata.conquery.sql.conversion.cqelement; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; +import com.bakdata.conquery.models.common.CDateSet; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.google.common.base.Preconditions; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.impl.DSL; + +public class CQExternalConverter implements NodeConverter { + + private static final String CQ_EXTERNAL_CTE_NAME = "external"; + + @Override + public Class getConversionClass() { + return CQExternal.class; + } + + @Override + public ConversionContext convert(CQExternal external, ConversionContext context) { + + SqlFunctionProvider functionProvider = context.getSqlDialect().getFunctionProvider(); + List unions = external.getValuesResolved() + .entrySet().stream() + .flatMap(entry -> createRowSelects(entry, functionProvider).stream()) + .toList(); + + Preconditions.checkArgument(!unions.isEmpty(), "Expecting at least 1 converted resolved row when converting a CQExternal"); + QueryStep externalStep = QueryStep.createUnionStep(unions, CQ_EXTERNAL_CTE_NAME, Collections.emptyList()); + return context.withQueryStep(externalStep); + } + + /** + * For each entry, we need to create a SELECT statement of static values for each pid -> date set. For dialects that support date multiranges, 1 row per ID + * is sufficient. For other dialects there can be multiple rows with the same pid -> date range from the date set. + */ + private static List createRowSelects(Map.Entry entry, SqlFunctionProvider functionProvider) { + + Field primaryColumn = DSL.field(DSL.val(entry.getKey())).coerce(Object.class).as(SharedAliases.PRIMARY_COLUMN.getAlias()); + SqlIdColumns ids = new SqlIdColumns(primaryColumn); + + List validityDateEntries = functionProvider.forCDateSet(entry.getValue(), SharedAliases.DATES_COLUMN); + return validityDateEntries.stream() + .map(validityDateEntry -> createRowSelect(ids, validityDateEntry, functionProvider)) + .collect(Collectors.toList()); + } + + /** + * Creates a SELECT statement of static values for each pid -> date entry, like + * 
{@code select 1 as "pid", '[2021-01-01,2022-01-01)'::daterange as "date_range"}
+ */ + private static QueryStep createRowSelect(SqlIdColumns ids, ColumnDateRange validityDate, SqlFunctionProvider functionProvider) { + + Selects selects = Selects.builder() + .ids(ids) + .validityDate(Optional.ofNullable(validityDate)) + .build(); + + // not all SQL dialects can create a SELECT statement without a FROM clause, + // so we ensure there is some no-op table to select the static values from + Table table = functionProvider.getNoOpTable(); + + return QueryStep.builder() + .selects(selects) + .fromTable(table) + .build(); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java index b4a7b7bb686..aa4bf1041f7 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java @@ -39,6 +39,9 @@ public class ConversionContext implements Context { @Nullable SqlQuery finalQuery; + @Nullable + QueryStep stratificationTable; + /** * An optional date restriction range. Is set when converting a {@link CQDateRestriction}. */ @@ -59,6 +62,10 @@ public boolean dateRestrictionActive() { return this.dateRestrictionRange != null; } + public boolean isWithStratification() { + return this.stratificationTable != null; + } + /** * Adds a query step to the list of {@link QueryStep} of this context. */ diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java index ad67c344499..478d8d90c01 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/DateAggregationCte.java @@ -27,7 +27,7 @@ public QueryStep convert(DateAggregationContext context, QueryStep previous) { builder = builder.cteName(dateAggregationTables.cteName(cteStep)) .predecessors(List.of(previous)); } - if (cteStep != DateAggregationCteStep.INVERT) { + if (cteStep != DateAggregationCteStep.INVERT && cteStep != DateAggregationCteStep.NODE_NO_OVERLAP) { builder = builder.fromTable(QueryStep.toTableLike(dateAggregationTables.getPredecessor(cteStep))); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java index 787d5f98508..e5c1119ac54 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/aggregation/InvertCte.java @@ -42,7 +42,7 @@ protected QueryStep.QueryStepBuilder convertStep(DateAggregationContext context) SqlIdColumns ids = context.getIds(); SqlIdColumns leftIds = ids.qualify(ROWS_LEFT_TABLE_NAME); SqlIdColumns rightIds = ids.qualify(ROWS_RIGHT_TABLE_NAME); - SqlIdColumns coalescedIds = SqlIdColumns.coalesce(List.of(leftIds, rightIds)); + SqlIdColumns coalescedIds = leftIds.coalesce(List.of(rightIds)); Selects invertSelects = getInvertSelects(rowNumberStep, coalescedIds, context); TableOnConditionStep fromTable = selfJoinWithShiftedRows(leftIds, rightIds, rowNumberStep); @@ -74,7 +74,7 @@ private Selects getInvertSelects(QueryStep rowNumberStep, SqlIdColumns coalesced .build(); } - private TableOnConditionStep selfJoinWithShiftedRows(SqlIdColumns leftPrimaryColumn, SqlIdColumns rightPrimaryColumn, QueryStep rowNumberStep) { + private TableOnConditionStep selfJoinWithShiftedRows(SqlIdColumns leftIds, SqlIdColumns rightIds, QueryStep rowNumberStep) { Field leftRowNumber = DSL.field(DSL.name(ROWS_LEFT_TABLE_NAME, RowNumberCte.ROW_NUMBER_FIELD_NAME), Integer.class) .plus(1); @@ -82,7 +82,7 @@ private TableOnConditionStep selfJoinWithShiftedRows(SqlIdColumns leftPr Condition[] joinConditions = Stream.concat( Stream.of(leftRowNumber.eq(rightRowNumber)), - SqlIdColumns.join(leftPrimaryColumn, rightPrimaryColumn).stream() + leftIds.join(rightIds).stream() ) .toArray(Condition[]::new); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java index 56381837583..0ada55f0764 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationFilterCte.java @@ -20,7 +20,7 @@ public ConceptCteStep cteStep() { @Override public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { - Selects aggregationFilterSelects = getAggregationFilterSelects(tableContext); + Selects aggregationFilterSelects = collectSelects(tableContext); List aggregationFilterConditions = tableContext.getSqlFilters().stream() .flatMap(conceptFilter -> conceptFilter.getWhereClauses().getGroupFilters().stream()) @@ -32,7 +32,7 @@ public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { .conditions(aggregationFilterConditions); } - private Selects getAggregationFilterSelects(CQTableContext tableContext) { + private Selects collectSelects(CQTableContext tableContext) { QueryStep previous = tableContext.getPrevious(); Selects previousSelects = previous.getQualifiedSelects(); @@ -45,6 +45,7 @@ private Selects getAggregationFilterSelects(CQTableContext tableContext) { return Selects.builder() .ids(previousSelects.getIds()) + .stratificationDate(previousSelects.getStratificationDate()) .validityDate(previousSelects.getValidityDate()) .sqlSelects(forAggregationFilterStep) .build(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java index 769706d9ef5..26ae3c67728 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/AggregationSelectCte.java @@ -1,32 +1,43 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import org.jooq.Field; class AggregationSelectCte extends ConnectorCte { @Override public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { - String predecessor = tableContext.getConnectorTables().getPredecessor(ConceptCteStep.AGGREGATION_SELECT); - SqlIdColumns ids = tableContext.getIds().qualify(predecessor); - List requiredInAggregationFilterStep = tableContext.allSqlSelects().stream() .flatMap(sqlSelects -> sqlSelects.getAggregationSelects().stream()) .toList(); + Selects predecessorSelects = tableContext.getPrevious().getQualifiedSelects(); + SqlIdColumns ids = predecessorSelects.getIds(); + Optional stratificationDate = predecessorSelects.getStratificationDate(); Selects aggregationSelectSelects = Selects.builder() .ids(ids) + .stratificationDate(stratificationDate) .sqlSelects(requiredInAggregationFilterStep) .build(); + List> groupByFields = Stream.concat( + ids.toFields().stream(), + stratificationDate.stream().flatMap(range -> range.toFields().stream()) + ) + .toList(); + return QueryStep.builder() .selects(aggregationSelectSelects) - .groupBy(ids.toFields()); + .groupBy(groupByFields); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java index 5400ed36e87..f3ac2380086 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQTableContext.java @@ -8,11 +8,9 @@ import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingContext; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.SqlTables; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; import lombok.Builder; @@ -27,8 +25,7 @@ class CQTableContext implements Context { Optional validityDate; List sqlSelects; List sqlFilters; - SqlTables connectorTables; - IntervalPackingContext intervalPackingContext; + ConceptConversionTables connectorTables; ConversionContext conversionContext; @With QueryStep previous; @@ -40,15 +37,4 @@ public List allSqlSelects() { return Stream.concat(sqlSelects.stream(), sqlFilters.stream().map(SqlFilters::getSelects)).toList(); } - public SqlIdColumns getIds() { - if (previous == null) { - return ids; - } - return previous.getQualifiedSelects().getIds(); - } - - public Optional getIntervalPackingContext() { - return Optional.ofNullable(intervalPackingContext); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java index d64d8da2f98..0840b875e3d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConnectorCte.java @@ -18,12 +18,12 @@ protected Optional convert(CQTableContext tableContext, Optional eventFilterConditions = tableContext.getSqlFilters().stream() - .flatMap(conceptFilter -> conceptFilter.getWhereClauses().getEventFilters().stream()) - .map(WhereCondition::condition) - .toList(); + Selects eventFilterSelects = colltectSelects(tableContext); + List eventFilterConditions = collectEventFilterConditions(tableContext); return QueryStep.builder() .selects(eventFilterSelects) .conditions(eventFilterConditions); @@ -35,24 +34,25 @@ public ConceptCteStep cteStep() { return ConceptCteStep.EVENT_FILTER; } - private Selects getEventFilterSelects(CQTableContext tableContext) { - String predecessorTableName = tableContext.getConnectorTables().getPredecessor(cteStep()); - SqlIdColumns ids = tableContext.getIds().qualify(predecessorTableName); + private Selects colltectSelects(CQTableContext tableContext) { - Optional validityDate = tableContext.getValidityDate(); - if (validityDate.isPresent()) { - validityDate = Optional.of(validityDate.get().qualify(predecessorTableName)); - } + String predecessorTableName = tableContext.getPrevious().getCteName(); + Selects predecessorSelects = tableContext.getPrevious().getQualifiedSelects(); + + SqlIdColumns ids = predecessorSelects.getIds(); + Optional validityDate = predecessorSelects.getValidityDate(); + Optional stratificationDate = predecessorSelects.getStratificationDate(); List eventFilterSelects = tableContext.allSqlSelects().stream() - .flatMap(sqlSelects -> collectForEventFilterStep(sqlSelects).stream()) + .flatMap(sqlSelects -> collectSelects(sqlSelects).stream()) .flatMap(sqlSelect -> referenceRequiredColumns(sqlSelect, predecessorTableName)) .toList(); return Selects.builder() .ids(ids) .validityDate(validityDate) + .stratificationDate(stratificationDate) .sqlSelects(eventFilterSelects) .build(); } @@ -60,10 +60,10 @@ private Selects getEventFilterSelects(CQTableContext tableContext) { /** * Collects the columns required in {@link ConceptCteStep#AGGREGATION_SELECT}, but also columns additional tables require (like the ones created by the * {@link SumDistinctSqlAggregator}). An additional predecessor can contain an N-ary tree of predecessors itself (like all {@link QueryStep}s), so we want to - * look for the deepest predeceasing QueryStep leafs and collect their {@link SqlSelects}, because they expect this CTE to contain all their + * look for the deepest preceding QueryStep leafs and collect their {@link SqlSelects}, because they expect this CTE to contain all their * {@link SqlSelect#requiredColumns()}. */ - private static List collectForEventFilterStep(SqlSelects sqlSelects) { + private static List collectSelects(SqlSelects sqlSelects) { return Stream.concat( sqlSelects.getAggregationSelects().stream(), sqlSelects.getAdditionalPredecessor().map(EventFilterCte::collectDeepestPredecessorsColumns).orElse(Stream.empty()) @@ -88,4 +88,33 @@ private static Stream> referenceRequiredColumns(SqlSelect return sqlSelect.requiredColumns().stream().map(column -> new ExtractingSqlSelect<>(predecessorTableName, column, Object.class)); } + private static List collectEventFilterConditions(CQTableContext tableContext) { + + List eventFilterConditions = tableContext.getSqlFilters().stream() + .flatMap(conceptFilter -> conceptFilter.getWhereClauses().getEventFilters().stream()) + .map(WhereCondition::condition) + .toList(); + + if (!tableContext.getConversionContext().isWithStratification()) { + return eventFilterConditions; + } + return addStratificationCondition(eventFilterConditions, tableContext); + } + + private static List addStratificationCondition(List eventFilterConditions, CQTableContext tableContext) { + Selects previousSelects = tableContext.getPrevious().getQualifiedSelects(); + Preconditions.checkArgument( + previousSelects.getStratificationDate().isPresent() && previousSelects.getValidityDate().isPresent(), + "Can't apply stratification for table %s".formatted(tableContext.getConnectorTables().getRootTable()) + ); + + // we filter every entry where stratification date range and validity date range do not overlap + SqlFunctionProvider functionProvider = tableContext.getConversionContext().getSqlDialect().getFunctionProvider(); + ColumnDateRange stratificationDate = previousSelects.getStratificationDate().get(); + ColumnDateRange validityDate = previousSelects.getValidityDate().get(); + Condition stratificationCondition = functionProvider.dateRestriction(stratificationDate, validityDate); + + return Stream.concat(Stream.of(stratificationCondition), eventFilterConditions.stream()).toList(); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java index ce78bab1ae5..0cac94b643a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/JoinBranchesCte.java @@ -4,6 +4,7 @@ import java.util.List; import java.util.Optional; +import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingContext; import com.bakdata.conquery.sql.conversion.dialect.IntervalPacker; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.LogicalOperation; @@ -55,13 +56,15 @@ protected QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { List queriesToJoin = new ArrayList<>(); queriesToJoin.add(tableContext.getPrevious()); + // validity date aggregation Optional validityDate; - if (tableContext.getIntervalPackingContext().isEmpty()) { + if (!tableContext.getConnectorTables().isWithIntervalPacking()) { validityDate = Optional.empty(); } else { + IntervalPackingContext intervalPackingContext = createIntervalPackingContext(tableContext); IntervalPacker intervalPacker = tableContext.getConversionContext().getSqlDialect().getIntervalPacker(); - QueryStep lastIntervalPackingStep = intervalPacker.aggregateAsValidityDate(tableContext.getIntervalPackingContext().get()); + QueryStep lastIntervalPackingStep = intervalPacker.aggregateAsValidityDate(intervalPackingContext); queriesToJoin.add(lastIntervalPackingStep); validityDate = lastIntervalPackingStep.getQualifiedSelects().getValidityDate(); @@ -71,18 +74,12 @@ protected QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { } } + // additional preceding tables tableContext.allSqlSelects().stream() .flatMap(sqlSelects -> sqlSelects.getAdditionalPredecessor().stream()) .forEach(queriesToJoin::add); - SqlIdColumns ids = QueryStepJoiner.coalesceIds(queriesToJoin); - List mergedSqlSelects = QueryStepJoiner.mergeSelects(queriesToJoin); - Selects selects = Selects.builder() - .ids(ids) - .validityDate(validityDate) - .sqlSelects(mergedSqlSelects) - .build(); - + Selects selects = collectSelects(validityDate, queriesToJoin, tableContext); TableLike fromTable = QueryStepJoiner.constructJoinedTable(queriesToJoin, LogicalOperation.AND, tableContext.getConversionContext()); return QueryStep.builder() @@ -91,4 +88,27 @@ protected QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { .predecessors(queriesToJoin); } + private static IntervalPackingContext createIntervalPackingContext(CQTableContext tableContext) { + Selects predcessorSelects = tableContext.getPrevious().getQualifiedSelects(); + return IntervalPackingContext.builder() + .ids(predcessorSelects.getIds()) + .daterange(tableContext.getValidityDate().get()) + .tables(tableContext.getConnectorTables()) + .build(); + } + + private static Selects collectSelects(Optional validityDate, List queriesToJoin, CQTableContext tableContext) { + + SqlIdColumns ids = QueryStepJoiner.coalesceIds(queriesToJoin); + List mergedSqlSelects = QueryStepJoiner.mergeSelects(queriesToJoin); + Optional stratificationDate = tableContext.getPrevious().getQualifiedSelects().getStratificationDate(); + + return Selects.builder() + .ids(ids) + .stratificationDate(stratificationDate) + .validityDate(validityDate) + .sqlSelects(mergedSqlSelects) + .build(); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java index befd8b8e070..2a6edc23339 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingCte.java @@ -2,14 +2,25 @@ import java.util.List; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; import org.jooq.Condition; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.TableLike; +import org.jooq.impl.DSL; class PreprocessingCte extends ConnectorCte { + @Override + public ConceptCteStep cteStep() { + return ConceptCteStep.PREPROCESSING; + } + public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { List forPreprocessing = tableContext.allSqlSelects().stream() @@ -28,15 +39,46 @@ public QueryStep.QueryStepBuilder convertStep(CQTableContext tableContext) { .map(WhereCondition::condition) .toList(); - return QueryStep.builder() - .selects(preprocessingSelects) - .conditions(conditions) - .fromTable(QueryStep.toTableLike(tableContext.getConnectorTables().getPredecessor(ConceptCteStep.PREPROCESSING))); + QueryStep.QueryStepBuilder builder = QueryStep.builder() + .selects(preprocessingSelects) + .conditions(conditions); + + if (!tableContext.getConversionContext().isWithStratification()) { + TableLike rootTable = QueryStep.toTableLike(tableContext.getConnectorTables().getPredecessor(ConceptCteStep.PREPROCESSING)); + return builder.fromTable(rootTable); + } + + return joinWithStratificationTable(forPreprocessing, conditions, tableContext); } - @Override - public ConceptCteStep cteStep() { - return ConceptCteStep.PREPROCESSING; + private static QueryStep.QueryStepBuilder joinWithStratificationTable( + List preprocessingSelects, + List conditions, + CQTableContext tableContext + ) { + QueryStep stratificationTable = tableContext.getConversionContext().getStratificationTable(); + + Selects stratificationSelects = stratificationTable.getQualifiedSelects(); + SqlIdColumns stratificationIds = stratificationSelects.getIds(); + SqlIdColumns rootTableIds = tableContext.getIds(); + List idConditions = stratificationIds.join(rootTableIds); + + // join full stratification with connector table on all ID's from prerequisite query + SqlFunctionProvider functionProvider = tableContext.getConversionContext().getSqlDialect().getFunctionProvider(); + Table connectorTable = DSL.table(DSL.name(tableContext.getConnectorTables().getPredecessor(ConceptCteStep.PREPROCESSING))); + TableLike joinedTable = functionProvider.innerJoin(connectorTable, stratificationTable, idConditions); + + Selects selects = Selects.builder() + .ids(stratificationSelects.getIds()) + .validityDate(tableContext.getValidityDate()) + .stratificationDate(stratificationSelects.getStratificationDate()) + .sqlSelects(preprocessingSelects) + .build(); + + return QueryStep.builder() + .selects(selects) + .fromTable(joinedTable) + .conditions(conditions); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePathGenerator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePathGenerator.java index 830eb47cfd9..54eb8923c2b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePathGenerator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/TablePathGenerator.java @@ -1,6 +1,11 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; -import static com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep.*; +import static com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep.EVENT_FILTER; +import static com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep.INTERVAL_PACKING_SELECTS; +import static com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep.JOIN_BRANCHES; +import static com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep.MANDATORY_STEPS; +import static com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep.UNIVERSAL_SELECTS; +import static com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep.UNNEST_DATE; import static com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep.INTERVAL_COMPLETE; import java.util.HashMap; @@ -62,9 +67,9 @@ private TablePathInfo collectConnectorTables(CQConcept cqConcept, CQTable cqTabl tableInfo.addWithDefaultMapping(MANDATORY_STEPS); tableInfo.setLastPredecessor(JOIN_BRANCHES); - boolean intervalPackingSelectsPresent = cqTable.getSelects().stream().anyMatch(Select::requiresIntervalPacking); + boolean eventDateSelectsPresent = cqTable.getSelects().stream().anyMatch(Select::isEventDateSelect); // no validity date aggregation possible nor necessary - if (cqTable.findValidityDate() == null || (!intervalPackingSelectsPresent && cqConcept.isExcludeFromTimeAggregation())) { + if (cqTable.findValidityDate() == null || (!cqConcept.isAggregateEventDates() && !eventDateSelectsPresent)) { return tableInfo; } @@ -72,7 +77,7 @@ private TablePathInfo collectConnectorTables(CQConcept cqConcept, CQTable cqTabl tableInfo.setContainsIntervalPacking(true); tableInfo.addMappings(IntervalPackingCteStep.getMappings(EVENT_FILTER, sqlDialect)); - if (!intervalPackingSelectsPresent) { + if (!eventDateSelectsPresent) { return tableInfo; } @@ -98,8 +103,8 @@ private TablePathInfo collectConceptTables(QueryStep predecessor, CQConcept cqCo tableInfo.setRootTable(predecessor.getCteName()); // last table of a single connector or merged and aggregated table of multiple connectors tableInfo.addRootTableMapping(UNIVERSAL_SELECTS); - // no interval packing selects present - if (cqConcept.getSelects().stream().noneMatch(Select::requiresIntervalPacking)) { + // no event date selects present + if (cqConcept.getSelects().stream().noneMatch(Select::isEventDateSelect)) { return tableInfo; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlDialect.java index 17e54c5d84b..e9880d0d27a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlDialect.java @@ -2,10 +2,14 @@ import java.util.List; +import com.bakdata.conquery.models.config.Dialect; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.cqelement.aggregation.AnsiSqlDateAggregator; import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.AnsiSqlIntervalPacker; +import com.bakdata.conquery.sql.conversion.forms.StratificationTableFactory; +import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.execution.DefaultSqlCDateSetParser; import com.bakdata.conquery.sql.execution.SqlCDateSetParser; import org.jooq.DSLContext; @@ -56,4 +60,9 @@ public SqlDateAggregator getDateAggregator() { return this.hanaSqlDateAggregator; } + @Override + public StratificationTableFactory getStratificationTableFactory(QueryStep base, ConversionContext context) { + return StratificationTableFactory.create(Dialect.HANA, base, context); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlDialect.java index 014308bbe24..ec8d2d81051 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlDialect.java @@ -2,10 +2,14 @@ import java.util.List; +import com.bakdata.conquery.models.config.Dialect; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.cqelement.aggregation.PostgreSqlDateAggregator; import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.PostgreSqlIntervalPacker; +import com.bakdata.conquery.sql.conversion.forms.StratificationTableFactory; +import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.execution.DefaultSqlCDateSetParser; import com.bakdata.conquery.sql.execution.SqlCDateSetParser; import org.jooq.DSLContext; @@ -61,4 +65,9 @@ public SqlDateAggregator getDateAggregator() { return this.postgresqlDateAggregator; } + @Override + public StratificationTableFactory getStratificationTableFactory(QueryStep base, ConversionContext context) { + return StratificationTableFactory.create(Dialect.POSTGRESQL, base, context); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java index 90672372f5d..5107a453ea1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java @@ -10,10 +10,15 @@ import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.cqelement.CQAndConverter; import com.bakdata.conquery.sql.conversion.cqelement.CQDateRestrictionConverter; +import com.bakdata.conquery.sql.conversion.cqelement.CQExternalConverter; import com.bakdata.conquery.sql.conversion.cqelement.CQNegationConverter; import com.bakdata.conquery.sql.conversion.cqelement.CQOrConverter; +import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.cqelement.concept.CQConceptConverter; +import com.bakdata.conquery.sql.conversion.forms.StratificationTableFactory; +import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.QueryStepTransformer; +import com.bakdata.conquery.sql.conversion.query.AbsoluteFormQueryConverter; import com.bakdata.conquery.sql.conversion.query.ConceptQueryConverter; import com.bakdata.conquery.sql.conversion.query.SecondaryIdQueryConverter; import com.bakdata.conquery.sql.conversion.supplier.DateNowSupplier; @@ -31,6 +36,8 @@ public interface SqlDialect { SqlDateAggregator getDateAggregator(); + StratificationTableFactory getStratificationTableFactory(QueryStep base, ConversionContext context); + List> getNodeConverters(); DSLContext getDSLContext(); @@ -46,14 +53,17 @@ default boolean supportsSingleColumnRanges() { } default List> getDefaultNodeConverters() { + QueryStepTransformer queryStepTransformer = new QueryStepTransformer(getDSLContext()); return List.of( new CQDateRestrictionConverter(), new CQAndConverter(), new CQOrConverter(), new CQNegationConverter(), new CQConceptConverter(), - new ConceptQueryConverter(new QueryStepTransformer(getDSLContext())), - new SecondaryIdQueryConverter() + new CQExternalConverter(), + new ConceptQueryConverter(queryStepTransformer), + new SecondaryIdQueryConverter(), + new AbsoluteFormQueryConverter(queryStepTransformer) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index a868d9b1adc..2f9c1b62b90 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -5,7 +5,9 @@ import java.util.List; import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; +import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.SqlTables; @@ -42,11 +44,26 @@ public interface SqlFunctionProvider { String getAnyCharRegex(); /** - * A date restriction condition is true if holds: dateRestrictionStart <= validityDateEnd and dateRestrictionEnd >= validityDateStart + * @return A dummy table that enables selection of static values. */ - Condition dateRestriction(ColumnDateRange dateRestrictionRange, ColumnDateRange validityFieldRange); + Table getNoOpTable(); - ColumnDateRange forDateRestriction(CDateRange dateRestriction); + /** + * A date restriction condition is true if holds: dateRestrictionStart < daterangeEnd and dateRestrictionEnd > daterangeStart. The ends of both ranges are + * exclusive. + */ + Condition dateRestriction(ColumnDateRange dateRestriction, ColumnDateRange daterange); + + /** + * Creates a {@link ColumnDateRange} as a SQL representation of the {@link CDateRange}. + */ + ColumnDateRange forCDateRange(CDateRange daterange); + + /** + * Creates a list of {@link ColumnDateRange}s for each {@link CDateRange} of the given {@link CDateSet}. Each {@link ColumnDateRange} will be aliased with + * the same given {@link SharedAliases}. + */ + List forCDateSet(CDateSet dateset, SharedAliases alias); /** * Creates a {@link ColumnDateRange} for a tables {@link CQTable}s validity date. @@ -89,6 +106,13 @@ public interface SqlFunctionProvider { */ Field daterangeStringAggregation(ColumnDateRange columnDateRange); + /** + * Combines the start and end column of a validity date entry into one compound string expression. + *

+ * Example: [2013-11-10,2013-11-11) + */ + Field daterangeStringExpression(ColumnDateRange columnDateRange); + Field dateDistance(ChronoUnit datePart, Field startDate, Field endDate); Field addDays(Field dateColumn, int amountOfDays); @@ -150,6 +174,16 @@ default TableOnConditionStep fullOuterJoin( .on(joinConditions.toArray(Condition[]::new)); } + default TableOnConditionStep leftJoin( + Table leftPartQueryBase, + QueryStep rightPartQS, + List joinConditions + ) { + return leftPartQueryBase + .leftJoin(DSL.name(rightPartQS.getCteName())) + .on(joinConditions.toArray(Condition[]::new)); + } + default Field toDateField(String dateExpression) { return DSL.toDate(dateExpression, DEFAULT_DATE_FORMAT); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java new file mode 100644 index 00000000000..a75d7039cbd --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java @@ -0,0 +1,42 @@ +package com.bakdata.conquery.sql.conversion.forms; + +import java.time.LocalDate; +import java.util.List; + +import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.conversion.model.Selects; +import org.jooq.Condition; + +interface AbsoluteStratification { + + /** + * Finds the date range the stratification range is bound by. It can be either bound by entity date, meaning entities validity date defines the min and max + * stratification date bounds. Otherwise, it is bound by forms date restriction range. + */ + ColumnDateRange findBounds(Range formDateRestriction, Selects baseStepSelects); + + /** + * Generates the correctly bound stratification date range. The generated series range will be bound by the given bounds range. + *

+ * A generated series will always span over full quarters and years, for example: generate_series(2012-01-01, 2013-01-01, interval '1 year') generates + * + * + * + * + * + * + * + *
2012-01-01
2013-01-01
+ * But if the entity date or the form's date restriction range is not "bigger", meaning bounds.startDdate > series.startDate and/or bounds.endDate < + * series.endDate, then start and or end date have to be overwritten to set the stratification range correctly. + */ + ColumnDateRange createStratificationDateRange(ColumnDateRange seriesRange, ColumnDateRange bounds); + + /** + * Defines the conditions we need to apply when joining a generated series set with entities IDs (and dates). + * When we have an entity date stratification, we only keep those stratification windows where the entity date and the stratification range overlap. + */ + List stratificationTableConditions(ColumnDateRange seriesRange, ColumnDateRange bounds); + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java new file mode 100644 index 00000000000..320992a4bb4 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java @@ -0,0 +1,42 @@ +package com.bakdata.conquery.sql.conversion.forms; + +import com.bakdata.conquery.models.forms.util.Resolution; +import com.bakdata.conquery.sql.conversion.model.CteStep; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@Getter +@RequiredArgsConstructor +public enum FormCteStep implements CteStep { + + // prerequisite + EXTRACT_IDS("extract_ids"), + + // stratification + QUARTER_SERIES("quarter_series"), + QUARTERS("quarters"), + YEAR_SERIES("year_series"), + YEARS("years"), + COMPLETE("complete"), + FULL_STRATIFICATION("full_stratification"); + + private final String suffix; + + public static FormCteStep stratificationCte(Resolution resolution) { + return switch (resolution) { + case COMPLETE -> FormCteStep.COMPLETE; + case YEARS -> FormCteStep.YEARS; + case QUARTERS -> FormCteStep.QUARTERS; + case DAYS -> throw new UnsupportedOperationException("Not implemented yet"); + }; + } + + public static FormCteStep seriesCte(Resolution resolution) { + return switch (resolution) { + case YEARS -> FormCteStep.YEAR_SERIES; + case QUARTERS -> FormCteStep.QUARTER_SERIES; + case DAYS, COMPLETE -> throw new UnsupportedOperationException("Not supported"); + }; + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/HanaStratificationTableFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/HanaStratificationTableFactory.java new file mode 100644 index 00000000000..3d09299094b --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/HanaStratificationTableFactory.java @@ -0,0 +1,101 @@ +package com.bakdata.conquery.sql.conversion.forms; + +import java.sql.Date; +import java.time.LocalDate; +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.forms.util.Resolution; +import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import lombok.Getter; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.TableLike; +import org.jooq.impl.DSL; + +@Getter +class HanaStratificationTableFactory extends StratificationTableFactory implements AbsoluteStratification { + + // HANA pre-generates start and end date when generating a date series + // see https://help.sap.com/docs/SAP_HANA_PLATFORM/4fe29514fd584807ac9f2a04f6754767/c8101037ad4344768db31e68e4d30eb4.html + private static final Field SERIES_START_FIELD = DSL.field(DSL.name("GENERATED_PERIOD_START"), Date.class); + private static final Field SERIES_END_FIELD = DSL.field(DSL.name("GENERATED_PERIOD_END"), Date.class); + + public HanaStratificationTableFactory(QueryStep baseStep, ConversionContext context) { + super(baseStep, context); + } + + @Override + public QueryStep createIntervalTable(Range formDateRestriction, Resolution resolution) { + + Table seriesTable = createGenerateSeriesTable(formDateRestriction, resolution); + + Selects baseStepSelects = getBaseStep().getQualifiedSelects(); + SqlIdColumns ids = baseStepSelects.getIds().withAbsoluteStratification(resolution, indexField(baseStepSelects.getIds())); + + ColumnDateRange bounds = findBounds(formDateRestriction, baseStepSelects); + ColumnDateRange seriesRange = ColumnDateRange.of(SERIES_START_FIELD, SERIES_END_FIELD); + ColumnDateRange yearRange = createStratificationDateRange(seriesRange, bounds); + + Selects selects = Selects.builder() + .ids(ids) + .stratificationDate(Optional.ofNullable(yearRange)) + .build(); + + List conditions = stratificationTableConditions(seriesRange, bounds); + List> tables = List.of(QueryStep.toTableLike(getBaseStep().getCteName()), seriesTable); + + return QueryStep.builder() + .cteName(FormCteStep.stratificationCte(resolution).getSuffix()) + .selects(selects) + .fromTables(tables) + .conditions(conditions) + .build(); + } + + @Override + public ColumnDateRange createStratificationDateRange(ColumnDateRange seriesRange, ColumnDateRange bounds) { + Field rangeStart = DSL.greatest(seriesRange.getStart(), bounds.getStart()); + Field rangeEnd = DSL.least(seriesRange.getEnd(), bounds.getEnd()); + return ColumnDateRange.of(rangeStart, rangeEnd).as(SharedAliases.STRATIFICATION_RANGE.getAlias()); + } + + @Override + public ColumnDateRange findBounds(Range formDateRestriction, Selects baseStepSelects) { + ColumnDateRange bounds; + if (isEntityDateStratification()) { + bounds = getFunctionProvider().toDualColumn(baseStepSelects.getValidityDate().get()); + } + else { + bounds = getFunctionProvider().forCDateRange(CDateRange.of(formDateRestriction)); + } + return bounds; + } + + @Override + public List stratificationTableConditions(ColumnDateRange seriesRange, ColumnDateRange bounds) { + if (!isEntityDateStratification()) { + return Collections.emptyList(); + } + return List.of(getFunctionProvider().dateRestriction(bounds, seriesRange)); + } + + private Table createGenerateSeriesTable(Range formDateRestriction, Resolution resolution) { + String resolutionExpression = toResolutionExpression(resolution).toUpperCase(); // uppercase not required, but HANA best-practice + Range adjustedRange = toGenerateSeriesBounds(formDateRestriction, resolution); + Field start = getFunctionProvider().toDateField(adjustedRange.getMin().toString()); + Field end = getFunctionProvider().toDateField(adjustedRange.getMax().toString()); + return DSL.table("SERIES_GENERATE_DATE({0}, {1}, {2})", DSL.val("INTERVAL %s".formatted(resolutionExpression)), start, end); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/PostgresStratificationTableFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/PostgresStratificationTableFactory.java new file mode 100644 index 00000000000..8ed4613cb82 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/PostgresStratificationTableFactory.java @@ -0,0 +1,162 @@ +package com.bakdata.conquery.sql.conversion.forms; + +import java.sql.Date; +import java.sql.Timestamp; +import java.time.LocalDate; +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.models.forms.util.Resolution; +import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; +import com.bakdata.conquery.sql.conversion.dialect.PostgreSqlFunctionProvider; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.TableLike; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; + +class PostgresStratificationTableFactory extends StratificationTableFactory implements AbsoluteStratification { + + public PostgresStratificationTableFactory(QueryStep baseStep, ConversionContext context) { + super(baseStep, context); + } + + @Override + public QueryStep createIntervalTable(Range formDateRestriction, Resolution resolution) { + + QueryStep seriesTableStep = createSeriesTableStep(resolution, formDateRestriction); + + Selects baseStepSelects = getBaseStep().getQualifiedSelects(); + Field index = indexField(baseStepSelects.getIds()); + SqlIdColumns ids = baseStepSelects.getIds().withAbsoluteStratification(resolution, index); + + ColumnDateRange seriesRange = seriesTableStep.getQualifiedSelects().getStratificationDate().orElseThrow( + () -> new IllegalStateException("Series table step should contain a stratification date") + ); + ColumnDateRange bounds = findBounds(formDateRestriction, baseStepSelects); + ColumnDateRange stratificationDate = createStratificationDateRange(seriesRange, bounds); + + Selects selects = Selects.builder() + .ids(ids) + .stratificationDate(Optional.ofNullable(stratificationDate)) + .build(); + + List conditions = stratificationTableConditions(seriesRange, bounds); + List> tables = List.of( + QueryStep.toTableLike(getBaseStep().getCteName()), + QueryStep.toTableLike(seriesTableStep.getCteName()) + ); + + return QueryStep.builder() + .cteName(FormCteStep.stratificationCte(resolution).getSuffix()) + .selects(selects) + .fromTables(tables) + .conditions(conditions) + .predecessors(List.of(seriesTableStep)) + .build(); + } + + @Override + protected PostgreSqlFunctionProvider getFunctionProvider() { + return (PostgreSqlFunctionProvider) super.getFunctionProvider(); + } + + /** + * Unlike HANA, Postgres does not create a start and end date when creating a date series. Instead, it just defines timestamps from start to end in a + * single row set. That's why we have to define start and end ourselves via SQL's lead() function. + */ + private QueryStep createSeriesTableStep(Resolution resolution, Range dateRange) { + + Table seriesTable = createSeries(dateRange, resolution); + + // series are generated as timestamps, so we have to cast + Field seriesField = getFunctionProvider().cast(DSL.field(DSL.name(SharedAliases.DATE_SERIES.getAlias()), Timestamp.class), SQLDataType.DATE); + Field startDate = seriesField.as(SharedAliases.DATE_START.getAlias()); + Field endDate = DSL.lead(seriesField).over().as(SharedAliases.DATE_END.getAlias()); + ColumnDateRange seriesRange = ColumnDateRange.of(startDate, endDate); + + // not actually required, but Selects expect at least 1 SqlIdColumn + Field rowNumber = DSL.rowNumber().over().coerce(Object.class); + SqlIdColumns ids = new SqlIdColumns(rowNumber); + + Selects selects = Selects.builder() + .ids(ids) + .stratificationDate(Optional.of(seriesRange)) + .build(); + + return QueryStep.builder() + .cteName(FormCteStep.seriesCte(resolution).getSuffix()) + .selects(selects) + .fromTable(seriesTable) + .build(); + } + + private Table createSeries(Range dateRange, Resolution resolution) { + + PostgreSqlFunctionProvider functionProvider = getFunctionProvider(); + Range adjustedRange = toGenerateSeriesBounds(dateRange, resolution); + Field start = functionProvider.cast( + DSL.field(DSL.val(adjustedRange.getMin().toString())), + SQLDataType.TIMESTAMP + ); + Field end = functionProvider.cast( + DSL.field(DSL.val(adjustedRange.getMax().toString())), + SQLDataType.TIMESTAMP + ); + + return DSL.table( + "generate_series({0}, {1}, {2} {3})", + start, + end, + DSL.keyword("interval"), + DSL.val(toResolutionExpression(resolution)) + ) + .as(SharedAliases.DATE_SERIES.getAlias()); + } + + @Override + public ColumnDateRange findBounds(Range formDateRestriction, Selects baseStepSelects) { + ColumnDateRange bounds; + if (isEntityDateStratification()) { + bounds = getFunctionProvider().toDualColumn(baseStepSelects.getValidityDate().get()); + } + else { + Field formDateRangeMin = getFunctionProvider().toDateField(formDateRestriction.getMin().toString()); + Field formDateRangeMax = getFunctionProvider().addDays(getFunctionProvider().toDateField(formDateRestriction.getMax().toString()), 1); + bounds = ColumnDateRange.of(formDateRangeMin, formDateRangeMax); + } + return bounds; + } + + @Override + public ColumnDateRange createStratificationDateRange(ColumnDateRange seriesRange, ColumnDateRange bounds) { + PostgreSqlFunctionProvider functionProvider = getFunctionProvider(); + Field rangeStart = DSL.greatest(seriesRange.getStart(), bounds.getStart()); + Field rangeEnd = DSL.least(seriesRange.getEnd(), bounds.getEnd()); + Field daterange = functionProvider.daterange(rangeStart, rangeEnd, "[)"); + return ColumnDateRange.of(daterange).as(SharedAliases.STRATIFICATION_RANGE.getAlias()); + } + + @Override + public List stratificationTableConditions(ColumnDateRange seriesRange, ColumnDateRange bounds) { + + PostgreSqlFunctionProvider functionProvider = getFunctionProvider(); + + // we need to filter the single entry with a null end date which is created because we use the SQL leap() function in createSeriesTableStep() + Field stratificationDateEnd = DSL.field(DSL.name(SharedAliases.DATE_END.getAlias())); + Condition endNotNull = DSL.condition(stratificationDateEnd.isNotNull()); + + Condition overlapCondition = isEntityDateStratification() ? functionProvider.dateRestriction(bounds, seriesRange) : DSL.noCondition(); + + return List.of(endNotNull, overlapCondition); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/README.md b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/README.md new file mode 100644 index 00000000000..309a8a25444 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/README.md @@ -0,0 +1,203 @@ +# Form conversion - how to apply stratification in SQL? + +This document outlines the procedure to apply stratification within SQL in the context of the form conversion process. + +## Prerequisite conversion + +The prerequisite query conversion produces a CTE, which will contain the IDs of those subjects relevant for the form. +Because this could be any kind of Query, the CTE might also contain a validity date and converted Selects. +Take this CTE representing a converted CQExternal as an example: + +**CTE:** `external` + +```sql +select '1' "primary_id", + TO_DATE('2001-12-01', 'yyyy-mm-dd') "dates_start", + TO_DATE('2016-12-02', 'yyyy-mm-dd') "dates_end" +from "DUMMY"; -- "DUMMY" is SAP HANAs built-in no-op table +``` + +## Absolute stratification + +This example is covering the following +testcase: `src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/ABS_EXPORT_FORM.test.json`. + +For an absolute form, we only care for the primary ID, so we extract the primary IDs (and discard the validity date). +We group by primary ID to keep only 1 entry per subject (a ≥`select distinct` would to the trick too). + +**CTE:** `extract_ids` + +```sql +select "primary_id" +from "external" +group by "primary_id" +``` + +Now, we want to create a resolution table for each resolution (`COMPLETE`, `YEAR`, `QUARTER`). + +**CTE:** `complete` + +```sql +select "primary_id", + 'COMPLETE' "resolution", + 1 "index", + TO_DATE('2012-01-16', 'yyyy-mm-dd') "stratification_range_start", + TO_DATE('2012-12-18', 'yyyy-mm-dd') "stratification_range_end" +from "extract_ids" +``` + +For an absolute form, the is only 1 complete range which spans over the given forms date +range `[2012-01-16,2012-12-18)`. Thus, the complete `stratification_range_start` and `stratification_range_end` are +simply the static values given by +the forms date range. + +A complete range shall have a `null` index, because it spans the complete range, but we set it to 1 to ensure we can +join tables on index. We do this, because a condition involving `null` in a join (e.g., `null = some_value` or +`null = null`) always evaluates to false, which would cause incorrect joining results. + +**CTE:** `years` + +```sql +select "primary_id", + 'YEARS' "resolution", + row_number() over (partition by "primary_id") "index", + greatest("GENERATED_PERIOD_START", TO_DATE('2012-01-16', 'yyyy-mm-dd')) "stratification_range_start", + least("GENERATED_PERIOD_END", TO_DATE('2012-12-18', 'yyyy-mm-dd')) "stratification_range_end" +from "extract_ids", SERIES_GENERATE_DATE('INTERVAL 1 YEAR', TO_DATE('2012-01-01', 'yyyy-mm-dd'), + TO_DATE('2013-01-01', 'yyyy-mm-dd')) +``` + +For `YEAR` and `QUARTER`, we generate a series over the whole forms date range. Therefore, the forms actual range +`[2012-01-16,2012-12-18)` has to be adjusted: the start and end range do not cover the interval of 1 year, thus the +generated series would be an empty set. That's why we set the start date of the generated series to the first day of +the year of the forms date range start: for `2012-01-16`, this is the `2012-01-01`. The end date is set to the first day +of the year of the forms date range end + 1 year: for `2012-12-17`, it's the `2013-01-01`. + +`SERIES_GENERATE_DATE('INTERVAL 1 YEAR', TO_DATE('2012-01-01', 'yyyy-mm-dd'), TO_DATE('2013-01-01', 'yyyy-mm-dd'))` +creates the following set: + +| GENERATED\_PERIOD\_START | GENERATED\_PERIOD\_END | +|:-------------------------|:-----------------------| +| 2012-01-01 | 2013-01-01 | + +For HANA, the two columns names are pre-generated - that's why we can use them directly in the select statement. + +Because the generated series start might be before the forms absolute date range start and/or the generated series end +after the forms absolute date range end, we cover these edge cases by using the `greatest()` and `least()` functions to +compute the correctly bound stratification dates. + +**CTE:** `quarters` + +```sql +select "primary_id", + 'QUARTER' "resolution", + row_number() over (partition by "primary_id") "index", + greatest("GENERATED_PERIOD_START", TO_DATE('2012-01-16', 'yyyy-mm-dd')) "stratification_range_start", + least("GENERATED_PERIOD_END", TO_DATE('2012-12-17', 'yyyy-mm-dd') "stratification_range_end" + from "extract_ids", + SERIES_GENERATE_DATE('INTERVAL 3 MONTH', TO_DATE('2012-01-01', 'yyyy-mm-dd'), + TO_DATE('2013-01-01', 'yyyy-mm-dd')) +``` + +Similar to the `YEAR` resolution, we generate a series, but this time with a changed interval of 3 month (1 quarter). +The generated series looks like this: + +| GENERATED\_PERIOD\_START | GENERATED\_PERIOD\_END | +|:-------------------------|:-----------------------| +| 2012-01-01 | 2012-04-01 | +| 2012-04-01 | 2012-07-01 | +| 2012-07-01 | 2012-10-01 | +| 2012-10-01 | 2013-01-01 | + +Again, we make sure the stratification dates have the correct bounds via `greatest()` and `least()`. + +**CTE:** `full_stratification` + +Now, we union all the resolution tables. + +```sql +select "complete"."primary_id", + "complete"."resolution", + "complete"."index", + "complete"."stratification_range_start", + "complete"."stratification_range_end" +from "complete" +union all +select "years"."primary_id", + "years"."resolution", + "years"."index", + "years"."stratification_range_start", + "years"."stratification_range_end" +from "years" +union all +select "quarters"."primary_id", + "quarters"."resolution", + "quarters"."index", + "quarters"."stratification_range_start", + "quarters"."stratification_range_end" +from "quarters" +``` + +| primary\_id | resolution | index | stratification\_range\_start | stratification\_range\_end | +|:------------|:-----------|:------|:-----------------------------|:---------------------------| +| 1 | COMPLETE | 1 | 2012-01-16 | 2012-12-18 | +| 1 | YEARS | 1 | 2012-01-16 | 2012-12-18 | +| 1 | QUARTERS | 1 | 2012-01-16 | 2012-04-01 | +| 1 | QUARTERS | 2 | 2012-04-01 | 2012-07-01 | +| 1 | QUARTERS | 3 | 2012-07-01 | 2012-10-01 | +| 1 | QUARTERS | 4 | 2012-10-01 | 2012-12-18 | + +## Feature conversion + +After we got our full stratification table, containing all stratification windows for each ID, we want to convert all +the features of the form, while using our stratification table as a starting point: + +1. When converting a concept and creating the `PREPROCESSING` CTE, which is the starting point of each concept + conversion, we join the concepts or respectively the connectors table with the stratification table for all IDs from + the stratification table. + +**CTE:** `preprocessing` + +```sql +select "full_stratification"."primary_id", + "full_stratification"."resolution", + "full_stratification"."index", + "full_stratification"."stratification_range_start", + "full_stratification"."stratification_range_end", + "vers_stamm"."date_start" "validity_date_start", + ADD_DAYS("vers_stamm"."date_end", 1) "validity_date_end", + "vers_stamm"."date_of_birth" +from "vers_stamm" + join "full_stratification" + on "full_stratification"."primary_id" = "vers_stamm"."pid" +``` + +2. In the `EVENT_FILTER` step, we filter all entries where the stratification range and the subjects validity date do + not overlap. This is important because we only want to compute aggregations for those ranges that satisfy this + condition. + +**CTE:** `event_filter` + +```sql +select "primary_id", + "resolution", + "index", + "stratification_range_start", + "stratification_range_end", + "validity_date_start", + "validity_date_end", + "date_of_birth" +from "preprocessing" +where "stratification_range_start" < "validity_date_end" + and "stratification_range_end" > "validity_date_start" +``` + +Besides grouping by ID, resolution, index and stratification range, the remaining concept conversion CTE process +remains as usual. If we have multiple features, we'll join the respective converted concept queries via an OR. + +## Left-join converted features with the full stratification table for the final select + +For an absolute form, we expect the final result to contain all stratification ranges for each ID of the respective +chosen resolutions. Because we filter all entries where stratification range and validity date do not overlap in each +concept conversion's event filter step, the converted feature(s) table might not contain all stratification ranges. +Thus, we left-join the table with the converted feature(s) back with the full stratification table. diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java new file mode 100644 index 00000000000..9ecc74e3ce2 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java @@ -0,0 +1,232 @@ +package com.bakdata.conquery.sql.conversion.forms; + +import java.time.LocalDate; +import java.time.Month; +import java.util.Iterator; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; +import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.config.Dialect; +import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery; +import com.bakdata.conquery.models.forms.util.Resolution; +import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.conversion.model.NameGenerator; +import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.google.common.base.Preconditions; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.jooq.Field; +import org.jooq.impl.DSL; + +@Getter(AccessLevel.PROTECTED) +@RequiredArgsConstructor +public abstract class StratificationTableFactory { + + private final QueryStep baseStep; + private final SqlFunctionProvider functionProvider; + private final NameGenerator nameGenerator; + + protected StratificationTableFactory(QueryStep baseStep, ConversionContext context) { + this.baseStep = baseStep; + this.functionProvider = context.getSqlDialect().getFunctionProvider(); + this.nameGenerator = context.getNameGenerator(); + } + + public static StratificationTableFactory create(Dialect dialect, QueryStep baseStep, ConversionContext context) { + return switch (dialect) { + case POSTGRESQL -> new PostgresStratificationTableFactory(baseStep, context); + case HANA -> new HanaStratificationTableFactory(baseStep, context); + case CLICKHOUSE -> throw new UnsupportedOperationException("Not implemented"); + }; + } + + public QueryStep createStratificationTable(AbsoluteFormQuery form) { + List tables = form.getResolutionsAndAlignmentMap().stream() + .map(ExportForm.ResolutionAndAlignment::getResolution) + .map(resolution -> createResolutionTable(form.getDateRange(), resolution)) + .toList(); + return unionResolutionTables(tables, getBaseStep()); + } + + /** + * True if there is an entity date in the base step. Compared to absolute mode, stratification window is bound by the entity date of each subject. + */ + protected boolean isEntityDateStratification() { + return baseStep.getSelects().getValidityDate().isPresent(); + } + + protected QueryStep unionResolutionTables(List unionSteps, QueryStep baseStep) { + + Preconditions.checkArgument(!unionSteps.isEmpty(), "Expecting at least 1 resolution table"); + + Iterator iterator = unionSteps.iterator(); + QueryStep lastResolutionTable = iterator.next(); + while (iterator.hasNext()) { + lastResolutionTable = iterator.next() + .toBuilder() + .predecessor(lastResolutionTable) + .build(); + } + + List withQualifiedSelects = unionSteps.stream() + .map(queryStep -> QueryStep.builder() + .selects(queryStep.getQualifiedSelects()) + .fromTable(QueryStep.toTableLike(queryStep.getCteName())) + .build()) + .toList(); + + return QueryStep.createUnionStep(withQualifiedSelects, FormCteStep.FULL_STRATIFICATION.getSuffix(), List.of(baseStep, lastResolutionTable)); + } + + protected QueryStep createResolutionTable(Range formDateRestriction, Resolution resolution) { + return switch (resolution) { + case COMPLETE -> createCompleteTable(formDateRestriction); + case YEARS, QUARTERS -> createIntervalTable(formDateRestriction, resolution); + case DAYS -> throw new UnsupportedOperationException("Resolution days not supported yet"); + }; + } + + protected QueryStep createCompleteTable(Range formDateRestriction) { + + Selects baseStepSelects = baseStep.getQualifiedSelects(); + + // complete range shall have a null index because it spans the complete range, but we set it to 1 to ensure we can join tables on index, + // because a condition involving null in a join (e.g., null = some_value or null = null) always evaluates to false + Field index = DSL.field(DSL.val(1, Integer.class)).as(SharedAliases.INDEX.getAlias()); + SqlIdColumns ids = baseStepSelects.getIds().withAbsoluteStratification(Resolution.COMPLETE, index); + + ColumnDateRange completeRange; + if (isEntityDateStratification()) { + completeRange = baseStepSelects.getValidityDate().get(); + } + // otherwise the complete range is the form's date range (absolute form) + else { + completeRange = functionProvider.forCDateRange(CDateRange.of(formDateRestriction)).as(SharedAliases.STRATIFICATION_RANGE.getAlias()); + } + + Selects selects = Selects.builder() + .ids(ids) + .stratificationDate(Optional.of(completeRange)) + .build(); + + return QueryStep.builder() + .cteName(FormCteStep.COMPLETE.getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(baseStep.getCteName())) + .build(); + } + + protected abstract QueryStep createIntervalTable(Range formDateRestriction, Resolution resolution); + + protected Field indexField(SqlIdColumns ids) { + + List> partitioningFields = + Stream.concat( + ids.toFields().stream(), + getBaseStep().getSelects().getValidityDate().stream().flatMap(validityDate -> validityDate.toFields().stream()) + ) + .collect(Collectors.toList()); + + return DSL.rowNumber() + .over(DSL.partitionBy(partitioningFields)) + .as(SharedAliases.INDEX.getAlias()); + } + + protected static String toResolutionExpression(Resolution resolution) { + return switch (resolution) { + case QUARTERS -> "3 month"; + case YEARS -> "1 year"; + case COMPLETE -> throw new UnsupportedOperationException("Generating series for a complete stratification range is not necessary"); + default -> throw new UnsupportedOperationException("Resolution %s currently not supported".formatted(resolution)); + }; + } + + /** + * Converts the given date range according to the given resolution to a range that is suited for a dateset-generating SQL function. + *

+ * Example: Given the date range [2012-06-16,2013-01-17] and the resolution YEARS, we cant pipe this directly into SAP HANA's SERIES_GENERATE_DATE command. + * Because it does not span the range of a whole year from start and end, the SERIES_GENERATE_DATE command would create an empty set. Instead, we have to + * jump to the start of the year of the start date and to the year + 1 of the end date. This will generate the expected series of + * 

+	 * 
+	 *   
+	 *     
+	 *     
+	 *   
+	 *   
+	 *     
+	 *     
+	 *   
+	 *   
+	 *     
+	 *     
+	 *   
+	 * 
GENERATED_PERIOD_STARTGENERATED_PERIOD_END
2012-01-012013-01-01
2013-01-012014-01-01

+	 *
+ */ + protected Range toGenerateSeriesBounds(Range formDateRestriction, Resolution resolution) { + return switch (resolution) { + case COMPLETE -> formDateRestriction; // no adjustment necessary + case YEARS -> Range.of(getYearStart(formDateRestriction.getMin()), getYearEnd(formDateRestriction.getMax())); + case QUARTERS -> Range.of(getQuarterStart(formDateRestriction.getMin()), getQuarterEnd(formDateRestriction.getMax())); + case DAYS -> throw new UnsupportedOperationException("DAYS resolution not supported yet"); + }; + } + + private static LocalDate getYearStart(LocalDate date) { + return LocalDate.of(date.getYear(), Month.JANUARY, 1); + } + + private static LocalDate getYearEnd(LocalDate date) { + return LocalDate.of(date.getYear() + 1, Month.JANUARY, 1); + } + + private static LocalDate getQuarterStart(LocalDate date) { + + Month startMonth = switch (date.getMonthValue()) { + case 1, 2, 3 -> Month.JANUARY; + case 4, 5, 6 -> Month.APRIL; + case 7, 8, 9 -> Month.JULY; + default -> Month.OCTOBER; + }; + + return LocalDate.of(date.getYear(), startMonth, 1); + } + + private static LocalDate getQuarterEnd(LocalDate date) { + + int year = date.getYear(); + Month startMonth; + + switch (date.getMonthValue()) { + case 1, 2, 3: + startMonth = Month.APRIL; // Start of Q2 + break; + case 4, 5, 6: + startMonth = Month.JULY; // Start of Q3 + break; + case 7, 8, 9: + startMonth = Month.OCTOBER; // Start of Q4 + break; + default: + // For Q4, increment the year and set month to January + startMonth = Month.JANUARY; + year++; + break; + } + + return LocalDate.of(year, startMonth, 1); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java index f3768691db2..902e8be75ff 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java @@ -7,12 +7,12 @@ import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; import lombok.Getter; +import org.jooq.Condition; import org.jooq.Field; @Getter public class ColumnDateRange implements SqlSelect { - private static final String DATE_RESTRICTION_COLUMN_NAME = "date_restriction"; private static final String VALIDITY_DATE_COLUMN_NAME_SUFFIX = "_validity_date"; private static final String START_SUFFIX = "_start"; private static final String END_SUFFIX = "_end"; @@ -52,10 +52,6 @@ public static ColumnDateRange of(Field startColumn, Field endColumn, return new ColumnDateRange(startColumn, endColumn, alias); } - public ColumnDateRange asDateRestrictionRange() { - return this.as(DATE_RESTRICTION_COLUMN_NAME); - } - public ColumnDateRange asValidityDateRange(String alias) { return this.as(alias + VALIDITY_DATE_COLUMN_NAME_SUFFIX); } @@ -105,4 +101,21 @@ public ColumnDateRange as(String alias) { ); } + public Condition join(ColumnDateRange right) { + if (this.isSingleColumnRange() != right.isSingleColumnRange()) { + throw new UnsupportedOperationException("Can only join ColumnDateRanges of same type"); + } + if (this.isSingleColumnRange()) { + return this.range.coerce(Object.class).eq(right.getRange()); + } + return this.start.eq(right.getStart()).and(end.eq(right.getEnd())); + } + + public Condition isNotNull() { + if (this.isSingleColumnRange()) { + return this.range.isNotNull(); + } + return this.start.isNotNull().and(this.end.isNotNull()); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ConceptConversionTables.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ConceptConversionTables.java deleted file mode 100644 index 8855afc2a09..00000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ConceptConversionTables.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.bakdata.conquery.sql.conversion.model; - -import java.util.Map; - -import com.bakdata.conquery.sql.conversion.cqelement.intervalpacking.IntervalPackingCteStep; -import lombok.Getter; - -@Getter -public class ConceptConversionTables extends SqlTables { - - /** - * Stores the name of the predecessor of the last CTE these tables contain. - */ - private final String lastPredecessor; - - /** - * True if these tables contain interval packing CTEs {@link IntervalPackingCteStep}. - */ - private final boolean withIntervalPacking; - - public ConceptConversionTables( - String rootTable, - Map cteNameMap, - Map predecessorMap, - String lastPredecessor, - boolean containsIntervalPacking - ) { - super(rootTable, cteNameMap, predecessorMap); - this.lastPredecessor = lastPredecessor; - this.withIntervalPacking = containsIntervalPacking; - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/LogicalOperation.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/LogicalOperation.java index 57220db7141..d54eb07538d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/LogicalOperation.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/LogicalOperation.java @@ -2,5 +2,6 @@ public enum LogicalOperation { AND, - OR + OR, + LEFT_JOIN } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java index 6749bfcdcf7..8e68c0e3845 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java @@ -72,6 +72,7 @@ public String joinedNodeName(LogicalOperation logicalOperation) { return switch (logicalOperation) { case AND -> "AND-%d".formatted(++andCount); case OR -> "OR-%d".formatted(++orCount); + case LEFT_JOIN -> throw new UnsupportedOperationException("Creating CTE names for LEFT_JOIN nodes is not supported"); }; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java index 31cf3eb4593..bb557398197 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java @@ -4,6 +4,7 @@ import java.util.List; import lombok.Builder; +import lombok.Singular; import lombok.Value; import org.jooq.Condition; import org.jooq.Field; @@ -20,7 +21,8 @@ public class QueryStep { String cteName; Selects selects; - TableLike fromTable; + @Singular + List> fromTables; @Builder.Default List conditions = Collections.emptyList(); /** @@ -36,8 +38,17 @@ public class QueryStep { /** * All {@link QueryStep}'s that shall be converted before this {@link QueryStep}. */ - @Builder.Default - List predecessors = Collections.emptyList(); + @Singular + List predecessors; + + public static QueryStep createUnionStep(List unionSteps, String cteName, List predecessors) { + return unionSteps.get(0) + .toBuilder() + .cteName(cteName) + .union(unionSteps.subList(1, unionSteps.size())) + .predecessors(predecessors) + .build(); + } public static TableLike toTableLike(String fromTableName) { return DSL.table(DSL.name(fromTableName)); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java index 620fe845b57..ea48e82b95e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepJoiner.java @@ -4,6 +4,7 @@ import java.util.List; import java.util.Optional; import java.util.stream.Collectors; +import java.util.stream.Stream; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; @@ -12,6 +13,7 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlDateAggregator; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; +import com.google.common.base.Preconditions; import org.jooq.Condition; import org.jooq.Record; import org.jooq.Table; @@ -56,12 +58,14 @@ public static QueryStep joinSteps( DateAggregationDates dateAggregationDates = DateAggregationDates.forSteps(queriesToJoin); if (dateAggregationAction == DateAggregationAction.BLOCK || dateAggregationDates.dateAggregationImpossible()) { - joinedStep = buildJoinedStep(ids, mergedSelects, Optional.empty(), joinedStepBuilder); + // for forms, date aggregation is allways blocked // TODO check if this is really correct + Optional stratificationDate = queriesToJoin.get(0).getQualifiedSelects().getStratificationDate(); + joinedStep = buildJoinedStep(ids, mergedSelects, Optional.empty(), stratificationDate, joinedStepBuilder); } // if there is only 1 child node containing a validity date, we just keep it as overall validity date for the joined node else if (dateAggregationDates.getValidityDates().size() == 1) { ColumnDateRange validityDate = dateAggregationDates.getValidityDates().get(0); - joinedStep = buildJoinedStep(ids, mergedSelects, Optional.of(validityDate), joinedStepBuilder); + joinedStep = buildJoinedStep(ids, mergedSelects, Optional.of(validityDate), Optional.empty(), joinedStepBuilder); } else { joinedStep = buildStepAndAggregateDates(ids, mergedSelects, joinedStepBuilder, dateAggregationDates, dateAggregationAction, context); @@ -70,13 +74,13 @@ else if (dateAggregationDates.getValidityDates().size() == 1) { } public static TableLike constructJoinedTable(List queriesToJoin, LogicalOperation logicalOperation, ConversionContext context) { - Table joinedQuery = getIntitialJoinTable(queriesToJoin); SqlFunctionProvider functionProvider = context.getSqlDialect().getFunctionProvider(); JoinType joinType = switch (logicalOperation) { case AND -> functionProvider::innerJoin; case OR -> functionProvider::fullOuterJoin; + case LEFT_JOIN -> functionProvider::leftJoin; }; for (int i = 0; i < queriesToJoin.size() - 1; i++) { @@ -87,7 +91,17 @@ public static TableLike constructJoinedTable(List queriesToJo SqlIdColumns leftIds = leftPartQS.getQualifiedSelects().getIds(); SqlIdColumns rightIds = rightPartQS.getQualifiedSelects().getIds(); - List joinConditions = SqlIdColumns.join(leftIds, rightIds); + List joinIdsCondition = leftIds.join(rightIds); + + Condition joinDateCondition = DSL.noCondition(); + // join on stratification date if present + if (leftPartQS.getSelects().getStratificationDate().isPresent() && rightPartQS.getSelects().getStratificationDate().isPresent()) { + ColumnDateRange leftStratificationDate = leftPartQS.getQualifiedSelects().getStratificationDate().get(); + ColumnDateRange rightStratificationDate = rightPartQS.getQualifiedSelects().getStratificationDate().get(); + joinDateCondition = leftStratificationDate.join(rightStratificationDate); + } + + List joinConditions = Stream.concat(joinIdsCondition.stream(), Stream.of(joinDateCondition)).collect(Collectors.toList()); joinedQuery = joinType.join(joinedQuery, rightPartQS, joinConditions); } @@ -103,7 +117,8 @@ public static List mergeSelects(List querySteps) { public static SqlIdColumns coalesceIds(List querySteps) { List ids = querySteps.stream().map(QueryStep::getQualifiedSelects).map(Selects::getIds).toList(); - return SqlIdColumns.coalesce(ids); + Preconditions.checkArgument(!ids.isEmpty(), "Need at least 1 query step in the list to coalesce Ids"); + return ids.get(0).coalesce(ids.subList(1, ids.size())); } private static Table getIntitialJoinTable(List queriesToJoin) { @@ -114,12 +129,14 @@ private static QueryStep buildJoinedStep( SqlIdColumns ids, List mergedSelects, Optional validityDate, + Optional stratificationDate, QueryStep.QueryStepBuilder builder ) { Selects selects = Selects.builder() .ids(ids) - .sqlSelects(mergedSelects) + .stratificationDate(stratificationDate) .validityDate(validityDate) + .sqlSelects(mergedSelects) .build(); return builder.selects(selects).build(); } @@ -134,7 +151,7 @@ private static QueryStep buildStepAndAggregateDates( ) { List withAllValidityDates = new ArrayList<>(mergedSelects); withAllValidityDates.addAll(dateAggregationDates.allStartsAndEnds()); - QueryStep joinedStep = buildJoinedStep(ids, withAllValidityDates, Optional.empty(), builder); + QueryStep joinedStep = buildJoinedStep(ids, withAllValidityDates, Optional.empty(), Optional.empty(), builder); SqlDateAggregator sqlDateAggregator = context.getSqlDialect().getDateAggregator(); return sqlDateAggregator.apply( diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java index a0b8a2c3398..bc7371bf22e 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java @@ -5,6 +5,7 @@ import org.jooq.CommonTableExpression; import org.jooq.DSLContext; +import org.jooq.Field; import org.jooq.Record; import org.jooq.Select; import org.jooq.SelectConditionStep; @@ -25,15 +26,18 @@ public QueryStepTransformer(DSLContext dslContext) { * Converts a given {@link QueryStep} into an executable SELECT statement. */ public Select toSelectQuery(QueryStep queryStep) { + SelectConditionStep queryBase = this.dslContext.with(constructPredecessorCteList(queryStep)) .select(queryStep.getSelects().all()) - .from(queryStep.getFromTable()) + .from(queryStep.getFromTables()) .where(queryStep.getConditions()); + + List> orderByFields = queryStep.getSelects().getIds().toFields(); if (queryStep.isGroupBy()) { - return queryBase.groupBy(queryStep.getGroupBy()); + return queryBase.groupBy(queryStep.getGroupBy()).orderBy(orderByFields); } else { - return queryBase; + return queryBase.orderBy(orderByFields); } } @@ -59,7 +63,7 @@ private CommonTableExpression toCte(QueryStep queryStep) { Select selectStep = this.dslContext .select(queryStep.getSelects().all()) - .from(queryStep.getFromTable()) + .from(queryStep.getFromTables()) .where(queryStep.getConditions()); if (queryStep.isGroupBy()) { @@ -71,7 +75,9 @@ private CommonTableExpression toCte(QueryStep queryStep) { // we only use the union as part of the date aggregation process - the entries of the UNION tables are all unique // thus we can use a UNION ALL because it's way faster than UNION selectStep = selectStep.unionAll( - this.dslContext.select(unionStep.getSelects().all()).from(unionStep.getFromTable()) + this.dslContext.select(unionStep.getSelects().all()) + .from(unionStep.getFromTables()) + .where(unionStep.getConditions()) ); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java index 5dca7792235..5b846f642f2 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/Selects.java @@ -19,6 +19,8 @@ public class Selects { SqlIdColumns ids; @Builder.Default Optional validityDate = Optional.empty(); + @Builder.Default + Optional stratificationDate = Optional.empty(); @Singular List sqlSelects; @@ -46,12 +48,17 @@ public Selects qualify(String qualifier) { builder = builder.validityDate(this.validityDate.map(_validityDate -> _validityDate.qualify(qualifier))); } + if (this.stratificationDate.isPresent()) { + builder = builder.stratificationDate(this.stratificationDate.map(_validityDate -> _validityDate.qualify(qualifier))); + } + return builder.build(); } public List> all() { return Stream.of( this.ids.toFields().stream(), + this.stratificationDate.stream().flatMap(range -> range.toFields().stream()), this.validityDate.stream().flatMap(range -> range.toFields().stream()), this.sqlSelects.stream().flatMap(sqlSelect -> sqlSelect.toFields().stream()) ) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java index 029c970b249..9b1eb067e6a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.sql.conversion.model; +import java.sql.Date; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -8,9 +9,9 @@ import javax.annotation.Nullable; +import com.bakdata.conquery.models.forms.util.Resolution; import com.bakdata.conquery.sql.conversion.SharedAliases; import lombok.Getter; -import lombok.NonNull; import org.jooq.Condition; import org.jooq.Field; import org.jooq.impl.DSL; @@ -23,68 +24,89 @@ public class SqlIdColumns implements Qualifiable { @Nullable private final Field secondaryId; - private final List> ids; - - public SqlIdColumns(Field primaryColumn, @NonNull Field secondaryId) { + public SqlIdColumns(Field primaryColumn, Field secondaryId) { this.primaryColumn = primaryColumn; this.secondaryId = secondaryId; - this.ids = Stream.concat(Stream.of(this.primaryColumn), Stream.ofNullable(this.secondaryId)).collect(Collectors.toList()); } public SqlIdColumns(Field primaryColumn) { this.primaryColumn = primaryColumn; this.secondaryId = null; - this.ids = List.of(this.primaryColumn); } @Override public SqlIdColumns qualify(String qualifier) { Field primaryColumn = QualifyingUtil.qualify(this.primaryColumn, qualifier); - if (this.secondaryId == null) { - return new SqlIdColumns(primaryColumn); - } - Field secondaryId = QualifyingUtil.qualify(this.secondaryId, qualifier); + Field secondaryId = this.secondaryId != null ? QualifyingUtil.qualify(this.secondaryId, qualifier) : null; return new SqlIdColumns(primaryColumn, secondaryId); } + public SqlIdColumns withAbsoluteStratification(Resolution resolution, Field index) { + Field resolutionField = DSL.field(DSL.val(resolution.toString())).as(SharedAliases.RESOLUTION.getAlias()); + // absolute stratification carries no event date + Field eventDate = null; + return new StratificationSqlIdColumns(this.primaryColumn, this.secondaryId, resolutionField, index, eventDate); + } + + public SqlIdColumns withRelativeStratification(Resolution resolution, Field index, Field eventDate) { + Field resolutionField = DSL.field(DSL.val(resolution.toString())).as(SharedAliases.RESOLUTION.getAlias()); + return new StratificationSqlIdColumns(this.primaryColumn, this.secondaryId, resolutionField, index, eventDate); + } + + public SqlIdColumns forFinalSelect() { + return this; + } + public Optional> getSecondaryId() { return Optional.ofNullable(this.secondaryId); } + public boolean isWithStratification() { + return false; + } + public List> toFields() { - return this.ids; + return Stream.concat(Stream.of(this.primaryColumn), Optional.ofNullable(this.secondaryId).stream()).collect(Collectors.toList()); } - public static List join(SqlIdColumns leftIds, SqlIdColumns rightIds) { - Condition joinPrimariesCondition = leftIds.getPrimaryColumn().eq(rightIds.getPrimaryColumn()); - Condition joinSecondariesCondition; - if (leftIds.getSecondaryId().isPresent() && rightIds.getSecondaryId().isPresent()) { - joinSecondariesCondition = leftIds.getSecondaryId().get().eq(rightIds.getSecondaryId().get()); - } - else { - joinSecondariesCondition = DSL.noCondition(); - } - return List.of(joinPrimariesCondition, joinSecondariesCondition); + public List join(SqlIdColumns rightIds) { + + // always join on primary columns + Condition joinPrimariesCondition = primaryColumn.eq(rightIds.getPrimaryColumn()); + + // join on secondary IDs if both are present + Condition joinSecondaries = getSecondaryId() + .flatMap(leftSecondaryId -> rightIds.getSecondaryId().map(leftSecondaryId::eq)) + .orElse(DSL.noCondition()); + + return List.of(joinPrimariesCondition, joinSecondaries); } - public static SqlIdColumns coalesce(List selectsIds) { + public SqlIdColumns coalesce(List selectsIds) { - List> primaryColumns = new ArrayList<>(selectsIds.size()); - List> secondaryIds = new ArrayList<>(selectsIds.size()); + List> primaryColumns = new ArrayList<>(); + List> secondaryIds = new ArrayList<>(); + + // add this ids + primaryColumns.add(this.primaryColumn); + getSecondaryId().ifPresent(secondaryIds::add); + + // add all other ids to coalesce with selectsIds.forEach(ids -> { primaryColumns.add(ids.getPrimaryColumn()); ids.getSecondaryId().ifPresent(secondaryIds::add); }); Field coalescedPrimaryColumn = coalesceFields(primaryColumns).as(SharedAliases.PRIMARY_COLUMN.getAlias()); - if (secondaryIds.isEmpty()) { - return new SqlIdColumns(coalescedPrimaryColumn); - } - Field coalescedSecondaryIds = coalesceFields(secondaryIds).as(SharedAliases.SECONDARY_ID.getAlias()); + Field coalescedSecondaryIds = !secondaryIds.isEmpty() + ? coalesceFields(secondaryIds).as(SharedAliases.SECONDARY_ID.getAlias()) + : null; + return new SqlIdColumns(coalescedPrimaryColumn, coalescedSecondaryIds); } - private static Field coalesceFields(List> fields) { + + protected static Field coalesceFields(List> fields) { if (fields.size() == 1) { return fields.get(0).coerce(Object.class); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlQuery.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlQuery.java index 5d179a69bfa..925f88efee3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlQuery.java @@ -1,13 +1,38 @@ package com.bakdata.conquery.sql.conversion.model; +import java.util.Collections; import java.util.List; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.jooq.Record; +import org.jooq.Select; +import org.jooq.conf.ParamType; -public interface SqlQuery { +@Getter +@AllArgsConstructor(access = AccessLevel.PRIVATE) +public class SqlQuery { - String getSql(); + private final String sql; + private final List resultInfos; - List getResultInfos(); + public SqlQuery(Select finalQuery, List resultInfos) { + this.sql = finalQuery.getSQL(ParamType.INLINED); + this.resultInfos = resultInfos; + } + + /** + * For testing purposes + */ + protected SqlQuery(String sql) { + this.sql = sql; + this.resultInfos = Collections.emptyList(); + } + + public SqlQuery overwriteResultInfos(List resultInfos) { + return new SqlQuery(sql, resultInfos); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java new file mode 100644 index 00000000000..902e2fa2778 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java @@ -0,0 +1,155 @@ +package com.bakdata.conquery.sql.conversion.model; + +import java.sql.Date; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import javax.annotation.Nullable; + +import com.bakdata.conquery.models.forms.util.Resolution; +import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.google.common.base.Preconditions; +import lombok.Getter; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.impl.DSL; + +@Getter +class StratificationSqlIdColumns extends SqlIdColumns { + + private final Field resolution; + + private final Field index; + + /** + * Optional event date. Only set for relative forms. + */ + @Nullable + private final Field eventDate; + + public StratificationSqlIdColumns( + Field primaryColumn, + Field secondaryId, + Field resolution, + Field index, + @Nullable Field eventDate + ) { + super(primaryColumn, secondaryId); + this.resolution = resolution; + this.index = index; + this.eventDate = eventDate; + } + + @Override + public SqlIdColumns qualify(String qualifier) { + Field primaryColumn = QualifyingUtil.qualify(getPrimaryColumn(), qualifier); + Field resolution = QualifyingUtil.qualify(this.resolution, qualifier); + Field index = QualifyingUtil.qualify(this.index, qualifier); + Field eventDate = null; + if (this.eventDate != null) { + eventDate = QualifyingUtil.qualify(this.eventDate, qualifier); + } + return new StratificationSqlIdColumns(primaryColumn, null, resolution, index, eventDate); + } + + /** + * Replaces the integer value of the {@link Resolution#COMPLETE} with null values. + */ + @Override + public SqlIdColumns forFinalSelect() { + Field withNulledCompleteIndex = DSL.when( + this.resolution.eq(DSL.val(Resolution.COMPLETE.toString().toUpperCase())), + DSL.field(DSL.val(null, Integer.class)) + ) + .otherwise(this.index) + .as(SharedAliases.INDEX.getAlias()); + return new StratificationSqlIdColumns(getPrimaryColumn(), null, this.resolution, withNulledCompleteIndex, this.eventDate); + } + + @Override + public boolean isWithStratification() { + return true; + } + + @Override + public List> toFields() { + return Stream.of( + getPrimaryColumn(), + this.resolution, + this.index, + this.eventDate + ) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + @Override + public List join(SqlIdColumns rightIds) { + + if (!rightIds.isWithStratification()) { + return super.join(rightIds); + } + + StratificationSqlIdColumns rightStratificationIds = (StratificationSqlIdColumns) rightIds; + Condition joinResolutionAndIndex = this.resolution.eq(rightStratificationIds.getResolution()).and(this.index.eq(rightStratificationIds.getIndex())); + + Condition joinEventDateCondition; + if (this.eventDate != null) { + joinEventDateCondition = this.eventDate.eq(rightStratificationIds.getEventDate()); + } + else { + joinEventDateCondition = DSL.noCondition(); + } + + return Stream.concat( + super.join(rightIds).stream(), + Stream.of(joinResolutionAndIndex, joinEventDateCondition) + ) + .collect(Collectors.toList()); + } + + @Override + public SqlIdColumns coalesce(List selectsIds) { + + Preconditions.checkArgument( + selectsIds.stream().allMatch(SqlIdColumns::isWithStratification), + "Can only coalesce SqlIdColumns if all are with stratification" + ); + + List> primaryColumns = new ArrayList<>(); + List> resolutions = new ArrayList<>(); + List> indices = new ArrayList<>(); + List> eventDates = new ArrayList<>(); + + // add this ids + primaryColumns.add(getPrimaryColumn()); + resolutions.add(this.resolution); + indices.add(this.index); + if (this.eventDate != null) { + eventDates.add(this.eventDate); + } + + selectsIds.forEach(ids -> { + StratificationSqlIdColumns stratificationIds = (StratificationSqlIdColumns) ids; + primaryColumns.add(stratificationIds.getPrimaryColumn()); + resolutions.add(stratificationIds.getResolution()); + indices.add(stratificationIds.getIndex()); + if (stratificationIds.getEventDate() != null) { + eventDates.add(stratificationIds.getEventDate()); + } + }); + + Field coalescedPrimaryColumn = coalesceFields(primaryColumns).as(SharedAliases.PRIMARY_COLUMN.getAlias()); + Field coalescedResolutions = coalesceFields(resolutions).coerce(String.class).as(SharedAliases.RESOLUTION.getAlias()); + Field coalescedIndices = coalesceFields(indices).coerce(Integer.class).as(SharedAliases.INDEX.getAlias()); + Field eventDate = null; + if (!eventDates.isEmpty()) { + eventDate = coalesceFields(eventDates).coerce(Date.class).as(SharedAliases.INDEX_DATE.getAlias()); + } + + return new StratificationSqlIdColumns(coalescedPrimaryColumn, null, coalescedResolutions, coalescedIndices, eventDate); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java new file mode 100644 index 00000000000..a789b052938 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java @@ -0,0 +1,140 @@ +package com.bakdata.conquery.sql.conversion.query; + +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.apiv1.query.Query; +import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery; +import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.conversion.forms.FormCteStep; +import com.bakdata.conquery.sql.conversion.forms.StratificationTableFactory; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.conversion.model.LogicalOperation; +import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; +import com.bakdata.conquery.sql.conversion.model.QueryStepTransformer; +import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.conversion.model.SqlQuery; +import com.google.common.base.Preconditions; +import lombok.RequiredArgsConstructor; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Select; +import org.jooq.TableLike; + +@RequiredArgsConstructor +public class AbsoluteFormQueryConverter implements NodeConverter { + + private final QueryStepTransformer queryStepTransformer; + + @Override + public Class getConversionClass() { + return AbsoluteFormQuery.class; + } + + @Override + public ConversionContext convert(AbsoluteFormQuery form, ConversionContext context) { + + // base population query conversion + QueryStep prerequisite = convertPrerequisite(form.getQuery(), context); + + // creating stratification tables + StratificationTableFactory tableFactory = context.getSqlDialect().getStratificationTableFactory(prerequisite, context); + QueryStep stratificationTable = tableFactory.createStratificationTable(form); + + // feature conversion + ConversionContext childContext = convertFeatures(form, context, stratificationTable); + + List queriesToJoin = childContext.getQuerySteps(); + // only 1 converted feature + if (queriesToJoin.size() == 1) { + QueryStep convertedFeature = queriesToJoin.get(0); + return createFinalSelect(form, stratificationTable, convertedFeature, childContext); + } + QueryStep joinedFeatures = QueryStepJoiner.joinSteps(queriesToJoin, LogicalOperation.OR, DateAggregationAction.BLOCK, context); + return createFinalSelect(form, stratificationTable, joinedFeatures, childContext); + } + + private static QueryStep convertPrerequisite(Query query, ConversionContext context) { + + ConversionContext withConvertedPrerequisite = context.getNodeConversions().convert(query, context); + Preconditions.checkArgument(withConvertedPrerequisite.getQuerySteps().size() == 1, "Base query conversion should produce exactly 1 QueryStep"); + QueryStep convertedPrerequisite = withConvertedPrerequisite.getQuerySteps().get(0); + + Selects prerequisiteSelects = convertedPrerequisite.getQualifiedSelects(); + // we only keep the primary column for the upcoming form + Selects selects = Selects.builder() + .ids(new SqlIdColumns(prerequisiteSelects.getIds().getPrimaryColumn())) + .build(); + + return QueryStep.builder() + .cteName(FormCteStep.EXTRACT_IDS.getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(convertedPrerequisite.getCteName())) + .groupBy(selects.getIds().toFields()) // group by primary column to ensure max. 1 entry per subject + .predecessors(List.of(convertedPrerequisite)) + .build(); + } + + private static ConversionContext convertFeatures(AbsoluteFormQuery form, ConversionContext context, QueryStep stratificationTable) { + ConversionContext childContext = context.createChildContext().withStratificationTable(stratificationTable); + for (ConceptQuery conceptQuery : form.getFeatures().getChildQueries()) { + childContext = context.getNodeConversions().convert(conceptQuery, childContext); + } + return childContext; + } + + /** + * Left-joins the full stratification table back with the converted feature tables to keep all the resolutions. + *

+ * When converting features, we filter out rows where a subjects validity date and the stratification date do not overlap. + * Thus, the pre-final step might not contain an entry for each expected stratification window. That's why we need to left-join the converted + * features with the full stratification table again. + */ + private ConversionContext createFinalSelect(AbsoluteFormQuery form, QueryStep stratificationTable, QueryStep convertedFeatures, ConversionContext context) { + + List queriesToJoin = List.of(stratificationTable, convertedFeatures); + TableLike joinedTable = QueryStepJoiner.constructJoinedTable(queriesToJoin, LogicalOperation.LEFT_JOIN, context); + + QueryStep finalStep = QueryStep.builder() + .cteName(null) // the final QueryStep won't be converted to a CTE + .selects(getFinalSelects(stratificationTable, convertedFeatures, context.getSqlDialect().getFunctionProvider())) + .fromTable(joinedTable) + .predecessors(queriesToJoin) + .build(); + + Select selectQuery = queryStepTransformer.toSelectQuery(finalStep); + return context.withFinalQuery(new SqlQuery(selectQuery, form.getResultInfos())); + } + + /** + * Selects the ID, resolution, index and date range from stratification table plus all explicit selects from the converted features step. + */ + private static Selects getFinalSelects(QueryStep stratificationTable, QueryStep convertedFeatures, SqlFunctionProvider functionProvider) { + + Selects preFinalSelects = convertedFeatures.getQualifiedSelects(); + + if (preFinalSelects.getStratificationDate().isEmpty() || !preFinalSelects.getIds().isWithStratification()) { + throw new IllegalStateException("Expecting the pre-final CTE to contain a stratification date, resolution and index"); + } + + Selects stratificationSelects = stratificationTable.getQualifiedSelects(); + SqlIdColumns ids = stratificationSelects.getIds().forFinalSelect(); + Field daterangeConcatenated = functionProvider.daterangeStringExpression(stratificationSelects.getStratificationDate().get()) + .as(SharedAliases.STRATIFICATION_RANGE.getAlias()); + + return Selects.builder() + .ids(ids) + .validityDate(Optional.empty()) + .stratificationDate(Optional.of(ColumnDateRange.of(daterangeConcatenated))) + .sqlSelects(preFinalSelects.getSqlSelects()) + .build(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptSqlQuery.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptSqlQuery.java deleted file mode 100644 index b7e7c2aca36..00000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptSqlQuery.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.bakdata.conquery.sql.conversion.query; - -import java.util.List; - -import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.sql.conversion.model.SqlQuery; -import lombok.Getter; -import org.jooq.Record; -import org.jooq.Select; -import org.jooq.conf.ParamType; - -@Getter -class ConceptSqlQuery implements SqlQuery { - - String sqlString; - List resultInfos; - - public ConceptSqlQuery(Select finalQuery, List resultInfos) { - this.sqlString = finalQuery.getSQL(ParamType.INLINED); - this.resultInfos = resultInfos; - } - - protected ConceptSqlQuery(String sqlString, List resultInfos) { - this.sqlString = sqlString; - this.resultInfos = resultInfos; - } - - @Override - public String getSql() { - return this.sqlString; - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdQueryConverter.java index 2b10605fc47..e5d9ee61978 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdQueryConverter.java @@ -3,6 +3,7 @@ import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; +import com.bakdata.conquery.sql.conversion.model.SqlQuery; import com.google.common.base.Preconditions; public class SecondaryIdQueryConverter implements NodeConverter { @@ -21,7 +22,7 @@ public ConversionContext convert(SecondaryIdQuery query, ConversionContext conte ); Preconditions.checkArgument(withConvertedQuery.getFinalQuery() != null, "The SecondaryIdQuery's query should be converted by now."); - SecondaryIdSqlQuery secondaryIdSqlQuery = SecondaryIdSqlQuery.overwriteResultInfos(withConvertedQuery.getFinalQuery(), query.getResultInfos()); + SqlQuery secondaryIdSqlQuery = withConvertedQuery.getFinalQuery().overwriteResultInfos(query.getResultInfos()); return withConvertedQuery.withFinalQuery(secondaryIdSqlQuery); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdSqlQuery.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdSqlQuery.java deleted file mode 100644 index aaa1e173990..00000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/SecondaryIdSqlQuery.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.bakdata.conquery.sql.conversion.query; - -import java.util.List; - -import com.bakdata.conquery.models.query.resultinfo.ResultInfo; -import com.bakdata.conquery.sql.conversion.model.SqlQuery; - -public class SecondaryIdSqlQuery extends ConceptSqlQuery { - - private SecondaryIdSqlQuery(String sqlString, List resultInfos) { - super(sqlString, resultInfos); - } - - public static SecondaryIdSqlQuery overwriteResultInfos(SqlQuery query, List resultInfos) { - return new SecondaryIdSqlQuery(query.getSql(), resultInfos); - } - -} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java index 7afa5afce28..3b643775228 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/LoadingUtil.java @@ -44,8 +44,7 @@ import com.bakdata.conquery.models.preproc.TableImportDescriptor; import com.bakdata.conquery.models.preproc.TableInputDescriptor; import com.bakdata.conquery.models.preproc.outputs.OutputDescription; -import com.bakdata.conquery.models.query.DistributedExecutionManager; -import com.bakdata.conquery.models.worker.DistributedNamespace; +import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.resources.ResourceConstants; import com.bakdata.conquery.resources.admin.rest.AdminDatasetResource; import com.bakdata.conquery.resources.hierarchies.HierarchyHelper; @@ -75,7 +74,7 @@ public static void importPreviousQueries(StandaloneSupport support, RequiredData ConceptQuery query = new ConceptQuery(new CQExternal(Arrays.asList("ID", "DATE_SET"), data, false)); - DistributedExecutionManager executionManager = ((DistributedNamespace) support.getNamespace()).getExecutionManager(); + ExecutionManager executionManager = support.getNamespace().getExecutionManager(); ManagedExecution managed = executionManager.createQuery(query, queryId, user, support.getNamespace().getDataset(), false); user.addPermission(managed.createPermission(AbilitySets.QUERY_CREATOR)); @@ -90,7 +89,7 @@ public static void importPreviousQueries(StandaloneSupport support, RequiredData Query query = ConqueryTestSpec.parseSubTree(support, queryNode, Query.class); UUID queryId = new UUID(0L, id++); - DistributedExecutionManager executionManager = ((DistributedNamespace) support.getNamespace()).getExecutionManager(); + ExecutionManager executionManager = support.getNamespace().getExecutionManager(); ManagedExecution managed = executionManager.createQuery(query, queryId, user, support.getNamespace().getDataset(), false); user.addPermission(ExecutionPermission.onInstance(AbilitySets.QUERY_CREATOR, managed.getId())); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java index 5f648e5ea9c..17fc0951ff2 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java @@ -12,7 +12,6 @@ import java.util.OptionalLong; import com.bakdata.conquery.apiv1.AdditionalMediaTypes; -import com.bakdata.conquery.apiv1.query.EditorQuery; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.integration.common.IntegrationUtils; import com.bakdata.conquery.integration.common.ResourceFile; @@ -20,6 +19,7 @@ import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.results.EntityResult; @@ -88,7 +88,7 @@ public void executeTest(StandaloneSupport standaloneSupport) throws IOException // check that getLastResultCount returns the correct size if (executionResult.streamResults(OptionalLong.empty()).noneMatch(MultilineEntityResult.class::isInstance)) { long lastResultCount; - if (executionResult instanceof EditorQuery editorQuery) { + if (executionResult instanceof ManagedQuery editorQuery) { lastResultCount = editorQuery.getLastResultCount(); } else { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java index 375fa9d1403..21a37e72670 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/SqlTestDataImporter.java @@ -4,6 +4,7 @@ import java.util.Collection; import java.util.List; +import com.bakdata.conquery.integration.common.LoadingUtil; import com.bakdata.conquery.integration.common.RequiredData; import com.bakdata.conquery.integration.common.RequiredSecondaryId; import com.bakdata.conquery.integration.common.RequiredTable; @@ -34,8 +35,14 @@ public void importQueryTestData(StandaloneSupport support, QueryTest test) throw } @Override - public void importFormTestData(StandaloneSupport support, FormTest formTest) throws Exception { - throw new UnsupportedOperationException("Not implemented yet."); + public void importFormTestData(StandaloneSupport support, FormTest test) throws Exception { + RequiredData content = test.getContent(); + importSecondaryIds(support, content.getSecondaryIds()); + importTables(support, content.getTables(), true); + importConcepts(support, test.getRawConcepts()); + importTableContents(support, content.getTables()); + waitUntilDone(support, () -> LoadingUtil.importIdMapping(support, content)); + waitUntilDone(support, () -> LoadingUtil.importPreviousQueries(support, content, support.getTestUser())); } @Override diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/TestDataImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/json/TestDataImporter.java index 9e8c389fb6f..5d29d2bbd6f 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/TestDataImporter.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/TestDataImporter.java @@ -27,6 +27,11 @@ public interface TestDataImporter { void importTableContents(StandaloneSupport support, Collection tables) throws Exception; + default void waitUntilDone(StandaloneSupport support, CheckedRunnable runnable) { + runnable.run(); + support.waitUntilWorkDone(); + } + @FunctionalInterface interface CheckedRunnable extends Runnable { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/WorkerTestDataImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/json/WorkerTestDataImporter.java index ae1fe008047..21a537e54bf 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/WorkerTestDataImporter.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/WorkerTestDataImporter.java @@ -96,11 +96,6 @@ public void importTableContents(StandaloneSupport support, Collection LoadingUtil.importTableContents(support, tables)); } - private void waitUntilDone(StandaloneSupport support, CheckedRunnable runnable) { - runnable.run(); - support.waitUntilWorkDone(); - } - private static void sendUpdateMatchingStatsMessage(StandaloneSupport support) { DistributedNamespace namespace = (DistributedNamespace) support.getNamespace(); namespace.getWorkerHandler().sendToAll(new UpdateMatchingStatsMessage(support.getNamespace().getStorage().getAllConcepts())); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java index 614efe4ff7f..6e7dfaa61eb 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.integration.sql.dialect; -import java.util.Collections; -import java.util.List; import java.util.stream.Stream; import com.bakdata.conquery.TestTags; @@ -13,13 +11,13 @@ import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.i18n.I18n; -import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.sql.DslContextFactory; import com.bakdata.conquery.sql.conversion.dialect.PostgreSqlDialect; import com.bakdata.conquery.sql.conversion.model.SqlQuery; import com.bakdata.conquery.sql.conversion.supplier.DateNowSupplier; import com.bakdata.conquery.sql.execution.ResultSetProcessorFactory; import com.bakdata.conquery.sql.execution.SqlExecutionService; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.assertj.core.api.Assertions; import org.jooq.DSLContext; @@ -80,11 +78,11 @@ public void shouldThrowException() { // This can be removed as soon as we switch to a full integration test including the REST API I18n.init(); SqlExecutionService executionService = new SqlExecutionService(dslContext, ResultSetProcessorFactory.create(testSqlDialect)); - SqlQuery validQuery = toSqlQuery("SELECT 1"); + SqlQuery validQuery = new TestSqlQuery("SELECT 1"); Assertions.assertThatNoException().isThrownBy(() -> executionService.execute(validQuery)); // executing an empty query should throw an SQL error - SqlQuery emptyQuery = toSqlQuery(""); + SqlQuery emptyQuery = new TestSqlQuery(""); Assertions.assertThatThrownBy(() -> executionService.execute(emptyQuery)) .isInstanceOf(ConqueryError.SqlError.class) .hasMessageContaining("$org.postgresql.util.PSQLException"); @@ -122,19 +120,11 @@ private static class PostgreSqlTestFunctionProvider implements TestFunctionProvi } - private static SqlQuery toSqlQuery(String query) { - return new SqlQuery() { - - @Override - public String getSql() { - return query; - } - - @Override - public List getResultInfos() { - return Collections.emptyList(); - } - }; + @Getter + private static class TestSqlQuery extends SqlQuery { + protected TestSqlQuery(String sql) { + super(sql); + } } } diff --git a/backend/src/test/resources/shared/alter.concept.json b/backend/src/test/resources/shared/alter.concept.json new file mode 100644 index 00000000000..a7383590efe --- /dev/null +++ b/backend/src/test/resources/shared/alter.concept.json @@ -0,0 +1,54 @@ +{ + "type": "TREE", + "name": "alter", + "label": "Alter", + "hidden": false, + "connectors": { + "name": "alter", + "label": "Alter", + "validityDates": [ + { + "name": "versichertenzeit", + "label": "Versichertenzeit", + "startColumn": "vers_stamm.date_start", + "endColumn": "vers_stamm.date_end" + }, + { + "name": "erster_tag", + "label": "Erster Tag", + "column": "vers_stamm.date_start" + }, + { + "name": "letzter_tag", + "label": "Letzter Tag", + "column": "vers_stamm.date_end" + } + ], + "table": "vers_stamm", + "filters": { + "type": "DATE_DISTANCE", + "name": "alterseinschränkung", + "label": "Alterseinschränkung", + "description": "Alter zur gegebenen Datumseinschränkung", + "column": "vers_stamm.geburtsdatum", + "timeUnit": "YEARS" + }, + "selects": [ + { + "type": "DATE_DISTANCE", + "name": "alter_select", + "label": "Alter", + "description": "Automatisch erzeugter Zusatzwert.", + "column": "vers_stamm.geburtsdatum", + "timeUnit": "YEARS", + "default": true + }, + { + "name": "liste_geburtsdatum", + "label": "Geburtsdatum", + "type": "FIRST", + "column": "vers_stamm.geburtsdatum" + } + ] + } +} diff --git a/backend/src/test/resources/shared/geschlecht.concept.json b/backend/src/test/resources/shared/geschlecht.concept.json new file mode 100644 index 00000000000..bc519e8c170 --- /dev/null +++ b/backend/src/test/resources/shared/geschlecht.concept.json @@ -0,0 +1,49 @@ +{ + "type": "TREE", + "name": "geschlecht", + "label": "Geschlecht", + "hidden": false, + "connectors": { + "name": "geschlecht", + "label": "Geschlecht", + "validityDates": [ + { + "name": "versichertenzeit", + "label": "Versichertenzeit", + "startColumn": "vers_stamm.date_start", + "endColumn": "vers_stamm.date_end" + }, + { + "name": "erster_tag", + "label": "Erster Tag", + "column": "vers_stamm.date_start" + }, + { + "name": "letzter_tag", + "label": "Letzter Tag", + "column": "vers_stamm.date_end" + } + ], + "table": "vers_stamm", + "filters": { + "type": "SELECT", + "name": "geschlecht", + "label": "Geschlecht", + "column": "vers_stamm.geschlecht", + "labels": { + "1": "Weiblich", + "2": "Männlich" + } + }, + "selects": [ + { + "type": "FIRST", + "column": "vers_stamm.geschlecht", + "name": "geschlecht_select", + "label": "Geschlecht", + "description": "Automatisch erzeugter Zusatzwert.", + "default": true + } + ] + } +} diff --git a/backend/src/test/resources/shared/two_connector.concept.json b/backend/src/test/resources/shared/two_connector.concept.json new file mode 100644 index 00000000000..3dc52a7f8fd --- /dev/null +++ b/backend/src/test/resources/shared/two_connector.concept.json @@ -0,0 +1,59 @@ +{ + "type": "TREE", + "name": "two_connector", + "label": "two_connector", + "hidden": false, + "connectors": [ + { + "name": "table1", + "label": "Table1", + "table": "vers_stamm", + "validityDates": { + "name": "versichertenzeit", + "label": "Versichertenzeit", + "startColumn": "vers_stamm.date_start", + "endColumn": "vers_stamm.date_end" + }, + "selects": [ + { + "type": "DATE_DISTANCE", + "name": "alter_select", + "label": "Ausgabe Alter", + "description": "Automatisch erzeugter Zusatzwert.", + "column": "vers_stamm.geburtsdatum", + "timeUnit": "YEARS", + "default": true + }, + { + "name": "liste_geburtsdatum", + "label": "Ausgabe Geburtsdatum", + "type": "FIRST", + "column": "vers_stamm.geburtsdatum" + } + ] + }, + { + "name": "table2", + "label": "Table2", + "table": "vers_stamm", + "validityDates": { + "name": "versichertenzeit", + "label": "Versichertenzeit", + "startColumn": "vers_stamm.date_start", + "endColumn": "vers_stamm.date_end" + }, + "selects": { + "name": "liste_geburtsdatum", + "label": "Ausgabe Geburtsdatum", + "type": "FIRST", + "column": "vers_stamm.geburtsdatum", + "default": true + } + } + ], + "selects": { + "name": "exists", + "type": "EXISTS", + "default": true + } +} diff --git a/backend/src/test/resources/shared/vers_stamm.csv b/backend/src/test/resources/shared/vers_stamm.csv new file mode 100644 index 00000000000..65f2be34810 --- /dev/null +++ b/backend/src/test/resources/shared/vers_stamm.csv @@ -0,0 +1,23 @@ +pid,date_start,date_end,geburtsdatum,geschlecht +1,2012-01-01,2012-12-31,1957-01-01,1 +2,2012-01-01,2012-12-31,1988-04-01,2 +3,2012-03-01,2012-12-31,1989-01-01,2 +4,2013-01-01,2013-12-31,1989-01-01,2 +5,2012-07-01,2012-12-31,1972-07-01,2 +6,2012-01-01,2012-12-31,1960-10-01,2 +7,2012-01-01,2012-12-31,1983-01-01,1 +8,2012-01-01,2012-12-31,1956-10-01,2 +9,2012-01-01,2012-12-31,1951-07-01,2 +10,2012-01-01,2012-12-31,1964-04-01,2 +11,2012-01-01,2012-12-31,1966-04-01,1 +12,2013-01-01,2013-12-31,1966-04-01,1 +13,2012-01-01,2012-12-31,1995-01-01,1 +14,2012-01-01,2012-12-31,1952-10-01,2 +15,2012-01-01,2012-12-31,1983-01-01,1 +16,2012-01-01,2012-12-31,1981-01-01,1 +17,2012-01-01,2012-12-31,1979-01-01,2 +18,2012-01-01,2012-12-31,1959-04-01,1 +19,2012-01-01,2012-12-31,1991-10-01,1 +20,2012-01-01,2012-12-31,1987-01-01,2 +21,2012-01-01,2012-12-31,1952-01-01,1 +22,2012-01-01,2012-10-01,1955-01-01,2 diff --git a/backend/src/test/resources/shared/vers_stamm.table.json b/backend/src/test/resources/shared/vers_stamm.table.json new file mode 100644 index 00000000000..c33e5bf70bb --- /dev/null +++ b/backend/src/test/resources/shared/vers_stamm.table.json @@ -0,0 +1,26 @@ +{ + "csv": "shared/vers_stamm.csv", + "name": "vers_stamm", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "date_start", + "type": "DATE" + }, + { + "name": "date_end", + "type": "DATE" + }, + { + "name": "geburtsdatum", + "type": "DATE" + }, + { + "name": "geschlecht", + "type": "STRING" + } + ] +} diff --git a/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/QUARTER YEAR.json b/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/QUARTER YEAR.json new file mode 100644 index 00000000000..7491eddc618 --- /dev/null +++ b/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/QUARTER YEAR.json @@ -0,0 +1,46 @@ +{ + "type": "FORM_TEST", + "label": "YEAR Alignment, QUARTER Resolution", + "expectedCsv": { + "results": "tests/sql/form/ABSOLUT/ALIGNMENT/quarter year.csv" + }, + "form": { + "type": "EXPORT_FORM", + "queryGroup": "00000000-0000-0000-0000-000000000001", + "resolution": "QUARTERS", + "alsoCreateCoarserSubdivisions": true, + "features": [ + { + "ids": [ + "alter" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "alter.alter", + "filters": [] + } + ] + } + ], + "timeMode": { + "value": "ABSOLUTE", + "alignmentHint": "YEAR", + "dateRange": { + "min": "2012-06-16", + "max": "2013-01-17" + } + } + }, + "concepts": [ + "/shared/alter.concept.json" + ], + "content": { + "tables": [ + "/shared/vers_stamm.table.json" + ], + "previousQueryResults": [ + "tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/query_results_1.csv" + ] + } +} diff --git a/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/YEAR QUARTER.json b/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/YEAR QUARTER.json new file mode 100644 index 00000000000..b3a741fabf7 --- /dev/null +++ b/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/YEAR QUARTER.json @@ -0,0 +1,46 @@ +{ + "type": "FORM_TEST", + "label": "Quarter Alignment, Year Resolution", + "expectedCsv": { + "results": "tests/sql/form/ABSOLUT/ALIGNMENT/year quarter expected.csv" + }, + "form": { + "type": "EXPORT_FORM", + "queryGroup": "00000000-0000-0000-0000-000000000001", + "resolution": "YEARS", + "alsoCreateCoarserSubdivisions": true, + "features": [ + { + "ids": [ + "alter" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "alter.alter", + "filters": [] + } + ] + } + ], + "timeMode": { + "value": "ABSOLUTE", + "alignmentHint": "QUARTER", + "dateRange": { + "min": "2012-06-16", + "max": "2012-12-17" + } + } + }, + "concepts": [ + "/shared/alter.concept.json" + ], + "content": { + "tables": [ + "/shared/vers_stamm.table.json" + ], + "previousQueryResults": [ + "tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/query_results_1.csv" + ] + } +} diff --git a/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/YEAR YEAR.json b/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/YEAR YEAR.json new file mode 100644 index 00000000000..df6f50f979f --- /dev/null +++ b/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/YEAR YEAR.json @@ -0,0 +1,46 @@ +{ + "type": "FORM_TEST", + "label": "YEAR Alignment, YEAR Resolution", + "expectedCsv": { + "results": "tests/sql/form/ABSOLUT/ALIGNMENT/year quarter expected.csv" + }, + "form": { + "type": "EXPORT_FORM", + "queryGroup": "00000000-0000-0000-0000-000000000001", + "resolution": "YEARS", + "alsoCreateCoarserSubdivisions": true, + "features": [ + { + "ids": [ + "alter" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "alter.alter", + "filters": [] + } + ] + } + ], + "timeMode": { + "value": "ABSOLUTE", + "alignmentHint": "YEAR", + "dateRange": { + "min": "2012-06-16", + "max": "2012-12-17" + } + } + }, + "concepts": [ + "/shared/alter.concept.json" + ], + "content": { + "tables": [ + "/shared/vers_stamm.table.json" + ], + "previousQueryResults": [ + "tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/query_results_1.csv" + ] + } +} diff --git a/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/quarter year.csv b/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/quarter year.csv new file mode 100644 index 00000000000..7abe6a68796 --- /dev/null +++ b/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/quarter year.csv @@ -0,0 +1,8 @@ +result,resolution,index,date_range,Alter +1,complete,,2012-06-16/2013-01-17,56 +1,year,1,2012-06-16/2012-12-31,55 +1,year,2,2013-01-01/2013-01-17, +1,quarter,1,2012-06-16/2012-06-30,55 +1,quarter,2,2012-07-01/2012-09-30,55 +1,quarter,3,2012-10-01/2012-12-31,55 +1,quarter,4,2013-01-01/2013-01-17, \ No newline at end of file diff --git a/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/year quarter expected.csv b/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/year quarter expected.csv new file mode 100644 index 00000000000..e658a4906c5 --- /dev/null +++ b/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/year quarter expected.csv @@ -0,0 +1,3 @@ +result,resolution,index,date_range,Alter +1,complete,,2012-06-16/2012-12-17,55 +1,year,1,2012-06-16/2012-12-17,55 diff --git a/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/MULTIPLE_FEATURES.json b/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/MULTIPLE_FEATURES.json new file mode 100644 index 00000000000..62a4a9ca025 --- /dev/null +++ b/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/MULTIPLE_FEATURES.json @@ -0,0 +1,58 @@ +{ + "type": "FORM_TEST", + "label": "ABS-EXPORT-FORM ADD DEFAULT SELECT Test", + "expectedCsv": { + "results": "tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/expected.csv" + }, + "form": { + "type": "EXPORT_FORM", + "queryGroup": "00000000-0000-0000-0000-000000000001", + "resolution": "QUARTERS", + "alsoCreateCoarserSubdivisions": true, + "features": [ + { + "ids": [ + "alter" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "alter.alter", + "filters": [] + } + ] + }, + { + "ids": [ + "geschlecht" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "geschlecht.geschlecht", + "filters": [] + } + ] + } + ], + "timeMode": { + "value": "ABSOLUTE", + "dateRange": { + "min": "2012-01-16", + "max": "2012-12-17" + } + } + }, + "concepts": [ + "/shared/alter.concept.json", + "/shared/geschlecht.concept.json" + ], + "content": { + "tables": [ + "/shared/vers_stamm.table.json" + ], + "previousQueryResults": [ + "tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/query_results_1.csv" + ] + } +} diff --git a/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/expected.csv b/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/expected.csv new file mode 100644 index 00000000000..44625d7b306 --- /dev/null +++ b/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/expected.csv @@ -0,0 +1,7 @@ +result,resolution,index,date_range,Alter,Geschlecht +1,complete,,2012-01-16/2012-12-17,55,1 +1,year,1,2012-01-16/2012-12-17,55,1 +1,quarter,1,2012-01-16/2012-03-31,55,1 +1,quarter,2,2012-04-01/2012-06-30,55,1 +1,quarter,3,2012-07-01/2012-09-30,55,1 +1,quarter,4,2012-10-01/2012-12-17,55,1 diff --git a/backend/src/test/resources/tests/sql/form/ABSOLUT/SECONDARY_ID/ABS_EXPORT_FORM_SECONDARY_ID.json b/backend/src/test/resources/tests/sql/form/ABSOLUT/SECONDARY_ID/ABS_EXPORT_FORM_SECONDARY_ID.json new file mode 100644 index 00000000000..1228386322e --- /dev/null +++ b/backend/src/test/resources/tests/sql/form/ABSOLUT/SECONDARY_ID/ABS_EXPORT_FORM_SECONDARY_ID.json @@ -0,0 +1,79 @@ +{ + "type": "FORM_TEST", + "label": "ABS-EXPORT-FORM SECONDARY_ID", + "expectedCsv": { + "results": "/tests/form/EXPORT_FORM/ABSOLUT/SECONDARY_ID/expected.csv" + }, + "form": { + "type": "EXPORT_FORM", + "queryGroup": "00000000-0000-0000-0000-000000000001", + "resolution": "QUARTERS", + "alsoCreateCoarserSubdivisions": true, + "features": [ + { + "ids": [ + "two_connector" + ], + "type": "CONCEPT", + "label": "explicitly set select", + "tables": [ + { + "id": "two_connector.table1", + "selects": "two_connector.table1.alter_select" + }, + { + "id": "two_connector.table2" + } + ] + } + ], + "timeMode": { + "value": "ABSOLUTE", + "dateRange": { + "min": "2012-01-16", + "max": "2012-12-17" + } + } + }, + "concepts": [ + "/shared/two_connector.concept.json", + "/tests/form/shared/abc.concept.json" + ], + "content": { + "secondaryIds": [ + { + "name": "secondary" + }, + { + "name": "ignored" + } + ], + "tables": [ + "/shared/vers_stamm.table.json", + "/tests/form/shared/abc.table.json" + ], + "previousQueries": [ + { + "type": "SECONDARY_ID_QUERY", + "secondaryId": "secondary", + "root": { + "type": "AND", + "children": [ + { + "type": "CONCEPT", + "excludeFromSecondaryId": false, + "ids": [ + "abc-concept.a" + ], + "tables": [ + { + "id": "abc-concept.connector" + } + ] + } + ] + } + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/form/ABSOLUT/SIMPLE/ABS_EXPORT_FORM.json b/backend/src/test/resources/tests/sql/form/ABSOLUT/SIMPLE/ABS_EXPORT_FORM.json new file mode 100644 index 00000000000..b14651aa12b --- /dev/null +++ b/backend/src/test/resources/tests/sql/form/ABSOLUT/SIMPLE/ABS_EXPORT_FORM.json @@ -0,0 +1,45 @@ +{ + "type": "FORM_TEST", + "label": "ABS-EXPORT-FORM ADD DEFAULT SELECT Test", + "expectedCsv": { + "results": "tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/expected.csv" + }, + "form": { + "type": "EXPORT_FORM", + "queryGroup": "00000000-0000-0000-0000-000000000001", + "resolution": "QUARTERS", + "alsoCreateCoarserSubdivisions": true, + "features": [ + { + "ids": [ + "alter" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "alter.alter", + "filters": [] + } + ] + } + ], + "timeMode": { + "value": "ABSOLUTE", + "dateRange": { + "min": "2012-01-16", + "max": "2012-12-17" + } + } + }, + "concepts": [ + "/shared/alter.concept.json" + ], + "content": { + "tables": [ + "/shared/vers_stamm.table.json" + ], + "previousQueryResults": [ + "tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/query_results_1.csv" + ] + } +} diff --git a/backend/src/test/resources/tests/sql/form/ABSOLUT/SIMPLE/ABS_EXPORT_FORM_WITH_SELECT.json b/backend/src/test/resources/tests/sql/form/ABSOLUT/SIMPLE/ABS_EXPORT_FORM_WITH_SELECT.json new file mode 100644 index 00000000000..f740dbc9a57 --- /dev/null +++ b/backend/src/test/resources/tests/sql/form/ABSOLUT/SIMPLE/ABS_EXPORT_FORM_WITH_SELECT.json @@ -0,0 +1,50 @@ +{ + "type": "FORM_TEST", + "label": "ABS-EXPORT-FORM WITH SELECT SET Test", + "expectedCsv": { + "results": "tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/expected_with_select.csv" + }, + "form": { + "type": "EXPORT_FORM", + "queryGroup": "00000000-0000-0000-0000-000000000001", + "resolution": "QUARTERS", + "alsoCreateCoarserSubdivisions": true, + "features": [ + { + "ids": [ + "two_connector" + ], + "type": "CONCEPT", + "label": "explicitly set select", + "tables": [ + { + "id": "two_connector.table1", + "selects": "two_connector.table1.liste_geburtsdatum" + }, + { + "id": "two_connector.table2" + } + ] + } + ], + "timeMode": { + "value": "ABSOLUTE", + "dateRange": { + "min": "2012-01-16", + "max": "2012-12-17" + } + }, + "values": "Some arbitrary data that is frontend/user provided" + }, + "concepts": [ + "/shared/two_connector.concept.json" + ], + "content": { + "tables": [ + "/shared/vers_stamm.table.json" + ], + "previousQueryResults": [ + "tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/query_results_1.csv" + ] + } +}