diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java index 182be96205..961f607a7d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java @@ -294,7 +294,7 @@ public Field addDays(Field dateColumn, Field amountOfDays) "ADD_DAYS", Date.class, dateColumn, - DSL.val(amountOfDays) + amountOfDays ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java index 4d8c6ef293..041b70b95a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java @@ -19,6 +19,7 @@ import com.bakdata.conquery.sql.conversion.query.ConceptQueryConverter; import com.bakdata.conquery.sql.conversion.query.EntityDateQueryConverter; import com.bakdata.conquery.sql.conversion.query.FormConversionHelper; +import com.bakdata.conquery.sql.conversion.query.RelativFormQueryConverter; import com.bakdata.conquery.sql.conversion.query.SecondaryIdQueryConverter; import com.bakdata.conquery.sql.conversion.supplier.DateNowSupplier; import com.bakdata.conquery.sql.conversion.supplier.SystemDateNowSupplier; @@ -64,7 +65,8 @@ default List> getDefaultNodeConverters() { new ConceptQueryConverter(queryStepTransformer), new SecondaryIdQueryConverter(), new AbsoluteFormQueryConverter(formConversionUtil), - new EntityDateQueryConverter(formConversionUtil) + new EntityDateQueryConverter(formConversionUtil), + new RelativFormQueryConverter(formConversionUtil) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index 14b08a540e..54eac06171 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -114,6 +114,9 @@ public interface SqlFunctionProvider { */ Field daterangeStringExpression(ColumnDateRange columnDateRange); + /** + * Calculates the date distance in the given {@link ChronoUnit} between an exclusive end date and an inclusive start date. + */ Field dateDistance(ChronoUnit datePart, Field startDate, Field endDate); Field addDays(Field dateColumn, Field amountOfDays); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java new file mode 100644 index 0000000000..d151fc6b93 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java @@ -0,0 +1,188 @@ +package com.bakdata.conquery.sql.conversion.forms; + +import java.sql.Date; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; +import com.bakdata.conquery.models.forms.util.Resolution; +import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.google.common.base.Preconditions; +import lombok.RequiredArgsConstructor; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.impl.DSL; + +@RequiredArgsConstructor +class AbsoluteStratification { + + private final int INDEX_START = 1; + private final int INDEX_END = 10_000; + + private final QueryStep baseStep; + private final StratificationFunctions stratificationFunctions; + + public QueryStep createStratificationTable(List resolutionAndAlignments) { + + QueryStep intSeriesStep = createIntSeriesStep(); + QueryStep indexStartStep = createIndexStartStep(); + + List resolutionTables = resolutionAndAlignments.stream() + .map(resolutionAndAlignment -> createResolutionTable(indexStartStep, resolutionAndAlignment)) + .toList(); + + List predecessors = List.of(baseStep, intSeriesStep, indexStartStep); + return StratificationTableFactory.unionResolutionTables(resolutionTables, predecessors); + } + + private QueryStep createIntSeriesStep() { + + // not actually required, but Selects expect at least 1 SqlIdColumn + Field rowNumber = DSL.rowNumber().over().coerce(Object.class); + SqlIdColumns ids = new SqlIdColumns(rowNumber); + + FieldWrapper seriesIndex = new FieldWrapper<>(stratificationFunctions.intSeriesField()); + + Selects selects = Selects.builder() + .ids(ids) + .sqlSelect(seriesIndex) + .build(); + + Table seriesTable = stratificationFunctions.generateIntSeries(INDEX_START, INDEX_END) + .as(SharedAliases.SERIES_INDEX.getAlias()); + + return QueryStep.builder() + .cteName(FormCteStep.INT_SERIES.getSuffix()) + .selects(selects) + .fromTable(seriesTable) + .build(); + } + + private QueryStep createIndexStartStep() { + + Selects baseStepSelects = baseStep.getQualifiedSelects(); + Preconditions.checkArgument(baseStepSelects.getStratificationDate().isPresent(), "The base step must have a stratification date set"); + ColumnDateRange bounds = baseStepSelects.getStratificationDate().get(); + + Field indexStart = stratificationFunctions.absoluteIndexStartDate(bounds).as(SharedAliases.INDEX_START.getAlias()); + Field yearStart = stratificationFunctions.lowerBoundYearStart(bounds).as(SharedAliases.YEAR_START.getAlias()); + Field yearEnd = stratificationFunctions.upperBoundYearEnd(bounds).as(SharedAliases.YEAR_END.getAlias()); + Field yearEndQuarterAligned = stratificationFunctions.upperBoundYearEndQuarterAligned(bounds).as(SharedAliases.YEAR_END_QUARTER_ALIGNED.getAlias()); + Field quarterStart = stratificationFunctions.lowerBoundQuarterStart(bounds).as(SharedAliases.QUARTER_START.getAlias()); + Field quarterEnd = stratificationFunctions.upperBoundQuarterEnd(bounds).as(SharedAliases.QUARTER_END.getAlias()); + + List> startDates = Stream.of( + indexStart, + yearStart, + yearEnd, + yearEndQuarterAligned, + quarterStart, + quarterEnd + ) + .map(FieldWrapper::new) + .toList(); + + Selects selects = Selects.builder() + .ids(baseStepSelects.getIds()) + .stratificationDate(Optional.of(bounds)) + .sqlSelects(startDates) + .build(); + + return QueryStep.builder() + .cteName(FormCteStep.INDEX_START.getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(baseStep.getCteName())) + .build(); + } + + private QueryStep createResolutionTable(QueryStep indexStartStep, ExportForm.ResolutionAndAlignment resolutionAndAlignment) { + return switch (resolutionAndAlignment.getResolution()) { + case COMPLETE -> createCompleteTable(); + case YEARS, QUARTERS, DAYS -> createIntervalTable(indexStartStep, resolutionAndAlignment); + }; + } + + private QueryStep createCompleteTable() { + + Selects baseStepSelects = baseStep.getQualifiedSelects(); + + // complete range shall have a null index because it spans the complete range, but we set it to 1 to ensure we can join tables on index, + // because a condition involving null in a join (e.g., null = some_value or null = null) always evaluates to false + Field index = DSL.field(DSL.val(1, Integer.class)).as(SharedAliases.INDEX.getAlias()); + SqlIdColumns ids = baseStepSelects.getIds().withAbsoluteStratification(Resolution.COMPLETE, index); + + ColumnDateRange completeRange = baseStepSelects.getStratificationDate().get(); + + Selects selects = Selects.builder() + .ids(ids) + .stratificationDate(Optional.of(completeRange)) + .build(); + + return QueryStep.builder() + .cteName(FormCteStep.COMPLETE.getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(baseStep.getCteName())) + .build(); + } + + private QueryStep createIntervalTable(QueryStep indexStartStep, ExportForm.ResolutionAndAlignment resolutionAndAlignment) { + + QueryStep countsCte = createCountsCte(indexStartStep, resolutionAndAlignment); + Preconditions.checkArgument(countsCte.getSelects().getStratificationDate().isPresent(), "The countsCte must have a stratification date set"); + Selects countsCteSelects = countsCte.getQualifiedSelects(); + + ColumnDateRange stratificationRange = stratificationFunctions.createStratificationRange( + resolutionAndAlignment, + countsCteSelects.getStratificationDate().get() + ); + + Field index = stratificationFunctions.index(countsCteSelects.getIds(), countsCte.getQualifiedSelects().getStratificationDate()); + SqlIdColumns ids = countsCteSelects.getIds().withAbsoluteStratification(resolutionAndAlignment.getResolution(), index); + + Selects selects = Selects.builder() + .ids(ids) + .stratificationDate(Optional.ofNullable(stratificationRange)) + .build(); + + Condition stopOnMaxResolutionWindowCount = stratificationFunctions.stopOnMaxResolutionWindowCount(resolutionAndAlignment); + + return QueryStep.builder() + .cteName(FormCteStep.stratificationCte(resolutionAndAlignment.getResolution()).getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(countsCte.getCteName())) + .fromTable(QueryStep.toTableLike(FormCteStep.INT_SERIES.getSuffix())) + .conditions(List.of(stopOnMaxResolutionWindowCount)) + .predecessor(countsCte) + .build(); + } + + private QueryStep createCountsCte(QueryStep indexStartStep, ExportForm.ResolutionAndAlignment resolutionAndAlignment) { + + Selects indexStartSelects = indexStartStep.getQualifiedSelects(); + Preconditions.checkArgument(indexStartSelects.getStratificationDate().isPresent(), "The indexStartStep must have a stratification date set"); + + Field resolutionWindowCount = stratificationFunctions.calculateResolutionWindowCount( + resolutionAndAlignment, + indexStartSelects.getStratificationDate().get() + ); + + Selects selects = indexStartSelects.toBuilder() + .sqlSelect(new FieldWrapper<>(resolutionWindowCount)) + .build(); + + return QueryStep.builder() + .cteName(FormCteStep.countsCte(resolutionAndAlignment.getResolution()).getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(indexStartStep.getCteName())) + .build(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/CombinationNotSupportedException.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/CombinationNotSupportedException.java index 0078f1290a..a29f4999c5 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/CombinationNotSupportedException.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/CombinationNotSupportedException.java @@ -1,6 +1,9 @@ package com.bakdata.conquery.sql.conversion.forms; +import com.bakdata.conquery.apiv1.forms.IndexPlacement; import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; +import com.bakdata.conquery.models.forms.util.CalendarUnit; +import com.bakdata.conquery.models.forms.util.Resolution; class CombinationNotSupportedException extends RuntimeException { @@ -11,4 +14,12 @@ public CombinationNotSupportedException(ExportForm.ResolutionAndAlignment resolu )); } + public CombinationNotSupportedException(IndexPlacement indexPlacement, CalendarUnit timeUnit) { + super("Combination of index placement %s and time unit %s not supported".formatted(indexPlacement, timeUnit)); + } + + public CombinationNotSupportedException(CalendarUnit timeUnit, Resolution resolution) { + super("Combination of time unit %s and resolution %s not supported".formatted(timeUnit, resolution)); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormConstants.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormConstants.java index a4ad99ef70..f1ca4ca05d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormConstants.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormConstants.java @@ -19,6 +19,21 @@ class FormConstants { */ public static Field INDEX_START = DSL.field(DSL.name(SharedAliases.INDEX_START.getAlias()), Date.class); + /** + * The index date corresponding to the {@link TemporalSamplerFactory} of a relative stratification. + */ + public static Field INDEX_SELECTOR = DSL.field(DSL.name(SharedAliases.INDEX_SELECTOR.getAlias()), Date.class); + + /** + * The index date from which we start when calculating a feature range. + */ + public static Field INDEX_START_NEGATIVE = DSL.field(DSL.name(SharedAliases.INDEX_START_NEGATIVE.getAlias()), Date.class); + + /** + * The index date from which we start when calculating an outcome range. + */ + public static Field INDEX_START_POSITIVE = DSL.field(DSL.name(SharedAliases.INDEX_START_POSITIVE.getAlias()), Date.class); + /** * The quarter start of the lower bound of an absolute stratification range. The stratification range this date is referring to can be an * {@link AbsoluteFormQuery#getDateRange()} or for an {@link EntityDateQuery} the respective entities date range bound by the diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java index e97f224a55..7dbd2baedb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java @@ -16,6 +16,11 @@ public enum FormCteStep implements CteStep { UNNEST_ENTITY_DATE_CTE("unnest_entity_date"), OVERWRITE_BOUNDS("overwrite_bounds"), + // relative form + UNNEST_DATES("unnest_dates"), + INDEX_SELECTOR("index_selector"), + TOTAL_BOUNDS("total_bounds"), + // stratification INDEX_START("index_start"), INT_SERIES("int_series"), diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormType.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormType.java new file mode 100644 index 0000000000..d448a6ba33 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormType.java @@ -0,0 +1,7 @@ +package com.bakdata.conquery.sql.conversion.forms; + +public enum FormType { + ABSOLUTE, + ENTITY_DATE, + RELATIVE +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/HanaStratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/HanaStratificationFunctions.java index 41361b1d70..62938726e1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/HanaStratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/HanaStratificationFunctions.java @@ -3,7 +3,9 @@ import static com.bakdata.conquery.sql.conversion.forms.Interval.MONTHS_PER_QUARTER; import java.sql.Date; +import java.time.temporal.ChronoUnit; +import com.bakdata.conquery.apiv1.query.concept.specific.temporal.TemporalSamplerFactory; import com.bakdata.conquery.sql.conversion.dialect.HanaSqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import lombok.Getter; @@ -12,6 +14,7 @@ import org.jooq.Record; import org.jooq.Table; import org.jooq.impl.DSL; +import org.jooq.impl.QOM; import org.jooq.impl.SQLDataType; @Getter @@ -35,7 +38,12 @@ protected Field lower(ColumnDateRange dateRange) { } @Override - protected Field upper(ColumnDateRange dateRange) { + protected Field inclusiveUpper(ColumnDateRange dateRange) { + return functionProvider.addDays(exclusiveUpper(dateRange), DSL.val(-1)); + } + + @Override + protected Field exclusiveUpper(ColumnDateRange dateRange) { // HANA does not support single-column ranges, so we can return start and end directly return dateRange.getEnd(); } @@ -54,12 +62,12 @@ public Field absoluteIndexStartDate(ColumnDateRange dateRange) { } @Override - public Field yearStart(ColumnDateRange dateRange) { + public Field lowerBoundYearStart(ColumnDateRange dateRange) { return jumpToYearStart(dateRange.getStart()); } @Override - public Field yearEnd(ColumnDateRange dateRange) { + public Field upperBoundYearEnd(ColumnDateRange dateRange) { return DSL.field( "SERIES_ROUND({0}, {1}, {2})", Date.class, @@ -70,24 +78,39 @@ public Field yearEnd(ColumnDateRange dateRange) { } @Override - public Field yearEndQuarterAligned(ColumnDateRange dateRange) { - Field nextYearStart = yearEnd(dateRange); - Field quartersInMonths = getMonthsInQuarters(dateRange.getStart(), Offset.MINUS_ONE); - return addMonths(nextYearStart, quartersInMonths); + public Field upperBoundYearEndQuarterAligned(ColumnDateRange dateRange) { + Field yearStartOfUpperBound = jumpToYearStart(dateRange.getEnd()); + Field quartersInMonths = getQuartersInMonths(dateRange.getStart(), Offset.MINUS_ONE); + Field yearEndQuarterAligned = addMonths(yearStartOfUpperBound, quartersInMonths); + // we add +1 year to the quarter aligned end if it is less than the upper bound we want to align + return DSL.when( + yearEndQuarterAligned.lessThan(dateRange.getEnd()), + shiftByInterval(yearEndQuarterAligned, Interval.ONE_YEAR_INTERVAL, DSL.val(1), Offset.NONE) + ) + .otherwise(yearEndQuarterAligned); + } + + @Override + public Field lowerBoundQuarterStart(ColumnDateRange dateRange) { + return jumpToQuarterStart(dateRange.getStart()); } @Override - public Field quarterStart(ColumnDateRange dateRange) { - Field yearStart = jumpToYearStart(dateRange.getStart()); - Field quartersInMonths = getMonthsInQuarters(dateRange.getStart(), Offset.MINUS_ONE); + protected Field jumpToQuarterStart(Field date) { + Field yearStart = jumpToYearStart(date); + Field quartersInMonths = getQuartersInMonths(date, Offset.MINUS_ONE); return addMonths(yearStart, quartersInMonths); } @Override - public Field quarterEnd(ColumnDateRange dateRange) { - Field yearStart = jumpToYearStart(dateRange.getEnd()); - Field inclusiveEnd = functionProvider.addDays(dateRange.getEnd(), DSL.val(-1)); - Field quartersInMonths = getMonthsInQuarters(inclusiveEnd, Offset.NONE); + public Field upperBoundQuarterEnd(ColumnDateRange dateRange) { + return jumpToNextQuarterStart(inclusiveUpper(dateRange)); + } + + @Override + protected Field jumpToNextQuarterStart(Field date) { + Field yearStart = jumpToYearStart(date); + Field quartersInMonths = getQuartersInMonths(date, Offset.NONE); return addMonths(yearStart, quartersInMonths); } @@ -104,6 +127,36 @@ public Table generateIntSeries(int start, int end) { return DSL.table("SERIES_GENERATE_INTEGER({0}, {1}, {2})", INCREMENT, adjustedStart, end); } + @Override + public Field indexSelectorField(TemporalSamplerFactory indexSelector, ColumnDateRange validityDate) { + return switch (indexSelector) { + case EARLIEST -> DSL.min(validityDate.getStart()); + case LATEST -> DSL.max(inclusiveUpper(validityDate)); + case RANDOM -> { + // we calculate a random int which is in range of the date distance between upper and lower bound + Field dateDistanceInDays = functionProvider.dateDistance(ChronoUnit.DAYS, validityDate.getStart(), validityDate.getEnd()); + Field randomAmountOfDays = DSL.function("RAND", Double.class).times(dateDistanceInDays); + Field flooredAsInt = functionProvider.cast(DSL.floor(randomAmountOfDays), SQLDataType.INTEGER); + // then we add this random amount (of days) to the start date + Field randomDateInRange = functionProvider.addDays(lower(validityDate), flooredAsInt); + // finally, we handle multiple ranges by randomizing which range we use to select a random date from + yield functionProvider.random(randomDateInRange); + } + }; + } + + @Override + public Field shiftByInterval(Field startDate, Interval interval, Field amount, Offset offset) { + Field multiplier = amount.plus(offset.getOffset()); + return switch (interval) { + case ONE_YEAR_INTERVAL -> DSL.function("ADD_YEARS", Date.class, startDate, multiplier.times(Interval.ONE_YEAR_INTERVAL.getAmount())); + case YEAR_AS_DAYS_INTERVAL -> addDays(startDate, multiplier.times(Interval.YEAR_AS_DAYS_INTERVAL.getAmount())); + case QUARTER_INTERVAL -> addMonths(startDate, multiplier.times(Interval.QUARTER_INTERVAL.getAmount())); + case NINETY_DAYS_INTERVAL -> addDays(startDate, multiplier.times(Interval.NINETY_DAYS_INTERVAL.getAmount())); + case ONE_DAY_INTERVAL -> addDays(startDate, multiplier.times(Interval.ONE_DAY_INTERVAL.getAmount())); + }; + } + private static Field addMonths(Field yearStart, Field amount) { return DSL.function("ADD_MONTHS", Date.class, yearStart, amount); } @@ -121,14 +174,7 @@ private Field calcEndDate(Field start, Interval interval) { } private Field calcDate(Field start, Interval interval, Offset offset) { - Field seriesIndex = intSeriesField().plus(offset.getOffset()); - return switch (interval) { - case ONE_YEAR_INTERVAL -> DSL.function("ADD_YEARS", Date.class, start, seriesIndex.times(Interval.ONE_YEAR_INTERVAL.getAmount())); - case YEAR_AS_DAYS_INTERVAL -> addDays(start, seriesIndex.times(Interval.YEAR_AS_DAYS_INTERVAL.getAmount())); - case QUARTER_INTERVAL -> addMonths(start, seriesIndex.times(Interval.QUARTER_INTERVAL.getAmount())); - case NINETY_DAYS_INTERVAL -> addDays(start, seriesIndex.times(Interval.NINETY_DAYS_INTERVAL.getAmount())); - case ONE_DAY_INTERVAL -> addDays(start, seriesIndex.times(Interval.ONE_DAY_INTERVAL.getAmount())); - }; + return shiftByInterval(start, interval, intSeriesField(), offset); } private static Field jumpToYearStart(Field date) { @@ -141,7 +187,7 @@ private static Field jumpToYearStart(Field date) { ); } - private Field getMonthsInQuarters(Field date, Offset offset) { + private Field getQuartersInMonths(Field date, Offset offset) { Field quarterExpression = functionProvider.yearQuarter(date); Field rightMostCharacter = DSL.function("RIGHT", String.class, quarterExpression, DSL.val(1)); Field amountOfQuarters = functionProvider.cast(rightMostCharacter, SQLDataType.INTEGER) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/PostgresStratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/PostgresStratificationFunctions.java index d3737c4834..4be86a1576 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/PostgresStratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/PostgresStratificationFunctions.java @@ -2,10 +2,13 @@ import static com.bakdata.conquery.sql.conversion.forms.FormConstants.SERIES_INDEX; +import java.math.BigDecimal; import java.sql.Date; import java.sql.Timestamp; +import java.time.temporal.ChronoUnit; import java.util.Map; +import com.bakdata.conquery.apiv1.query.concept.specific.temporal.TemporalSamplerFactory; import com.bakdata.conquery.sql.conversion.dialect.PostgreSqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import lombok.Getter; @@ -16,6 +19,7 @@ import org.jooq.Record; import org.jooq.Table; import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; @Getter @RequiredArgsConstructor @@ -33,45 +37,67 @@ class PostgresStratificationFunctions extends StratificationFunctions { private final PostgreSqlFunctionProvider functionProvider; + @Override + public ColumnDateRange ofStartAndEnd(Field start, Field end) { + return ColumnDateRange.of(functionProvider.daterange(start, end, "[)")); + } + @Override public Field absoluteIndexStartDate(ColumnDateRange dateRange) { return lower(dateRange); } @Override - public Field yearStart(ColumnDateRange dateRange) { + public Field lowerBoundYearStart(ColumnDateRange dateRange) { return castExpressionToDate(jumpToYearStart(lower(dateRange))); } @Override - public Field yearEnd(ColumnDateRange dateRange) { + public Field upperBoundYearEnd(ColumnDateRange dateRange) { return DSL.field( "{0} + {1} {2}", Date.class, - dateTruncate(DSL.val("year"), upper(dateRange)), + dateTruncate(DSL.val("year"), inclusiveUpper(dateRange)), INTERVAL_KEYWORD, INTERVAL_MAP.get(Interval.ONE_YEAR_INTERVAL) ); } @Override - public Field yearEndQuarterAligned(ColumnDateRange dateRange) { - Field quarter = functionProvider.extract(DatePart.QUARTER, lower(dateRange)); - Field nextYearStart = yearEnd(dateRange); - return addQuarters(nextYearStart, quarter, Offset.MINUS_ONE); + public Field upperBoundYearEndQuarterAligned(ColumnDateRange dateRange) { + Field lowerBoundQuarter = functionProvider.extract(DatePart.QUARTER, lower(dateRange)); + Field upperBound = inclusiveUpper(dateRange); + Field yearStartOfUpperBound = castExpressionToDate(jumpToYearStart(upperBound)); + Field yearEndQuarterAligned = addQuarters(yearStartOfUpperBound, lowerBoundQuarter, Offset.MINUS_ONE); + // we add +1 year to the quarter aligned end if it is less than the upper bound we want to align + return DSL.when( + yearEndQuarterAligned.lessThan(upperBound), + shiftByInterval(yearEndQuarterAligned, Interval.ONE_YEAR_INTERVAL, DSL.val(1), Offset.NONE) + ) + .otherwise(yearEndQuarterAligned); + } + + @Override + public Field lowerBoundQuarterStart(ColumnDateRange dateRange) { + return jumpToQuarterStart(lower(dateRange)); } @Override - public Field quarterStart(ColumnDateRange dateRange) { - Field quarter = functionProvider.extract(DatePart.QUARTER, lower(dateRange)); - return addQuarters(jumpToYearStart(lower(dateRange)), quarter, Offset.MINUS_ONE); + protected Field jumpToQuarterStart(Field date) { + Field quarter = functionProvider.extract(DatePart.QUARTER, date); + return addQuarters(jumpToYearStart(date), quarter, Offset.MINUS_ONE); } @Override - public Field quarterEnd(ColumnDateRange dateRange) { - Field yearStart = dateTruncate(DSL.val("year"), upper(dateRange)); - Field quarterEndInclusive = upper(dateRange).minus(1); - Field quarter = functionProvider.extract(DatePart.QUARTER, quarterEndInclusive); + public Field upperBoundQuarterEnd(ColumnDateRange dateRange) { + Field inclusiveEnd = inclusiveUpper(dateRange); + return jumpToNextQuarterStart(inclusiveEnd); + } + + @Override + protected Field jumpToNextQuarterStart(Field date) { + Field yearStart = dateTruncate(DSL.val("year"), date); + Field quarter = functionProvider.extract(DatePart.QUARTER, date); return addQuarters(yearStart, quarter, Offset.NONE); } @@ -85,6 +111,31 @@ public Table generateIntSeries(int from, int to) { return DSL.table("generate_series({0}, {1})", from, to); } + @Override + public Field indexSelectorField(TemporalSamplerFactory indexSelector, ColumnDateRange validityDate) { + return switch (indexSelector) { + case EARLIEST -> DSL.min(lower(validityDate)); + // upper returns the exclusive end date, we want to inclusive one, so we add -1 day + case LATEST -> DSL.max(inclusiveUpper(validityDate)); + case RANDOM -> { + // we calculate a random int which is in range of the date distance between upper and lower bound + Field dateDistanceInDays = functionProvider.dateDistance(ChronoUnit.DAYS, lower(validityDate), exclusiveUpper(validityDate)); + Field randomAmountOfDays = DSL.rand().times(dateDistanceInDays); + Field flooredAsInt = functionProvider.cast(DSL.floor(randomAmountOfDays), SQLDataType.INTEGER); + // then we add this random amount (of days) to the start date + Field randomDateInRange = functionProvider.addDays(lower(validityDate), flooredAsInt); + // finally, we handle multiple ranges by randomizing which range we use to select a random date from + yield functionProvider.random(randomDateInRange); + } + }; + } + + @Override + public Field shiftByInterval(Field startDate, Interval interval, Field amount, Offset offset) { + Field intervalExpression = INTERVAL_MAP.get(interval); + return addMultipliedInterval(startDate, intervalExpression, amount, offset); + } + @Override protected Field lower(ColumnDateRange dateRange) { checkIsSingleColumnRange(dateRange); @@ -92,7 +143,13 @@ protected Field lower(ColumnDateRange dateRange) { } @Override - protected Field upper(ColumnDateRange dateRange) { + protected Field inclusiveUpper(ColumnDateRange dateRange) { + checkIsSingleColumnRange(dateRange); + return exclusiveUpper(dateRange).minus(1); + } + + @Override + protected Field exclusiveUpper(ColumnDateRange dateRange) { checkIsSingleColumnRange(dateRange); return DSL.function("upper", Date.class, dateRange.getRange()); } @@ -100,24 +157,23 @@ protected Field upper(ColumnDateRange dateRange) { @Override protected ColumnDateRange calcRange(Field start, Interval interval) { Field intervalExpression = INTERVAL_MAP.get(interval); - return ColumnDateRange.of(functionProvider.daterange( + return ofStartAndEnd( calcStartDate(start, intervalExpression), - calcEndDate(start, intervalExpression), - "[)" - )); + calcEndDate(start, intervalExpression) + ); } private Field calcStartDate(Field start, Field intervalExpression) { Field intSeriesField = intSeriesField(); - return multiplyByInterval(start, intervalExpression, intSeriesField, Offset.MINUS_ONE); + return addMultipliedInterval(start, intervalExpression, intSeriesField, Offset.MINUS_ONE); } private Field calcEndDate(Field start, Field intervalExpression) { Field intSeriesField = intSeriesField(); - return multiplyByInterval(start, intervalExpression, intSeriesField, Offset.NONE); + return addMultipliedInterval(start, intervalExpression, intSeriesField, Offset.NONE); } - private Field multiplyByInterval(Field start, Field intervalExpression, Field amount, Offset offset) { + private Field addMultipliedInterval(Field start, Field intervalExpression, Field amount, Offset offset) { Field multiplier = amount.plus(offset.getOffset()); Field shiftedDate = DSL.field( "{0} + {1} {2} * {3}", @@ -136,7 +192,7 @@ private Field dateTruncate(Field field, Field date) { } private Field addQuarters(Field start, Field amountOfQuarters, Offset offset) { - return multiplyByInterval(start, INTERVAL_MAP.get(Interval.QUARTER_INTERVAL), amountOfQuarters, offset); + return addMultipliedInterval(start, INTERVAL_MAP.get(Interval.QUARTER_INTERVAL), amountOfQuarters, offset); } private Field jumpToYearStart(Field date) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/README.md b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/README.md index 9ea523fd93..81ac0c26e9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/README.md +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/README.md @@ -11,8 +11,10 @@ to dialect, but the overall process is the same. This document is using Postgres 1. [For absolute forms](#absolute-forms) 2. [For entity date queries](#entity-date) 3. [Stratification tables](#stratification-tables) -3. [Feature conversion](#feature-conversion) -4. [Left-join converted features](#left-join-converted-features-with-the-full-stratification-table-for-the-final-select) +3. [Relative stratification](#relative-stratification) +4. [Full stratification table](#full-stratification-table) +5. [Feature conversion](#feature-conversion) +6. [Left-join converted features](#left-join-converted-features-with-the-full-stratification-table-for-the-final-select) ## Prerequisite conversion @@ -32,18 +34,19 @@ from "DUMMY"; -- "DUMMY" is SAP HANAs built-in no-op table ### Absolute forms -For an absolute form, we only care for the primary ID, so we extract the primary IDs and discard all other fields from -the prerequisite query. The `stratification_bounds` represent the absolute forms date range. They define the required -complete stratification window. We group by primary ID to keep only distinct entries for each entity and discard any -duplicated entries which, for example, might occur due to a preceding secondary id query. +For an absolute form, we only care for the primary ID, so we extract the primary IDs and discard the validitiy date. For entity date queries, it's kept. The `stratification_bounds` represent the absolute forms date +range. They define the required complete stratification window. We group by primary ID (and validity date, if present) +to keep only distinct entries for each entity and discard any duplicated entries which, for example, might occur due +to a preceding secondary id query. **CTE:** `extract_ids` ```sql select "primary_id", + "validity_date", -- the validity date is only kept in case we convert an entity date query daterange('2012-06-01', '2012-09-30', '[]') as "stratification_bounds" from "external" -group by "primary_id" +group by "primary_id", "validity_date" ``` ### Entity date @@ -189,9 +192,124 @@ in our case. The result looks like this: | 1 | YEARS | 2 | \[2013-04-01,2014-04-01\) | | 1 | YEARS | 3 | \[2014-04-01,2014-12-18\) | -**CTE:** `full_stratification` +## Relative stratification + +Like for entity date queries, we need to extract the primary ID and the corresponding validity date for each distinct +entity. + +**CTE:** `extract_ids` + +```sql +select "primary_id", + unnest("validity_date") as "validity_date" -- unnesting is only required for dialects with multiranges +from "external" +group by "primary_id", "validity_date" +``` + +Next, we need to find the index selector date: For each validity date of an entity, we calculate either: + +- the `EARLIEST` date of the given range +- the `LATEST` date of the given range +- or a `RANDOM` date within the given range + depending on the relative forms + [index selector](../../../apiv1/query/concept/specific/temporal/TemporalSamplerFactory.java). + +**CTE:** `index_selector` + +```sql +select "primary_id", + min(lower("dates")) as "index_selector" -- example for EARLIEST +from "extract_ids" +group by "primary_id" +``` + +Using the index selector date, we can now define the index start dates from where the feature and/or outcome ranges of +the relative form start. Their exact calculation depends on the +[index placement](../../../apiv1/forms/IndexPlacement.java) of the relative form. + +For the `BEFORE` and `AFTER` placement, the positive start (outcome range) and negative start (feature range) is the +same. Only for the `NEUTRAL` placement, the start dates differ. + +We take the `BEFORE` placement with the time unit `QUARTERS` as an example: We jump to the next quarters start of the +index selector date. From these index start dates, we will start in the next step when calculating the stratification +windows of the feature and outcome range. + +**CTE:** `index_start` + +```sql +select "primary_id", + "index_selector", + (date_trunc('year', "index_selector") + + interval '3 months' * (extract(quarter from ("index_selector" + -1)) + 0))::date as "index_start_positive", + (date_trunc('year', "index_selector") + + interval '3 months' * (extract(quarter from ("index_selector" + -1)) + 0))::date as "index_start_negative" +from "index_selector" +``` + +The last step before calculating the actual stratification windows is to calculate the min and max date of the +stratification for each entity, which is basically the lower bound of the feature range and the upper bound of the +outcome range. Assuming a time count before of 6 quarters and a time count after of 2 quarters, the calculation looks +like the following: + +**CTE:** `total_bounds` + +```sql +select "primary_id", + daterange( + ("index_start_negative" + interval '3 months' * (-6 + 0))::date, + ("index_start_positive" + interval '3 months' * (2 + 0))::date, + '[)' + ) as "stratification_bounds", + "index_selector", + "index_start_positive", + "index_start_negative" +from "index_start" +``` + +We will intersect this range with the calculated ranges in the next step, because calculated ranges always span over +whole intervals (`YEARS`, `QUARTERS`), but must be ultimately bound by the complete min and max dates. In the following +example, we take a look at how the `YEARS` resolution is calculated. We still assume a time count before of 6 quarters +and a time count after of 2 quarters. This means we need to jump 2 years back for the feature range and 1 year forward +for the outcome range. Similar to absolute stratification, we create a set of date ranges by manipulating the start +dates by adding positive or negative time intervals times an index. The index is again taken from a generated integer +series. + +**CTE:** `years` + +```sql +-- feature range +select "primary_id", + 'YEARS' as "resolution", + "index", + "index_selector", + daterange( + ("index_start_negative" + interval '1 year' * ("index" + 0))::date, + ("index_start_negative" + interval '1 year' * ("index" + 1))::date, + '[)' + ) * "stratification_bounds" as "stratification_bounds" +from "total_bounds", + generate_series(-2, -1) as "index" -- -2 -> we jump 2 years back +union all +-- outcome range +select "primary_id", + 'YEARS' as "resolution", + "index", + "index_selector", + daterange( + ("index_start_positive" + interval '1 year' * ("index" + -1))::date, + ("index_start_positive" + interval '1 year' * ("index" + 0))::date, + '[)' + ) * "stratification_bounds" as "stratification_bounds" +from "total_bounds", + generate_series(1, 1) as "index" -- 1 -> we jump 1 year forward +``` + +Using the `stratification_bounds`, representing the min and max stratification date calculated in the previous step, +we intersect this range with our calculated time frame to generate stratification windows which are correctly bound. + +## Full stratification table -Now, we union all the resolution tables. +Now, we union all the resolution tables to obtain the full stratification table. ```sql select "complete"."primary_id", diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java new file mode 100644 index 0000000000..baaa89b7f6 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java @@ -0,0 +1,313 @@ +package com.bakdata.conquery.sql.conversion.forms; + +import static com.bakdata.conquery.sql.conversion.forms.FormConstants.INDEX_SELECTOR; +import static com.bakdata.conquery.sql.conversion.forms.FormConstants.INDEX_START_NEGATIVE; +import static com.bakdata.conquery.sql.conversion.forms.FormConstants.INDEX_START_POSITIVE; + +import java.sql.Date; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import com.bakdata.conquery.apiv1.forms.IndexPlacement; +import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; +import com.bakdata.conquery.apiv1.query.concept.specific.temporal.TemporalSamplerFactory; +import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.models.forms.managed.RelativeFormQuery; +import com.bakdata.conquery.models.forms.util.CalendarUnit; +import com.bakdata.conquery.models.forms.util.Resolution; +import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.google.common.base.Preconditions; +import lombok.RequiredArgsConstructor; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.impl.DSL; + +@RequiredArgsConstructor +class RelativeStratification { + + private final QueryStep baseStep; + private final StratificationFunctions stratificationFunctions; + private final SqlFunctionProvider functionProvider; + + public QueryStep createRelativeStratificationTable(RelativeFormQuery form) { + + // we want to create the stratification for each distinct validity date range of an entity, + // so we first need to unnest the validity date in case it is a multirange + QueryStep withUnnestedValidityDate = functionProvider.unnestValidityDate(baseStep, FormCteStep.UNNEST_DATES.getSuffix()); + + QueryStep indexSelectorStep = createIndexSelectorStep(form, withUnnestedValidityDate); + QueryStep indexStartStep = createIndexStartStep(form, indexSelectorStep); + QueryStep totalBoundsStep = createTotalBoundsStep(form, indexStartStep); + + List tables = form.getResolutionsAndAlignmentMap().stream() + .map(ExportForm.ResolutionAndAlignment::getResolution) + .map(resolution -> createResolutionTable(totalBoundsStep, resolution, form)) + .toList(); + + List predecessors = new ArrayList<>(); + predecessors.add(baseStep); + if (baseStep != withUnnestedValidityDate) { + predecessors.add(withUnnestedValidityDate); + } + predecessors.addAll(List.of(indexSelectorStep, indexStartStep, totalBoundsStep)); + + return StratificationTableFactory.unionResolutionTables(tables, predecessors); + } + + /** + * Creates {@link QueryStep} containing the date select for the corresponding {@link TemporalSamplerFactory} of the relative form. + */ + private QueryStep createIndexSelectorStep(RelativeFormQuery form, QueryStep prerequisite) { + + Selects predecessorSelects = prerequisite.getQualifiedSelects(); + ColumnDateRange validityDate = predecessorSelects.getValidityDate() + .orElseThrow(() -> new IllegalStateException("Expecting a validity date to be present")); + Field indexDate = stratificationFunctions.indexSelectorField(form.getIndexSelector(), validityDate) + .as(SharedAliases.INDEX_SELECTOR.getAlias()); + + Selects selects = Selects.builder() + .ids(predecessorSelects.getIds()) + .sqlSelect(new FieldWrapper<>(indexDate)) + .build(); + + return QueryStep.builder() + .cteName(FormCteStep.INDEX_SELECTOR.getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(prerequisite.getCteName())) + .groupBy(predecessorSelects.getIds().toFields()) + .build(); + } + + /** + * Creates {@link QueryStep} containing the start date selects ({@link FormConstants#INDEX_START_POSITIVE} and {@link FormConstants#INDEX_START_NEGATIVE}) + * from where the feature and/or outcome ranges of the relative form start. Their placement depends on the relative forms {@link IndexPlacement}. + */ + private QueryStep createIndexStartStep(RelativeFormQuery form, QueryStep indexSelectorStep) { + + List> indexStartFields = stratificationFunctions.indexStartFields(form.getIndexPlacement(), form.getTimeUnit()).stream() + .map(FieldWrapper::new) + .toList(); + + // add index start fields to qualified selects of previous step + Selects selects = indexSelectorStep.getQualifiedSelects() + .toBuilder() + .sqlSelects(indexStartFields) + .build(); + + return QueryStep.builder() + .cteName(FormCteStep.INDEX_START.getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(indexSelectorStep.getCteName())) + .build(); + } + + /** + * Creates a {@link QueryStep} containing the minimum and maximum stratification date for each entity. + */ + private QueryStep createTotalBoundsStep(RelativeFormQuery form, QueryStep indexStartStep) { + + Interval interval = getInterval(form.getTimeUnit(), Resolution.COMPLETE); + Range intRange = toGenerateSeriesBounds(form, Resolution.COMPLETE); + + Field minStratificationDate = stratificationFunctions.shiftByInterval(INDEX_START_NEGATIVE, interval, DSL.val(intRange.getMin()), Offset.NONE); + Field maxStratificationDate = stratificationFunctions.shiftByInterval(INDEX_START_POSITIVE, interval, DSL.val(intRange.getMax()), Offset.NONE); + ColumnDateRange minAndMaxStratificationDate = stratificationFunctions.ofStartAndEnd(minStratificationDate, maxStratificationDate) + .as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); + + // add min and max stratification date to qualified selects of previous step + Selects selects = indexStartStep.getQualifiedSelects() + .toBuilder() + .stratificationDate(Optional.of(minAndMaxStratificationDate)) + .build(); + + return QueryStep.builder() + .cteName(FormCteStep.TOTAL_BOUNDS.getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(indexStartStep.getCteName())) + .build(); + } + + private QueryStep createResolutionTable(QueryStep indexStartStep, Resolution resolution, RelativeFormQuery form) { + return switch (resolution) { + case COMPLETE -> createCompleteTable(indexStartStep, form); + case YEARS, QUARTERS, DAYS -> createIntervalTable(indexStartStep, resolution, form); + }; + } + + private QueryStep createCompleteTable(QueryStep totalBoundsStep, RelativeFormQuery form) { + + Selects predecessorSelects = totalBoundsStep.getQualifiedSelects(); + Interval interval = getInterval(form.getTimeUnit(), Resolution.COMPLETE); + Range intRange = toGenerateSeriesBounds(form, Resolution.COMPLETE); + + QueryStep featureTable = form.getTimeCountBefore() > 0 ? createCompleteFeatureTable(predecessorSelects, interval, intRange, totalBoundsStep) : null; + QueryStep outcomeTable = form.getTimeCountAfter() > 0 ? createCompleteOutcomeTable(predecessorSelects, interval, intRange, totalBoundsStep) : null; + + return QueryStep.createUnionStep( + Stream.concat(Stream.ofNullable(outcomeTable), Stream.ofNullable(featureTable)).toList(), + FormCteStep.COMPLETE.getSuffix(), + Collections.emptyList() + ); + } + + private QueryStep createCompleteFeatureTable(Selects predecessorSelects, Interval interval, Range intRange, QueryStep totalBoundsStep) { + Field featureIndex = DSL.field(DSL.val(-1)).as(SharedAliases.INDEX.getAlias()); + SqlIdColumns featureIds = predecessorSelects.getIds().withRelativeStratification(Resolution.COMPLETE, featureIndex, INDEX_SELECTOR); + Field rangeStart = stratificationFunctions.shiftByInterval(INDEX_START_NEGATIVE, interval, DSL.val(intRange.getMin()), Offset.NONE); + return createIntervalStep(featureIds, rangeStart, INDEX_START_NEGATIVE, Optional.empty(), totalBoundsStep); + } + + private QueryStep createCompleteOutcomeTable(Selects predecessorSelects, Interval interval, Range intRange, QueryStep totalBoundsStep) { + Field outcomeIndex = DSL.field(DSL.val(1)).as(SharedAliases.INDEX.getAlias()); + SqlIdColumns outcomeIds = predecessorSelects.getIds().withRelativeStratification(Resolution.COMPLETE, outcomeIndex, INDEX_SELECTOR); + Field rangeEnd = stratificationFunctions.shiftByInterval(INDEX_START_POSITIVE, interval, DSL.val(intRange.getMax()), Offset.NONE); + return createIntervalStep(outcomeIds, INDEX_START_POSITIVE, rangeEnd, Optional.empty(), totalBoundsStep); + } + + private QueryStep createIntervalTable(QueryStep totalBoundsStep, Resolution resolution, RelativeFormQuery form) { + + Field seriesIndex = stratificationFunctions.intSeriesField(); + Selects predecessorSelects = totalBoundsStep.getQualifiedSelects(); + SqlIdColumns ids = predecessorSelects.getIds().withRelativeStratification(resolution, seriesIndex, INDEX_SELECTOR); + Interval interval = getInterval(form.getTimeUnit(), resolution); + Range bounds = toGenerateSeriesBounds(form, resolution); + + QueryStep timeBeforeStep = createFeatureTable(totalBoundsStep, interval, seriesIndex, bounds, ids); + QueryStep timeAfterStep = createOutcomeTable(totalBoundsStep, interval, seriesIndex, bounds, ids); + + return QueryStep.createUnionStep( + List.of(timeBeforeStep, timeAfterStep), + FormCteStep.stratificationCte(resolution).getSuffix(), + Collections.emptyList() + ); + } + + private QueryStep createOutcomeTable(QueryStep totalBoundsStep, Interval interval, Field seriesIndex, Range bounds, SqlIdColumns ids) { + Field outcomeRangeStart = stratificationFunctions.shiftByInterval(INDEX_START_POSITIVE, interval, seriesIndex, Offset.MINUS_ONE); + Field outcomeRangeEnd = stratificationFunctions.shiftByInterval(INDEX_START_POSITIVE, interval, seriesIndex, Offset.NONE); + Table outcomeSeries = stratificationFunctions.generateIntSeries(1, bounds.getMax()).as(SharedAliases.INDEX.getAlias()); + return createIntervalStep(ids, outcomeRangeStart, outcomeRangeEnd, Optional.of(outcomeSeries), totalBoundsStep); + } + + private QueryStep createFeatureTable(QueryStep totalBoundsStep, Interval interval, Field seriesIndex, Range bounds, SqlIdColumns ids) { + Field featureRangeStart = stratificationFunctions.shiftByInterval(INDEX_START_NEGATIVE, interval, seriesIndex, Offset.NONE); + Field featureRangeEnd = stratificationFunctions.shiftByInterval(INDEX_START_NEGATIVE, interval, seriesIndex, Offset.ONE); + Table featureSeries = stratificationFunctions.generateIntSeries(bounds.getMin(), -1).as(SharedAliases.INDEX.getAlias()); + return createIntervalStep(ids, featureRangeStart, featureRangeEnd, Optional.of(featureSeries), totalBoundsStep); + } + + private QueryStep createIntervalStep( + SqlIdColumns ids, + Field rangeStart, + Field rangeEnd, + Optional> seriesTable, + QueryStep predecessor + ) { + Preconditions.checkArgument( + predecessor.getSelects().getStratificationDate().isPresent(), + "Expecting %s to contain a stratification date representing the min and max stratification bounds" + ); + ColumnDateRange finalRange = functionProvider.intersection( + stratificationFunctions.ofStartAndEnd(rangeStart, rangeEnd), + predecessor.getQualifiedSelects().getStratificationDate().get() + ) + .as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); + + Selects selects = Selects.builder() + .ids(ids) + .stratificationDate(Optional.of(finalRange)) + .build(); + + QueryStep.QueryStepBuilder queryStep = QueryStep.builder() + .selects(selects) + .fromTable(QueryStep.toTableLike(predecessor.getCteName())); + + seriesTable.ifPresent(queryStep::fromTable); + return queryStep.build(); + } + + /** + * Adjusts the {@link RelativeFormQuery#getTimeCountBefore()} && {@link RelativeFormQuery#getTimeCountAfter()} bounds, so they fit the SQL approach. + * Take time unit QUARTERS and Resolution YEARS as an example: If the time counts are not divisible by 4 (because 1 year == 4 quarters), we need to round + * up for each starting year. 5 Quarters mean 2 years we have to consider when creating the stratification. + */ + private static Range toGenerateSeriesBounds(RelativeFormQuery relativeForm, Resolution resolution) { + + int timeCountBefore; + int timeCountAfter; + + switch (relativeForm.getTimeUnit()) { + case QUARTERS -> { + if (resolution == Resolution.YEARS) { + timeCountBefore = divideAndRoundUp(relativeForm.getTimeCountBefore(), 4); + timeCountAfter = divideAndRoundUp(relativeForm.getTimeCountAfter(), 4); + } + else { + timeCountBefore = relativeForm.getTimeCountBefore(); + timeCountAfter = relativeForm.getTimeCountAfter(); + } + } + case DAYS -> { + switch (resolution) { + case COMPLETE, DAYS -> { + timeCountBefore = relativeForm.getTimeCountBefore(); + timeCountAfter = relativeForm.getTimeCountAfter(); + } + case YEARS -> { + timeCountBefore = divideAndRoundUp(relativeForm.getTimeCountBefore(), Interval.YEAR_AS_DAYS_INTERVAL.getAmount()); + timeCountAfter = divideAndRoundUp(relativeForm.getTimeCountAfter(), Interval.YEAR_AS_DAYS_INTERVAL.getAmount()); + } + case QUARTERS -> { + timeCountBefore = divideAndRoundUp(relativeForm.getTimeCountBefore(), Interval.NINETY_DAYS_INTERVAL.getAmount()); + timeCountAfter = divideAndRoundUp(relativeForm.getTimeCountAfter(), Interval.NINETY_DAYS_INTERVAL.getAmount()); + } + default -> throw new CombinationNotSupportedException(relativeForm.getTimeUnit(), resolution); + } + } + default -> throw new CombinationNotSupportedException(relativeForm.getTimeUnit(), resolution); + } + + return Range.of( + - timeCountBefore, + timeCountAfter + ); + } + + private static int divideAndRoundUp(int numerator, int denominator) { + if (denominator == 0) { + throw new IllegalArgumentException("Denominator cannot be zero."); + } + return (int) Math.ceil((double) numerator / denominator); + } + + /** + * @return The interval expression which will be multiplied by the {@link StratificationFunctions#intSeriesField()} and added to the + * {@link SharedAliases#INDEX_START_NEGATIVE} or {@link SharedAliases#INDEX_START_POSITIVE}. + */ + private static Interval getInterval(CalendarUnit timeUnit, Resolution resolution) { + return switch (timeUnit) { + case QUARTERS -> switch (resolution) { + case COMPLETE, QUARTERS -> Interval.QUARTER_INTERVAL; + case YEARS -> Interval.ONE_YEAR_INTERVAL; + case DAYS -> Interval.ONE_DAY_INTERVAL; + }; + case DAYS -> switch (resolution) { + case COMPLETE, DAYS -> Interval.ONE_DAY_INTERVAL; + case YEARS -> Interval.YEAR_AS_DAYS_INTERVAL; + case QUARTERS -> Interval.NINETY_DAYS_INTERVAL; + }; + default -> throw new CombinationNotSupportedException(timeUnit, resolution); + }; + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java index 7de94b1ff3..a997cf7dc9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.sql.conversion.forms; import static com.bakdata.conquery.sql.conversion.forms.FormConstants.DAY_ALIGNED_COUNT; +import static com.bakdata.conquery.sql.conversion.forms.FormConstants.INDEX_SELECTOR; import static com.bakdata.conquery.sql.conversion.forms.FormConstants.INDEX_START; import static com.bakdata.conquery.sql.conversion.forms.FormConstants.QUARTER_ALIGNED_COUNT; import static com.bakdata.conquery.sql.conversion.forms.FormConstants.QUARTER_END; @@ -20,7 +21,11 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import com.bakdata.conquery.ConqueryConstants; +import com.bakdata.conquery.apiv1.forms.IndexPlacement; import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; +import com.bakdata.conquery.apiv1.query.concept.specific.temporal.TemporalSamplerFactory; +import com.bakdata.conquery.models.forms.util.CalendarUnit; import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.HanaSqlFunctionProvider; @@ -28,7 +33,6 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import org.jetbrains.annotations.NotNull; import org.jooq.Condition; import org.jooq.Field; import org.jooq.Record; @@ -45,6 +49,10 @@ static StratificationFunctions create(ConversionContext context) { }; } + public ColumnDateRange ofStartAndEnd(Field start, Field end) { + return ColumnDateRange.of(start, end); // needs to be overwritten for dialects that support single-column ranges + } + protected abstract SqlFunctionProvider getFunctionProvider(); /** @@ -53,9 +61,14 @@ static StratificationFunctions create(ConversionContext context) { protected abstract Field lower(ColumnDateRange dateRange); /** - * Extract the upper bound from a given daterange. + * Extract the inclusive upper bound from a given daterange. + */ + protected abstract Field inclusiveUpper(ColumnDateRange dateRange); + + /** + * Extract the exclusive upper bound from a given daterange. */ - protected abstract Field upper(ColumnDateRange dateRange); + protected abstract Field exclusiveUpper(ColumnDateRange dateRange); /** * Calculates the start and end date based on the given start date and an interval expression. @@ -70,28 +83,38 @@ static StratificationFunctions create(ConversionContext context) { /** * Determines the start of the year based on the lower bound of the provided date range. */ - public abstract Field yearStart(ColumnDateRange dateRange); + public abstract Field lowerBoundYearStart(ColumnDateRange dateRange); /** * Determines the exclusive end (first day of the next year) of the upper bound of the provided date range. */ - public abstract Field yearEnd(ColumnDateRange dateRange); + public abstract Field upperBoundYearEnd(ColumnDateRange dateRange); /** - * Determines the start of the next year based on the upper bound of the provided date range, but aligned on the quarter of the lower bound of the + * Determines the end of the upper bound of the provided date range, but aligned on the quarter of the lower bound of the * provided daterange. */ - public abstract Field yearEndQuarterAligned(ColumnDateRange dateRange); + public abstract Field upperBoundYearEndQuarterAligned(ColumnDateRange dateRange); /** * Calculates the start of the quarter using the lower bound of the provided date range. */ - public abstract Field quarterStart(ColumnDateRange dateRange); + public abstract Field lowerBoundQuarterStart(ColumnDateRange dateRange); + + /** + * Calculates the start of the quarter of the given date. + */ + protected abstract Field jumpToQuarterStart(Field date); /** * Calculates the exclusive end (first day of the next quarter) of the upper bound of the provided date range. */ - public abstract Field quarterEnd(ColumnDateRange dateRange); + public abstract Field upperBoundQuarterEnd(ColumnDateRange dateRange); + + /** + * Calculates the start of the next quarter of the given date. + */ + protected abstract Field jumpToNextQuarterStart(Field date); /** * The int field generated by the {@link #generateIntSeries(int, int)} @@ -103,6 +126,66 @@ static StratificationFunctions create(ConversionContext context) { */ public abstract Table generateIntSeries(int start, int end); + /** + * Generates a date field representing the {@link TemporalSamplerFactory} using the given validity date range. + */ + public abstract Field indexSelectorField(TemporalSamplerFactory indexSelector, ColumnDateRange validityDate); + + /** + * Shift's a start date by an interval times an amount. The offset will we added to the amount before multiplying. + */ + public abstract Field shiftByInterval(Field startDate, Interval interval, Field amount, Offset offset); + + /** + * Generates the start and end field for the respective {@link IndexPlacement} and {@link CalendarUnit timeUnit}. + */ + public List> indexStartFields(IndexPlacement indexPlacement, CalendarUnit timeUnit) { + + Field positiveStart; + Field negativeStart; + + switch (timeUnit) { + case QUARTERS -> { + switch (indexPlacement) { + case BEFORE -> { + Field nextQuarterStart = jumpToNextQuarterStart(INDEX_SELECTOR); + positiveStart = nextQuarterStart; + negativeStart = nextQuarterStart; + } + case AFTER -> { + Field quarterStart = jumpToQuarterStart(INDEX_SELECTOR); + positiveStart = quarterStart; + negativeStart = quarterStart; + } + case NEUTRAL -> { + positiveStart = jumpToNextQuarterStart(INDEX_SELECTOR); + negativeStart = jumpToQuarterStart(INDEX_SELECTOR); + } + default -> throw new CombinationNotSupportedException(indexPlacement, timeUnit); + } + } + case DAYS -> { + switch (indexPlacement) { + case BEFORE, AFTER -> { + positiveStart = INDEX_SELECTOR; + negativeStart = INDEX_SELECTOR; + } + case NEUTRAL -> { + positiveStart = getFunctionProvider().addDays(INDEX_SELECTOR, DSL.val(1)); + negativeStart = INDEX_SELECTOR; + } + default -> throw new CombinationNotSupportedException(indexPlacement, timeUnit); + } + } + default -> throw new CombinationNotSupportedException(indexPlacement, timeUnit); + } + + return List.of( + positiveStart.as(SharedAliases.INDEX_START_POSITIVE.getAlias()), + negativeStart.as(SharedAliases.INDEX_START_NEGATIVE.getAlias()) + ); + } + /** * Calculates the count of the required resolution windows based on the provided resolution, alignment, and date range. * @@ -115,7 +198,7 @@ public Field calculateResolutionWindowCount(ExportForm.ResolutionAndAli case COMPLETE -> DSL.val(1); case YEARS -> calculateResolutionWindowForYearResolution(resolutionAndAlignment, bounds, functionProvider); case QUARTERS -> calculateResolutionWindowForQuarterResolution(resolutionAndAlignment, bounds, functionProvider); - case DAYS -> functionProvider.dateDistance(ChronoUnit.DAYS, lower(bounds), upper(bounds)) + case DAYS -> functionProvider.dateDistance(ChronoUnit.DAYS, lower(bounds), exclusiveUpper(bounds)) .as(SharedAliases.DAY_ALIGNED_COUNT.getAlias()); }; } @@ -138,7 +221,7 @@ public ColumnDateRange createStratificationRange(ExportForm.ResolutionAndAlignme } /** - * The index field for the corresponding resolution index {@link com.bakdata.conquery.ConqueryConstants#CONTEXT_INDEX_INFO}. + * The index field for the corresponding resolution index {@link ConqueryConstants#CONTEXT_INDEX_INFO}. */ public Field index(SqlIdColumns ids, Optional stratificationBounds) { @@ -177,7 +260,7 @@ private Field calculateResolutionWindowForQuarterResolution( case QUARTER -> functionProvider.dateDistance(ChronoUnit.MONTHS, QUARTER_START, QUARTER_END) .divide(MONTHS_PER_QUARTER) .as(SharedAliases.QUARTER_ALIGNED_COUNT.getAlias()); - case DAY -> functionProvider.dateDistance(ChronoUnit.DAYS, lower(bounds), upper(bounds)) + case DAY -> functionProvider.dateDistance(ChronoUnit.DAYS, lower(bounds), exclusiveUpper(bounds)) .plus(89) .divide(DAYS_PER_QUARTER) .as(SharedAliases.DAY_ALIGNED_COUNT.getAlias()); @@ -195,7 +278,7 @@ private Field calculateResolutionWindowForYearResolution( .as(SharedAliases.YEAR_ALIGNED_COUNT.getAlias()); case QUARTER -> functionProvider.dateDistance(ChronoUnit.YEARS, QUARTER_START, YEAR_END_QUARTER_ALIGNED) .as(SharedAliases.QUARTER_ALIGNED_COUNT.getAlias()); - case DAY -> functionProvider.dateDistance(ChronoUnit.DAYS, lower(bounds), upper(bounds)) + case DAY -> functionProvider.dateDistance(ChronoUnit.DAYS, lower(bounds), exclusiveUpper(bounds)) .plus(364) .divide(DAYS_PER_YEAR) .as(SharedAliases.DAY_ALIGNED_COUNT.getAlias()); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java index 7f2c280380..c02cdb9f45 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java @@ -1,29 +1,17 @@ package com.bakdata.conquery.sql.conversion.forms; -import java.sql.Date; import java.util.List; -import java.util.Optional; import java.util.stream.Stream; import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; -import com.bakdata.conquery.models.forms.util.Resolution; -import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.models.forms.managed.RelativeFormQuery; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; import com.google.common.base.Preconditions; import lombok.AccessLevel; import lombok.Getter; import lombok.RequiredArgsConstructor; -import org.jooq.Condition; -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Table; -import org.jooq.impl.DSL; @Getter(AccessLevel.PROTECTED) @RequiredArgsConstructor @@ -42,162 +30,17 @@ public StratificationTableFactory(QueryStep baseStep, ConversionContext context) this.functionProvider = context.getSqlDialect().getFunctionProvider(); } - public QueryStep createStratificationTable(List resolutionAndAlignments) { - - QueryStep intSeriesStep = createIntSeriesStep(); - QueryStep indexStartStep = createIndexStartStep(); - - List tables = resolutionAndAlignments.stream() - .map(resolutionAndAlignment -> createResolutionTable(indexStartStep, resolutionAndAlignment)) - .toList(); - - List predecessors = List.of(baseStep, intSeriesStep, indexStartStep); - return unionResolutionTables(tables, predecessors); - } - - private QueryStep createIntSeriesStep() { - - // not actually required, but Selects expect at least 1 SqlIdColumn - Field rowNumber = DSL.rowNumber().over().coerce(Object.class); - SqlIdColumns ids = new SqlIdColumns(rowNumber); - - FieldWrapper seriesIndex = new FieldWrapper<>(stratificationFunctions.intSeriesField()); - - Selects selects = Selects.builder() - .ids(ids) - .sqlSelect(seriesIndex) - .build(); - - Table seriesTable = stratificationFunctions.generateIntSeries(INDEX_START, INDEX_END) - .as(SharedAliases.SERIES_INDEX.getAlias()); - - return QueryStep.builder() - .cteName(FormCteStep.INT_SERIES.getSuffix()) - .selects(selects) - .fromTable(seriesTable) - .build(); - } - - private QueryStep createIndexStartStep() { - - Selects baseStepSelects = baseStep.getQualifiedSelects(); - Preconditions.checkArgument(baseStepSelects.getStratificationDate().isPresent(), "The base step must have a stratification date set"); - ColumnDateRange bounds = baseStepSelects.getStratificationDate().get(); - - Field indexStart = stratificationFunctions.absoluteIndexStartDate(bounds).as(SharedAliases.INDEX_START.getAlias()); - Field yearStart = stratificationFunctions.yearStart(bounds).as(SharedAliases.YEAR_START.getAlias()); - Field yearEnd = stratificationFunctions.yearEnd(bounds).as(SharedAliases.YEAR_END.getAlias()); - Field yearEndQuarterAligned = stratificationFunctions.yearEndQuarterAligned(bounds).as(SharedAliases.YEAR_END_QUARTER_ALIGNED.getAlias()); - Field quarterStart = stratificationFunctions.quarterStart(bounds).as(SharedAliases.QUARTER_START.getAlias()); - Field quarterEnd = stratificationFunctions.quarterEnd(bounds).as(SharedAliases.QUARTER_END.getAlias()); - - List> startDates = Stream.of( - indexStart, - yearStart, - yearEnd, - yearEndQuarterAligned, - quarterStart, - quarterEnd - ) - .map(FieldWrapper::new) - .toList(); - - Selects selects = Selects.builder() - .ids(baseStepSelects.getIds()) - .stratificationDate(Optional.of(bounds)) - .sqlSelects(startDates) - .build(); - - return QueryStep.builder() - .cteName(FormCteStep.INDEX_START.getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(baseStep.getCteName())) - .build(); - } - - private QueryStep createResolutionTable(QueryStep indexStartStep, ExportForm.ResolutionAndAlignment resolutionAndAlignment) { - return switch (resolutionAndAlignment.getResolution()) { - case COMPLETE -> createCompleteTable(); - case YEARS, QUARTERS, DAYS -> createIntervalTable(indexStartStep, resolutionAndAlignment); - }; - } - - private QueryStep createCompleteTable() { - - Selects baseStepSelects = baseStep.getQualifiedSelects(); - - // complete range shall have a null index because it spans the complete range, but we set it to 1 to ensure we can join tables on index, - // because a condition involving null in a join (e.g., null = some_value or null = null) always evaluates to false - Field index = DSL.val(1, Integer.class).as(SharedAliases.INDEX.getAlias()); - SqlIdColumns ids = baseStepSelects.getIds().withAbsoluteStratification(Resolution.COMPLETE, index); - - ColumnDateRange completeRange = baseStepSelects.getStratificationDate().get(); - - Selects selects = Selects.builder() - .ids(ids) - .stratificationDate(Optional.of(completeRange)) - .build(); - - return QueryStep.builder() - .cteName(FormCteStep.COMPLETE.getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(baseStep.getCteName())) - .build(); - } - - private QueryStep createIntervalTable(QueryStep indexStartStep, ExportForm.ResolutionAndAlignment resolutionAndAlignment) { - - QueryStep countsCte = createCountsCte(indexStartStep, resolutionAndAlignment); - Preconditions.checkArgument(countsCte.getSelects().getStratificationDate().isPresent(), "The countsCte must have a stratification date set"); - Selects countsCteSelects = countsCte.getQualifiedSelects(); - - ColumnDateRange stratificationRange = stratificationFunctions.createStratificationRange( - resolutionAndAlignment, - countsCteSelects.getStratificationDate().get() - ); - - Field index = stratificationFunctions.index(countsCteSelects.getIds(), countsCte.getQualifiedSelects().getStratificationDate()); - SqlIdColumns ids = countsCteSelects.getIds().withAbsoluteStratification(resolutionAndAlignment.getResolution(), index); - - Selects selects = Selects.builder() - .ids(ids) - .stratificationDate(Optional.ofNullable(stratificationRange)) - .build(); - - Condition stopOnMaxResolutionWindowCount = stratificationFunctions.stopOnMaxResolutionWindowCount(resolutionAndAlignment); - - return QueryStep.builder() - .cteName(FormCteStep.stratificationCte(resolutionAndAlignment.getResolution()).getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(countsCte.getCteName())) - .fromTable(QueryStep.toTableLike(FormCteStep.INT_SERIES.getSuffix())) - .conditions(List.of(stopOnMaxResolutionWindowCount)) - .predecessor(countsCte) - .build(); + public QueryStep createRelativeStratificationTable(RelativeFormQuery form) { + RelativeStratification relativeStratification = new RelativeStratification(baseStep, stratificationFunctions, functionProvider); + return relativeStratification.createRelativeStratificationTable(form); } - private QueryStep createCountsCte(QueryStep indexStartStep, ExportForm.ResolutionAndAlignment resolutionAndAlignment) { - - Selects indexStartSelects = indexStartStep.getQualifiedSelects(); - Preconditions.checkArgument(indexStartSelects.getStratificationDate().isPresent(), "The indexStartStep must have a stratification date set"); - - Field resolutionWindowCount = stratificationFunctions.calculateResolutionWindowCount( - resolutionAndAlignment, - indexStartSelects.getStratificationDate().get() - ); - - Selects selects = indexStartSelects.toBuilder() - .sqlSelect(new FieldWrapper<>(resolutionWindowCount)) - .build(); - - return QueryStep.builder() - .cteName(FormCteStep.countsCte(resolutionAndAlignment.getResolution()).getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(indexStartStep.getCteName())) - .build(); + public QueryStep createAbsoluteStratificationTable(List resolutionAndAlignments) { + AbsoluteStratification absoluteStratification = new AbsoluteStratification(baseStep, stratificationFunctions); + return absoluteStratification.createStratificationTable(resolutionAndAlignments); } - private QueryStep unionResolutionTables(List unionSteps, List predecessors) { + protected static QueryStep unionResolutionTables(List unionSteps, List predecessors) { Preconditions.checkArgument(!unionSteps.isEmpty(), "Expecting at least 1 resolution table"); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java index 0ce8f4aebb..2b0e2ecdd1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java @@ -1,17 +1,17 @@ package com.bakdata.conquery.sql.conversion.query; -import java.time.LocalDate; import java.util.List; import java.util.Optional; import com.bakdata.conquery.apiv1.query.Query; -import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.forms.FormCteStep; +import com.bakdata.conquery.sql.conversion.forms.FormType; +import com.bakdata.conquery.sql.conversion.forms.StratificationTableFactory; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; @@ -31,25 +31,33 @@ public Class getConversionClass() { @Override public ConversionContext convert(AbsoluteFormQuery form, ConversionContext context) { - QueryStep prerequisite = convertPrerequisite(form.getQuery(), form.getDateRange(), context); + + QueryStep convertedPrerequisite = convertPrerequisite(form, context); + StratificationTableFactory tableFactory = new StratificationTableFactory(convertedPrerequisite, context); + QueryStep stratificationTable = tableFactory.createAbsoluteStratificationTable(form.getResolutionsAndAlignmentMap()); + return formHelper.convertForm( - prerequisite, - form.getResolutionsAndAlignmentMap(), + FormType.ABSOLUTE, + stratificationTable, form.getFeatures(), form.getResultInfos(), context ); } - private static QueryStep convertPrerequisite(Query query, Range formDateRange, ConversionContext context) { + /** + * Converts the given {@link Query} and creates another {@link QueryStep} on top which extracts only the primary id. The form's date range is set + * as stratification range. + */ + private static QueryStep convertPrerequisite(AbsoluteFormQuery absoluteForm, ConversionContext context) { - ConversionContext withConvertedPrerequisite = context.getNodeConversions().convert(query, context); + ConversionContext withConvertedPrerequisite = context.getNodeConversions().convert(absoluteForm.getQuery(), context); Preconditions.checkArgument(withConvertedPrerequisite.getQuerySteps().size() == 1, "Base query conversion should produce exactly 1 QueryStep"); QueryStep convertedPrerequisite = withConvertedPrerequisite.getQuerySteps().get(0); ColumnDateRange bounds = context.getSqlDialect() .getFunctionProvider() - .forCDateRange(CDateRange.of(formDateRange)).as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); + .forCDateRange(CDateRange.of(absoluteForm.getDateRange())).as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); Selects prerequisiteSelects = convertedPrerequisite.getQualifiedSelects(); // we only keep the primary column for the upcoming form diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java index 6f279297eb..808c672a63 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java @@ -2,24 +2,20 @@ import java.util.List; import java.util.Optional; -import java.util.stream.Collectors; -import java.util.stream.Stream; -import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.models.forms.managed.EntityDateQuery; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.forms.FormCteStep; +import com.bakdata.conquery.sql.conversion.forms.FormType; +import com.bakdata.conquery.sql.conversion.forms.StratificationTableFactory; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.google.common.base.Preconditions; import lombok.RequiredArgsConstructor; -import org.jooq.Condition; -import org.jooq.Field; @RequiredArgsConstructor public class EntityDateQueryConverter implements NodeConverter { @@ -34,51 +30,20 @@ public Class getConversionClass() { @Override public ConversionContext convert(EntityDateQuery entityDateQuery, ConversionContext context) { - QueryStep prerequisite = convertPrerequisite(entityDateQuery.getQuery(), context); + QueryStep prerequisite = formHelper.convertPrerequisite(entityDateQuery.getQuery(), context); QueryStep withOverwrittenValidityDateBounds = overwriteBounds(prerequisite, entityDateQuery, context); + StratificationTableFactory tableFactory = new StratificationTableFactory(withOverwrittenValidityDateBounds, context); + QueryStep stratificationTable = tableFactory.createAbsoluteStratificationTable(entityDateQuery.getResolutionsAndAlignments()); return formHelper.convertForm( - withOverwrittenValidityDateBounds, - entityDateQuery.getResolutionsAndAlignments(), + FormType.ENTITY_DATE, + stratificationTable, entityDateQuery.getFeatures(), entityDateQuery.getResultInfos(), context ); } - private static QueryStep convertPrerequisite(Query query, ConversionContext context) { - - ConversionContext withConvertedPrerequisite = context.getNodeConversions().convert(query, context); - Preconditions.checkArgument(withConvertedPrerequisite.getQuerySteps().size() == 1, "Base query conversion should produce exactly 1 QueryStep"); - QueryStep convertedPrerequisite = withConvertedPrerequisite.getQuerySteps().get(0); - - Selects prerequisiteSelects = convertedPrerequisite.getQualifiedSelects(); - // we keep the primary column and the validity date - Selects selects = Selects.builder() - .ids(new SqlIdColumns(prerequisiteSelects.getIds().getPrimaryColumn())) - .validityDate(prerequisiteSelects.getValidityDate()) - .build(); - - // we want to keep each primary column and the corresponding distinct validity date ranges - List> groupByFields = Stream.concat( - Stream.of(prerequisiteSelects.getIds().getPrimaryColumn()), - prerequisiteSelects.getValidityDate().stream().flatMap(validityDate -> validityDate.toFields().stream()) - ) - .collect(Collectors.toList()); - - // filter out entries with a null validity date - Condition dateNotNullCondition = prerequisiteSelects.getValidityDate().get().isNotNull(); - - return QueryStep.builder() - .cteName(FormCteStep.EXTRACT_IDS.getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(convertedPrerequisite.getCteName())) - .conditions(List.of(dateNotNullCondition)) - .groupBy(groupByFields) - .predecessors(List.of(convertedPrerequisite)) - .build(); - } - /** * Computes the intersection of the entity date and the entity date query's daterange. */ diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java index 94cd4545a8..39e3286c44 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java @@ -2,16 +2,20 @@ import java.util.List; import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; -import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; +import com.bakdata.conquery.apiv1.forms.FeatureGroup; import com.bakdata.conquery.apiv1.query.ArrayConceptQuery; import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.forms.StratificationTableFactory; +import com.bakdata.conquery.sql.conversion.forms.FormCteStep; +import com.bakdata.conquery.sql.conversion.forms.FormType; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.ConqueryJoinType; import com.bakdata.conquery.sql.conversion.model.QueryStep; @@ -20,29 +24,64 @@ import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; import com.bakdata.conquery.sql.conversion.model.SqlQuery; +import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; import com.google.common.base.Preconditions; import lombok.RequiredArgsConstructor; +import org.jooq.Condition; import org.jooq.Field; import org.jooq.Record; import org.jooq.Select; import org.jooq.TableLike; +import org.jooq.impl.DSL; @RequiredArgsConstructor public class FormConversionHelper { private final QueryStepTransformer queryStepTransformer; + /** + * Converts the given {@link Query} and creates another {@link QueryStep} on top which extracts only the primary id and the validity dates. + */ + public QueryStep convertPrerequisite(Query query, ConversionContext context) { + + ConversionContext withConvertedPrerequisite = context.getNodeConversions().convert(query, context); + Preconditions.checkArgument(withConvertedPrerequisite.getQuerySteps().size() == 1, "Base query conversion should produce exactly 1 QueryStep"); + QueryStep convertedPrerequisite = withConvertedPrerequisite.getQuerySteps().get(0); + + Selects prerequisiteSelects = convertedPrerequisite.getQualifiedSelects(); + // we keep the primary column and the validity date + Selects selects = Selects.builder() + .ids(new SqlIdColumns(prerequisiteSelects.getIds().getPrimaryColumn())) + .validityDate(prerequisiteSelects.getValidityDate()) + .build(); + + // we want to keep each primary column and the corresponding distinct validity date ranges + List> groupByFields = Stream.concat( + Stream.of(prerequisiteSelects.getIds().getPrimaryColumn()), + prerequisiteSelects.getValidityDate().stream().flatMap(validityDate -> validityDate.toFields().stream()) + ) + .collect(Collectors.toList()); + + // filter out entries with a null validity date + Condition dateNotNullCondition = prerequisiteSelects.getValidityDate().get().isNotNull(); + + return QueryStep.builder() + .cteName(FormCteStep.EXTRACT_IDS.getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(convertedPrerequisite.getCteName())) + .conditions(List.of(dateNotNullCondition)) + .groupBy(groupByFields) + .predecessors(List.of(convertedPrerequisite)) + .build(); + } + public ConversionContext convertForm( - QueryStep convertedPrerequisite, - List resolutionAndAlignments, + FormType formType, + QueryStep stratificationTable, ArrayConceptQuery features, List resultInfos, ConversionContext context ) { - // create stratification table - StratificationTableFactory tableFactory = new StratificationTableFactory(convertedPrerequisite, context); - QueryStep stratificationTable = tableFactory.createStratificationTable(resolutionAndAlignments); - // feature conversion ConversionContext childContext = context.createChildContext().withStratificationTable(stratificationTable); for (ConceptQuery conceptQuery : features.getChildQueries()) { @@ -52,7 +91,7 @@ public ConversionContext convertForm( // child context contains the converted feature's QuerySteps List queriesToJoin = childContext.getQuerySteps(); QueryStep joinedFeatures = QueryStepJoiner.joinSteps(queriesToJoin, ConqueryJoinType.OUTER_JOIN, DateAggregationAction.BLOCK, context); - return createFinalSelect(stratificationTable, joinedFeatures, resultInfos, context); + return createFinalSelect(formType, stratificationTable, joinedFeatures, resultInfos, context); } /** @@ -62,7 +101,8 @@ public ConversionContext convertForm( * Thus, the pre-final step might not contain an entry for each expected stratification window. That's why we need to left-join the converted * features with the full stratification table again. */ - public ConversionContext createFinalSelect( + private ConversionContext createFinalSelect( + FormType formType, QueryStep stratificationTable, QueryStep convertedFeatures, List resultInfos, @@ -75,10 +115,11 @@ public ConversionContext createFinalSelect( List queriesToJoin = List.of(stratificationTable, convertedFeatures); TableLike joinedTable = QueryStepJoiner.constructJoinedTable(queriesToJoin, ConqueryJoinType.LEFT_JOIN, context); + SqlFunctionProvider functionProvider = context.getSqlDialect().getFunctionProvider(); QueryStep finalStep = QueryStep.builder() .cteName(null) // the final QueryStep won't be converted to a CTE - .selects(getFinalSelects(stratificationTable, convertedFeatures, context.getSqlDialect().getFunctionProvider())) + .selects(getFinalSelects(formType, stratificationTable, convertedFeatures, functionProvider)) .fromTable(joinedTable) .predecessors(queriesToJoin) .build(); @@ -89,9 +130,14 @@ public ConversionContext createFinalSelect( /** * Selects the ID, resolution, index and date range from stratification table plus all explicit selects from the converted features step. + * For {@link FormType#RELATIVE}, the {@link FeatureGroup} will be set too. */ - private static Selects getFinalSelects(QueryStep stratificationTable, QueryStep convertedFeatures, SqlFunctionProvider functionProvider) { - + private static Selects getFinalSelects( + FormType formType, + QueryStep stratificationTable, + QueryStep convertedFeatures, + SqlFunctionProvider functionProvider + ) { Selects preFinalSelects = convertedFeatures.getQualifiedSelects(); Selects stratificationSelects = stratificationTable.getQualifiedSelects(); @@ -99,10 +145,22 @@ private static Selects getFinalSelects(QueryStep stratificationTable, QueryStep Field daterangeConcatenated = functionProvider.daterangeStringExpression(stratificationSelects.getStratificationDate().get()) .as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); - return Selects.builder() - .ids(ids) - .validityDate(Optional.empty()) - .stratificationDate(Optional.of(ColumnDateRange.of(daterangeConcatenated))) + Selects.SelectsBuilder selects = Selects.builder() + .ids(ids) + .validityDate(Optional.empty()) + .stratificationDate(Optional.of(ColumnDateRange.of(daterangeConcatenated))); + + if (formType != FormType.RELATIVE) { + return selects.sqlSelects(preFinalSelects.getSqlSelects()).build(); + } + + // relative forms have FeatureGroup information after the stratification date and before all other selects + Field indexField = DSL.field(DSL.name(stratificationTable.getCteName(), SharedAliases.INDEX.getAlias()), Integer.class); + Field scope = DSL.when(indexField.isNull().or(indexField.lessThan(0)), DSL.val(FeatureGroup.FEATURE.toString())) + .otherwise(DSL.val(FeatureGroup.OUTCOME.toString())) + .as(SharedAliases.OBSERVATION_SCOPE.getAlias()); + + return selects.sqlSelect(new FieldWrapper<>(scope)) .sqlSelects(preFinalSelects.getSqlSelects()) .build(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java new file mode 100644 index 0000000000..544d1e0548 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java @@ -0,0 +1,37 @@ +package com.bakdata.conquery.sql.conversion.query; + +import com.bakdata.conquery.models.forms.managed.RelativeFormQuery; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; +import com.bakdata.conquery.sql.conversion.forms.FormType; +import com.bakdata.conquery.sql.conversion.forms.StratificationTableFactory; +import com.bakdata.conquery.sql.conversion.model.QueryStep; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class RelativFormQueryConverter implements NodeConverter { + + private final FormConversionHelper formHelper; + + @Override + public Class getConversionClass() { + return RelativeFormQuery.class; + } + + @Override + public ConversionContext convert(RelativeFormQuery form, ConversionContext context) { + + QueryStep convertedPrerequisite = formHelper.convertPrerequisite(form.getQuery(), context); + StratificationTableFactory tableFactory = new StratificationTableFactory(convertedPrerequisite, context); + QueryStep stratificationTable = tableFactory.createRelativeStratificationTable(form); + + return formHelper.convertForm( + FormType.RELATIVE, + stratificationTable, + form.getFeatures(), + form.getResultInfos(), + context + ); + } + +} diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/SIMPLE/REL_EXPORT_FORM.test.json b/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/SIMPLE/REL_EXPORT_FORM.test.json index 1a632787bd..705d6af418 100644 --- a/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/SIMPLE/REL_EXPORT_FORM.test.json +++ b/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/SIMPLE/REL_EXPORT_FORM.test.json @@ -26,8 +26,8 @@ "value": "RELATIVE", "indexSelector": "EARLIEST", "timeUnit": "QUARTERS", - "timeCountAfter": 4, - "timeCountBefore": 4, + "timeCountAfter": 2, + "timeCountBefore": 6, "indexPlacement": "BEFORE" } }, @@ -45,4 +45,4 @@ "tests/form/EXPORT_FORM/RELATIVE/SIMPLE/query_results_1.csv" ] } -} \ No newline at end of file +} diff --git a/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/SIMPLE/expected.csv b/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/SIMPLE/expected.csv index 7a1f84a651..ccbc9d058b 100644 --- a/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/SIMPLE/expected.csv +++ b/backend/src/test/resources/tests/form/EXPORT_FORM/RELATIVE/SIMPLE/expected.csv @@ -1,25 +1,27 @@ result,resolution,index,event_date,date_range,scope,Alter -1,complete,,2012-06-01,2011-07-01/2012-06-30,feature_date_range,55 +1,complete,,2012-06-01,2011-01-01/2012-06-30,feature_date_range,55 +1,year,-2,2012-06-01,2011-01-01/2011-06-30,feature_date_range, 1,year,-1,2012-06-01,2011-07-01/2012-06-30,feature_date_range,55 +1,quarter,-6,2012-06-01,2011-01-01/2011-03-31,feature_date_range, +1,quarter,-5,2012-06-01,2011-04-01/2011-06-30,feature_date_range, 1,quarter,-4,2012-06-01,2011-07-01/2011-09-30,feature_date_range, 1,quarter,-3,2012-06-01,2011-10-01/2011-12-31,feature_date_range, 1,quarter,-2,2012-06-01,2012-01-01/2012-03-31,feature_date_range,55 1,quarter,-1,2012-06-01,2012-04-01/2012-06-30,feature_date_range,55 -1,complete,,2012-06-01,2012-07-01/2013-06-30,outcome_date_range,56 -1,year,1,2012-06-01,2012-07-01/2013-06-30,outcome_date_range,56 +1,complete,,2012-06-01,2012-07-01/2012-12-31,outcome_date_range,55 +1,year,1,2012-06-01,2012-07-01/2012-12-31,outcome_date_range,55 1,quarter,1,2012-06-01,2012-07-01/2012-09-30,outcome_date_range,55 1,quarter,2,2012-06-01,2012-10-01/2012-12-31,outcome_date_range,55 -1,quarter,3,2012-06-01,2013-01-01/2013-03-31,outcome_date_range, -1,quarter,4,2012-06-01,2013-04-01/2013-06-30,outcome_date_range, -23,complete,,2012-06-01,2011-07-01/2012-06-30,feature_date_range, +23,complete,,2012-06-01,2011-01-01/2012-06-30,feature_date_range, +23,year,-2,2012-06-01,2011-01-01/2011-06-30,feature_date_range, 23,year,-1,2012-06-01,2011-07-01/2012-06-30,feature_date_range, +23,quarter,-6,2012-06-01,2011-01-01/2011-03-31,feature_date_range, +23,quarter,-5,2012-06-01,2011-04-01/2011-06-30,feature_date_range, 23,quarter,-4,2012-06-01,2011-07-01/2011-09-30,feature_date_range, 23,quarter,-3,2012-06-01,2011-10-01/2011-12-31,feature_date_range, 23,quarter,-2,2012-06-01,2012-01-01/2012-03-31,feature_date_range, 23,quarter,-1,2012-06-01,2012-04-01/2012-06-30,feature_date_range, -23,complete,,2012-06-01,2012-07-01/2013-06-30,outcome_date_range, -23,year,1,2012-06-01,2012-07-01/2013-06-30,outcome_date_range, +23,complete,,2012-06-01,2012-07-01/2012-12-31,outcome_date_range, +23,year,1,2012-06-01,2012-07-01/2012-12-31,outcome_date_range, 23,quarter,1,2012-06-01,2012-07-01/2012-09-30,outcome_date_range, 23,quarter,2,2012-06-01,2012-10-01/2012-12-31,outcome_date_range, -23,quarter,3,2012-06-01,2013-01-01/2013-03-31,outcome_date_range, -23,quarter,4,2012-06-01,2013-04-01/2013-06-30,outcome_date_range, \ No newline at end of file diff --git a/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/QUARTER YEAR.json b/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/QUARTER YEAR.json index 4b51086d1b..12e4037993 100644 --- a/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/QUARTER YEAR.json +++ b/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/QUARTER YEAR.json @@ -28,7 +28,7 @@ "alignmentHint": "YEAR", "dateRange": { "min": "2012-06-16", - "max": "2014-01-17" + "max": "2013-12-31" } } }, diff --git a/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/YEAR QUARTER.json b/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/YEAR QUARTER.json index 9903386e9b..5a3433bdcb 100644 --- a/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/YEAR QUARTER.json +++ b/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/YEAR QUARTER.json @@ -28,7 +28,7 @@ "alignmentHint": "QUARTER", "dateRange": { "min": "2012-06-16", - "max": "2014-12-17" + "max": "2015-01-01" } } }, diff --git a/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/quarter_year.csv b/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/quarter_year.csv index de42f67958..0b9125cb54 100644 --- a/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/quarter_year.csv +++ b/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/quarter_year.csv @@ -1,8 +1,7 @@ result,resolution,index,date_range,Alter -1,complete,,2012-06-16/2014-01-17,57 +1,complete,,2012-06-16/2013-12-31,56 1,year,1,2012-06-16/2012-12-31,55 1,year,2,2013-01-01/2013-12-31, -1,year,3,2014-01-01/2014-01-17, 1,quarter,1,2012-06-16/2012-06-30,55 1,quarter,2,2012-07-01/2012-09-30,55 1,quarter,3,2012-10-01/2012-12-31,55 @@ -10,4 +9,3 @@ result,resolution,index,date_range,Alter 1,quarter,5,2013-04-01/2013-06-30, 1,quarter,6,2013-07-01/2013-09-30, 1,quarter,7,2013-10-01/2013-12-31, -1,quarter,8,2014-01-01/2014-01-17, diff --git a/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/year_quarter_expected.csv b/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/year_quarter_expected.csv index 87eb32785c..a974ba7d6e 100644 --- a/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/year_quarter_expected.csv +++ b/backend/src/test/resources/tests/sql/form/ABSOLUT/ALIGNMENT/year_quarter_expected.csv @@ -1,5 +1,5 @@ result,resolution,index,date_range,Alter -1,complete,,2012-06-16/2014-12-17,57 +1,complete,,2012-06-16/2015-01-01,58 1,year,1,2012-06-16/2013-03-31,56 1,year,2,2013-04-01/2014-03-31, -1,year,3,2014-04-01/2014-12-17, +1,year,3,2014-04-01/2015-01-01, diff --git a/backend/src/test/resources/tests/sql/form/RELATIVE/DAYS_NEUTRAL/DAYS.json b/backend/src/test/resources/tests/sql/form/RELATIVE/DAYS_NEUTRAL/DAYS.json new file mode 100644 index 0000000000..82054082ed --- /dev/null +++ b/backend/src/test/resources/tests/sql/form/RELATIVE/DAYS_NEUTRAL/DAYS.json @@ -0,0 +1,45 @@ +{ + "type": "FORM_TEST", + "label": "REL-EXPORT-FORM Test", + "expectedCsv": { + "results": "tests/sql/form/RELATIVE/DAYS_NEUTRAL/expected.csv" + }, + "form": { + "type": "EXPORT_FORM", + "queryGroup": "00000000-0000-0000-0000-000000000001", + "resolution": "QUARTERS", + "alsoCreateCoarserSubdivisions": true, + "features": [ + { + "ids": [ + "alter" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "alter.alter" + } + ] + } + ], + "timeMode": { + "value": "RELATIVE", + "indexSelector": "LATEST", + "timeUnit": "DAYS", + "timeCountAfter": 91, + "timeCountBefore": 366, + "indexPlacement": "NEUTRAL" + } + }, + "concepts": [ + "/shared/alter.concept.json" + ], + "content": { + "tables": [ + "/shared/vers_stamm.table.json" + ], + "previousQueryResults": [ + "tests/form/EXPORT_FORM/RELATIVE/SIMPLE/query_results_1.csv" + ] + } +} diff --git a/backend/src/test/resources/tests/sql/form/RELATIVE/DAYS_NEUTRAL/expected.csv b/backend/src/test/resources/tests/sql/form/RELATIVE/DAYS_NEUTRAL/expected.csv new file mode 100644 index 0000000000..db0604dd89 --- /dev/null +++ b/backend/src/test/resources/tests/sql/form/RELATIVE/DAYS_NEUTRAL/expected.csv @@ -0,0 +1,25 @@ +result,resolution,index,event_date,date_range,scope,Alter +1,complete,,2016-12-01,2015-12-01/2016-11-30,feature_date_range, +1,year,-2,2016-12-01,2015-12-01/2015-12-01,feature_date_range, +1,year,-1,2016-12-01,2015-12-02/2016-11-30,feature_date_range, +1,quarter,-5,2016-12-01,2015-12-01/2015-12-06,feature_date_range, +1,quarter,-4,2016-12-01,2015-12-07/2016-03-05,feature_date_range, +1,quarter,-3,2016-12-01,2016-03-06/2016-06-03,feature_date_range, +1,quarter,-2,2016-12-01,2016-06-04/2016-09-01,feature_date_range, +1,quarter,-1,2016-12-01,2016-09-02/2016-11-30,feature_date_range, +1,complete,,2016-12-01,2016-12-02/2017-03-02,outcome_date_range, +1,year,1,2016-12-01,2016-12-02/2017-03-02,outcome_date_range, +1,quarter,1,2016-12-01,2016-12-02/2017-03-01,outcome_date_range, +1,quarter,2,2016-12-01,2017-03-02/2017-03-02,outcome_date_range, +23,complete,,2016-12-01,2015-12-01/2016-11-30,feature_date_range, +23,year,-2,2016-12-01,2015-12-01/2015-12-01,feature_date_range, +23,year,-1,2016-12-01,2015-12-02/2016-11-30,feature_date_range, +23,quarter,-5,2016-12-01,2015-12-01/2015-12-06,feature_date_range, +23,quarter,-4,2016-12-01,2015-12-07/2016-03-05,feature_date_range, +23,quarter,-3,2016-12-01,2016-03-06/2016-06-03,feature_date_range, +23,quarter,-2,2016-12-01,2016-06-04/2016-09-01,feature_date_range, +23,quarter,-1,2016-12-01,2016-09-02/2016-11-30,feature_date_range, +23,complete,,2016-12-01,2016-12-02/2017-03-02,outcome_date_range, +23,year,1,2016-12-01,2016-12-02/2017-03-02,outcome_date_range, +23,quarter,1,2016-12-01,2016-12-02/2017-03-01,outcome_date_range, +23,quarter,2,2016-12-01,2017-03-02/2017-03-02,outcome_date_range, diff --git a/backend/src/test/resources/tests/sql/form/RELATIVE/QUARTERS_AFTER/REL_EXPORT_FORM.json b/backend/src/test/resources/tests/sql/form/RELATIVE/QUARTERS_AFTER/REL_EXPORT_FORM.json new file mode 100644 index 0000000000..708ecc5a77 --- /dev/null +++ b/backend/src/test/resources/tests/sql/form/RELATIVE/QUARTERS_AFTER/REL_EXPORT_FORM.json @@ -0,0 +1,45 @@ +{ + "type": "FORM_TEST", + "label": "REL-EXPORT-FORM Test", + "expectedCsv": { + "results": "tests/sql/form/RELATIVE/QUARTERS_AFTER/expected.csv" + }, + "form": { + "type": "EXPORT_FORM", + "queryGroup": "00000000-0000-0000-0000-000000000001", + "resolution": "QUARTERS", + "alsoCreateCoarserSubdivisions": true, + "features": [ + { + "ids": [ + "alter" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "alter.alter" + } + ] + } + ], + "timeMode": { + "value": "RELATIVE", + "indexSelector": "EARLIEST", + "timeUnit": "QUARTERS", + "timeCountAfter": 2, + "timeCountBefore": 6, + "indexPlacement": "AFTER" + } + }, + "concepts": [ + "/shared/alter.concept.json" + ], + "content": { + "tables": [ + "/shared/vers_stamm.table.json" + ], + "previousQueryResults": [ + "tests/form/EXPORT_FORM/RELATIVE/SIMPLE/query_results_1.csv" + ] + } +} diff --git a/backend/src/test/resources/tests/sql/form/RELATIVE/QUARTERS_AFTER/expected.csv b/backend/src/test/resources/tests/sql/form/RELATIVE/QUARTERS_AFTER/expected.csv new file mode 100644 index 0000000000..8c5c569e1b --- /dev/null +++ b/backend/src/test/resources/tests/sql/form/RELATIVE/QUARTERS_AFTER/expected.csv @@ -0,0 +1,27 @@ +result,resolution,index,event_date,date_range,scope,Alter +1,complete,,2012-06-01,2010-10-01/2012-03-31,feature_date_range,55 +1,year,-2,2012-06-01,2010-10-01/2011-03-31,feature_date_range, +1,year,-1,2012-06-01,2011-04-01/2012-03-31,feature_date_range,55 +1,quarter,-6,2012-06-01,2010-10-01/2010-12-31,feature_date_range, +1,quarter,-5,2012-06-01,2011-01-01/2011-03-31,feature_date_range, +1,quarter,-4,2012-06-01,2011-04-01/2011-06-30,feature_date_range, +1,quarter,-3,2012-06-01,2011-07-01/2011-09-30,feature_date_range, +1,quarter,-2,2012-06-01,2011-10-01/2011-12-31,feature_date_range, +1,quarter,-1,2012-06-01,2012-01-01/2012-03-31,feature_date_range,55 +1,complete,,2012-06-01,2012-04-01/2012-09-30,outcome_date_range,55 +1,year,1,2012-06-01,2012-04-01/2012-09-30,outcome_date_range,55 +1,quarter,1,2012-06-01,2012-04-01/2012-06-30,outcome_date_range,55 +1,quarter,2,2012-06-01,2012-07-01/2012-09-30,outcome_date_range,55 +23,complete,,2012-06-01,2010-10-01/2012-03-31,feature_date_range, +23,year,-2,2012-06-01,2010-10-01/2011-03-31,feature_date_range, +23,year,-1,2012-06-01,2011-04-01/2012-03-31,feature_date_range, +23,quarter,-6,2012-06-01,2010-10-01/2010-12-31,feature_date_range, +23,quarter,-5,2012-06-01,2011-01-01/2011-03-31,feature_date_range, +23,quarter,-4,2012-06-01,2011-04-01/2011-06-30,feature_date_range, +23,quarter,-3,2012-06-01,2011-07-01/2011-09-30,feature_date_range, +23,quarter,-2,2012-06-01,2011-10-01/2011-12-31,feature_date_range, +23,quarter,-1,2012-06-01,2012-01-01/2012-03-31,feature_date_range, +23,complete,,2012-06-01,2012-04-01/2012-09-30,outcome_date_range, +23,year,1,2012-06-01,2012-04-01/2012-09-30,outcome_date_range, +23,quarter,1,2012-06-01,2012-04-01/2012-06-30,outcome_date_range, +23,quarter,2,2012-06-01,2012-07-01/2012-09-30,outcome_date_range, diff --git a/backend/src/test/resources/tests/sql/form/RELATIVE/QUARTERS_NEUTRAL/REL_EXPORT_FORM.json b/backend/src/test/resources/tests/sql/form/RELATIVE/QUARTERS_NEUTRAL/REL_EXPORT_FORM.json new file mode 100644 index 0000000000..11bff2e768 --- /dev/null +++ b/backend/src/test/resources/tests/sql/form/RELATIVE/QUARTERS_NEUTRAL/REL_EXPORT_FORM.json @@ -0,0 +1,45 @@ +{ + "type": "FORM_TEST", + "label": "REL-EXPORT-FORM Test", + "expectedCsv": { + "results": "tests/sql/form/RELATIVE/QUARTERS_NEUTRAL/expected.csv" + }, + "form": { + "type": "EXPORT_FORM", + "queryGroup": "00000000-0000-0000-0000-000000000001", + "resolution": "QUARTERS", + "alsoCreateCoarserSubdivisions": true, + "features": [ + { + "ids": [ + "alter" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "alter.alter" + } + ] + } + ], + "timeMode": { + "value": "RELATIVE", + "indexSelector": "LATEST", + "timeUnit": "QUARTERS", + "timeCountAfter": 2, + "timeCountBefore": 6, + "indexPlacement": "NEUTRAL" + } + }, + "concepts": [ + "/shared/alter.concept.json" + ], + "content": { + "tables": [ + "/shared/vers_stamm.table.json" + ], + "previousQueryResults": [ + "tests/form/EXPORT_FORM/RELATIVE/SIMPLE/query_results_1.csv" + ] + } +} diff --git a/backend/src/test/resources/tests/sql/form/RELATIVE/QUARTERS_NEUTRAL/expected.csv b/backend/src/test/resources/tests/sql/form/RELATIVE/QUARTERS_NEUTRAL/expected.csv new file mode 100644 index 0000000000..26b644f303 --- /dev/null +++ b/backend/src/test/resources/tests/sql/form/RELATIVE/QUARTERS_NEUTRAL/expected.csv @@ -0,0 +1,27 @@ +result,resolution,index,event_date,date_range,scope,Alter +1,complete,,2016-12-01,2015-04-01/2016-09-30,feature_date_range, +1,year,-2,2016-12-01,2015-04-01/2015-09-30,feature_date_range, +1,year,-1,2016-12-01,2015-10-01/2016-09-30,feature_date_range, +1,quarter,-6,2016-12-01,2015-04-01/2015-06-30,feature_date_range, +1,quarter,-5,2016-12-01,2015-07-01/2015-09-30,feature_date_range, +1,quarter,-4,2016-12-01,2015-10-01/2015-12-31,feature_date_range, +1,quarter,-3,2016-12-01,2016-01-01/2016-03-31,feature_date_range, +1,quarter,-2,2016-12-01,2016-04-01/2016-06-30,feature_date_range, +1,quarter,-1,2016-12-01,2016-07-01/2016-09-30,feature_date_range, +1,complete,,2016-12-01,2017-01-01/2017-06-30,outcome_date_range, +1,year,1,2016-12-01,2017-01-01/2017-06-30,outcome_date_range, +1,quarter,1,2016-12-01,2017-01-01/2017-03-31,outcome_date_range, +1,quarter,2,2016-12-01,2017-04-01/2017-06-30,outcome_date_range, +23,complete,,2016-12-01,2015-04-01/2016-09-30,feature_date_range, +23,year,-2,2016-12-01,2015-04-01/2015-09-30,feature_date_range, +23,year,-1,2016-12-01,2015-10-01/2016-09-30,feature_date_range, +23,quarter,-6,2016-12-01,2015-04-01/2015-06-30,feature_date_range, +23,quarter,-5,2016-12-01,2015-07-01/2015-09-30,feature_date_range, +23,quarter,-4,2016-12-01,2015-10-01/2015-12-31,feature_date_range, +23,quarter,-3,2016-12-01,2016-01-01/2016-03-31,feature_date_range, +23,quarter,-2,2016-12-01,2016-04-01/2016-06-30,feature_date_range, +23,quarter,-1,2016-12-01,2016-07-01/2016-09-30,feature_date_range, +23,complete,,2016-12-01,2017-01-01/2017-06-30,outcome_date_range, +23,year,1,2016-12-01,2017-01-01/2017-06-30,outcome_date_range, +23,quarter,1,2016-12-01,2017-01-01/2017-03-31,outcome_date_range, +23,quarter,2,2016-12-01,2017-04-01/2017-06-30,outcome_date_range, diff --git a/backend/src/test/resources/tests/sql/form/RELATIVE/SECONDARY_ID/SECONDARY_ID.json b/backend/src/test/resources/tests/sql/form/RELATIVE/SECONDARY_ID/SECONDARY_ID.json new file mode 100644 index 0000000000..729eaf19ae --- /dev/null +++ b/backend/src/test/resources/tests/sql/form/RELATIVE/SECONDARY_ID/SECONDARY_ID.json @@ -0,0 +1,74 @@ +{ + "type": "FORM_TEST", + "label": "REL-EXPORT-FORM SECONDARY_ID", + "expectedCsv": { + "results": "/tests/form/EXPORT_FORM/RELATIVE/SECONDARY_ID/expected.csv" + }, + "form": { + "type": "EXPORT_FORM", + "queryGroup": "00000000-0000-0000-0000-000000000001", + "resolution": "QUARTERS", + "alsoCreateCoarserSubdivisions": true, + "features": [ + { + "ids": [ + "two_connector" + ], + "type": "CONCEPT", + "label": "explicitly set select", + "tables": [ + { + "id": "two_connector.table1", + "selects": "two_connector.table1.alter_select" + }, + { + "id": "two_connector.table2" + } + ] + } + ], + "timeMode": { + "value": "RELATIVE", + "indexSelector": "EARLIEST", + "timeUnit": "QUARTERS", + "indexPlacement": "BEFORE", + "timeCountBefore": 4 + } + }, + "concepts": [ + "/shared/two_connector.concept.json", + "/tests/form/shared/abc.concept.json" + ], + "content": { + "secondaryIds": [ + "/tests/form/shared/secondary.sid.json" + ], + "tables": [ + "/shared/vers_stamm.table.json", + "/tests/form/shared/abc.table.json" + ], + "previousQueries": [ + { + "type": "SECONDARY_ID_QUERY", + "secondaryId": "secondary", + "root": { + "type": "AND", + "children": [ + { + "type": "CONCEPT", + "excludeFromSecondaryId": false, + "ids": [ + "abc-concept.a" + ], + "tables": [ + { + "id": "abc-concept.connector" + } + ] + } + ] + } + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/form/RELATIVE/SIMPLE/REL_EXPORT_FORM.json b/backend/src/test/resources/tests/sql/form/RELATIVE/SIMPLE/REL_EXPORT_FORM.json new file mode 100644 index 0000000000..32a5bc3e57 --- /dev/null +++ b/backend/src/test/resources/tests/sql/form/RELATIVE/SIMPLE/REL_EXPORT_FORM.json @@ -0,0 +1,45 @@ +{ + "type": "FORM_TEST", + "label": "REL-EXPORT-FORM Test", + "expectedCsv": { + "results": "tests/form/EXPORT_FORM/RELATIVE/SIMPLE/expected.csv" + }, + "form": { + "type": "EXPORT_FORM", + "queryGroup": "00000000-0000-0000-0000-000000000001", + "resolution": "QUARTERS", + "alsoCreateCoarserSubdivisions": true, + "features": [ + { + "ids": [ + "alter" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "alter.alter" + } + ] + } + ], + "timeMode": { + "value": "RELATIVE", + "indexSelector": "EARLIEST", + "timeUnit": "QUARTERS", + "timeCountAfter": 2, + "timeCountBefore": 6, + "indexPlacement": "BEFORE" + } + }, + "concepts": [ + "/shared/alter.concept.json" + ], + "content": { + "tables": [ + "/shared/vers_stamm.table.json" + ], + "previousQueryResults": [ + "tests/form/EXPORT_FORM/RELATIVE/SIMPLE/query_results_1.csv" + ] + } +}