Skip to content

Commit

Permalink
Fix date restriction and validity date conversion (#3325)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jnsrnhld
jnsrnhld authored Mar 6, 2024
1 parent 63c6f0e commit 7e75d0d
Show file tree
Hide file tree
Showing 9 changed files with 296 additions and 127 deletions.
18 changes: 10 additions & 8 deletions ...c/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,7 @@ private CQTableContext createTableContext(String conceptLabel, CQConcept cqConce
String tableName = connector.getTable().getName();

Field<Object> primaryColumn = DSL.field(DSL.name(conversionContext.getConfig().getPrimaryColumn()));
Optional<ColumnDateRange> tablesValidityDate = convertValidityDate(cqTable, tableName, functionProvider);
Optional<ColumnDateRange> tablesValidityDate = convertValidityDate(cqTable, tableName, conversionContext);
SqlTables connectorTables = ConnectorCteStep.createTables(conceptConnectorLabel, tableName, nameGenerator);

// validity date
Expand Down Expand Up @@ -180,15 +180,17 @@ private CQTableContext createTableContext(String conceptLabel, CQConcept cqConce
.build();
}

private static Optional<ColumnDateRange> convertValidityDate(CQTable cqTable, String label, SqlFunctionProvider functionProvider) {
private static Optional<ColumnDateRange> convertValidityDate(CQTable cqTable, String connectorLabel, ConversionContext context) {
if (Objects.isNull(cqTable.findValidityDate())) {
return Optional.empty();
}
ColumnDateRange validityDate = functionProvider.daterange(
cqTable.findValidityDate(),
cqTable.getConnector().getTable().getName(),
label
);
ColumnDateRange validityDate;
if (context.getDateRestrictionRange() != null) {
validityDate = context.getSqlDialect().getFunctionProvider().forTablesValidityDate(cqTable, context.getDateRestrictionRange(), connectorLabel);
}
else {
validityDate = context.getSqlDialect().getFunctionProvider().forTablesValidityDate(cqTable, connectorLabel);
}
return Optional.of(validityDate);
}

Expand Down Expand Up @@ -250,7 +252,7 @@ private static Optional<SqlFilters> getDateRestriction(ConversionContext context
}

SqlFunctionProvider functionProvider = context.getSqlDialect().getFunctionProvider();
ColumnDateRange dateRestriction = functionProvider.daterange(context.getDateRestrictionRange())
ColumnDateRange dateRestriction = functionProvider.forDateRestriction(context.getDateRestrictionRange())
.asDateRestrictionRange();

List<SqlSelect> dateRestrictionSelects = dateRestriction.toFields().stream()
Expand Down
100 changes: 64 additions & 36 deletions ...nd/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import java.util.List;
import java.util.stream.Collectors;

import com.bakdata.conquery.apiv1.query.concept.filter.CQTable;
import com.bakdata.conquery.models.common.daterange.CDateRange;
import com.bakdata.conquery.models.datasets.Column;
import com.bakdata.conquery.models.datasets.concepts.ValidityDate;
Expand Down Expand Up @@ -61,50 +62,30 @@ public Condition dateRestriction(ColumnDateRange dateRestriction, ColumnDateRang
}

@Override
public ColumnDateRange daterange(CDateRange dateRestriction) {

String startDateExpression = MIN_DATE_VALUE;
String endDateExpression = MAX_DATE_VALUE;

if (dateRestriction.hasLowerBound()) {
startDateExpression = dateRestriction.getMin().toString();
}
if (dateRestriction.hasUpperBound()) {
endDateExpression = dateRestriction.getMax().toString();
}
public ColumnDateRange forDateRestriction(CDateRange dateRestriction) {
return toColumnDateRange(dateRestriction).asDateRestrictionRange();
}

return ColumnDateRange.of(toDateField(startDateExpression), toDateField(endDateExpression))
.asDateRestrictionRange();
@Override
public ColumnDateRange forTablesValidityDate(CQTable cqTable, String alias) {
return toColumnDateRange(cqTable).asValidityDateRange(alias);
}

@Override
public ColumnDateRange daterange(ValidityDate validityDate, String qualifier, String label) {
public ColumnDateRange forTablesValidityDate(CQTable cqTable, CDateRange dateRestriction, String alias) {

Column startColumn;
Column endColumn;
ColumnDateRange validityDate = toColumnDateRange(cqTable);
ColumnDateRange restriction = toColumnDateRange(dateRestriction);

if (validityDate.getEndColumn() != null) {
startColumn = validityDate.getStartColumn();
endColumn = validityDate.getEndColumn();
}
else {
startColumn = validityDate.getColumn();
endColumn = validityDate.getColumn();
}
Field<Date> lowerBound = DSL.when(validityDate.getStart().lessThan(restriction.getStart()), restriction.getStart())
.otherwise(validityDate.getStart());

Field<Date> rangeStart = DSL.coalesce(
DSL.field(DSL.name(qualifier, startColumn.getName()), Date.class),
toDateField(MIN_DATE_VALUE)
);
// when aggregating date ranges, we want to treat the last day of the range as excluded,
// so when using the date value of the end column, we add +1 day as end of the date range
Field<Date> rangeEnd = DSL.coalesce(
addDays(DSL.field(DSL.name(qualifier, endColumn.getName()), Date.class), 1),
toDateField(MAX_DATE_VALUE)
);
Field<Date> maxDate = toDateField(MAX_DATE_VALUE); // we want to add +1 day to the end date - except when it's the max date already
Field<Date> restrictionUpperBound = DSL.when(restriction.getEnd().eq(maxDate), maxDate).otherwise(addDays(restriction.getEnd(), 1));
Field<Date> upperBound = DSL.when(validityDate.getEnd().greaterThan(restriction.getEnd()), restrictionUpperBound)
.otherwise(validityDate.getEnd());

return ColumnDateRange.of(rangeStart, rangeEnd)
.asValidityDateRange(label);
return ColumnDateRange.of(lowerBound, upperBound).as(alias);
}

@Override
Expand Down Expand Up @@ -267,4 +248,51 @@ private Field<String> toVarcharField(Field<Date> startDate, Param<Integer> dateE
);
}

private ColumnDateRange toColumnDateRange(CDateRange dateRestriction) {

String startDateExpression = MIN_DATE_VALUE;
String endDateExpression = MAX_DATE_VALUE;

if (dateRestriction.hasLowerBound()) {
startDateExpression = dateRestriction.getMin().toString();
}
if (dateRestriction.hasUpperBound()) {
endDateExpression = dateRestriction.getMax().toString();
}

return ColumnDateRange.of(toDateField(startDateExpression), toDateField(endDateExpression));
}

private ColumnDateRange toColumnDateRange(CQTable cqTable) {

ValidityDate validityDate = cqTable.findValidityDate();
String tableName = cqTable.getConnector().getTable().getName();

Column startColumn;
Column endColumn;

// if no end column is present, the only existing column is both start and end of the date range
if (validityDate.getEndColumn() == null) {
startColumn = validityDate.getColumn();
endColumn = validityDate.getColumn();
}
else {
startColumn = validityDate.getStartColumn();
endColumn = validityDate.getEndColumn();
}

Field<Date> rangeStart = DSL.coalesce(
DSL.field(DSL.name(tableName, startColumn.getName()), Date.class),
toDateField(MIN_DATE_VALUE)
);
// when aggregating date ranges, we want to treat the last day of the range as excluded,
// so when using the date value of the end column, we add +1 day as end of the date range
Field<Date> rangeEnd = DSL.coalesce(
addDays(DSL.field(DSL.name(tableName, endColumn.getName()), Date.class), 1),
toDateField(MAX_DATE_VALUE)
);

return ColumnDateRange.of(rangeStart, rangeEnd);
}

}
143 changes: 88 additions & 55 deletions ...src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import java.util.List;
import java.util.stream.Collectors;

import com.bakdata.conquery.apiv1.query.concept.filter.CQTable;
import com.bakdata.conquery.models.common.daterange.CDateRange;
import com.bakdata.conquery.models.datasets.Column;
import com.bakdata.conquery.models.datasets.concepts.ValidityDate;
Expand Down Expand Up @@ -64,68 +65,28 @@ public Condition dateRestriction(ColumnDateRange dateRestriction, ColumnDateRang
}

@Override
public ColumnDateRange daterange(CDateRange dateRestriction) {

String startDateExpression = MINUS_INFINITY_DATE_VALUE;
String endDateExpression = INFINITY_DATE_VALUE;

if (dateRestriction.hasLowerBound()) {
startDateExpression = dateRestriction.getMin().toString();
}
if (dateRestriction.hasUpperBound()) {
endDateExpression = dateRestriction.getMax().toString();
}

Field<Object> dateRestrictionRange = DSL.function(
"daterange",
Object.class,
toDateField(startDateExpression),
toDateField(endDateExpression),
DSL.val("[]")
);

return ColumnDateRange.of(dateRestrictionRange)
.asDateRestrictionRange();
public ColumnDateRange forDateRestriction(CDateRange dateRestriction) {
return toColumnDateRange(dateRestriction).asDateRestrictionRange();
}

@Override
public ColumnDateRange daterange(ValidityDate validityDate, String qualifier, String label) {
public ColumnDateRange forTablesValidityDate(CQTable cqTable, String alias) {
return toColumnDateRange(cqTable).asValidityDateRange(alias);
}

Field<?> dateRange;
@Override
public ColumnDateRange forTablesValidityDate(CQTable cqTable, CDateRange dateRestriction, String alias) {

if (validityDate.getEndColumn() != null) {
ColumnDateRange validityDate = toColumnDateRange(cqTable);
ColumnDateRange restriction = toColumnDateRange(dateRestriction);

Field<?> startColumn = DSL.coalesce(
DSL.field(DSL.name(qualifier, validityDate.getStartColumn().getName())),
toDateField(MINUS_INFINITY_DATE_VALUE)
);
Field<?> endColumn = DSL.coalesce(
DSL.field(DSL.name(qualifier, validityDate.getEndColumn().getName())),
toDateField(INFINITY_DATE_VALUE)
);

dateRange = daterange(startColumn, endColumn, "[]");
}
else {
Column validityDateColumn = validityDate.getColumn();
dateRange = switch (validityDateColumn.getType()) {
// if validityDateColumn is a DATE_RANGE we can make use of Postgres' integrated daterange type.
case DATE_RANGE -> DSL.field(validityDateColumn.getName());
// if the validity date column is not of daterange type, we construct it manually
case DATE -> {
Field<Date> column = DSL.field(DSL.name(qualifier, validityDate.getColumn().getName()), Date.class);
Field<Date> startColumn = DSL.coalesce(column, toDateField(MINUS_INFINITY_DATE_VALUE));
Field<Date> endColumn = DSL.coalesce(column, toDateField(INFINITY_DATE_VALUE));
yield daterange(startColumn, endColumn, "[]");
}
default -> throw new IllegalArgumentException(
"Given column type '%s' can't be converted to a proper date restriction.".formatted(validityDateColumn.getType())
);
};
}
Field<Object> intersection = DSL.field(
"{0} * {1}", // intersection of both ranges
validityDate.getRange(),
restriction.getRange()
);

return ColumnDateRange.of(dateRange)
.asValidityDateRange(label);
return ColumnDateRange.of(intersection).asValidityDateRange(alias);
}

@Override
Expand Down Expand Up @@ -247,4 +208,76 @@ public Field<Date> toDateField(String dateValue) {
return DSL.field("{0}::{1}", Date.class, DSL.val(dateValue), DSL.keyword("date"));
}

private ColumnDateRange toColumnDateRange(CDateRange dateRestriction) {
String startDateExpression = MINUS_INFINITY_DATE_VALUE;
String endDateExpression = INFINITY_DATE_VALUE;

if (dateRestriction.hasLowerBound()) {
startDateExpression = dateRestriction.getMin().toString();
}
if (dateRestriction.hasUpperBound()) {
endDateExpression = dateRestriction.getMax().toString();
}

Field<Object> dateRestrictionRange = DSL.function(
"daterange",
Object.class,
toDateField(startDateExpression),
toDateField(endDateExpression),
DSL.val("[]")
);

return ColumnDateRange.of(dateRestrictionRange);
}

private ColumnDateRange toColumnDateRange(CQTable cqTable) {
ValidityDate validityDate = cqTable.findValidityDate();
String tableName = cqTable.getConnector().getTable().getName();

Field<?> dateRange;

if (validityDate.getEndColumn() != null) {

Field<?> startColumn = DSL.coalesce(
DSL.field(DSL.name(tableName, validityDate.getStartColumn().getName())),
toDateField(MINUS_INFINITY_DATE_VALUE)
);
Field<?> endColumn = DSL.coalesce(
DSL.field(DSL.name(tableName, validityDate.getEndColumn().getName())),
toDateField(INFINITY_DATE_VALUE)
);

return ColumnDateRange.of(daterange(startColumn, endColumn, "[]"));
}

Column validityDateColumn = validityDate.getColumn();
dateRange = switch (validityDateColumn.getType()) {
// if validityDateColumn is a DATE_RANGE we can make use of Postgres' integrated daterange type, but the upper bound is exclusive by default
case DATE_RANGE -> {
Field<Object> daterange = DSL.field(DSL.name(validityDateColumn.getName()));
Field<Date> startColumn = DSL.coalesce(
DSL.function("lower", Date.class, daterange),
toDateField(MINUS_INFINITY_DATE_VALUE)
);
Field<Date> endColumn = DSL.coalesce(
DSL.function("upper", Date.class, daterange),
toDateField(INFINITY_DATE_VALUE)
);
yield daterange(startColumn, endColumn, "[]");
}
// if the validity date column is not of daterange type, we construct it manually
case DATE -> {
Field<Date> column = DSL.field(DSL.name(tableName, validityDate.getColumn().getName()), Date.class);
Field<Date> startColumn = DSL.coalesce(column, toDateField(MINUS_INFINITY_DATE_VALUE));
Field<Date> endColumn = DSL.coalesce(column, toDateField(INFINITY_DATE_VALUE));
yield daterange(startColumn, endColumn, "[]");
}
default -> throw new IllegalArgumentException(
"Given column type '%s' can't be converted to a proper date restriction.".formatted(validityDateColumn.getType())
);
};

return ColumnDateRange.of(dateRange);
}

}
15 changes: 12 additions & 3 deletions backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@
import java.time.temporal.ChronoUnit;
import java.util.List;

import com.bakdata.conquery.apiv1.query.concept.filter.CQTable;
import com.bakdata.conquery.models.common.daterange.CDateRange;
import com.bakdata.conquery.models.datasets.concepts.ValidityDate;
import com.bakdata.conquery.sql.conversion.model.ColumnDateRange;
import com.bakdata.conquery.sql.conversion.model.QueryStep;
import org.jooq.Condition;
Expand Down Expand Up @@ -46,9 +46,18 @@ public interface SqlFunctionProvider {
*/
Condition dateRestriction(ColumnDateRange dateRestrictionRange, ColumnDateRange validityFieldRange);

ColumnDateRange daterange(CDateRange dateRestriction);
ColumnDateRange forDateRestriction(CDateRange dateRestriction);

ColumnDateRange daterange(ValidityDate validityDate, String qualifier, String label);
/**
* Creates a {@link ColumnDateRange} for a tables {@link CQTable}s validity date.
*/
ColumnDateRange forTablesValidityDate(CQTable cqTable, String alias);

/**
* Creates a {@link ColumnDateRange} for a tables {@link CQTable}s validity date. The validity dates bounds will be restricted by the given date
* restriction.
*/
ColumnDateRange forTablesValidityDate(CQTable cqTable, CDateRange dateRestriction, String alias);

ColumnDateRange aggregated(ColumnDateRange columnDateRange);

Expand Down
Loading

0 comments on commit 7e75d0d

Please sign in to comment.