Merge branch 'develop' into feature/query-stats-followup

backend/pom.xml

@@ -212,7 +212,7 @@
         <dependency>
             <groupId>org.apache.shiro</groupId>
             <artifactId>shiro-core</artifactId>
-            <version>1.10.1</version>
+            <version>1.13.0</version>
             <exclusions>
                 <!-- All these transitive deps are already bundled in shiro core-->
                 <exclusion>

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java

@@ -19,6 +19,11 @@
 import com.bakdata.conquery.models.identifiable.ids.specific.FilterId;
 import com.bakdata.conquery.models.query.QueryResolveContext;
 import com.bakdata.conquery.models.query.queryplan.filter.FilterNode;
+import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext;
+import com.bakdata.conquery.sql.conversion.model.SqlTables;
+import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
 import com.fasterxml.jackson.core.JsonParser;
@@ -66,6 +71,14 @@ public FilterNode<?> createNode() {
 		return getFilter().createFilterNode(getValue());
 	}
 
+	public SqlFilters convertToSqlFilter(ConversionContext context, SqlTables<ConceptCteStep> conceptTables) {
+		FilterContext<VALUE> filterContext = new FilterContext<>(value, context, conceptTables);
+		SqlFilters sqlFilters = filter.convertToSqlFilter(filterContext);
+		if (context.isNegation()) {
+			return new SqlFilters(sqlFilters.getSelects(), sqlFilters.getWhereClauses().negated());
+		}
+		return sqlFilters;
+	}
 
 	@NoArgsConstructor
 	@CPSType(id = FrontendFilterType.Fields.MULTI_SELECT, base = FilterValue.class)
@@ -228,4 +241,4 @@ public GroupFilterValue deserialize(JsonParser p, DeserializationContext ctxt) t
 		}
 
 	}
-}
+}

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Connector.java

@@ -14,6 +14,7 @@
 import com.bakdata.conquery.models.datasets.Column;
 import com.bakdata.conquery.models.datasets.Dataset;
 import com.bakdata.conquery.models.datasets.Table;
+import com.bakdata.conquery.models.datasets.concepts.conditions.CTCondition;
 import com.bakdata.conquery.models.datasets.concepts.filters.Filter;
 import com.bakdata.conquery.models.datasets.concepts.select.Select;
 import com.bakdata.conquery.models.identifiable.IdMap;
@@ -88,6 +89,9 @@ public Collection<Filter<?>> getFilters() {
 	@CheckForNull
 	public abstract Column getColumn();
 
+	@CheckForNull
+	public abstract CTCondition getCondition();
+
 	@JsonIgnore
 	public List<Select> getDefaultSelects() {
 		return getSelects()

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/AndCondition.java

@@ -9,23 +9,31 @@
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeNode;
 import com.bakdata.conquery.models.exceptions.ConceptConfigurationException;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext;
+import com.bakdata.conquery.sql.conversion.model.filter.ConditionType;
+import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition;
+import com.bakdata.conquery.sql.conversion.model.filter.WhereConditionWrapper;
 import com.bakdata.conquery.util.CalculatedValue;
 import lombok.Getter;
 import lombok.Setter;
+import org.jooq.Condition;
 
 /**
  * This condition connects multiple conditions with an and.
  */
-@CPSType(id="AND", base=CTCondition.class)
+@CPSType(id = "AND", base = CTCondition.class)
 public class AndCondition implements CTCondition {
 
-	@Setter @Getter @Valid @NotEmpty
+	@Setter
+	@Getter
+	@Valid
+	@NotEmpty
 	private List<CTCondition> conditions;
 
 	@Override
 	public boolean matches(String value, CalculatedValue<Map<String, Object>> rowMap) throws ConceptConfigurationException {
-		for(CTCondition cond:conditions) {
-			if(!cond.matches(value, rowMap)) {
+		for (CTCondition cond : conditions) {
+			if (!cond.matches(value, rowMap)) {
 				return false;
 			}
 		}
@@ -34,8 +42,20 @@ public boolean matches(String value, CalculatedValue<Map<String, Object>> rowMap
 
 	@Override
 	public void init(ConceptTreeNode node) throws ConceptConfigurationException {
-		for(CTCondition cond:conditions) {
+		for (CTCondition cond : conditions) {
 			cond.init(node);
 		}
 	}
+
+	@Override
+	public WhereCondition convertToSqlCondition(CTConditionContext context) {
+		return conditions.stream()
+						 .map(condition -> condition.convertToSqlCondition(context))
+						 .map(WhereCondition::condition)
+						 .reduce(Condition::and)
+						 .map(condition -> new WhereConditionWrapper(condition, ConditionType.PREPROCESSING))
+						 .orElseThrow(
+								 () -> new IllegalStateException("At least one condition is required to convert %s to a SQL condition.".formatted(getClass()))
+						 );
+	}
 }

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/CTCondition.java

@@ -5,8 +5,11 @@
 import com.bakdata.conquery.io.cps.CPSBase;
 import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeNode;
 import com.bakdata.conquery.models.exceptions.ConceptConfigurationException;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext;
+import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition;
 import com.bakdata.conquery.util.CalculatedValue;
 import com.fasterxml.jackson.annotation.JsonTypeInfo;
+
 /**
  * A general condition that serves as a guard for concept tree nodes.
  */
@@ -18,4 +21,6 @@ public default void init(ConceptTreeNode node) throws ConceptConfigurationExcept
 
 	public boolean matches(String value, CalculatedValue<Map<String, Object>> rowMap) throws ConceptConfigurationException;
 
+	WhereCondition convertToSqlCondition(CTConditionContext context);
+
 }

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/ColumnEqualCondition.java

@@ -6,13 +6,18 @@
 import javax.validation.constraints.NotEmpty;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext;
+import com.bakdata.conquery.sql.conversion.model.filter.MultiSelectCondition;
+import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition;
 import com.bakdata.conquery.util.CalculatedValue;
 import com.bakdata.conquery.util.CollectionsUtil;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.Setter;
+import org.jooq.Field;
+import org.jooq.impl.DSL;
 
 /**
  * This condition requires the value of another column to be equal to a given value.
@@ -39,4 +44,10 @@ public boolean matches(String value, CalculatedValue<Map<String, Object>> rowMap
 		}
 		return values.contains(checkedValue.toString());
 	}
-}
+
+	@Override
+	public WhereCondition convertToSqlCondition(CTConditionContext context) {
+		Field<String> field = DSL.field(DSL.name(context.getConnectorTable().getName(), column), String.class);
+		return new MultiSelectCondition(field, values.toArray(String[]::new), context.getFunctionProvider());
+	}
+}

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/EqualCondition.java

@@ -6,12 +6,17 @@
 import javax.validation.constraints.NotEmpty;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext;
+import com.bakdata.conquery.sql.conversion.model.filter.MultiSelectCondition;
+import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition;
 import com.bakdata.conquery.util.CalculatedValue;
 import com.bakdata.conquery.util.CollectionsUtil;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import lombok.AllArgsConstructor;
 import lombok.Getter;
 import lombok.Setter;
+import org.jooq.Field;
+import org.jooq.impl.DSL;
 
 /**
  * This condition requires each value to be exactly as given in the list.
@@ -28,10 +33,14 @@ public static EqualCondition create(Set<String> values) {
 		return new EqualCondition(CollectionsUtil.createSmallestSet(values));
 	}
 
-
-
 	@Override
 	public boolean matches(String value, CalculatedValue<Map<String, Object>> rowMap) {
 		return values.contains(value);
 	}
-}
+
+	@Override
+	public WhereCondition convertToSqlCondition(CTConditionContext context) {
+		Field<String> field = DSL.field(DSL.name(context.getConnectorTable().getName(), context.getConnectorColumn().getName()), String.class);
+		return new MultiSelectCondition(field, values.toArray(String[]::new), context.getFunctionProvider());
+	}
+}

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/GroovyCondition.java

@@ -10,6 +10,8 @@
 import com.bakdata.conquery.models.common.Range;
 import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeNode;
 import com.bakdata.conquery.models.exceptions.ConceptConfigurationException;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext;
+import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition;
 import com.bakdata.conquery.util.CalculatedValue;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import groovy.lang.GroovyShell;
@@ -74,6 +76,11 @@ public boolean matches(String value, CalculatedValue<Map<String, Object>> rowMap
 		}
 	}
 
+	@Override
+	public WhereCondition convertToSqlCondition(CTConditionContext context) {
+		throw new UnsupportedOperationException("SQL conversion of CTCondition %s not supported yet.".formatted(getClass()));
+	}
+
 
 	public abstract static class ConditionScript extends Script {
 

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/IsPresentCondition.java

@@ -3,10 +3,16 @@
 import java.util.Map;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext;
+import com.bakdata.conquery.sql.conversion.model.filter.ConditionType;
+import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition;
+import com.bakdata.conquery.sql.conversion.model.filter.WhereConditionWrapper;
 import com.bakdata.conquery.util.CalculatedValue;
 import lombok.Getter;
 import lombok.NonNull;
 import lombok.Setter;
+import org.jooq.Condition;
+import org.jooq.impl.DSL;
 
 /**
  * This condition requires that the selected Column has a value.
@@ -22,4 +28,10 @@ public class IsPresentCondition implements CTCondition {
 	public boolean matches(String value, CalculatedValue<Map<String, Object>> rowMap) {
 		return rowMap.getValue().containsKey(column);
 	}
-}
+
+	@Override
+	public WhereCondition convertToSqlCondition(CTConditionContext context) {
+		Condition condition = DSL.field(DSL.name(context.getConnectorTable().getName(), column)).isNotNull();
+		return new WhereConditionWrapper(condition, ConditionType.PREPROCESSING);
+	}
+}

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/NotCondition.java

@@ -7,6 +7,8 @@
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeNode;
 import com.bakdata.conquery.models.exceptions.ConceptConfigurationException;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext;
+import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition;
 import com.bakdata.conquery.util.CalculatedValue;
 import lombok.Getter;
 import lombok.Setter;
@@ -29,4 +31,10 @@ public boolean matches(String value, CalculatedValue<Map<String, Object>> rowMap
 	public void init(ConceptTreeNode node) throws ConceptConfigurationException {
 		condition.init(node);
 	}
+
+	@Override
+	public WhereCondition convertToSqlCondition(CTConditionContext context) {
+		WhereCondition whereCondition = condition.convertToSqlCondition(context);
+		return whereCondition.negate();
+	}
 }

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/OrCondition.java

@@ -9,33 +9,53 @@
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeNode;
 import com.bakdata.conquery.models.exceptions.ConceptConfigurationException;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext;
+import com.bakdata.conquery.sql.conversion.model.filter.ConditionType;
+import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition;
+import com.bakdata.conquery.sql.conversion.model.filter.WhereConditionWrapper;
 import com.bakdata.conquery.util.CalculatedValue;
 import lombok.Getter;
 import lombok.Setter;
+import org.jooq.Condition;
 
 /**
  * This condition connects multiple conditions with an or.
  */
-@CPSType(id="OR", base=CTCondition.class)
+@CPSType(id = "OR", base = CTCondition.class)
 public class OrCondition implements CTCondition {
 
-	@Setter @Getter @Valid @NotEmpty
+	@Setter
+	@Getter
+	@Valid
+	@NotEmpty
 	private List<CTCondition> conditions;
-	
+
 	@Override
 	public boolean matches(String value, CalculatedValue<Map<String, Object>> rowMap) throws ConceptConfigurationException {
-		for(CTCondition cond:conditions) {
-			if(cond.matches(value, rowMap)) {
+		for (CTCondition cond : conditions) {
+			if (cond.matches(value, rowMap)) {
 				return true;
 			}
 		}
 		return false;
 	}
-	
+
 	@Override
 	public void init(ConceptTreeNode node) throws ConceptConfigurationException {
-		for(CTCondition cond:conditions) {
+		for (CTCondition cond : conditions) {
 			cond.init(node);
 		}
 	}
+
+	@Override
+	public WhereCondition convertToSqlCondition(CTConditionContext context) {
+		return conditions.stream()
+						 .map(condition -> condition.convertToSqlCondition(context))
+						 .map(WhereCondition::condition)
+						 .reduce(Condition::or)
+						 .map(condition -> new WhereConditionWrapper(condition, ConditionType.PREPROCESSING))
+						 .orElseThrow(
+								 () -> new IllegalStateException("At least one condition is required to convert %s to a SQL condition.".formatted(getClass()))
+						 );
+	}
 }

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/conditions/PrefixCondition.java

@@ -1,34 +1,51 @@
 package com.bakdata.conquery.models.datasets.concepts.conditions;
 
+import java.util.Arrays;
 import java.util.Map;
+import java.util.stream.Collectors;
 
 import javax.validation.constraints.NotEmpty;
 
 import com.bakdata.conquery.io.cps.CPSType;
+import com.bakdata.conquery.sql.conversion.cqelement.concept.CTConditionContext;
+import com.bakdata.conquery.sql.conversion.model.filter.ConditionType;
+import com.bakdata.conquery.sql.conversion.model.filter.WhereCondition;
+import com.bakdata.conquery.sql.conversion.model.filter.WhereConditionWrapper;
 import com.bakdata.conquery.util.CalculatedValue;
 import lombok.Getter;
 import lombok.Setter;
 import lombok.ToString;
+import org.jooq.Condition;
+import org.jooq.Field;
+import org.jooq.impl.DSL;
 
 /**
  * This condition requires each value to start with one of the given values.
  */
-@CPSType(id="PREFIX_LIST", base=CTCondition.class)
+@CPSType(id = "PREFIX_LIST", base = CTCondition.class)
 @ToString
 public class PrefixCondition implements CTCondition {
 
-	@Setter @Getter @NotEmpty
+	@Setter
+	@Getter
+	@NotEmpty
 	private String[] prefixes;
 
 	@Override
 	public boolean matches(String value, CalculatedValue<Map<String, Object>> rowMap) {
-		for(String p:prefixes) {
-			if(value.startsWith(p)) {
+		for (String p : prefixes) {
+			if (value.startsWith(p)) {
 				return true;
 			}
 		}
 		return false;
 	}
 
-
+	@Override
+	public WhereCondition convertToSqlCondition(CTConditionContext context) {
+		Field<String> field = DSL.field(DSL.name(context.getConnectorTable().getName(), context.getConnectorColumn().getName()), String.class);
+		String pattern = Arrays.stream(prefixes).collect(Collectors.joining("|", "", "%"));
+		Condition condition = context.getFunctionProvider().likeRegex(field, pattern);
+		return new WhereConditionWrapper(condition, ConditionType.PREPROCESSING);
+	}
 }