Merge branch 'develop' into feature/global_indexservice

backend/pom.xml

@@ -389,5 +389,25 @@
             <artifactId>ngdbc</artifactId>
             <version>2.17.10</version>
         </dependency>
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>simpleclient_dropwizard</artifactId>
+            <version>0.16.0</version>
+        </dependency>
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>prometheus-metrics-simpleclient-bridge</artifactId>
+            <version>1.2.1</version>
+        </dependency>
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>prometheus-metrics-exporter-servlet-jakarta</artifactId>
+            <version>1.2.1</version>
+        </dependency>
+        <dependency>
+            <groupId>io.prometheus</groupId>
+            <artifactId>prometheus-metrics-core</artifactId>
+            <version>1.2.1</version>
+        </dependency>
     </dependencies>
 </project>

backend/src/main/java/com/bakdata/conquery/Conquery.java

@@ -9,6 +9,7 @@
 import com.bakdata.conquery.commands.ShardNode;
 import com.bakdata.conquery.io.jackson.Jackson;
 import com.bakdata.conquery.io.jackson.MutableInjectableValues;
+import com.bakdata.conquery.metrics.prometheus.PrometheusBundle;
 import com.bakdata.conquery.mode.Manager;
 import com.bakdata.conquery.mode.ManagerProvider;
 import com.bakdata.conquery.mode.cluster.ClusterManagerProvider;
@@ -82,6 +83,8 @@ public void initialize(Bootstrap<?> bootstrap) {
 						bootstrap.getConfigurationSourceProvider(), StringSubstitutor.createInterpolator()));
 			}
 		});
+
+		bootstrap.addBundle(new PrometheusBundle());
 	}
 
 	@Override

backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java

@@ -56,6 +56,7 @@
 import com.bakdata.conquery.models.datasets.SecondaryIdDescription;
 import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.error.ConqueryError;
+import com.bakdata.conquery.models.exceptions.ValidatorHelper;
 import com.bakdata.conquery.models.execution.ExecutionState;
 import com.bakdata.conquery.models.execution.ManagedExecution;
 import com.bakdata.conquery.models.i18n.I18n;
@@ -84,6 +85,7 @@
 import com.google.common.collect.MutableClassToInstanceMap;
 import jakarta.inject.Inject;
 import jakarta.servlet.http.HttpServletRequest;
+import jakarta.validation.Validator;
 import jakarta.ws.rs.BadRequestException;
 import jakarta.ws.rs.core.Response;
 import jakarta.ws.rs.core.UriBuilder;
@@ -102,6 +104,8 @@ public class QueryProcessor {
 	private MetaStorage storage;
 	@Inject
 	private ConqueryConfig config;
+	@Inject
+	private Validator validator;
 
 
 
@@ -339,6 +343,9 @@ public FullExecutionStatus getSingleEntityExport(Subject subject, UriBuilder uri
 		final EntityPreviewForm form =
 				EntityPreviewForm.create(entity, idKind, dateRange, sources, previewConfig.getSelects(), previewConfig.getTimeStratifiedSelects(), datasetRegistry);
 
+		// Validate our own form because we provide it directly to the processor, which does not validate.
+		ValidatorHelper.failOnError(log, validator.validate(form));
+
 		// TODO make sure that subqueries are also system
 		// TODO do not persist system queries
 		final EntityPreviewExecution execution = (EntityPreviewExecution) postQuery(dataset, form, subject, true);

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java

@@ -99,6 +99,7 @@ public static CQConcept forSelect(Select select) {
 			table.setConnector(((Connector) select.getHolder()));
 
 			table.setSelects(List.of(select));
+			table.setConcept(cqConcept);
 		}
 		else {
 			cqConcept.setTables(((Concept<?>) select.getHolder())

backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java

@@ -4,8 +4,6 @@
 import java.util.Objects;
 import java.util.OptionalInt;
 
-import jakarta.validation.Validator;
-
 import com.bakdata.conquery.io.storage.xodus.stores.CachedStore;
 import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore;
 import com.bakdata.conquery.models.config.StoreFactory;
@@ -19,6 +17,7 @@
 import com.bakdata.conquery.models.worker.WorkerToBucketsMap;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
+import jakarta.validation.Validator;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -34,7 +33,7 @@ public class NamespaceStorage extends NamespacedStorage {
 	protected CachedStore<String, Integer> entity2Bucket;
 
 	public NamespaceStorage(StoreFactory storageFactory, String pathName, Validator validator) {
-		super(storageFactory, pathName, validator);
+		super(storageFactory, pathName);
 	}
 
 

backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java

@@ -3,7 +3,6 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
-import java.util.Set;
 
 import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore;
 import com.bakdata.conquery.models.config.StoreFactory;
@@ -15,16 +14,13 @@
 import com.bakdata.conquery.models.datasets.concepts.Concept;
 import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept;
-import com.bakdata.conquery.models.exceptions.ValidatorHelper;
 import com.bakdata.conquery.models.identifiable.CentralRegistry;
 import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId;
 import com.bakdata.conquery.models.identifiable.ids.specific.ImportId;
 import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId;
 import com.bakdata.conquery.models.identifiable.ids.specific.TableId;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
-import jakarta.validation.ConstraintViolation;
-import jakarta.validation.Validator;
 import lombok.Getter;
 import lombok.SneakyThrows;
 import lombok.ToString;
@@ -48,19 +44,15 @@ public abstract class NamespacedStorage extends ConqueryStorage {
 	@Getter
 	private final StoreFactory storageFactory;
 
-	@Getter
-	private final Validator validator;
-
 	protected SingletonStore<Dataset> dataset;
 	protected IdentifiableStore<SecondaryIdDescription> secondaryIds;
 	protected IdentifiableStore<Table> tables;
 	protected IdentifiableStore<Import> imports;
 	protected IdentifiableStore<Concept<?>> concepts;
 
-	public NamespacedStorage(StoreFactory storageFactory, String pathName, Validator validator) {
+	public NamespacedStorage(StoreFactory storageFactory, String pathName) {
 		this.pathName = pathName;
 		this.storageFactory = storageFactory;
-		this.validator = validator;
 	}
 
 	public void openStores(ObjectMapper objectMapper) {
@@ -123,13 +115,6 @@ private void decorateConceptStore(IdentifiableStore<Concept<?>> store) {
 
 			concept.initElements();
 
-			if (log.isTraceEnabled()) {
-				// Validating concepts is quite slow, so we only validate when requested.
-				final Set<ConstraintViolation<Concept<?>>> violations = validator.validate(concept);
-
-				ValidatorHelper.failOnError(log, violations);
-			}
-
 			concept.getSelects().forEach(centralRegistry::register);
 			for (Connector connector : concept.getConnectors()) {
 				centralRegistry.register(connector);

backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java

@@ -26,7 +26,7 @@ public class WorkerStorage extends NamespacedStorage {
 	private IdentifiableStore<CBlock> cBlocks;
 
 	public WorkerStorage(StoreFactory storageFactory, Validator validator, String pathName) {
-		super(storageFactory, pathName, validator);
+		super(storageFactory, pathName);
 	}
 
 	@Override

backend/src/main/java/com/bakdata/conquery/metrics/prometheus/PrometheusBundle.java

@@ -0,0 +1,28 @@
+package com.bakdata.conquery.metrics.prometheus;
+
+import com.bakdata.conquery.models.config.ConqueryConfig;
+import io.dropwizard.core.ConfiguredBundle;
+import io.dropwizard.core.setup.Bootstrap;
+import io.dropwizard.core.setup.Environment;
+import io.prometheus.client.CollectorRegistry;
+import io.prometheus.client.dropwizard.DropwizardExports;
+import io.prometheus.metrics.exporter.servlet.jakarta.PrometheusMetricsServlet;
+import io.prometheus.metrics.simpleclient.bridge.SimpleclientCollector;
+import org.eclipse.jetty.servlet.ServletHolder;
+
+public class PrometheusBundle implements ConfiguredBundle<ConqueryConfig> {
+
+	@Override
+	public void initialize(Bootstrap<?> bootstrap) {
+		// Init simple_client https://www.robustperception.io/exposing-dropwizard-metrics-to-prometheus/
+		CollectorRegistry.defaultRegistry.register(new DropwizardExports(bootstrap.getMetricRegistry()));
+		// Init client_java bridge: https://prometheus.github.io/client_java/migration/simpleclient/
+		SimpleclientCollector.builder().register();
+	}
+
+	@Override
+	public void run(ConqueryConfig configuration, Environment environment) throws Exception {
+
+		environment.getAdminContext().addServlet(new ServletHolder(new PrometheusMetricsServlet()), "/metrics-prometheus");
+	}
+}

backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConfig.java

@@ -2,9 +2,11 @@
 
 import lombok.Builder;
 import lombok.Data;
+import lombok.extern.jackson.Jacksonized;
 
 @Data
 @Builder
+@Jacksonized
 public class DatabaseConfig {
 
 	private static final String DEFAULT_PRIMARY_COLUMN = "pid";

backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java

@@ -3,15 +3,20 @@
 import java.util.Map;
 
 import com.bakdata.conquery.models.datasets.Dataset;
+import com.fasterxml.jackson.annotation.JsonIgnore;
+import io.dropwizard.validation.ValidationMethod;
+import jakarta.validation.Valid;
 import lombok.AccessLevel;
 import lombok.AllArgsConstructor;
 import lombok.Builder;
 import lombok.Data;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
+import lombok.extern.jackson.Jacksonized;
 
 @Data
 @Builder
+@Jacksonized
 @NoArgsConstructor
 @AllArgsConstructor
 public class SqlConnectorConfig {
@@ -27,10 +32,19 @@ public class SqlConnectorConfig {
 	 * Keys must match the name of existing {@link Dataset}s.
 	 */
 	@Getter(AccessLevel.PRIVATE)
-	private Map<String, DatabaseConfig> databaseConfigs;
+	private Map<String, @Valid DatabaseConfig> databaseConfigs;
 
 	public DatabaseConfig getDatabaseConfig(Dataset dataset) {
 		return databaseConfigs.get(dataset.getName());
 	}
 
+	@JsonIgnore
+	@ValidationMethod(message = "At lease 1 DatabaseConfig has to be present if SqlConnector config is enabled")
+	public boolean isValidSqlConnectorConfig() {
+		if (!enabled) {
+			return true;
+		}
+		return databaseConfigs != null && !databaseConfigs.isEmpty();
+	}
+
 }

backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java

@@ -41,7 +41,7 @@ public class Table extends Labeled<TableId> implements NamespacedIdentifiable<Ta
 	 */
 	@Nullable
 	@JsonManagedReference
-	private Column primaryColum;
+	private Column primaryColumn;
 
 	@ValidationMethod(message = "More than one column map to the same secondaryId")
 	@JsonIgnore

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/Select.java

@@ -61,8 +61,8 @@ public Dataset getDataset() {
 	@JsonIgnore
 	public abstract List<Column> getRequiredColumns();
 
-	@JsonIgnore @Getter(lazy=true)
-	private final ResultType resultType = createAggregator().getResultType();
+	@JsonIgnore
+	public abstract ResultType<?> getResultType();
 
 	public abstract Aggregator<?> createAggregator();
 

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java

@@ -3,6 +3,7 @@
 import java.util.Collections;
 import java.util.Set;
 
+import com.bakdata.conquery.apiv1.query.TableExportQuery;
 import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept;
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.models.datasets.concepts.ConceptElement;
@@ -12,6 +13,7 @@
 import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.ConceptElementsAggregator;
 import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.ConceptValuesAggregator;
 import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo;
+import com.bakdata.conquery.models.types.ResultType;
 import com.bakdata.conquery.models.types.SemanticType;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import io.dropwizard.validation.ValidationMethod;
@@ -31,7 +33,7 @@ public class ConceptColumnSelect extends UniversalSelect {
 
 	@Override
 	public Aggregator<?> createAggregator() {
-		if(isAsIds()){
+		if (isAsIds()) {
 			return new ConceptElementsAggregator(((TreeConcept) getHolder().findConcept()));
 		}
 
@@ -55,6 +57,13 @@ public boolean isHolderTreeConcept() {
 		return getHolder().findConcept() instanceof TreeConcept;
 	}
 
+	@Override
+	public ResultType<?> getResultType() {
+		if (isAsIds()) {
+			return new ResultType.ListT<>(new ResultType.StringT((val, settings) -> TableExportQuery.printValue(getHolder().findConcept(), val, settings)));
+		}
 
+		return new ResultType.ListT<>(ResultType.StringT.INSTANCE);
+	}
 
 }

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDateUnionSelect.java

@@ -9,6 +9,7 @@
 import com.bakdata.conquery.models.datasets.concepts.select.concept.UniversalSelect;
 import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator;
 import com.bakdata.conquery.models.query.queryplan.aggregators.specific.EventDateUnionAggregator;
+import com.bakdata.conquery.models.types.ResultType;
 import com.bakdata.conquery.sql.conversion.model.aggregator.EventDateUnionSqlAggregator;
 import com.bakdata.conquery.sql.conversion.model.select.SelectContext;
 import com.bakdata.conquery.sql.conversion.model.select.SqlSelects;
@@ -39,4 +40,8 @@ public boolean isEventDateSelect() {
 		return true;
 	}
 
+	@Override
+	public ResultType<?> getResultType() {
+		return new ResultType.ListT(ResultType.DateRangeT.INSTANCE);
+	}
 }

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/EventDurationSumSelect.java

@@ -5,6 +5,7 @@
 import com.bakdata.conquery.models.datasets.concepts.select.concept.UniversalSelect;
 import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator;
 import com.bakdata.conquery.models.query.queryplan.aggregators.specific.EventDurationSumAggregator;
+import com.bakdata.conquery.models.types.ResultType;
 import com.bakdata.conquery.sql.conversion.model.aggregator.EventDurationSumSqlAggregator;
 import com.bakdata.conquery.sql.conversion.model.select.SelectContext;
 import com.bakdata.conquery.sql.conversion.model.select.SqlSelects;
@@ -39,4 +40,9 @@ public SqlSelects convertToSqlSelects(SelectContext selectContext) {
 	public boolean isEventDateSelect() {
 		return true;
 	}
+
+	@Override
+	public ResultType<?> getResultType() {
+		return ResultType.IntegerT.INSTANCE;
+	}
 }

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/ExistsSelect.java

@@ -9,8 +9,9 @@
 import com.bakdata.conquery.models.datasets.concepts.select.Select;
 import com.bakdata.conquery.models.datasets.concepts.select.concept.UniversalSelect;
 import com.bakdata.conquery.models.query.queryplan.aggregators.specific.ExistsAggregator;
-import com.bakdata.conquery.sql.conversion.model.select.SelectContext;
+import com.bakdata.conquery.models.types.ResultType;
 import com.bakdata.conquery.sql.conversion.model.aggregator.ExistsSqlAggregator;
+import com.bakdata.conquery.sql.conversion.model.select.SelectContext;
 import com.bakdata.conquery.sql.conversion.model.select.SqlSelects;
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import lombok.Getter;
@@ -34,4 +35,9 @@ public SqlSelects convertToSqlSelects(SelectContext selectContext) {
 	private Set<Table> collectRequiredTables() {
 		return this.getHolder().findConcept().getConnectors().stream().map(Connector::getTable).collect(Collectors.toSet());
 	}
+
+	@Override
+	public ResultType<?> getResultType() {
+		return ResultType.BooleanT.INSTANCE;
+	}
 }

backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/specific/QuarterSelect.java

@@ -6,6 +6,7 @@
 import com.bakdata.conquery.models.datasets.concepts.select.concept.UniversalSelect;
 import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator;
 import com.bakdata.conquery.models.query.queryplan.aggregators.specific.QuarterAggregator;
+import com.bakdata.conquery.models.types.ResultType;
 import com.fasterxml.jackson.annotation.JsonCreator;
 import jakarta.validation.constraints.NotNull;
 import lombok.Data;
@@ -26,4 +27,9 @@ public class QuarterSelect extends UniversalSelect {
 	public Aggregator<?> createAggregator() {
 		return new QuarterAggregator(sample);
 	}
+
+	@Override
+	public ResultType<?> getResultType() {
+		return ResultType.StringT.INSTANCE;
+	}
 }