Merge branch 'develop' into feature/prometheus-metrics

ingef · May 27, 2024 · 383afbb · 383afbb
2 parents 3acd39a + eb3d43d
commit 383afbb
Show file tree

Hide file tree

Showing 102 changed files with 1,129 additions and 770 deletions.
diff --git a/backend/pom.xml b/backend/pom.xml
@@ -303,7 +303,7 @@
         <dependency>
             <groupId>com.nimbusds</groupId>
             <artifactId>oauth2-oidc-sdk</artifactId>
-            <version>9.1</version>
+            <version>11.12</version>
         </dependency>
         <dependency>
             <groupId>org.keycloak</groupId>

diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendConfiguration.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendConfiguration.java
@@ -2,25 +2,22 @@
 
 import java.net.URL;
 import java.time.LocalDate;
-import java.util.Map;
+import java.util.List;
 
 import com.bakdata.conquery.models.config.FrontendConfig;
 import com.bakdata.conquery.models.config.IdColumnConfig;
 
 /**
  * API Response for the dynamic configuration of the frontend
  *
- * @param version             backend version
- * @param formBackendVersions mapping of form backend ids to their versions (version can be null)
+ * @param versions            mapping of backend and form backend ids to their versions (version can be null)
  * @param currency            currency representation
  * @param queryUpload         identifier specific column configuration for the query upload
  * @param manualUrl           url to a user manual
  * @param contactEmail        typical a mailto-url
  */
 public record FrontendConfiguration(
-		String version,
-
-		Map<String, String> formBackendVersions,
+		List<VersionContainer> versions,
 		FrontendConfig.CurrencyConfig currency,
 		IdColumnConfig queryUpload,
 		URL manualUrl,

diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/VersionContainer.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/VersionContainer.java
@@ -0,0 +1,12 @@
+package com.bakdata.conquery.apiv1.frontend;
+
+import java.time.ZonedDateTime;
+
+import lombok.NonNull;
+
+public record VersionContainer(
+		@NonNull String name,
+		String version,
+		ZonedDateTime buildTime
+) {
+}
diff --git a/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java b/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java
@@ -7,14 +7,6 @@
 import java.util.UUID;
 import java.util.function.Function;
 
-import jakarta.ws.rs.client.Client;
-import jakarta.ws.rs.client.Entity;
-import jakarta.ws.rs.client.Invocation;
-import jakarta.ws.rs.client.WebTarget;
-import jakarta.ws.rs.core.GenericType;
-import jakarta.ws.rs.core.MediaType;
-import jakarta.ws.rs.core.Response;
-
 import com.bakdata.conquery.apiv1.forms.ExternalForm;
 import com.bakdata.conquery.models.auth.entities.User;
 import com.bakdata.conquery.models.auth.permissions.Ability;
@@ -24,6 +16,13 @@
 import com.codahale.metrics.health.HealthCheck;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import io.dropwizard.health.check.http.HttpHealthCheck;
+import jakarta.ws.rs.client.Client;
+import jakarta.ws.rs.client.Entity;
+import jakarta.ws.rs.client.Invocation;
+import jakarta.ws.rs.client.WebTarget;
+import jakarta.ws.rs.core.GenericType;
+import jakarta.ws.rs.core.MediaType;
+import jakarta.ws.rs.core.Response;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -129,8 +128,8 @@ public HealthCheck createHealthCheck() {
 		return new HttpHealthCheck(getHealthTarget.getUri().toString(), client);
 	}
 
-	public String getVersion() {
-		return getVersionTarget.request(MediaType.APPLICATION_JSON_TYPE).get(FormBackendVersion.class).getVersion();
+	public FormBackendVersion getVersion() {
+		return getVersionTarget.request(MediaType.APPLICATION_JSON_TYPE).get(FormBackendVersion.class);
 	}
 
 	public ExternalTaskState cancelTask(UUID taskId) {

diff --git a/backend/src/main/java/com/bakdata/conquery/io/external/form/FormBackendVersion.java b/backend/src/main/java/com/bakdata/conquery/io/external/form/FormBackendVersion.java
@@ -1,8 +1,10 @@
 package com.bakdata.conquery.io.external.form;
 
-import lombok.Data;
+import java.time.ZonedDateTime;
 
-@Data
-public class FormBackendVersion {
-	private String version;
+
+public record FormBackendVersion(
+		String version,
+		ZonedDateTime buildTime
+) {
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java
@@ -4,8 +4,6 @@
 import java.util.Objects;
 import java.util.OptionalInt;
 
-import jakarta.validation.Validator;
-
 import com.bakdata.conquery.io.storage.xodus.stores.CachedStore;
 import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore;
 import com.bakdata.conquery.models.config.StoreFactory;
@@ -19,6 +17,7 @@
 import com.bakdata.conquery.models.worker.WorkerToBucketsMap;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
+import jakarta.validation.Validator;
 import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
@@ -34,7 +33,7 @@ public class NamespaceStorage extends NamespacedStorage {
 	protected CachedStore<String, Integer> entity2Bucket;
 
 	public NamespaceStorage(StoreFactory storageFactory, String pathName, Validator validator) {
-		super(storageFactory, pathName, validator);
+		super(storageFactory, pathName);
 	}
 
 

diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java
@@ -3,7 +3,6 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
-import java.util.Set;
 
 import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore;
 import com.bakdata.conquery.models.config.StoreFactory;
@@ -15,16 +14,13 @@
 import com.bakdata.conquery.models.datasets.concepts.Concept;
 import com.bakdata.conquery.models.datasets.concepts.Connector;
 import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept;
-import com.bakdata.conquery.models.exceptions.ValidatorHelper;
 import com.bakdata.conquery.models.identifiable.CentralRegistry;
 import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId;
 import com.bakdata.conquery.models.identifiable.ids.specific.ImportId;
 import com.bakdata.conquery.models.identifiable.ids.specific.SecondaryIdDescriptionId;
 import com.bakdata.conquery.models.identifiable.ids.specific.TableId;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.ImmutableList;
-import jakarta.validation.ConstraintViolation;
-import jakarta.validation.Validator;
 import lombok.Getter;
 import lombok.SneakyThrows;
 import lombok.ToString;
@@ -48,19 +44,15 @@ public abstract class NamespacedStorage extends ConqueryStorage {
 	@Getter
 	private final StoreFactory storageFactory;
 
-	@Getter
-	private final Validator validator;
-
 	protected SingletonStore<Dataset> dataset;
 	protected IdentifiableStore<SecondaryIdDescription> secondaryIds;
 	protected IdentifiableStore<Table> tables;
 	protected IdentifiableStore<Import> imports;
 	protected IdentifiableStore<Concept<?>> concepts;
 
-	public NamespacedStorage(StoreFactory storageFactory, String pathName, Validator validator) {
+	public NamespacedStorage(StoreFactory storageFactory, String pathName) {
 		this.pathName = pathName;
 		this.storageFactory = storageFactory;
-		this.validator = validator;
 	}
 
 	public void openStores(ObjectMapper objectMapper) {
@@ -123,13 +115,6 @@ private void decorateConceptStore(IdentifiableStore<Concept<?>> store) {
 
 			concept.initElements();
 
-			if (log.isTraceEnabled()) {
-				// Validating concepts is quite slow, so we only validate when requested.
-				final Set<ConstraintViolation<Concept<?>>> violations = validator.validate(concept);
-
-				ValidatorHelper.failOnError(log, violations);
-			}
-
 			concept.getSelects().forEach(centralRegistry::register);
 			for (Connector connector : concept.getConnectors()) {
 				centralRegistry.register(connector);

diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java
@@ -26,7 +26,7 @@ public class WorkerStorage extends NamespacedStorage {
 	private IdentifiableStore<CBlock> cBlocks;
 
 	public WorkerStorage(StoreFactory storageFactory, Validator validator, String pathName) {
-		super(storageFactory, pathName, validator);
+		super(storageFactory, pathName);
 	}
 
 	@Override

diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java
@@ -10,39 +10,30 @@
 import com.bakdata.conquery.mode.ManagerProvider;
 import com.bakdata.conquery.mode.NamespaceHandler;
 import com.bakdata.conquery.models.config.ConqueryConfig;
-import com.bakdata.conquery.models.config.SqlConnectorConfig;
 import com.bakdata.conquery.models.worker.DatasetRegistry;
 import com.bakdata.conquery.models.worker.LocalNamespace;
 import com.bakdata.conquery.models.worker.ShardNodeInformation;
-import com.bakdata.conquery.sql.DslContextFactory;
-import com.bakdata.conquery.sql.SqlContext;
-import com.bakdata.conquery.sql.conversion.dialect.HanaSqlDialect;
-import com.bakdata.conquery.sql.conversion.dialect.PostgreSqlDialect;
-import com.bakdata.conquery.sql.conversion.dialect.SqlDialect;
-import com.bakdata.conquery.sql.execution.ResultSetProcessorFactory;
-import com.bakdata.conquery.sql.execution.SqlExecutionService;
+import com.bakdata.conquery.sql.conversion.dialect.SqlDialectFactory;
 import io.dropwizard.core.setup.Environment;
-import org.jooq.DSLContext;
 
 public class LocalManagerProvider implements ManagerProvider {
 
 	private static final Supplier<Collection<ShardNodeInformation>> EMPTY_NODE_PROVIDER = Collections::emptyList;
 
-	public DelegateManager<LocalNamespace> provideManager(ConqueryConfig config, Environment environment) {
+	private final SqlDialectFactory dialectFactory;
 
-		InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, environment.getValidator());
+	public LocalManagerProvider() {
+		this.dialectFactory = new SqlDialectFactory();
+	}
 
-		SqlConnectorConfig sqlConnectorConfig = config.getSqlConnectorConfig();
-		DSLContext dslContext = DslContextFactory.create(sqlConnectorConfig);
-		SqlDialect sqlDialect = createSqlDialect(sqlConnectorConfig, dslContext);
-		SqlContext sqlContext = new SqlContext(sqlConnectorConfig, sqlDialect);
+	public LocalManagerProvider(SqlDialectFactory dialectFactory) {
+		this.dialectFactory = dialectFactory;
+	}
 
-		SqlExecutionService sqlExecutionService = new SqlExecutionService(
-				sqlDialect.getDSLContext(),
-				ResultSetProcessorFactory.create(sqlDialect)
-		);
+	public DelegateManager<LocalNamespace> provideManager(ConqueryConfig config, Environment environment) {
 
-		NamespaceHandler<LocalNamespace> namespaceHandler = new LocalNamespaceHandler(config, creator, sqlContext, sqlExecutionService);
+		InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, environment.getValidator());
+		NamespaceHandler<LocalNamespace> namespaceHandler = new LocalNamespaceHandler(config, creator, dialectFactory);
 		DatasetRegistry<LocalNamespace> datasetRegistry = ManagerProvider.createLocalDatasetRegistry(namespaceHandler, config, creator);
 		creator.init(datasetRegistry);
 
@@ -59,10 +50,4 @@ public DelegateManager<LocalNamespace> provideManager(ConqueryConfig config, Env
 		);
 	}
 
-	protected SqlDialect createSqlDialect(SqlConnectorConfig sqlConnectorConfig, DSLContext dslContext) {
-		return switch (sqlConnectorConfig.getDialect()) {
-			case POSTGRESQL -> new PostgreSqlDialect(dslContext);
-			case HANA -> new HanaSqlDialect(dslContext);
-		};
-	}
 }
diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java
@@ -6,33 +6,57 @@
 import com.bakdata.conquery.mode.NamespaceHandler;
 import com.bakdata.conquery.mode.NamespaceSetupData;
 import com.bakdata.conquery.models.config.ConqueryConfig;
+import com.bakdata.conquery.models.config.DatabaseConfig;
+import com.bakdata.conquery.models.config.SqlConnectorConfig;
 import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId;
 import com.bakdata.conquery.models.index.IndexService;
 import com.bakdata.conquery.models.query.ExecutionManager;
 import com.bakdata.conquery.models.worker.LocalNamespace;
-import com.bakdata.conquery.sql.SqlContext;
+import com.bakdata.conquery.sql.DSLContextWrapper;
+import com.bakdata.conquery.sql.DslContextFactory;
 import com.bakdata.conquery.sql.conquery.SqlExecutionManager;
+import com.bakdata.conquery.sql.conversion.SqlConverter;
+import com.bakdata.conquery.sql.conversion.dialect.SqlDialect;
+import com.bakdata.conquery.sql.conversion.dialect.SqlDialectFactory;
+import com.bakdata.conquery.sql.execution.ResultSetProcessorFactory;
+import com.bakdata.conquery.sql.execution.SqlExecutionResult;
 import com.bakdata.conquery.sql.execution.SqlExecutionService;
 import lombok.RequiredArgsConstructor;
+import lombok.extern.slf4j.Slf4j;
+import org.jooq.DSLContext;
 
 @RequiredArgsConstructor
+@Slf4j
 public class LocalNamespaceHandler implements NamespaceHandler<LocalNamespace> {
 
 	private final ConqueryConfig config;
 	private final InternalObjectMapperCreator mapperCreator;
-	private final SqlContext sqlContext;
-	private final SqlExecutionService sqlExecutionService;
+	private final SqlDialectFactory dialectFactory;
 
 	@Override
 	public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaStorage metaStorage, IndexService indexService) {
+
 		NamespaceSetupData namespaceData = NamespaceHandler.createNamespaceSetup(namespaceStorage, config, mapperCreator, indexService);
-		ExecutionManager executionManager = new SqlExecutionManager(sqlContext, sqlExecutionService, metaStorage);
+
+		SqlConnectorConfig sqlConnectorConfig = config.getSqlConnectorConfig();
+		DatabaseConfig databaseConfig = sqlConnectorConfig.getDatabaseConfig(namespaceStorage.getDataset());
+
+		DSLContextWrapper dslContextWrapper = DslContextFactory.create(databaseConfig, sqlConnectorConfig);
+		DSLContext dslContext = dslContextWrapper.getDslContext();
+		SqlDialect sqlDialect = dialectFactory.createSqlDialect(databaseConfig.getDialect());
+
+		SqlConverter sqlConverter = new SqlConverter(sqlDialect, dslContext, databaseConfig);
+		SqlExecutionService sqlExecutionService = new SqlExecutionService(dslContext, ResultSetProcessorFactory.create(sqlDialect));
+		ExecutionManager<SqlExecutionResult> executionManager = new SqlExecutionManager(sqlConverter, sqlExecutionService, metaStorage);
+		SqlStorageHandler sqlStorageHandler = new SqlStorageHandler(sqlExecutionService);
+
 		return new LocalNamespace(
 				namespaceData.getPreprocessMapper(),
 				namespaceData.getCommunicationMapper(),
 				namespaceStorage,
 				executionManager,
-				sqlExecutionService,
+				dslContextWrapper,
+				sqlStorageHandler,
 				namespaceData.getJobManager(),
 				namespaceData.getFilterSearch(),
 				namespaceData.getIndexService(),