Merge branch 'develop' into feature/prometheus-metrics

.github/workflows/codeql-analysis.yml

@@ -44,10 +44,12 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v2
 
-    - name: set java 17
-      uses: actions/setup-java@v1
+    - name: Set up JDK
+      uses: actions/setup-java@v4
       with:
-        java-version: 17
+        distribution: 'temurin'
+        java-version: '17'
+        overwrite-settings: false
 
       # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL

.github/workflows/test_cypress.yml

@@ -17,26 +17,28 @@ jobs:
     timeout-minutes: 10
     steps:
       - name: Cache local Maven repository
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         with:
           path: ~/.m2/repository
           key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
           restore-keys: |
             ${{ runner.os }}-maven-
 
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
 
       - name: Setup Node
-        uses: actions/setup-node@v3
+        uses: actions/setup-node@v4
         with:
           node-version: 18
 
       - name: Set up JDK
-        uses: actions/setup-java@v1
+        uses: actions/setup-java@v4
         with:
-          java-version: 17
+          distribution: 'temurin'
+          java-version: '17'
+          overwrite-settings: false
 
       - name: Build Backend
         run: ./scripts/build_backend_no_version.sh
@@ -53,14 +55,16 @@ jobs:
           start: bash ./scripts/run_e2e_all.sh
           wait-on: "http://localhost:8000"
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: cypress-screenshots
           path: cypress/screenshots
+          if-no-files-found: ignore
 
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         if: failure()
         with:
           name: cypress-videos
           path: cypress/videos
+          if-no-files-found: ignore

backend/src/main/java/com/bakdata/conquery/apiv1/ApiV1.java

@@ -9,7 +9,7 @@
 import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider;
 import com.bakdata.conquery.metrics.ActiveUsersFilter;
 import com.bakdata.conquery.models.auth.basic.JWTokenHandler;
-import com.bakdata.conquery.models.auth.web.DefaultAuthFilter;
+import com.bakdata.conquery.models.auth.web.AuthFilter;
 import com.bakdata.conquery.models.forms.frontendconfiguration.FormConfigProcessor;
 import com.bakdata.conquery.models.forms.frontendconfiguration.FormProcessor;
 import com.bakdata.conquery.resources.ResourcesProvider;
@@ -63,8 +63,8 @@ protected void configure() {
 		 * We use the same instance of the filter for the api servlet and the admin servlet to have a single
 		 * point for authentication.
 		 */
-		jersey.register(DefaultAuthFilter.class);
-		DefaultAuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, jersey.getResourceConfig());
+		jersey.register(AuthFilter.class);
+		AuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, jersey.getResourceConfig());
 
 
 		jersey.register(IdParamConverter.Provider.INSTANCE);

backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java

@@ -4,7 +4,6 @@
 import java.math.BigDecimal;
 
 import javax.annotation.Nonnull;
-import jakarta.validation.constraints.NotNull;
 
 import com.bakdata.conquery.apiv1.frontend.FrontendFilterType;
 import com.bakdata.conquery.io.cps.CPSBase;
@@ -32,13 +31,15 @@
 import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
 import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
 import com.fasterxml.jackson.databind.exc.InvalidTypeIdException;
+import jakarta.validation.constraints.NotNull;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
 import lombok.NoArgsConstructor;
 import lombok.RequiredArgsConstructor;
 import lombok.Setter;
 import lombok.SneakyThrows;
 import lombok.ToString;
+import org.jooq.Condition;
 
 @Getter
 @Setter
@@ -67,14 +68,19 @@ public FilterNode<?> createNode() {
 	}
 
 	public SqlFilters convertToSqlFilter(SqlIdColumns ids, ConversionContext context, ConceptConversionTables tables) {
-		FilterContext<VALUE> filterContext = new FilterContext<>(ids, value, context, tables);
+		FilterContext<VALUE> filterContext = FilterContext.forConceptConversion(ids, value, context, tables);
 		SqlFilters sqlFilters = filter.convertToSqlFilter(filterContext);
 		if (context.isNegation()) {
 			return new SqlFilters(sqlFilters.getSelects(), sqlFilters.getWhereClauses().negated());
 		}
 		return sqlFilters;
 	}
 
+	public Condition convertForTableExport(SqlIdColumns ids, ConversionContext context) {
+		FilterContext<VALUE> filterContext = FilterContext.forTableExport(ids, value, context);
+		return filter.convertForTableExport(filterContext);
+	}
+
 	@NoArgsConstructor
 	@CPSType(id = FrontendFilterType.Fields.MULTI_SELECT, base = FilterValue.class)
 	@ToString(callSuper = true)

backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java

@@ -29,6 +29,7 @@
 import com.bakdata.conquery.models.messages.network.specific.AddShardNode;
 import com.bakdata.conquery.models.messages.network.specific.RegisterWorker;
 import com.bakdata.conquery.models.messages.network.specific.UpdateJobManagerStatus;
+import com.bakdata.conquery.models.worker.IdResolveContext;
 import com.bakdata.conquery.models.worker.Worker;
 import com.bakdata.conquery.models.worker.WorkerInformation;
 import com.bakdata.conquery.models.worker.Workers;
@@ -51,6 +52,7 @@
 import org.apache.mina.core.session.IoSession;
 import org.apache.mina.filter.FilterEvent;
 import org.apache.mina.transport.socket.nio.NioSocketConnector;
+import org.jetbrains.annotations.NotNull;
 
 /**
  * This node holds a shard of data (in so called {@link Worker}s for the different datasets in conquery.
@@ -64,6 +66,7 @@ public class ShardNode extends ConqueryCommand implements IoHandler, Managed {
 	public static final String DEFAULT_NAME = "shard-node";
 
 	private NioSocketConnector connector;
+	private ConnectFuture future;
 	private JobManager jobManager;
 	private Validator validator;
 	private ConqueryConfig config;
@@ -88,7 +91,6 @@ protected void run(Environment environment, Namespace namespace, ConqueryConfig
 		this.environment = environment;
 		this.config = config;
 
-		connector = new NioSocketConnector();
 
 		jobManager = new JobManager(getName(), config.isFailOnError());
 		environment.lifecycle().manage(this);
@@ -106,6 +108,7 @@ protected void run(Environment environment, Namespace namespace, ConqueryConfig
 				getConfig().getQueries().getSecondaryIdSubPlanRetention()
 		);
 
+
 		final Collection<WorkerStorage> workerStorages = config.getStorage().discoverWorkerStorages();
 
 
@@ -268,6 +271,8 @@ private static void scheduleIdleLogger(ScheduledExecutorService scheduler, IoSes
 	public void sessionClosed(IoSession session) {
 		setLocation(session);
 		log.info("Disconnected from ManagerNode.");
+
+		scheduler.schedule(this::connectToCluster, 2, TimeUnit.SECONDS);
 	}
 
 	@Override
@@ -291,7 +296,9 @@ public void messageSent(IoSession session, Object message) {
 	@Override
 	public void inputClosed(IoSession session) {
 		setLocation(session);
-		log.info("Session closed.");
+		log.info("Input closed.");
+		session.closeNow();
+		scheduler.schedule(this::disconnectFromCluster, 0, TimeUnit.SECONDS);
 	}
 
 	@Override
@@ -306,24 +313,27 @@ public void start() throws Exception {
 			value.getJobManager().addSlowJob(new SimpleJob("Update Bucket Manager", value.getBucketManager()::fullUpdate));
 		}
 
-		ObjectMapper om = createInternalObjectMapper(View.InternalCommunication.class);
+		scheduler.schedule(this::connectToCluster, 0, TimeUnit.MINUTES);
 
-		BinaryJacksonCoder coder = new BinaryJacksonCoder(workers, validator, om);
-		connector.getFilterChain().addLast("codec", new CQProtocolCodecFilter(new ChunkWriter(coder), new ChunkReader(coder, om)));
-		connector.setHandler(this);
-		connector.getSessionConfig().setAll(config.getCluster().getMina());
 
+	}
+
+	private void connectToCluster() {
 		InetSocketAddress address = new InetSocketAddress(
 				config.getCluster().getManagerURL().getHostAddress(),
 				config.getCluster().getPort()
 		);
 
+		disconnectFromCluster();
+
+		connector = getClusterConnector(workers);
+
 		while (true) {
 			try {
 				log.info("Trying to connect to {}", address);
 
 				// Try opening a connection (Note: This fails immediately instead of waiting a minute to try and connect)
-				ConnectFuture future = connector.connect(address);
+				future = connector.connect(address);
 
 				future.awaitUninterruptibly();
 
@@ -339,9 +349,24 @@ public void start() throws Exception {
 			catch (RuntimeIoException e) {
 				log.warn("Failed to connect to {}", address, e);
 			}
+			catch (InterruptedException e) {
+				log.warn("Interrupted while trying to connector to cluster, giving up.", e);
+				break;
+			}
 		}
+	}
 
+	@NotNull
+	private NioSocketConnector getClusterConnector(IdResolveContext workers) {
+		ObjectMapper om = createInternalObjectMapper(View.InternalCommunication.class);
+
+		NioSocketConnector connector = new NioSocketConnector();
 
+		BinaryJacksonCoder coder = new BinaryJacksonCoder(workers, validator, om);
+		connector.getFilterChain().addLast("codec", new CQProtocolCodecFilter(new ChunkWriter(coder), new ChunkReader(coder, om)));
+		connector.setHandler(this);
+		connector.getSessionConfig().setAll(config.getCluster().getMina());
+		return connector;
 	}
 
 	@Override
@@ -350,12 +375,23 @@ public void stop() throws Exception {
 
 		workers.stop();
 
+		disconnectFromCluster();
+	}
+
+	private void disconnectFromCluster() {
+		if (future != null) {
+			future.cancel();
+		}
+
 		//after the close command was send
 		if (context != null) {
 			context.awaitClose();
 		}
-		log.info("Connection was closed by ManagerNode");
-		connector.dispose();
+
+		if (connector != null) {
+			log.info("Connection was closed by ManagerNode");
+			connector.dispose();
+		}
 	}
 
 	public boolean isBusy() {

backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java

@@ -15,7 +15,7 @@
 import com.bakdata.conquery.models.auth.entities.User;
 import com.bakdata.conquery.models.auth.permissions.Ability;
 import com.bakdata.conquery.models.auth.permissions.ConqueryPermission;
-import com.bakdata.conquery.models.auth.web.DefaultAuthFilter;
+import com.bakdata.conquery.models.auth.web.AuthFilter;
 import com.bakdata.conquery.models.auth.web.RedirectingAuthFilter;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.config.auth.AuthenticationRealmFactory;
@@ -85,18 +85,18 @@ public AuthorizationController(MetaStorage storage, ConqueryConfig config, Envir
 		this.adminServlet = adminServlet;
 
 		if (adminServlet != null) {
-			adminServlet.getJerseyConfig().register(DefaultAuthFilter.class);
-			DefaultAuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, adminServlet.getJerseyConfig());
+			adminServlet.getJerseyConfig().register(AuthFilter.class);
+			AuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, adminServlet.getJerseyConfig());
 
 			// The binding is necessary here because the RedirectingAuthFitler delegates to the DefaultAuthfilter at the moment
 			adminServlet.getJerseyConfigUI().register(new AbstractBinder() {
 				@Override
 				protected void configure() {
-					bindAsContract(DefaultAuthFilter.class);
+					bindAsContract(AuthFilter.class);
 				}
 			});
 			adminServlet.getJerseyConfigUI().register(RedirectingAuthFilter.class);
-			DefaultAuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, adminServlet.getJerseyConfigUI());
+			AuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, adminServlet.getJerseyConfigUI());
 		}
 
 		unprotectedAuthAdmin = AuthServlet.generalSetup(environment.metrics(), config, environment.admin(), environment.getObjectMapper());

backend/src/main/java/com/bakdata/conquery/models/auth/basic/JWTokenHandler.java

@@ -9,10 +9,9 @@
 import com.auth0.jwt.JWT;
 import com.auth0.jwt.algorithms.Algorithm;
 import com.auth0.jwt.exceptions.JWTDecodeException;
-import com.bakdata.conquery.models.auth.web.DefaultAuthFilter;
+import com.bakdata.conquery.models.auth.web.AuthFilter;
 import io.dropwizard.auth.oauth.OAuthCredentialAuthFilter;
 import io.dropwizard.util.Duration;
-import jakarta.inject.Named;
 import jakarta.ws.rs.container.ContainerRequestContext;
 import jakarta.ws.rs.core.HttpHeaders;
 import lombok.experimental.UtilityClass;
@@ -21,7 +20,6 @@
 import org.apache.commons.lang3.time.DateUtils;
 import org.apache.shiro.authc.AuthenticationToken;
 import org.apache.shiro.authc.BearerToken;
-import org.jvnet.hk2.annotations.Service;
 
 @UtilityClass
 @Slf4j
@@ -100,7 +98,7 @@ public static String generateTokenSecret() {
 	 * @param request
 	 * @return
 	 */
-	public static class JWTokenExtractor implements DefaultAuthFilter.TokenExtractor {
+	public static class JWTokenExtractor implements AuthFilter.TokenExtractor {
 
 		@Override
 		public AuthenticationToken apply(ContainerRequestContext request) {

backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java

@@ -3,7 +3,7 @@
 import com.bakdata.conquery.io.cps.CPSType;
 import com.bakdata.conquery.models.auth.AuthorizationController;
 import com.bakdata.conquery.models.auth.ConqueryAuthenticationRealm;
-import com.bakdata.conquery.models.auth.web.DefaultAuthFilter;
+import com.bakdata.conquery.models.auth.web.AuthFilter;
 import com.bakdata.conquery.models.config.ConqueryConfig;
 import com.bakdata.conquery.models.config.auth.AuthenticationRealmFactory;
 import com.bakdata.conquery.models.identifiable.ids.specific.UserId;
@@ -19,10 +19,10 @@ public class DevAuthConfig implements AuthenticationRealmFactory {
 	@Override
 	public ConqueryAuthenticationRealm createRealm(Environment environment, ConqueryConfig config, AuthorizationController authorizationController) {
 
-		DefaultAuthFilter.registerTokenExtractor(UserIdTokenExtractor.class, environment.jersey().getResourceConfig());
+		AuthFilter.registerTokenExtractor(UserIdTokenExtractor.class, environment.jersey().getResourceConfig());
 		if (authorizationController.getAdminServlet() != null) {
-			DefaultAuthFilter.registerTokenExtractor(UserIdTokenExtractor.class, authorizationController.getAdminServlet().getJerseyConfig());
-			DefaultAuthFilter.registerTokenExtractor(UserIdTokenExtractor.class, authorizationController.getAdminServlet().getJerseyConfigUI());
+			AuthFilter.registerTokenExtractor(UserIdTokenExtractor.class, authorizationController.getAdminServlet().getJerseyConfig());
+			AuthFilter.registerTokenExtractor(UserIdTokenExtractor.class, authorizationController.getAdminServlet().getJerseyConfigUI());
 		}
 
 		// Use the first defined user als the default user. This is usually the superuser if the DevelopmentAuthorizationConfig is set

backend/src/main/java/com/bakdata/conquery/models/auth/develop/UserIdTokenExtractor.java

@@ -1,6 +1,6 @@
 package com.bakdata.conquery.models.auth.develop;
 
-import com.bakdata.conquery.models.auth.web.DefaultAuthFilter;
+import com.bakdata.conquery.models.auth.web.AuthFilter;
 import com.bakdata.conquery.models.identifiable.ids.specific.UserId;
 import jakarta.inject.Named;
 import jakarta.ws.rs.container.ContainerRequestContext;
@@ -15,7 +15,7 @@
 @RequiredArgsConstructor
 @Service
 @Named("user-id")
-public class UserIdTokenExtractor implements DefaultAuthFilter.TokenExtractor {
+public class UserIdTokenExtractor implements AuthFilter.TokenExtractor {
 
 	private static final String UID_QUERY_STRING_PARAMETER = "access_token";