diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 570eb425f6..7d8fe7ddd0 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -44,10 +44,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v2 - - name: set java 17 - uses: actions/setup-java@v1 + - name: Set up JDK + uses: actions/setup-java@v4 with: - java-version: 17 + distribution: 'temurin' + java-version: '17' + overwrite-settings: false # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/test_cypress.yml b/.github/workflows/test_cypress.yml index cabccfb095..8990da08ab 100644 --- a/.github/workflows/test_cypress.yml +++ b/.github/workflows/test_cypress.yml @@ -17,26 +17,28 @@ jobs: timeout-minutes: 10 steps: - name: Cache local Maven repository - uses: actions/cache@v2 + uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-maven- - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - name: Setup Node - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: 18 - name: Set up JDK - uses: actions/setup-java@v1 + uses: actions/setup-java@v4 with: - java-version: 17 + distribution: 'temurin' + java-version: '17' + overwrite-settings: false - name: Build Backend run: ./scripts/build_backend_no_version.sh @@ -53,14 +55,16 @@ jobs: start: bash ./scripts/run_e2e_all.sh wait-on: "http://localhost:8000" - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 if: failure() with: name: cypress-screenshots path: cypress/screenshots + if-no-files-found: ignore - - uses: actions/upload-artifact@v2 + - uses: actions/upload-artifact@v4 if: failure() with: name: cypress-videos path: cypress/videos + if-no-files-found: ignore diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/ApiV1.java b/backend/src/main/java/com/bakdata/conquery/apiv1/ApiV1.java index a9cfc96914..ffc6ef6f4a 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/ApiV1.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/ApiV1.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider; import com.bakdata.conquery.metrics.ActiveUsersFilter; import com.bakdata.conquery.models.auth.basic.JWTokenHandler; -import com.bakdata.conquery.models.auth.web.DefaultAuthFilter; +import com.bakdata.conquery.models.auth.web.AuthFilter; import com.bakdata.conquery.models.forms.frontendconfiguration.FormConfigProcessor; import com.bakdata.conquery.models.forms.frontendconfiguration.FormProcessor; import com.bakdata.conquery.resources.ResourcesProvider; @@ -63,8 +63,8 @@ protected void configure() { * We use the same instance of the filter for the api servlet and the admin servlet to have a single * point for authentication. */ - jersey.register(DefaultAuthFilter.class); - DefaultAuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, jersey.getResourceConfig()); + jersey.register(AuthFilter.class); + AuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, jersey.getResourceConfig()); jersey.register(IdParamConverter.Provider.INSTANCE); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java index aa399f50c5..31a4eabb84 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java @@ -4,7 +4,6 @@ import java.math.BigDecimal; import javax.annotation.Nonnull; -import jakarta.validation.constraints.NotNull; import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; import com.bakdata.conquery.io.cps.CPSBase; @@ -32,6 +31,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.deser.std.StdDeserializer; import com.fasterxml.jackson.databind.exc.InvalidTypeIdException; +import jakarta.validation.constraints.NotNull; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.NoArgsConstructor; @@ -39,6 +39,7 @@ import lombok.Setter; import lombok.SneakyThrows; import lombok.ToString; +import org.jooq.Condition; @Getter @Setter @@ -67,7 +68,7 @@ public FilterNode createNode() { } public SqlFilters convertToSqlFilter(SqlIdColumns ids, ConversionContext context, ConceptConversionTables tables) { - FilterContext filterContext = new FilterContext<>(ids, value, context, tables); + FilterContext filterContext = FilterContext.forConceptConversion(ids, value, context, tables); SqlFilters sqlFilters = filter.convertToSqlFilter(filterContext); if (context.isNegation()) { return new SqlFilters(sqlFilters.getSelects(), sqlFilters.getWhereClauses().negated()); @@ -75,6 +76,11 @@ public SqlFilters convertToSqlFilter(SqlIdColumns ids, ConversionContext context return sqlFilters; } + public Condition convertForTableExport(SqlIdColumns ids, ConversionContext context) { + FilterContext filterContext = FilterContext.forTableExport(ids, value, context); + return filter.convertForTableExport(filterContext); + } + @NoArgsConstructor @CPSType(id = FrontendFilterType.Fields.MULTI_SELECT, base = FilterValue.class) @ToString(callSuper = true) diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java index cb4e7b8b87..2293959fec 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java @@ -29,6 +29,7 @@ import com.bakdata.conquery.models.messages.network.specific.AddShardNode; import com.bakdata.conquery.models.messages.network.specific.RegisterWorker; import com.bakdata.conquery.models.messages.network.specific.UpdateJobManagerStatus; +import com.bakdata.conquery.models.worker.IdResolveContext; import com.bakdata.conquery.models.worker.Worker; import com.bakdata.conquery.models.worker.WorkerInformation; import com.bakdata.conquery.models.worker.Workers; @@ -51,6 +52,7 @@ import org.apache.mina.core.session.IoSession; import org.apache.mina.filter.FilterEvent; import org.apache.mina.transport.socket.nio.NioSocketConnector; +import org.jetbrains.annotations.NotNull; /** * This node holds a shard of data (in so called {@link Worker}s for the different datasets in conquery. @@ -64,6 +66,7 @@ public class ShardNode extends ConqueryCommand implements IoHandler, Managed { public static final String DEFAULT_NAME = "shard-node"; private NioSocketConnector connector; + private ConnectFuture future; private JobManager jobManager; private Validator validator; private ConqueryConfig config; @@ -88,7 +91,6 @@ protected void run(Environment environment, Namespace namespace, ConqueryConfig this.environment = environment; this.config = config; - connector = new NioSocketConnector(); jobManager = new JobManager(getName(), config.isFailOnError()); environment.lifecycle().manage(this); @@ -106,6 +108,7 @@ protected void run(Environment environment, Namespace namespace, ConqueryConfig getConfig().getQueries().getSecondaryIdSubPlanRetention() ); + final Collection workerStorages = config.getStorage().discoverWorkerStorages(); @@ -268,6 +271,8 @@ private static void scheduleIdleLogger(ScheduledExecutorService scheduler, IoSes public void sessionClosed(IoSession session) { setLocation(session); log.info("Disconnected from ManagerNode."); + + scheduler.schedule(this::connectToCluster, 2, TimeUnit.SECONDS); } @Override @@ -291,7 +296,9 @@ public void messageSent(IoSession session, Object message) { @Override public void inputClosed(IoSession session) { setLocation(session); - log.info("Session closed."); + log.info("Input closed."); + session.closeNow(); + scheduler.schedule(this::disconnectFromCluster, 0, TimeUnit.SECONDS); } @Override @@ -306,24 +313,27 @@ public void start() throws Exception { value.getJobManager().addSlowJob(new SimpleJob("Update Bucket Manager", value.getBucketManager()::fullUpdate)); } - ObjectMapper om = createInternalObjectMapper(View.InternalCommunication.class); + scheduler.schedule(this::connectToCluster, 0, TimeUnit.MINUTES); - BinaryJacksonCoder coder = new BinaryJacksonCoder(workers, validator, om); - connector.getFilterChain().addLast("codec", new CQProtocolCodecFilter(new ChunkWriter(coder), new ChunkReader(coder, om))); - connector.setHandler(this); - connector.getSessionConfig().setAll(config.getCluster().getMina()); + } + + private void connectToCluster() { InetSocketAddress address = new InetSocketAddress( config.getCluster().getManagerURL().getHostAddress(), config.getCluster().getPort() ); + disconnectFromCluster(); + + connector = getClusterConnector(workers); + while (true) { try { log.info("Trying to connect to {}", address); // Try opening a connection (Note: This fails immediately instead of waiting a minute to try and connect) - ConnectFuture future = connector.connect(address); + future = connector.connect(address); future.awaitUninterruptibly(); @@ -339,9 +349,24 @@ public void start() throws Exception { catch (RuntimeIoException e) { log.warn("Failed to connect to {}", address, e); } + catch (InterruptedException e) { + log.warn("Interrupted while trying to connector to cluster, giving up.", e); + break; + } } + } + @NotNull + private NioSocketConnector getClusterConnector(IdResolveContext workers) { + ObjectMapper om = createInternalObjectMapper(View.InternalCommunication.class); + + NioSocketConnector connector = new NioSocketConnector(); + BinaryJacksonCoder coder = new BinaryJacksonCoder(workers, validator, om); + connector.getFilterChain().addLast("codec", new CQProtocolCodecFilter(new ChunkWriter(coder), new ChunkReader(coder, om))); + connector.setHandler(this); + connector.getSessionConfig().setAll(config.getCluster().getMina()); + return connector; } @Override @@ -350,12 +375,23 @@ public void stop() throws Exception { workers.stop(); + disconnectFromCluster(); + } + + private void disconnectFromCluster() { + if (future != null) { + future.cancel(); + } + //after the close command was send if (context != null) { context.awaitClose(); } - log.info("Connection was closed by ManagerNode"); - connector.dispose(); + + if (connector != null) { + log.info("Connection was closed by ManagerNode"); + connector.dispose(); + } } public boolean isBusy() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java index f36044bb6b..a6490c4b73 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationController.java @@ -15,7 +15,7 @@ import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; -import com.bakdata.conquery.models.auth.web.DefaultAuthFilter; +import com.bakdata.conquery.models.auth.web.AuthFilter; import com.bakdata.conquery.models.auth.web.RedirectingAuthFilter; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.auth.AuthenticationRealmFactory; @@ -85,18 +85,18 @@ public AuthorizationController(MetaStorage storage, ConqueryConfig config, Envir this.adminServlet = adminServlet; if (adminServlet != null) { - adminServlet.getJerseyConfig().register(DefaultAuthFilter.class); - DefaultAuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, adminServlet.getJerseyConfig()); + adminServlet.getJerseyConfig().register(AuthFilter.class); + AuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, adminServlet.getJerseyConfig()); // The binding is necessary here because the RedirectingAuthFitler delegates to the DefaultAuthfilter at the moment adminServlet.getJerseyConfigUI().register(new AbstractBinder() { @Override protected void configure() { - bindAsContract(DefaultAuthFilter.class); + bindAsContract(AuthFilter.class); } }); adminServlet.getJerseyConfigUI().register(RedirectingAuthFilter.class); - DefaultAuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, adminServlet.getJerseyConfigUI()); + AuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, adminServlet.getJerseyConfigUI()); } unprotectedAuthAdmin = AuthServlet.generalSetup(environment.metrics(), config, environment.admin(), environment.getObjectMapper()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/JWTokenHandler.java b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/JWTokenHandler.java index e29fbc5df6..04930ca201 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/JWTokenHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/JWTokenHandler.java @@ -9,10 +9,9 @@ import com.auth0.jwt.JWT; import com.auth0.jwt.algorithms.Algorithm; import com.auth0.jwt.exceptions.JWTDecodeException; -import com.bakdata.conquery.models.auth.web.DefaultAuthFilter; +import com.bakdata.conquery.models.auth.web.AuthFilter; import io.dropwizard.auth.oauth.OAuthCredentialAuthFilter; import io.dropwizard.util.Duration; -import jakarta.inject.Named; import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.core.HttpHeaders; import lombok.experimental.UtilityClass; @@ -21,7 +20,6 @@ import org.apache.commons.lang3.time.DateUtils; import org.apache.shiro.authc.AuthenticationToken; import org.apache.shiro.authc.BearerToken; -import org.jvnet.hk2.annotations.Service; @UtilityClass @Slf4j @@ -100,7 +98,7 @@ public static String generateTokenSecret() { * @param request * @return */ - public static class JWTokenExtractor implements DefaultAuthFilter.TokenExtractor { + public static class JWTokenExtractor implements AuthFilter.TokenExtractor { @Override public AuthenticationToken apply(ContainerRequestContext request) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java b/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java index cb887b9b9f..d28e350c44 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/develop/DevAuthConfig.java @@ -3,7 +3,7 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.auth.AuthorizationController; import com.bakdata.conquery.models.auth.ConqueryAuthenticationRealm; -import com.bakdata.conquery.models.auth.web.DefaultAuthFilter; +import com.bakdata.conquery.models.auth.web.AuthFilter; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.auth.AuthenticationRealmFactory; import com.bakdata.conquery.models.identifiable.ids.specific.UserId; @@ -19,10 +19,10 @@ public class DevAuthConfig implements AuthenticationRealmFactory { @Override public ConqueryAuthenticationRealm createRealm(Environment environment, ConqueryConfig config, AuthorizationController authorizationController) { - DefaultAuthFilter.registerTokenExtractor(UserIdTokenExtractor.class, environment.jersey().getResourceConfig()); + AuthFilter.registerTokenExtractor(UserIdTokenExtractor.class, environment.jersey().getResourceConfig()); if (authorizationController.getAdminServlet() != null) { - DefaultAuthFilter.registerTokenExtractor(UserIdTokenExtractor.class, authorizationController.getAdminServlet().getJerseyConfig()); - DefaultAuthFilter.registerTokenExtractor(UserIdTokenExtractor.class, authorizationController.getAdminServlet().getJerseyConfigUI()); + AuthFilter.registerTokenExtractor(UserIdTokenExtractor.class, authorizationController.getAdminServlet().getJerseyConfig()); + AuthFilter.registerTokenExtractor(UserIdTokenExtractor.class, authorizationController.getAdminServlet().getJerseyConfigUI()); } // Use the first defined user als the default user. This is usually the superuser if the DevelopmentAuthorizationConfig is set diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/develop/UserIdTokenExtractor.java b/backend/src/main/java/com/bakdata/conquery/models/auth/develop/UserIdTokenExtractor.java index 3c5f513e51..cd906eac6c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/develop/UserIdTokenExtractor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/develop/UserIdTokenExtractor.java @@ -1,6 +1,6 @@ package com.bakdata.conquery.models.auth.develop; -import com.bakdata.conquery.models.auth.web.DefaultAuthFilter; +import com.bakdata.conquery.models.auth.web.AuthFilter; import com.bakdata.conquery.models.identifiable.ids.specific.UserId; import jakarta.inject.Named; import jakarta.ws.rs.container.ContainerRequestContext; @@ -15,7 +15,7 @@ @RequiredArgsConstructor @Service @Named("user-id") -public class UserIdTokenExtractor implements DefaultAuthFilter.TokenExtractor { +public class UserIdTokenExtractor implements AuthFilter.TokenExtractor { private static final String UID_QUERY_STRING_PARAMETER = "access_token"; diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/StringPermissionBuilder.java b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/StringPermissionBuilder.java index a906d4514d..09b53ba686 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/StringPermissionBuilder.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/permissions/StringPermissionBuilder.java @@ -1,17 +1,17 @@ package com.bakdata.conquery.models.auth.permissions; -import com.bakdata.conquery.io.cps.CPSBase; -import com.bakdata.conquery.resources.admin.rest.AdminProcessor; - import java.util.Set; import java.util.stream.Collectors; +import com.bakdata.conquery.io.cps.CPSBase; +import com.bakdata.conquery.resources.admin.rest.UIProcessor; + /** * Base class with utility functions to build permissions. * Subclasses should wrap their call to the {@code instancePermission} functions with their own string representation. - * {@link com.bakdata.conquery.io.cps.CPSType} is used by {@link AdminProcessor#preparePermissionTemplate()} to generate + * {@link com.bakdata.conquery.io.cps.CPSType} is used by {@link UIProcessor#preparePermissionTemplate()} to generate * a view ob possible Permissions for creation. - * Therefore every builder should have a static INSTANCE-Field. + * Therefore, every builder should have a static INSTANCE-Field. */ @CPSBase public abstract class StringPermissionBuilder { diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthCookieFilter.java b/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthCookieFilter.java index a6ab815dbf..25ab71e036 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthCookieFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthCookieFilter.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.models.auth.web; +import static com.bakdata.conquery.models.auth.web.AuthCookieFilter.PRIORITY; + import java.io.IOException; import com.bakdata.conquery.models.config.ConqueryConfig; @@ -31,10 +33,12 @@ @Slf4j @PreMatching // Chain this filter before the Authentication filter -@Priority(Priorities.AUTHENTICATION-100) +@Priority(PRIORITY) @RequiredArgsConstructor(onConstructor_ = {@Inject}) public class AuthCookieFilter implements ContainerRequestFilter, ContainerResponseFilter { + public static final int PRIORITY = Priorities.AUTHENTICATION - 100; + public static final String ACCESS_TOKEN = "access_token"; private static final String PREFIX = "bearer"; @@ -42,7 +46,7 @@ public class AuthCookieFilter implements ContainerRequestFilter, ContainerRespon /** * The filter tries to extract a token from a cookie and puts it into the - * authorization header of the request. This simplifies the token retrival + * authorization header of the request. This simplifies the token retrieval * process for the realms. */ @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/web/DefaultAuthFilter.java b/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthFilter.java similarity index 89% rename from backend/src/main/java/com/bakdata/conquery/models/auth/web/DefaultAuthFilter.java rename to backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthFilter.java index a1793ebb27..24a0273966 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/web/DefaultAuthFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/web/AuthFilter.java @@ -8,7 +8,6 @@ import com.bakdata.conquery.models.auth.ConqueryAuthenticator; import com.bakdata.conquery.models.auth.entities.Subject; import com.google.common.base.Function; -import io.dropwizard.auth.AuthFilter; import io.dropwizard.auth.DefaultUnauthorizedHandler; import jakarta.annotation.Priority; import jakarta.inject.Inject; @@ -17,9 +16,6 @@ import jakarta.ws.rs.container.ContainerRequestContext; import jakarta.ws.rs.container.PreMatching; import jakarta.ws.rs.core.SecurityContext; -import jakarta.ws.rs.ext.Provider; -import lombok.RequiredArgsConstructor; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.shiro.authc.AuthenticationException; import org.apache.shiro.authc.AuthenticationToken; @@ -32,17 +28,19 @@ * security relevant information for protected resources. The request is first * submitted to the registered {@link ConqueryAuthenticationRealm}s for the * token extraction, then the extracted tokens are submitted these realms - * through Dropwizards {@link AuthFilter} and Shiro. + * through Dropwizards {@link io.dropwizard.auth.AuthFilter} and Shiro. */ @Slf4j @PreMatching -@Priority(Priorities.AUTHENTICATION) -public class DefaultAuthFilter extends AuthFilter { +@Priority(AuthFilter.PRIORITY) +public class AuthFilter extends io.dropwizard.auth.AuthFilter { + + public static final int PRIORITY = Priorities.AUTHENTICATION; private final IterableProvider tokenExtractors; @Inject - public DefaultAuthFilter(IterableProvider tokenExtractors) { + public AuthFilter(IterableProvider tokenExtractors) { this.tokenExtractors = tokenExtractors; this.authenticator = new ConqueryAuthenticator(); this.unauthorizedHandler = new DefaultUnauthorizedHandler(); @@ -102,7 +100,7 @@ public static void registerTokenExtractor(Class extrac * Authenticating realms need to be able to extract a token from a request. How * it performs the extraction is implementation dependent. Anyway the realm * should NOT alter the request. This function is called prior to the - * authentication process in the {@link DefaultAuthFilter}. After the token + * authentication process in the {@link AuthFilter}. After the token * extraction process the Token is resubmitted to the realm from the AuthFilter * to the {@link ConqueryAuthenticator} which dispatches it to shiro. * diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/web/RedirectingAuthFilter.java b/backend/src/main/java/com/bakdata/conquery/models/auth/web/RedirectingAuthFilter.java index c1ed9cca47..4d4a214bfd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/web/RedirectingAuthFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/web/RedirectingAuthFilter.java @@ -8,7 +8,6 @@ import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.resources.admin.ui.model.UIView; -import io.dropwizard.auth.AuthFilter; import jakarta.annotation.Priority; import jakarta.inject.Inject; import jakarta.ws.rs.BadRequestException; @@ -40,10 +39,12 @@ * the login schema can be chosen. */ @Slf4j -@Priority(Priorities.AUTHENTICATION - 1) +@Priority(RedirectingAuthFilter.PRIORITY) @PreMatching @RequiredArgsConstructor(onConstructor_ = {@Inject}) -public class RedirectingAuthFilter extends AuthFilter { +public class RedirectingAuthFilter extends io.dropwizard.auth.AuthFilter { + + public static final int PRIORITY = Priorities.AUTHENTICATION; public static final String REDIRECT_URI = "redirect_uri"; /** @@ -61,7 +62,7 @@ public class RedirectingAuthFilter extends AuthFilter /** * The Filter that checks if a request was authenticated */ - private final DefaultAuthFilter delegate; + private final AuthFilter delegate; public static void registerLoginInitiator(ResourceConfig resourceConfig, LoginInitiator initiator, final String name) { resourceConfig.register(new AbstractBinder() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/web/csrf/CsrfTokenCheckFilter.java b/backend/src/main/java/com/bakdata/conquery/models/auth/web/csrf/CsrfTokenCheckFilter.java new file mode 100644 index 0000000000..910af59848 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/web/csrf/CsrfTokenCheckFilter.java @@ -0,0 +1,58 @@ +package com.bakdata.conquery.models.auth.web.csrf; + +import java.io.IOException; +import java.util.Optional; + +import com.bakdata.conquery.models.auth.web.AuthCookieFilter; +import jakarta.annotation.Priority; +import jakarta.ws.rs.ForbiddenException; +import jakarta.ws.rs.HttpMethod; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.container.ContainerRequestFilter; +import jakarta.ws.rs.core.Cookie; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +/** + * Implementation of the Double-Submit-Cookie Pattern. + * Checks if tokens in cookie and header match if a cookie is present. + * Otherwise, the request is refused. + */ +@Priority(AuthCookieFilter.PRIORITY - 100) +@Slf4j +public class CsrfTokenCheckFilter implements ContainerRequestFilter { + public static final String CSRF_TOKEN_HEADER = "X-Csrf-Token"; + + @Override + public void filter(ContainerRequestContext requestContext) throws IOException { + final String + cookieTokenHash = + Optional.ofNullable(requestContext.getCookies().get(CsrfTokenSetFilter.CSRF_COOKIE_NAME)).map(Cookie::getValue).orElse(null); + final String headerToken = requestContext.getHeaders().getFirst(CSRF_TOKEN_HEADER); + + final String method = requestContext.getMethod(); + + if (HttpMethod.GET.equals(method) || HttpMethod.HEAD.equals(method) || HttpMethod.OPTIONS.equals(method)) { + log.trace("Skipping csrf check because request is not state changing (method={})", method); + return; + } + + if (cookieTokenHash == null) { + log.trace("Request had no csrf token set. Accepting request"); + return; + } + + if (StringUtils.isBlank(headerToken)) { + log.warn("Request contained csrf cookie but the header token was empty"); + throw new ForbiddenException("CSRF Attempt"); + } + + if (!CsrfTokenSetFilter.checkHash(headerToken, cookieTokenHash)) { + log.warn("Request csrf cookie and header did not match"); + log.trace("header-token={} cookie-token-hash={}", headerToken, cookieTokenHash); + throw new ForbiddenException("CSRF Attempt"); + } + + log.trace("Csrf check successful"); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/web/csrf/CsrfTokenSetFilter.java b/backend/src/main/java/com/bakdata/conquery/models/auth/web/csrf/CsrfTokenSetFilter.java new file mode 100644 index 0000000000..939db8a213 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/web/csrf/CsrfTokenSetFilter.java @@ -0,0 +1,122 @@ +package com.bakdata.conquery.models.auth.web.csrf; + +import java.io.IOException; +import java.security.SecureRandom; +import java.util.Base64; +import java.util.Random; + +import com.bakdata.conquery.models.auth.web.AuthCookieFilter; +import com.google.common.base.Stopwatch; +import com.password4j.Hash; +import com.password4j.PBKDF2Function; +import com.password4j.Password; +import com.password4j.types.Hmac; +import jakarta.annotation.Priority; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.container.ContainerRequestFilter; +import jakarta.ws.rs.container.ContainerResponseContext; +import jakarta.ws.rs.container.ContainerResponseFilter; +import jakarta.ws.rs.core.HttpHeaders; +import jakarta.ws.rs.core.NewCookie; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.RandomStringUtils; +import org.apache.commons.lang3.StringUtils; + +/** + * Implementation of the Double-Submit-Cookie Pattern. + * This filter generates a random token which is injected in to the response. + *
    + *
  • In a Set-Cookie header, so that browser requests send the token via cookie back to us
    • + *
  • In the response payload. This filter sets a request property, which is eventually provided to freemarker. + * Freemarker then writes the token into payload (see base.html.ftl)
    • + *
+ * + * This filter is only installed on the ui side of the admin servlet and should to be evaluated before any authentication. + * Hence, its priority is lower. + */ +@Slf4j +@Priority(AuthCookieFilter.PRIORITY - 100) +public class CsrfTokenSetFilter implements ContainerRequestFilter, ContainerResponseFilter { + + public static final String CSRF_COOKIE_NAME = "csrf_token"; + public static final String CSRF_TOKEN_PROPERTY = "csrf_token"; + public static final int TOKEN_LENGTH = 30; + + /** + * This needs to be fast, because the hash is computed on every api request and it is only short-lived. + */ + private final static PBKDF2Function HASH_FUNCTION = PBKDF2Function.getInstance(Hmac.SHA256, 1000, 256); + public static final int COOKIE_MAX_AGE = 3600 /* seconds */; + public static final int SALT_LENGTH = 32; + + private final Random random = new SecureRandom(); + + @Override + public void filter(ContainerRequestContext requestContext) throws IOException { + final String token = RandomStringUtils.random(TOKEN_LENGTH, 0, 0, true, true, + null, random + ); + requestContext.setProperty(CSRF_TOKEN_PROPERTY, token); + } + + @Override + public void filter(ContainerRequestContext requestContext, ContainerResponseContext responseContext) throws IOException { + final String csrfToken = getCsrfTokenProperty(requestContext); + + if (StringUtils.isBlank(csrfToken)) { + log.warn("No csrf-token was registered for this request. Skipping csrf-cookie creation."); + return; + } + + final String csrfTokenHash = getTokenHash(csrfToken); + + log.trace("Hashed token for cookie. token='{}' hash='{}'", csrfToken, csrfTokenHash); + + responseContext.getHeaders() + .add(HttpHeaders.SET_COOKIE, new NewCookie( + CSRF_COOKIE_NAME, + csrfTokenHash, + "/", + null, + 0, + null, + COOKIE_MAX_AGE, + null, + requestContext.getSecurityContext().isSecure(), + false + )); + } + + private static String getTokenHash(String csrfToken) { + final Stopwatch stopwatch = log.isTraceEnabled() ? Stopwatch.createStarted() : null; + + final Hash hash = Password.hash(csrfToken).addRandomSalt(SALT_LENGTH).with(HASH_FUNCTION); + + log.trace("Generated token in {}", stopwatch); + + final String encodedSalt = Base64.getEncoder().encodeToString(hash.getSaltBytes()); + + // Use '_' as join char, because it is not part of the standard base64 encoding (in base64url though) + return String.join("_", encodedSalt, hash.getResult()); + } + + public static boolean checkHash(String token, String hash) { + int delimIdx; + if ((delimIdx = hash.indexOf("_")) == -1 || delimIdx == hash.length() - 1) { + throw new IllegalArgumentException("The provided hash must be of this form: _, was: " + hash); + } + final String encodedSalt = hash.substring(0, delimIdx); + final String saltedHash = hash.substring(delimIdx + 1); + + final byte[] salt = Base64.getDecoder().decode(encodedSalt); + final Stopwatch stopwatch = log.isTraceEnabled() ? Stopwatch.createStarted() : null; + final boolean decision = Password.check(token, saltedHash).addSalt(salt).with(HASH_FUNCTION); + + log.trace("Checked token in {}", stopwatch); + return decision; + } + + public static String getCsrfTokenProperty(ContainerRequestContext requestContext) { + return (String) requestContext.getProperty(CSRF_TOKEN_PROPERTY); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/web/csrf/ReadMe.md b/backend/src/main/java/com/bakdata/conquery/models/auth/web/csrf/ReadMe.md new file mode 100644 index 0000000000..d1d36264e0 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/web/csrf/ReadMe.md @@ -0,0 +1,24 @@ +# CSRF-Filter + +This filters prevents cross-site-request-forgery attempts which are conducted by luring a browser user to a malicious +website. +The user's browser needs be authenticated via an authentication cookie. +Because we have two servlets (UI + API) for the admin-end there are two different filters. + +The prevention method used is the +stateless [Signed Double-Submit Cookie](https://cheatsheetseries.owasp.org/cheatsheets/Cross-Site_Request_Forgery_Prevention_Cheat_Sheet.html#signed-double-submit-cookie-recommended). + +## [CsrfTokenSetFilter](./CsrfTokenSetFilter.java) + +This filter is installed on the UI-servlet. It generates a csrf-token and injects the plaintext token into the payload +and the signed/hashed token into a cookie. +The UI-servlet provides all html resources for the browser. + +## [CsrfTokenCheckFilter](./CsrfTokenCheckFilter.java) + +This filter is installed on the API-servlet. It extracts the double submitted token from a requests and validates them. +If the request did not contain a csrf token cookie (e.g. request was initiated by cURL) the filter does nothing. + +This filter is not installed on the UI-servlet. Even though this servlet's responses contain sensitive data, it only +provides GET-endpoints, which are not state altering and thus don't need csrf protection. On the API-servlet side we +validate all requests for simplicity. diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java index e6e83b723b..7393a58f01 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java @@ -11,6 +11,8 @@ @AllArgsConstructor public class SqlConnectorConfig { + public static final String DEFAULT_PRIMARY_COLUMN = "pid"; + boolean enabled; private Dialect dialect; @@ -26,5 +28,5 @@ public class SqlConnectorConfig { private String jdbcConnectionUrl; - private String primaryColumn = "pid"; + private String primaryColumn = DEFAULT_PRIMARY_COLUMN; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/auth/IntrospectionDelegatingRealmFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/auth/IntrospectionDelegatingRealmFactory.java index 6220c3ba40..3501c01c67 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/auth/IntrospectionDelegatingRealmFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/auth/IntrospectionDelegatingRealmFactory.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.models.auth.basic.JWTokenHandler; import com.bakdata.conquery.models.auth.oidc.IntrospectionDelegatingRealm; import com.bakdata.conquery.models.auth.oidc.keycloak.KeycloakApi; -import com.bakdata.conquery.models.auth.web.DefaultAuthFilter; +import com.bakdata.conquery.models.auth.web.AuthFilter; import com.fasterxml.jackson.annotation.JsonIgnore; import com.nimbusds.oauth2.sdk.auth.ClientAuthentication; import com.nimbusds.oauth2.sdk.auth.ClientSecretBasic; @@ -49,7 +49,7 @@ public class IntrospectionDelegatingRealmFactory extends Configuration { public ConqueryAuthenticationRealm createRealm(Environment environment, AuthorizationController authorizationController) { // Register token extractor for JWT Tokens - DefaultAuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, environment.jersey().getResourceConfig()); + AuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, environment.jersey().getResourceConfig()); // At start up, try tp retrieve the idp client api object if possible. If the idp service is not up don't fail start up. authClient = getAuthClient(false); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/auth/LocalAuthenticationConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/auth/LocalAuthenticationConfig.java index 4ea4956d42..cde3dea9fa 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/auth/LocalAuthenticationConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/auth/LocalAuthenticationConfig.java @@ -13,7 +13,7 @@ import com.bakdata.conquery.models.auth.basic.JWTokenHandler; import com.bakdata.conquery.models.auth.basic.LocalAuthenticationRealm; import com.bakdata.conquery.models.auth.basic.UserAuthenticationManagementProcessor; -import com.bakdata.conquery.models.auth.web.DefaultAuthFilter; +import com.bakdata.conquery.models.auth.web.AuthFilter; import com.bakdata.conquery.models.auth.web.RedirectingAuthFilter; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.XodusConfig; @@ -79,7 +79,7 @@ boolean isStorageEncrypted() { @Override public ConqueryAuthenticationRealm createRealm(Environment environment, ConqueryConfig config, AuthorizationController authorizationController) { // Token extractor is not needed because this realm depends on the ConqueryTokenRealm - DefaultAuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, environment.jersey().getResourceConfig()); + AuthFilter.registerTokenExtractor(JWTokenHandler.JWTokenExtractor.class, environment.jersey().getResourceConfig()); log.info("Performing benchmark for default hash function (bcrypt) with max_milliseconds={}", BCRYPT_MAX_MILLISECONDS); final BenchmarkResult result = SystemChecker.benchmarkBcrypt(BCRYPT_MAX_MILLISECONDS); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java index e3f4e1bd85..8439974550 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/Table.java @@ -6,6 +6,7 @@ import java.util.stream.Stream; import javax.annotation.CheckForNull; +import javax.annotation.Nullable; import com.bakdata.conquery.io.jackson.serializer.NsIdRef; import com.bakdata.conquery.io.storage.NamespacedStorage; @@ -34,7 +35,13 @@ public class Table extends Labeled implements NamespacedIdentifiable filterContext) throw new UnsupportedOperationException("SQL conversion of filter %s not implemented yet.".formatted(getClass())); } + @JsonIgnore + public Condition convertForTableExport(FilterContext filterContext) { + throw new UnsupportedOperationException("SQL conversion of filter %s not implemented yet".formatted(getClass())); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java index 5685ba240c..864e939826 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java @@ -7,10 +7,12 @@ import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectFilterUtil; +import com.bakdata.conquery.sql.conversion.model.filter.MultiSelectCondition; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.Setter; +import org.jooq.Condition; /** * This filter represents a select in the front end. This means that the user can select one or more values from a list of values. @@ -39,4 +41,8 @@ public SqlFilters convertToSqlFilter(FilterContext filterContext) { return SelectFilterUtil.convert(this, filterContext, filterContext.getValue()); } + @Override + public Condition convertForTableExport(FilterContext filterContext) { + return MultiSelectCondition.onColumn(getColumn(), filterContext.getValue(), filterContext.getSqlDialect().getFunctionProvider()).condition(); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java index 0214fe9b6d..fafaf8ddf3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java @@ -19,10 +19,12 @@ import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.model.aggregator.CountSqlAggregator; +import com.bakdata.conquery.sql.conversion.model.filter.CountCondition; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import jakarta.validation.constraints.NotNull; import lombok.Data; import lombok.NoArgsConstructor; +import org.jooq.Condition; @CPSType(id = "COUNT", base = Filter.class) @NoArgsConstructor @@ -76,4 +78,8 @@ public SqlFilters convertToSqlFilter(FilterContext filterContex return CountSqlAggregator.create(this, filterContext).getSqlFilters(); } + @Override + public Condition convertForTableExport(FilterContext filterContext) { + return CountCondition.onColumn(column, filterContext.getValue()).condition(); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java index 82f5df279a..f9090ad6b4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java @@ -16,11 +16,13 @@ import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.model.aggregator.DateDistanceSqlAggregator; +import com.bakdata.conquery.sql.conversion.model.filter.DateDistanceCondition; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import jakarta.validation.constraints.NotNull; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.jooq.Condition; /** * This filter represents a select in the front end. This means that the user can select one or more values from a list of values. @@ -56,4 +58,8 @@ public SqlFilters convertToSqlFilter(FilterContext filterContex return DateDistanceSqlAggregator.create(this, filterContext).getSqlFilters(); } + @Override + public Condition convertForTableExport(FilterContext filterContext) { + return DateDistanceCondition.onColumn(getColumn(), getTimeUnit(), filterContext).condition(); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java index 2886f0d4fc..84d779bd71 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java @@ -21,6 +21,7 @@ import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.model.aggregator.FlagSqlAggregator; +import com.bakdata.conquery.sql.conversion.model.filter.FlagCondition; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -28,6 +29,7 @@ import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.ToString; +import org.jooq.Condition; /** * Implements a MultiSelect type filter, where an event can meet multiple criteria (as opposed to {@link MultiSelectFilter} which is restricted to one value per event). @@ -98,4 +100,8 @@ public SqlFilters convertToSqlFilter(FilterContext filterContext) { return FlagSqlAggregator.create(this, filterContext).getSqlFilters(); } + @Override + public Condition convertForTableExport(FilterContext filterContext) { + return FlagCondition.onColumn(getFlags(), filterContext.getValue()).condition(); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/MultiSelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/MultiSelectFilter.java index 338ddf7ab1..e5451b0be3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/MultiSelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/MultiSelectFilter.java @@ -7,8 +7,10 @@ import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectFilterUtil; +import com.bakdata.conquery.sql.conversion.model.filter.MultiSelectCondition; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.fasterxml.jackson.annotation.JsonIgnore; +import org.jooq.Condition; /** * This filter represents a select in the front end. This means that the user can select one or more values from a list of values. @@ -38,4 +40,8 @@ public SqlFilters convertToSqlFilter(FilterContext filterContext) { return SelectFilterUtil.convert(this, filterContext, filterContext.getValue()); } + @Override + public Condition convertForTableExport(FilterContext filterContext) { + return MultiSelectCondition.onColumn(getColumn(), filterContext.getValue(), filterContext.getSqlDialect().getFunctionProvider()).condition(); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java index 6e56e83d7d..fdf746d7d1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java @@ -18,10 +18,12 @@ import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.model.aggregator.NumberSqlAggregator; +import com.bakdata.conquery.sql.conversion.model.filter.NumberCondition; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import lombok.Getter; import lombok.Setter; import lombok.extern.slf4j.Slf4j; +import org.jooq.Condition; /** * This filter represents a filter on an integer columnof each event. @@ -62,4 +64,8 @@ public SqlFilters convertToSqlFilter(FilterContext filterContext) { return NumberSqlAggregator.create(this, filterContext).getSqlFilters(); } + @Override + public Condition convertForTableExport(FilterContext filterContext) { + return NumberCondition.onColumn(getColumn(), filterContext.getValue()).condition(); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SingleSelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SingleSelectFilter.java index 0e0210626a..012be75080 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SingleSelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SingleSelectFilter.java @@ -7,8 +7,10 @@ import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; import com.bakdata.conquery.sql.conversion.cqelement.concept.SelectFilterUtil; +import com.bakdata.conquery.sql.conversion.model.filter.MultiSelectCondition; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import net.minidev.json.annotate.JsonIgnore; +import org.jooq.Condition; /** * This filter represents a select in the front end. This means that the user can select one or more values from a list of values.", @@ -34,4 +36,12 @@ public SqlFilters convertToSqlFilter(FilterContext filterContext) { return SelectFilterUtil.convert(this, filterContext, new String[]{filterContext.getValue()}); } + @Override + public Condition convertForTableExport(FilterContext filterContext) { + return MultiSelectCondition.onColumn( + getColumn(), + new String[]{filterContext.getValue()}, + filterContext.getSqlDialect().getFunctionProvider() + ).condition(); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java index a3747018b2..9749afae68 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java @@ -34,11 +34,13 @@ import com.bakdata.conquery.sql.conversion.model.aggregator.SumDistinctSqlAggregator; import com.bakdata.conquery.sql.conversion.model.aggregator.SumSqlAggregator; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; +import com.bakdata.conquery.sql.conversion.model.filter.SumCondition; import com.fasterxml.jackson.annotation.JsonIgnore; import jakarta.validation.constraints.NotNull; import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; +import org.jooq.Condition; /** * This filter represents a filter on the sum of one integer column. @@ -114,6 +116,11 @@ public SqlFilters convertToSqlFilter(FilterContext filterContext) { return SumSqlAggregator.create(this, filterContext).getSqlFilters(); } + @Override + public Condition convertForTableExport(FilterContext filterContext) { + return SumCondition.onColumn(getColumn(), getSubtractColumn(), filterContext.getValue()).condition(); + } + @JsonIgnore private ColumnAggregator getAggregator() { if (getSubtractColumn() == null) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java index 66bb7b4c34..7453e12f0d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DateUnionSelect.java @@ -12,6 +12,7 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.DateUnionAggregator; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -20,6 +21,7 @@ @Getter @NoArgsConstructor(onConstructor_ = @JsonCreator) @CPSType(id = "DATE_UNION", base = Select.class) +@JsonIgnoreProperties("categorical") public class DateUnionSelect extends Select implements DaterangeSelect { @NsIdRef diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java index 4da14a7a6b..2f4f918c73 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/connector/specific/DurationSumSelect.java @@ -12,6 +12,7 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.DurationSumAggregator; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -20,6 +21,7 @@ @Getter @NoArgsConstructor(onConstructor_ = @JsonCreator) @CPSType(id = "DURATION_SUM", base = Select.class) +@JsonIgnoreProperties("categorical") public class DurationSumSelect extends Select implements DaterangeSelect { @NsIdRef diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java index 8696ba7edd..28e8ecf939 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java @@ -13,6 +13,8 @@ import com.bakdata.conquery.io.jersey.RESTServer; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.web.AuthCookieFilter; +import com.bakdata.conquery.models.auth.web.csrf.CsrfTokenCheckFilter; +import com.bakdata.conquery.models.auth.web.csrf.CsrfTokenSetFilter; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.models.worker.DatasetRegistry; @@ -120,7 +122,8 @@ protected void configure() { .register(IdRefPathParamConverterProvider.class) .register(new MultiPartFeature()) .register(IdParamConverter.Provider.INSTANCE) - .register(AuthCookieFilter.class); + .register(AuthCookieFilter.class) + .register(CsrfTokenCheckFilter.class); jerseyConfigUI.register(new ViewMessageBodyWriter(manager.getEnvironment().metrics(), Collections.singleton(Freemarker.HTML_RENDERER))) @@ -136,7 +139,8 @@ protected void configure() { }) .register(AdminPermissionFilter.class) .register(IdRefPathParamConverterProvider.class) - .register(AuthCookieFilter.class); + .register(AuthCookieFilter.class) + .register(CsrfTokenSetFilter.class); } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UIProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UIProcessor.java index 78b0448a2e..17c89ccdbf 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UIProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UIProcessor.java @@ -68,8 +68,8 @@ public MetaStorage getStorage() { return adminProcessor.getStorage(); } - public UIContext getUIContext() { - return new UIContext(adminProcessor.getNodeProvider()); + public UIContext getUIContext(String csrfToken) { + return new UIContext(adminProcessor.getNodeProvider(), csrfToken); } public Set> getLoadedIndexes() { diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/AdminUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/AdminUIResource.java index c4fefb8b73..c503612d9c 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/AdminUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/AdminUIResource.java @@ -4,6 +4,7 @@ import java.util.Objects; import com.bakdata.conquery.models.auth.entities.Subject; +import com.bakdata.conquery.models.auth.web.csrf.CsrfTokenSetFilter; import com.bakdata.conquery.models.config.auth.AuthenticationConfig; import com.bakdata.conquery.resources.ResourceConstants; import com.bakdata.conquery.resources.admin.rest.UIProcessor; @@ -27,34 +28,36 @@ public class AdminUIResource { private final UIProcessor uiProcessor; - + @Context + private ContainerRequestContext requestContext; @GET public View getIndex() { - return new UIView<>("index.html.ftl", uiProcessor.getUIContext()); + return new UIView<>("index.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext))); } @GET @Path("script") public View getScript() { - return new UIView<>("script.html.ftl", uiProcessor.getUIContext()); + return new UIView<>("script.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext))); } @GET @Path("jobs") public View getJobs() { - return new UIView<>("jobs.html.ftl", uiProcessor.getUIContext(), uiProcessor.getAdminProcessor().getJobs()); + return new UIView<>("jobs.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), uiProcessor.getAdminProcessor() + .getJobs()); } @GET @Path("queries") public View getQueries() { - return new UIView<>("queries.html.ftl", uiProcessor.getUIContext()); + return new UIView<>("queries.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext))); } @GET @Path("logout") - public Response logout(@Context ContainerRequestContext requestContext, @Auth Subject user) { + public Response logout(@Auth Subject user) { // Invalidate all cookies. At the moment the adminEnd uses cookies only for authentication, so this does not interfere with other things final NewCookie[] expiredCookies = requestContext.getCookies().keySet().stream().map(AuthenticationConfig::expireCookie).toArray(NewCookie[]::new); final URI logout = user.getAuthenticationInfo().getFrontChannelLogout(); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/AuthOverviewUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/AuthOverviewUIResource.java index f9ab29d6a5..969ec5d2c6 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/AuthOverviewUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/AuthOverviewUIResource.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.resources.admin.ui; +import com.bakdata.conquery.models.auth.web.csrf.CsrfTokenSetFilter; import com.bakdata.conquery.resources.ResourceConstants; import com.bakdata.conquery.resources.admin.rest.UIProcessor; import com.bakdata.conquery.resources.admin.ui.model.UIView; @@ -8,6 +9,8 @@ import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; import jakarta.ws.rs.Produces; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; import lombok.RequiredArgsConstructor; @@ -19,8 +22,8 @@ public class AuthOverviewUIResource { protected final UIProcessor uiProcessor; @GET - public View getOverview() { - return new UIView<>("authOverview.html.ftl", uiProcessor.getUIContext(), uiProcessor.getAuthOverview()); + public View getOverview(@Context ContainerRequestContext request) { + return new UIView<>("authOverview.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(request)), uiProcessor.getAuthOverview()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/ConceptsUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/ConceptsUIResource.java index fd7029ba17..ef1f88cecb 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/ConceptsUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/ConceptsUIResource.java @@ -4,6 +4,7 @@ import static com.bakdata.conquery.resources.ResourceConstants.DATASET; import com.bakdata.conquery.io.jersey.ExtraMimeTypes; +import com.bakdata.conquery.models.auth.web.csrf.CsrfTokenSetFilter; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.resources.admin.rest.UIProcessor; @@ -15,6 +16,8 @@ import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -35,12 +38,14 @@ public class ConceptsUIResource { protected Concept concept; @PathParam(DATASET) protected Dataset dataset; + @Context + ContainerRequestContext request; @GET public View getConceptView() { return new UIView<>( "concept.html.ftl", - uiProcessor.getUIContext(), + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(request)), concept ); } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/DatasetsUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/DatasetsUIResource.java index fef955311a..0e855e5786 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/DatasetsUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/DatasetsUIResource.java @@ -6,6 +6,7 @@ import java.util.Collection; import java.util.stream.Collectors; +import com.bakdata.conquery.models.auth.web.csrf.CsrfTokenSetFilter; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.Import; import com.bakdata.conquery.models.datasets.SecondaryIdDescription; @@ -16,6 +17,7 @@ import com.bakdata.conquery.models.index.search.SearchIndex; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.resources.admin.rest.UIProcessor; +import com.bakdata.conquery.resources.admin.ui.model.UIContext; import com.bakdata.conquery.resources.admin.ui.model.UIView; import io.dropwizard.views.common.View; import jakarta.inject.Inject; @@ -23,6 +25,8 @@ import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; import lombok.AllArgsConstructor; import lombok.Data; @@ -45,13 +49,16 @@ public class DatasetsUIResource { private final UIProcessor uiProcessor; + @Context + private ContainerRequestContext requestContext; + @GET @Produces(MediaType.TEXT_HTML) public View listDatasetsUI() { return new UIView<>( "datasets.html.ftl", - uiProcessor.getUIContext(), + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), uiProcessor.getDatasetRegistry().getAllDatasets() ); } @@ -63,7 +70,7 @@ public View getDataset(@PathParam(DATASET) Dataset dataset) { final Namespace namespace = uiProcessor.getDatasetRegistry().get(dataset.getId()); return new UIView<>( "dataset.html.ftl", - uiProcessor.getUIContext(), + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), new DatasetInfos( namespace.getDataset(), namespace.getStorage().getSecondaryIds(), @@ -102,11 +109,13 @@ public View getDataset(@PathParam(DATASET) Dataset dataset) { @Path("{" + DATASET + "}/mapping") public View getIdMapping(@PathParam(DATASET) Dataset dataset) { final Namespace namespace = uiProcessor.getDatasetRegistry().get(dataset.getId()); - EntityIdMap mapping = namespace.getStorage().getIdMapping(); + final EntityIdMap mapping = namespace.getStorage().getIdMapping(); + final UIContext uiContext = uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)); + if (mapping != null && mapping.getInternalToPrint() != null) { - return new UIView<>("idmapping.html.ftl", uiProcessor.getUIContext(), mapping.getInternalToPrint()); + return new UIView<>("idmapping.html.ftl", uiContext, mapping.getInternalToPrint()); } - return new UIView<>("add_idmapping.html.ftl", uiProcessor.getUIContext(), namespace.getDataset().getId()); + return new UIView<>("add_idmapping.html.ftl", uiContext, namespace.getDataset().getId()); } @Data diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/GroupUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/GroupUIResource.java index 7b4eb1e68d..c138afb3b6 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/GroupUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/GroupUIResource.java @@ -1,20 +1,22 @@ package com.bakdata.conquery.resources.admin.ui; +import static com.bakdata.conquery.resources.ResourceConstants.GROUPS_PATH_ELEMENT; +import static com.bakdata.conquery.resources.ResourceConstants.GROUP_ID; + import com.bakdata.conquery.models.auth.entities.Group; +import com.bakdata.conquery.models.auth.web.csrf.CsrfTokenSetFilter; import com.bakdata.conquery.resources.admin.rest.UIProcessor; import com.bakdata.conquery.resources.admin.ui.model.UIView; import io.dropwizard.views.common.View; -import lombok.RequiredArgsConstructor; - import jakarta.inject.Inject; import jakarta.ws.rs.GET; import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; - -import static com.bakdata.conquery.resources.ResourceConstants.GROUPS_PATH_ELEMENT; -import static com.bakdata.conquery.resources.ResourceConstants.GROUP_ID; +import lombok.RequiredArgsConstructor; @Produces(MediaType.TEXT_HTML) @Path(GROUPS_PATH_ELEMENT) @@ -22,10 +24,13 @@ public class GroupUIResource { protected final UIProcessor uiProcessor; + @Context + private ContainerRequestContext requestContext; @GET public View getGroups() { - return new UIView<>("groups.html.ftl", uiProcessor.getUIContext(), uiProcessor.getAdminProcessor().getAllGroups()); + return new UIView<>("groups.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), uiProcessor.getAdminProcessor() + .getAllGroups()); } /** @@ -37,6 +42,6 @@ public View getGroups() { @Path("{" + GROUP_ID + "}") @GET public View getGroup(@PathParam(GROUP_ID) Group group) { - return new UIView<>("group.html.ftl", uiProcessor.getUIContext(), uiProcessor.getGroupContent(group)); + return new UIView<>("group.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), uiProcessor.getGroupContent(group)); } } \ No newline at end of file diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/IndexServiceUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/IndexServiceUIResource.java index 557b8e1dec..dd1ee509fd 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/IndexServiceUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/IndexServiceUIResource.java @@ -4,17 +4,19 @@ import java.util.Set; -import jakarta.inject.Inject; -import jakarta.ws.rs.GET; -import jakarta.ws.rs.Path; -import jakarta.ws.rs.Produces; -import jakarta.ws.rs.core.MediaType; - +import com.bakdata.conquery.models.auth.web.csrf.CsrfTokenSetFilter; import com.bakdata.conquery.models.index.IndexKey; import com.bakdata.conquery.resources.admin.rest.UIProcessor; import com.bakdata.conquery.resources.admin.ui.model.UIView; import com.google.common.cache.CacheStats; import io.dropwizard.views.common.View; +import jakarta.inject.Inject; +import jakarta.ws.rs.GET; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.Produces; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.core.Context; +import jakarta.ws.rs.core.MediaType; import lombok.Data; import lombok.RequiredArgsConstructor; @@ -24,12 +26,14 @@ public class IndexServiceUIResource { private final UIProcessor uiProcessor; + @Context + private ContainerRequestContext requestContext; @GET @Path(INDEX_SERVICE_PATH_ELEMENT) public View getIndexService() { final IndexServiceUIContent content = new IndexServiceUIContent(uiProcessor.getIndexServiceStatistics(), uiProcessor.getLoadedIndexes()); - return new UIView<>("indexService.html.ftl", uiProcessor.getUIContext(), content); + return new UIView<>("indexService.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), content); } /** diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/RoleUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/RoleUIResource.java index ff7ebfc8e4..dce61a5219 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/RoleUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/RoleUIResource.java @@ -4,6 +4,7 @@ import static com.bakdata.conquery.resources.ResourceConstants.ROLE_ID; import com.bakdata.conquery.models.auth.entities.Role; +import com.bakdata.conquery.models.auth.web.csrf.CsrfTokenSetFilter; import com.bakdata.conquery.resources.admin.rest.UIProcessor; import com.bakdata.conquery.resources.admin.ui.model.UIView; import io.dropwizard.views.common.View; @@ -12,6 +13,8 @@ import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; import lombok.RequiredArgsConstructor; @@ -21,10 +24,13 @@ public class RoleUIResource { protected final UIProcessor uiProcessor; + @Context + private ContainerRequestContext requestContext; @GET public View getRoles() { - return new UIView<>("roles.html.ftl", uiProcessor.getUIContext(), uiProcessor.getAdminProcessor().getAllRoles()); + return new UIView<>("roles.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), uiProcessor.getAdminProcessor() + .getAllRoles()); } /** @@ -37,6 +43,6 @@ public View getRoles() { @Path("{" + ROLE_ID + "}") @GET public View getRole(@PathParam(ROLE_ID) Role role) { - return new UIView<>("role.html.ftl", uiProcessor.getUIContext(), uiProcessor.getRoleContent(role)); + return new UIView<>("role.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), uiProcessor.getRoleContent(role)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/TablesUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/TablesUIResource.java index 1aa4fbe0d8..2ede9a049c 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/TablesUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/TablesUIResource.java @@ -2,6 +2,7 @@ import static com.bakdata.conquery.resources.ResourceConstants.*; +import com.bakdata.conquery.models.auth.web.csrf.CsrfTokenSetFilter; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.Import; import com.bakdata.conquery.models.datasets.Table; @@ -13,6 +14,8 @@ import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -33,13 +36,15 @@ public class TablesUIResource { private Dataset dataset; @PathParam(TABLE) private Table table; + @Context + private ContainerRequestContext requestContext; @GET public View getTableView() { return new UIView<>( "table.html.ftl", - uiProcessor.getUIContext(), + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), uiProcessor.getTableStatistics(table) ); } @@ -50,7 +55,7 @@ public View getImportView(@PathParam(IMPORT_ID) Import imp) { return new UIView<>( "import.html.ftl", - uiProcessor.getUIContext(), + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), uiProcessor.getImportStatistics(imp) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/UserUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/UserUIResource.java index 892528a8d1..e8833a6b74 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/UserUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/UserUIResource.java @@ -4,6 +4,7 @@ import static com.bakdata.conquery.resources.ResourceConstants.USER_ID; import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.auth.web.csrf.CsrfTokenSetFilter; import com.bakdata.conquery.resources.admin.rest.UIProcessor; import com.bakdata.conquery.resources.admin.ui.model.UIView; import io.dropwizard.views.common.View; @@ -12,6 +13,8 @@ import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; import lombok.RequiredArgsConstructor; @@ -21,10 +24,13 @@ public class UserUIResource { protected final UIProcessor uiProcessor; + @Context + private ContainerRequestContext requestContext; @GET public View getUsers() { - return new UIView<>("users.html.ftl", uiProcessor.getUIContext(), uiProcessor.getAdminProcessor().getAllUsers()); + return new UIView<>("users.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), uiProcessor.getAdminProcessor() + .getAllUsers()); } /** @@ -36,6 +42,6 @@ public View getUsers() { @Path("{" + USER_ID + "}") @GET public View getUser(@PathParam(USER_ID) User user) { - return new UIView<>("user.html.ftl", uiProcessor.getUIContext(), uiProcessor.getUserContent(user)); + return new UIView<>("user.html.ftl", uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), uiProcessor.getUserContent(user)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/ConnectorUIResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/ConnectorUIResource.java index 4168fa7ab2..c846a92b30 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/ConnectorUIResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/ConnectorUIResource.java @@ -4,6 +4,7 @@ import static com.bakdata.conquery.resources.ResourceConstants.DATASET; import com.bakdata.conquery.io.jersey.ExtraMimeTypes; +import com.bakdata.conquery.models.auth.web.csrf.CsrfTokenSetFilter; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.resources.admin.rest.UIProcessor; @@ -14,6 +15,8 @@ import jakarta.ws.rs.Path; import jakarta.ws.rs.PathParam; import jakarta.ws.rs.Produces; +import jakarta.ws.rs.container.ContainerRequestContext; +import jakarta.ws.rs.core.Context; import jakarta.ws.rs.core.MediaType; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -34,12 +37,14 @@ public class ConnectorUIResource { protected Connector connector; @PathParam(DATASET) protected Dataset dataset; + @Context + private ContainerRequestContext requestContext; @GET public View getConnectorView() { return new UIView<>( "connector.html.ftl", - uiProcessor.getUIContext(), + uiProcessor.getUIContext(CsrfTokenSetFilter.getCsrfTokenProperty(requestContext)), connector ); } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/UIContext.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/UIContext.java index 83d48c9271..ef6fe0ccd7 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/UIContext.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/UIContext.java @@ -23,6 +23,9 @@ public class UIContext { @Getter public final TemplateModel staticUriElem = STATIC_URI_ELEMENTS; + @Getter + public final String csrfToken; + public Map getShardNodes() { return shardNodeSupplier.get().stream().collect(Collectors.toMap( ShardNodeInformation::getRemoteAddress, diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SharedAliases.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SharedAliases.java index ef65266db7..3264595ab9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SharedAliases.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SharedAliases.java @@ -31,7 +31,10 @@ public enum SharedAliases { INDEX_START_POSITIVE("index_start_positive"), INDEX_START_NEGATIVE("index_start_negative"), STRATIFICATION_BOUNDS("stratification_bounds"), - OBSERVATION_SCOPE("scope"); + OBSERVATION_SCOPE("scope"), + + // full export form + SOURCE("source"); private final String alias; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java index 3c45d3a86a..351432d17c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQExternalConverter.java @@ -52,7 +52,7 @@ public ConversionContext convert(CQExternal external, ConversionContext context) */ private static List createRowSelects(Map.Entry entry, SqlFunctionProvider functionProvider) { - Field primaryColumn = DSL.field(DSL.val(entry.getKey())).coerce(Object.class).as(SharedAliases.PRIMARY_COLUMN.getAlias()); + Field primaryColumn = DSL.val(entry.getKey()).coerce(Object.class).as(SharedAliases.PRIMARY_COLUMN.getAlias()); SqlIdColumns ids = new SqlIdColumns(primaryColumn); List validityDateEntries = functionProvider.forCDateSet(entry.getValue(), SharedAliases.DATES_COLUMN); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java index b268af4992..e9f79f501f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -35,6 +35,7 @@ import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; +import com.bakdata.conquery.util.TablePrimaryColumnUtil; import com.google.common.base.Preconditions; import org.jooq.Condition; import org.jooq.Field; @@ -169,7 +170,7 @@ private CQTableContext createTableContext(TablePathGenerator pathGenerator, CQCo private static SqlIdColumns convertIds(CQConcept cqConcept, CQTable cqTable, ConversionContext conversionContext) { - Field primaryColumn = DSL.field(DSL.name(conversionContext.getConfig().getPrimaryColumn())); + Field primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(cqTable.getConnector().getTable(), conversionContext.getConfig()); if (cqConcept.isExcludeFromSecondaryId() || conversionContext.getSecondaryIdDescription() == null @@ -196,12 +197,13 @@ private static Optional convertValidityDate(CQTable cqTable, St if (Objects.isNull(cqTable.findValidityDate())) { return Optional.empty(); } + SqlFunctionProvider functionProvider = context.getSqlDialect().getFunctionProvider(); ColumnDateRange validityDate; if (context.getDateRestrictionRange() != null) { - validityDate = context.getSqlDialect().getFunctionProvider().forTablesValidityDate(cqTable, context.getDateRestrictionRange(), connectorLabel); + validityDate = functionProvider.forValidityDate(cqTable.findValidityDate(), context.getDateRestrictionRange()).asValidityDateRange(connectorLabel); } else { - validityDate = context.getSqlDialect().getFunctionProvider().forTablesValidityDate(cqTable, connectorLabel); + validityDate = functionProvider.forValidityDate(cqTable.findValidityDate()).asValidityDateRange(connectorLabel); } return Optional.of(validityDate); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java index 9a18097e51..d93f92596b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FilterContext.java @@ -1,20 +1,40 @@ package com.bakdata.conquery.sql.conversion.cqelement.concept; +import javax.annotation.Nullable; + import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; import lombok.Value; @Value +@AllArgsConstructor(access = AccessLevel.PRIVATE) public class FilterContext implements Context { + SqlIdColumns ids; + /** * A filter value ({@link FilterValue#getValue()}) */ - SqlIdColumns ids; V value; + ConversionContext conversionContext; + + /** + * Not present if this context is for table export. + */ + @Nullable ConceptConversionTables tables; + public static FilterContext forConceptConversion(SqlIdColumns ids, V value, ConversionContext conversionContext, ConceptConversionTables tables) { + return new FilterContext<>(ids, value, conversionContext, tables); + } + + public static FilterContext forTableExport(SqlIdColumns ids, V value, ConversionContext conversionContext) { + return new FilterContext<>(ids, value, conversionContext, null); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java index 6085286ad5..d4b234c69b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/IntervalPackingSelectsCte.java @@ -64,7 +64,7 @@ private static QueryStep create( List predecessors = List.of(); QueryStep actualPredecessor = predecessor; if (predecessor.getQualifiedSelects().getValidityDate().get().isSingleColumnRange()) { - actualPredecessor = functionProvider.unnestValidityDate(predecessor, tables); + actualPredecessor = functionProvider.unnestValidityDate(predecessor, tables.cteName(ConceptCteStep.UNNEST_DATE)); predecessors = List.of(actualPredecessor); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java index 37b05b16fc..62e1d8d66b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java @@ -7,8 +7,6 @@ import java.util.Objects; import java.util.stream.Collectors; -import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; -import com.bakdata.conquery.models.common.CDate; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; @@ -17,7 +15,6 @@ import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.SqlTables; import org.jooq.Condition; import org.jooq.DataType; import org.jooq.Field; @@ -107,25 +104,25 @@ public List forCDateSet(CDateSet dateset, SharedAliases alias) } @Override - public ColumnDateRange forTablesValidityDate(CQTable cqTable, String alias) { - return toColumnDateRange(cqTable).asValidityDateRange(alias); + public ColumnDateRange forValidityDate(ValidityDate validityDate) { + return toColumnDateRange(validityDate); } @Override - public ColumnDateRange forTablesValidityDate(CQTable cqTable, CDateRange dateRestriction, String alias) { + public ColumnDateRange forValidityDate(ValidityDate validityDate, CDateRange dateRestriction) { - ColumnDateRange validityDate = toColumnDateRange(cqTable); + ColumnDateRange validityDateRange = toColumnDateRange(validityDate); ColumnDateRange restriction = toColumnDateRange(dateRestriction); - Field lowerBound = DSL.when(validityDate.getStart().lessThan(restriction.getStart()), restriction.getStart()) - .otherwise(validityDate.getStart()); + Field lowerBound = DSL.when(validityDateRange.getStart().lessThan(restriction.getStart()), restriction.getStart()) + .otherwise(validityDateRange.getStart()); Field maxDate = toDateField(MAX_DATE_VALUE); // we want to add +1 day to the end date - except when it's the max date already Field restrictionUpperBound = DSL.when(restriction.getEnd().eq(maxDate), maxDate).otherwise(addDays(restriction.getEnd(), DSL.val(1))); - Field upperBound = DSL.when(validityDate.getEnd().greaterThan(restriction.getEnd()), restrictionUpperBound) - .otherwise(validityDate.getEnd()); + Field upperBound = DSL.when(validityDateRange.getEnd().greaterThan(restriction.getEnd()), restrictionUpperBound) + .otherwise(validityDateRange.getEnd()); - return ColumnDateRange.of(lowerBound, upperBound).as(alias); + return ColumnDateRange.of(lowerBound, upperBound); } @Override @@ -151,7 +148,7 @@ public ColumnDateRange intersection(ColumnDateRange left, ColumnDateRange right) } @Override - public QueryStep unnestValidityDate(QueryStep predecessor, SqlTables sqlTables) { + public QueryStep unnestValidityDate(QueryStep predecessor, String cteName) { // HANA does not support single column datemultiranges return predecessor; } @@ -169,7 +166,7 @@ public Field daterangeStringAggregation(ColumnDateRange columnDateRange) ); // encapsulate all ranges (including empty ranges) within curly braces - return DSL.when(stringAggregation.isNull(), DSL.field(DSL.val("{}"))) + return DSL.when(stringAggregation.isNull(), DSL.val("{}")) .otherwise(DSL.field("'{' || {0} || '}'", String.class, stringAggregation)); } @@ -295,7 +292,7 @@ public Field addDays(Field dateColumn, Field amountOfDays) "ADD_DAYS", Date.class, dateColumn, - DSL.val(amountOfDays) + amountOfDays ); } @@ -325,10 +322,9 @@ private ColumnDateRange toColumnDateRange(CDateRange dateRestriction) { return ColumnDateRange.of(toDateField(startDateExpression), toDateField(endDateExpression)); } - private ColumnDateRange toColumnDateRange(CQTable cqTable) { + private ColumnDateRange toColumnDateRange(ValidityDate validityDate) { - ValidityDate validityDate = cqTable.findValidityDate(); - String tableName = cqTable.getConnector().getTable().getName(); + String tableName = validityDate.getConnector().getTable().getName(); Column startColumn; Column endColumn; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java index b76608d442..b68ab67c6b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java @@ -6,17 +6,15 @@ import java.util.Optional; import java.util.stream.Collectors; -import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.sql.conversion.SharedAliases; -import com.bakdata.conquery.sql.conversion.cqelement.concept.ConceptCteStep; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlTables; +import com.google.common.base.Preconditions; import org.jooq.ArrayAggOrderByStep; import org.jooq.Condition; import org.jooq.DataType; @@ -110,18 +108,15 @@ public List forCDateSet(CDateSet dateset, SharedAliases alias) } @Override - public ColumnDateRange forTablesValidityDate(CQTable cqTable, String alias) { - return toColumnDateRange(cqTable).asValidityDateRange(alias); + public ColumnDateRange forValidityDate(ValidityDate validityDate) { + return toColumnDateRange(validityDate); } @Override - public ColumnDateRange forTablesValidityDate(CQTable cqTable, CDateRange dateRestriction, String alias) { - - ColumnDateRange validityDate = toColumnDateRange(cqTable); + public ColumnDateRange forValidityDate(ValidityDate validityDate, CDateRange dateRestriction) { + ColumnDateRange validityDateRange = toColumnDateRange(validityDate); ColumnDateRange restriction = toColumnDateRange(dateRestriction); - ColumnDateRange intersection = intersection(validityDate, restriction); - - return intersection.asValidityDateRange(alias); + return intersection(validityDateRange, restriction); } @Override @@ -147,7 +142,7 @@ public ColumnDateRange intersection(ColumnDateRange left, ColumnDateRange right) } @Override - public QueryStep unnestValidityDate(QueryStep predecessor, SqlTables sqlTables) { + public QueryStep unnestValidityDate(QueryStep predecessor, String cteName) { Preconditions.checkArgument( predecessor.getSelects().getValidityDate().isPresent(), @@ -164,7 +159,7 @@ public QueryStep unnestValidityDate(QueryStep predecessor, SqlTables sqlTables) .build(); return QueryStep.builder() - .cteName(sqlTables.cteName(ConceptCteStep.UNNEST_DATE)) + .cteName(cteName) .selects(selects) .fromTable(QueryStep.toTableLike(predecessor.getCteName())) .build(); @@ -181,7 +176,7 @@ public Field daterangeStringExpression(ColumnDateRange columnDateRange) if (!columnDateRange.isSingleColumnRange()) { throw new UnsupportedOperationException("All column date ranges should have been converted to single column ranges."); } - Field aggregatedValidityDate = DSL.field("{0}::{1}", String.class, columnDateRange.getRange(), DSL.keyword("varchar")); + Field aggregatedValidityDate = DSL.field("({0})::{1}", String.class, columnDateRange.getRange(), DSL.keyword("varchar")); return replace(aggregatedValidityDate, INFINITY_DATE_VALUE, INFINITY_SIGN); } @@ -319,9 +314,9 @@ private ColumnDateRange toColumnDateRange(CDateRange dateRestriction) { return ColumnDateRange.of(dateRestrictionRange); } - private ColumnDateRange toColumnDateRange(CQTable cqTable) { - ValidityDate validityDate = cqTable.findValidityDate(); - String tableName = cqTable.getConnector().getTable().getName(); + private ColumnDateRange toColumnDateRange(ValidityDate validityDate) { + + String tableName = validityDate.getConnector().getTable().getName(); Field dateRange; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java index 552cc1bc06..4f17b904ea 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java @@ -17,7 +17,11 @@ import com.bakdata.conquery.sql.conversion.model.QueryStepTransformer; import com.bakdata.conquery.sql.conversion.query.AbsoluteFormQueryConverter; import com.bakdata.conquery.sql.conversion.query.ConceptQueryConverter; +import com.bakdata.conquery.sql.conversion.query.EntityDateQueryConverter; +import com.bakdata.conquery.sql.conversion.query.FormConversionHelper; +import com.bakdata.conquery.sql.conversion.query.RelativFormQueryConverter; import com.bakdata.conquery.sql.conversion.query.SecondaryIdQueryConverter; +import com.bakdata.conquery.sql.conversion.query.TableExportQueryConverter; import com.bakdata.conquery.sql.conversion.supplier.DateNowSupplier; import com.bakdata.conquery.sql.conversion.supplier.SystemDateNowSupplier; import com.bakdata.conquery.sql.execution.SqlCDateSetParser; @@ -48,7 +52,10 @@ default boolean supportsSingleColumnRanges() { } default List> getDefaultNodeConverters() { + QueryStepTransformer queryStepTransformer = new QueryStepTransformer(getDSLContext()); + FormConversionHelper formConversionUtil = new FormConversionHelper(queryStepTransformer); + return List.of( new CQDateRestrictionConverter(), new CQAndConverter(), @@ -58,7 +65,10 @@ default List> getDefaultNodeConverters() { new CQExternalConverter(), new ConceptQueryConverter(queryStepTransformer), new SecondaryIdQueryConverter(), - new AbsoluteFormQueryConverter(queryStepTransformer) + new AbsoluteFormQueryConverter(formConversionUtil), + new EntityDateQueryConverter(formConversionUtil), + new RelativFormQueryConverter(formConversionUtil), + new TableExportQueryConverter(queryStepTransformer) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index 4400b4feef..91e05823bc 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -7,10 +7,10 @@ import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.SqlTables; import org.jooq.Condition; import org.jooq.DataType; import org.jooq.Field; @@ -66,15 +66,15 @@ public interface SqlFunctionProvider { List forCDateSet(CDateSet dateset, SharedAliases alias); /** - * Creates a {@link ColumnDateRange} for a tables {@link CQTable}s validity date. + * Creates a {@link ColumnDateRange} for a tables {@link ValidityDate}. */ - ColumnDateRange forTablesValidityDate(CQTable cqTable, String alias); + ColumnDateRange forValidityDate(ValidityDate validityDate); /** * Creates a {@link ColumnDateRange} for a tables {@link CQTable}s validity date. The validity dates bounds will be restricted by the given date * restriction. */ - ColumnDateRange forTablesValidityDate(CQTable cqTable, CDateRange dateRestriction, String alias); + ColumnDateRange forValidityDate(ValidityDate validityDate, CDateRange dateRestriction); ColumnDateRange aggregated(ColumnDateRange columnDateRange); @@ -93,7 +93,7 @@ public interface SqlFunctionProvider { * @return A QueryStep containing an unnested validity date with 1 row per single daterange for each id. For dialects that don't support single column * multiranges, the given predecessor will be returned as is. */ - QueryStep unnestValidityDate(QueryStep predecessor, SqlTables sqlTables); + QueryStep unnestValidityDate(QueryStep predecessor, String cteName); /** * Aggregates the start and end columns of the validity date of entries into one compound string expression. @@ -115,6 +115,9 @@ public interface SqlFunctionProvider { */ Field daterangeStringExpression(ColumnDateRange columnDateRange); + /** + * Calculates the date distance in the given {@link ChronoUnit} between an exclusive end date and an inclusive start date. + */ Field dateDistance(ChronoUnit datePart, Field startDate, Field endDate); Field addDays(Field dateColumn, Field amountOfDays); @@ -194,6 +197,10 @@ default Field replace(Field target, String old, String _new) { return DSL.function("replace", String.class, target, DSL.val(old), DSL.val(_new)); } + default Field encloseInCurlyBraces(Field stringExpression) { + return DSL.field("'{' || {0} || '}'", String.class, stringExpression); + } + default Field prefixStringAggregation(Field field, String prefix) { return DSL.field( "'[' || {0}({1}, {2}) || ']'", diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java new file mode 100644 index 0000000000..d151fc6b93 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/AbsoluteStratification.java @@ -0,0 +1,188 @@ +package com.bakdata.conquery.sql.conversion.forms; + +import java.sql.Date; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; +import com.bakdata.conquery.models.forms.util.Resolution; +import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.google.common.base.Preconditions; +import lombok.RequiredArgsConstructor; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.impl.DSL; + +@RequiredArgsConstructor +class AbsoluteStratification { + + private final int INDEX_START = 1; + private final int INDEX_END = 10_000; + + private final QueryStep baseStep; + private final StratificationFunctions stratificationFunctions; + + public QueryStep createStratificationTable(List resolutionAndAlignments) { + + QueryStep intSeriesStep = createIntSeriesStep(); + QueryStep indexStartStep = createIndexStartStep(); + + List resolutionTables = resolutionAndAlignments.stream() + .map(resolutionAndAlignment -> createResolutionTable(indexStartStep, resolutionAndAlignment)) + .toList(); + + List predecessors = List.of(baseStep, intSeriesStep, indexStartStep); + return StratificationTableFactory.unionResolutionTables(resolutionTables, predecessors); + } + + private QueryStep createIntSeriesStep() { + + // not actually required, but Selects expect at least 1 SqlIdColumn + Field rowNumber = DSL.rowNumber().over().coerce(Object.class); + SqlIdColumns ids = new SqlIdColumns(rowNumber); + + FieldWrapper seriesIndex = new FieldWrapper<>(stratificationFunctions.intSeriesField()); + + Selects selects = Selects.builder() + .ids(ids) + .sqlSelect(seriesIndex) + .build(); + + Table seriesTable = stratificationFunctions.generateIntSeries(INDEX_START, INDEX_END) + .as(SharedAliases.SERIES_INDEX.getAlias()); + + return QueryStep.builder() + .cteName(FormCteStep.INT_SERIES.getSuffix()) + .selects(selects) + .fromTable(seriesTable) + .build(); + } + + private QueryStep createIndexStartStep() { + + Selects baseStepSelects = baseStep.getQualifiedSelects(); + Preconditions.checkArgument(baseStepSelects.getStratificationDate().isPresent(), "The base step must have a stratification date set"); + ColumnDateRange bounds = baseStepSelects.getStratificationDate().get(); + + Field indexStart = stratificationFunctions.absoluteIndexStartDate(bounds).as(SharedAliases.INDEX_START.getAlias()); + Field yearStart = stratificationFunctions.lowerBoundYearStart(bounds).as(SharedAliases.YEAR_START.getAlias()); + Field yearEnd = stratificationFunctions.upperBoundYearEnd(bounds).as(SharedAliases.YEAR_END.getAlias()); + Field yearEndQuarterAligned = stratificationFunctions.upperBoundYearEndQuarterAligned(bounds).as(SharedAliases.YEAR_END_QUARTER_ALIGNED.getAlias()); + Field quarterStart = stratificationFunctions.lowerBoundQuarterStart(bounds).as(SharedAliases.QUARTER_START.getAlias()); + Field quarterEnd = stratificationFunctions.upperBoundQuarterEnd(bounds).as(SharedAliases.QUARTER_END.getAlias()); + + List> startDates = Stream.of( + indexStart, + yearStart, + yearEnd, + yearEndQuarterAligned, + quarterStart, + quarterEnd + ) + .map(FieldWrapper::new) + .toList(); + + Selects selects = Selects.builder() + .ids(baseStepSelects.getIds()) + .stratificationDate(Optional.of(bounds)) + .sqlSelects(startDates) + .build(); + + return QueryStep.builder() + .cteName(FormCteStep.INDEX_START.getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(baseStep.getCteName())) + .build(); + } + + private QueryStep createResolutionTable(QueryStep indexStartStep, ExportForm.ResolutionAndAlignment resolutionAndAlignment) { + return switch (resolutionAndAlignment.getResolution()) { + case COMPLETE -> createCompleteTable(); + case YEARS, QUARTERS, DAYS -> createIntervalTable(indexStartStep, resolutionAndAlignment); + }; + } + + private QueryStep createCompleteTable() { + + Selects baseStepSelects = baseStep.getQualifiedSelects(); + + // complete range shall have a null index because it spans the complete range, but we set it to 1 to ensure we can join tables on index, + // because a condition involving null in a join (e.g., null = some_value or null = null) always evaluates to false + Field index = DSL.field(DSL.val(1, Integer.class)).as(SharedAliases.INDEX.getAlias()); + SqlIdColumns ids = baseStepSelects.getIds().withAbsoluteStratification(Resolution.COMPLETE, index); + + ColumnDateRange completeRange = baseStepSelects.getStratificationDate().get(); + + Selects selects = Selects.builder() + .ids(ids) + .stratificationDate(Optional.of(completeRange)) + .build(); + + return QueryStep.builder() + .cteName(FormCteStep.COMPLETE.getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(baseStep.getCteName())) + .build(); + } + + private QueryStep createIntervalTable(QueryStep indexStartStep, ExportForm.ResolutionAndAlignment resolutionAndAlignment) { + + QueryStep countsCte = createCountsCte(indexStartStep, resolutionAndAlignment); + Preconditions.checkArgument(countsCte.getSelects().getStratificationDate().isPresent(), "The countsCte must have a stratification date set"); + Selects countsCteSelects = countsCte.getQualifiedSelects(); + + ColumnDateRange stratificationRange = stratificationFunctions.createStratificationRange( + resolutionAndAlignment, + countsCteSelects.getStratificationDate().get() + ); + + Field index = stratificationFunctions.index(countsCteSelects.getIds(), countsCte.getQualifiedSelects().getStratificationDate()); + SqlIdColumns ids = countsCteSelects.getIds().withAbsoluteStratification(resolutionAndAlignment.getResolution(), index); + + Selects selects = Selects.builder() + .ids(ids) + .stratificationDate(Optional.ofNullable(stratificationRange)) + .build(); + + Condition stopOnMaxResolutionWindowCount = stratificationFunctions.stopOnMaxResolutionWindowCount(resolutionAndAlignment); + + return QueryStep.builder() + .cteName(FormCteStep.stratificationCte(resolutionAndAlignment.getResolution()).getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(countsCte.getCteName())) + .fromTable(QueryStep.toTableLike(FormCteStep.INT_SERIES.getSuffix())) + .conditions(List.of(stopOnMaxResolutionWindowCount)) + .predecessor(countsCte) + .build(); + } + + private QueryStep createCountsCte(QueryStep indexStartStep, ExportForm.ResolutionAndAlignment resolutionAndAlignment) { + + Selects indexStartSelects = indexStartStep.getQualifiedSelects(); + Preconditions.checkArgument(indexStartSelects.getStratificationDate().isPresent(), "The indexStartStep must have a stratification date set"); + + Field resolutionWindowCount = stratificationFunctions.calculateResolutionWindowCount( + resolutionAndAlignment, + indexStartSelects.getStratificationDate().get() + ); + + Selects selects = indexStartSelects.toBuilder() + .sqlSelect(new FieldWrapper<>(resolutionWindowCount)) + .build(); + + return QueryStep.builder() + .cteName(FormCteStep.countsCte(resolutionAndAlignment.getResolution()).getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(indexStartStep.getCteName())) + .build(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/CombinationNotSupportedException.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/CombinationNotSupportedException.java index 0078f1290a..a29f4999c5 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/CombinationNotSupportedException.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/CombinationNotSupportedException.java @@ -1,6 +1,9 @@ package com.bakdata.conquery.sql.conversion.forms; +import com.bakdata.conquery.apiv1.forms.IndexPlacement; import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; +import com.bakdata.conquery.models.forms.util.CalendarUnit; +import com.bakdata.conquery.models.forms.util.Resolution; class CombinationNotSupportedException extends RuntimeException { @@ -11,4 +14,12 @@ public CombinationNotSupportedException(ExportForm.ResolutionAndAlignment resolu )); } + public CombinationNotSupportedException(IndexPlacement indexPlacement, CalendarUnit timeUnit) { + super("Combination of index placement %s and time unit %s not supported".formatted(indexPlacement, timeUnit)); + } + + public CombinationNotSupportedException(CalendarUnit timeUnit, Resolution resolution) { + super("Combination of time unit %s and resolution %s not supported".formatted(timeUnit, resolution)); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormConstants.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormConstants.java index a4ad99ef70..f1ca4ca05d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormConstants.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormConstants.java @@ -19,6 +19,21 @@ class FormConstants { */ public static Field INDEX_START = DSL.field(DSL.name(SharedAliases.INDEX_START.getAlias()), Date.class); + /** + * The index date corresponding to the {@link TemporalSamplerFactory} of a relative stratification. + */ + public static Field INDEX_SELECTOR = DSL.field(DSL.name(SharedAliases.INDEX_SELECTOR.getAlias()), Date.class); + + /** + * The index date from which we start when calculating a feature range. + */ + public static Field INDEX_START_NEGATIVE = DSL.field(DSL.name(SharedAliases.INDEX_START_NEGATIVE.getAlias()), Date.class); + + /** + * The index date from which we start when calculating an outcome range. + */ + public static Field INDEX_START_POSITIVE = DSL.field(DSL.name(SharedAliases.INDEX_START_POSITIVE.getAlias()), Date.class); + /** * The quarter start of the lower bound of an absolute stratification range. The stratification range this date is referring to can be an * {@link AbsoluteFormQuery#getDateRange()} or for an {@link EntityDateQuery} the respective entities date range bound by the diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java index b624aab42f..7dbd2baedb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormCteStep.java @@ -12,6 +12,15 @@ public enum FormCteStep implements CteStep { // prerequisite EXTRACT_IDS("extract_ids"), + // entity date + UNNEST_ENTITY_DATE_CTE("unnest_entity_date"), + OVERWRITE_BOUNDS("overwrite_bounds"), + + // relative form + UNNEST_DATES("unnest_dates"), + INDEX_SELECTOR("index_selector"), + TOTAL_BOUNDS("total_bounds"), + // stratification INDEX_START("index_start"), INT_SERIES("int_series"), diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormType.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormType.java new file mode 100644 index 0000000000..d448a6ba33 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/FormType.java @@ -0,0 +1,7 @@ +package com.bakdata.conquery.sql.conversion.forms; + +public enum FormType { + ABSOLUTE, + ENTITY_DATE, + RELATIVE +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/HanaStratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/HanaStratificationFunctions.java index 395f108ebd..62938726e1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/HanaStratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/HanaStratificationFunctions.java @@ -3,7 +3,9 @@ import static com.bakdata.conquery.sql.conversion.forms.Interval.MONTHS_PER_QUARTER; import java.sql.Date; +import java.time.temporal.ChronoUnit; +import com.bakdata.conquery.apiv1.query.concept.specific.temporal.TemporalSamplerFactory; import com.bakdata.conquery.sql.conversion.dialect.HanaSqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import lombok.Getter; @@ -12,6 +14,7 @@ import org.jooq.Record; import org.jooq.Table; import org.jooq.impl.DSL; +import org.jooq.impl.QOM; import org.jooq.impl.SQLDataType; @Getter @@ -35,7 +38,12 @@ protected Field lower(ColumnDateRange dateRange) { } @Override - protected Field upper(ColumnDateRange dateRange) { + protected Field inclusiveUpper(ColumnDateRange dateRange) { + return functionProvider.addDays(exclusiveUpper(dateRange), DSL.val(-1)); + } + + @Override + protected Field exclusiveUpper(ColumnDateRange dateRange) { // HANA does not support single-column ranges, so we can return start and end directly return dateRange.getEnd(); } @@ -54,12 +62,12 @@ public Field absoluteIndexStartDate(ColumnDateRange dateRange) { } @Override - public Field yearStart(ColumnDateRange dateRange) { + public Field lowerBoundYearStart(ColumnDateRange dateRange) { return jumpToYearStart(dateRange.getStart()); } @Override - public Field nextYearStart(ColumnDateRange dateRange) { + public Field upperBoundYearEnd(ColumnDateRange dateRange) { return DSL.field( "SERIES_ROUND({0}, {1}, {2})", Date.class, @@ -70,23 +78,39 @@ public Field nextYearStart(ColumnDateRange dateRange) { } @Override - public Field yearEndQuarterAligned(ColumnDateRange dateRange) { - Field nextYearStart = nextYearStart(dateRange); - Field quartersInMonths = getMonthsInQuarters(dateRange.getStart(), Offset.MINUS_ONE); - return addMonths(nextYearStart, quartersInMonths); + public Field upperBoundYearEndQuarterAligned(ColumnDateRange dateRange) { + Field yearStartOfUpperBound = jumpToYearStart(dateRange.getEnd()); + Field quartersInMonths = getQuartersInMonths(dateRange.getStart(), Offset.MINUS_ONE); + Field yearEndQuarterAligned = addMonths(yearStartOfUpperBound, quartersInMonths); + // we add +1 year to the quarter aligned end if it is less than the upper bound we want to align + return DSL.when( + yearEndQuarterAligned.lessThan(dateRange.getEnd()), + shiftByInterval(yearEndQuarterAligned, Interval.ONE_YEAR_INTERVAL, DSL.val(1), Offset.NONE) + ) + .otherwise(yearEndQuarterAligned); + } + + @Override + public Field lowerBoundQuarterStart(ColumnDateRange dateRange) { + return jumpToQuarterStart(dateRange.getStart()); } @Override - public Field quarterStart(ColumnDateRange dateRange) { - Field yearStart = yearStart(dateRange); - Field quartersInMonths = getMonthsInQuarters(dateRange.getStart(), Offset.MINUS_ONE); + protected Field jumpToQuarterStart(Field date) { + Field yearStart = jumpToYearStart(date); + Field quartersInMonths = getQuartersInMonths(date, Offset.MINUS_ONE); return addMonths(yearStart, quartersInMonths); } @Override - public Field nextQuartersStart(ColumnDateRange dateRange) { - Field yearStart = jumpToYearStart(dateRange.getEnd()); - Field quartersInMonths = getMonthsInQuarters(dateRange.getEnd(), Offset.NONE); + public Field upperBoundQuarterEnd(ColumnDateRange dateRange) { + return jumpToNextQuarterStart(inclusiveUpper(dateRange)); + } + + @Override + protected Field jumpToNextQuarterStart(Field date) { + Field yearStart = jumpToYearStart(date); + Field quartersInMonths = getQuartersInMonths(date, Offset.NONE); return addMonths(yearStart, quartersInMonths); } @@ -103,6 +127,36 @@ public Table generateIntSeries(int start, int end) { return DSL.table("SERIES_GENERATE_INTEGER({0}, {1}, {2})", INCREMENT, adjustedStart, end); } + @Override + public Field indexSelectorField(TemporalSamplerFactory indexSelector, ColumnDateRange validityDate) { + return switch (indexSelector) { + case EARLIEST -> DSL.min(validityDate.getStart()); + case LATEST -> DSL.max(inclusiveUpper(validityDate)); + case RANDOM -> { + // we calculate a random int which is in range of the date distance between upper and lower bound + Field dateDistanceInDays = functionProvider.dateDistance(ChronoUnit.DAYS, validityDate.getStart(), validityDate.getEnd()); + Field randomAmountOfDays = DSL.function("RAND", Double.class).times(dateDistanceInDays); + Field flooredAsInt = functionProvider.cast(DSL.floor(randomAmountOfDays), SQLDataType.INTEGER); + // then we add this random amount (of days) to the start date + Field randomDateInRange = functionProvider.addDays(lower(validityDate), flooredAsInt); + // finally, we handle multiple ranges by randomizing which range we use to select a random date from + yield functionProvider.random(randomDateInRange); + } + }; + } + + @Override + public Field shiftByInterval(Field startDate, Interval interval, Field amount, Offset offset) { + Field multiplier = amount.plus(offset.getOffset()); + return switch (interval) { + case ONE_YEAR_INTERVAL -> DSL.function("ADD_YEARS", Date.class, startDate, multiplier.times(Interval.ONE_YEAR_INTERVAL.getAmount())); + case YEAR_AS_DAYS_INTERVAL -> addDays(startDate, multiplier.times(Interval.YEAR_AS_DAYS_INTERVAL.getAmount())); + case QUARTER_INTERVAL -> addMonths(startDate, multiplier.times(Interval.QUARTER_INTERVAL.getAmount())); + case NINETY_DAYS_INTERVAL -> addDays(startDate, multiplier.times(Interval.NINETY_DAYS_INTERVAL.getAmount())); + case ONE_DAY_INTERVAL -> addDays(startDate, multiplier.times(Interval.ONE_DAY_INTERVAL.getAmount())); + }; + } + private static Field addMonths(Field yearStart, Field amount) { return DSL.function("ADD_MONTHS", Date.class, yearStart, amount); } @@ -120,14 +174,7 @@ private Field calcEndDate(Field start, Interval interval) { } private Field calcDate(Field start, Interval interval, Offset offset) { - Field seriesIndex = intSeriesField().plus(offset.getOffset()); - return switch (interval) { - case ONE_YEAR_INTERVAL -> DSL.function("ADD_YEARS", Date.class, start, seriesIndex.times(Interval.ONE_YEAR_INTERVAL.getAmount())); - case YEAR_AS_DAYS_INTERVAL -> addDays(start, seriesIndex.times(Interval.YEAR_AS_DAYS_INTERVAL.getAmount())); - case QUARTER_INTERVAL -> addMonths(start, seriesIndex.times(Interval.QUARTER_INTERVAL.getAmount())); - case NINETY_DAYS_INTERVAL -> addDays(start, seriesIndex.times(Interval.NINETY_DAYS_INTERVAL.getAmount())); - case ONE_DAY_INTERVAL -> addDays(start, seriesIndex.times(Interval.ONE_DAY_INTERVAL.getAmount())); - }; + return shiftByInterval(start, interval, intSeriesField(), offset); } private static Field jumpToYearStart(Field date) { @@ -140,7 +187,7 @@ private static Field jumpToYearStart(Field date) { ); } - private Field getMonthsInQuarters(Field date, Offset offset) { + private Field getQuartersInMonths(Field date, Offset offset) { Field quarterExpression = functionProvider.yearQuarter(date); Field rightMostCharacter = DSL.function("RIGHT", String.class, quarterExpression, DSL.val(1)); Field amountOfQuarters = functionProvider.cast(rightMostCharacter, SQLDataType.INTEGER) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/PostgresStratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/PostgresStratificationFunctions.java index ec04e2e875..4be86a1576 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/PostgresStratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/PostgresStratificationFunctions.java @@ -2,10 +2,13 @@ import static com.bakdata.conquery.sql.conversion.forms.FormConstants.SERIES_INDEX; +import java.math.BigDecimal; import java.sql.Date; import java.sql.Timestamp; +import java.time.temporal.ChronoUnit; import java.util.Map; +import com.bakdata.conquery.apiv1.query.concept.specific.temporal.TemporalSamplerFactory; import com.bakdata.conquery.sql.conversion.dialect.PostgreSqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import lombok.Getter; @@ -16,6 +19,7 @@ import org.jooq.Record; import org.jooq.Table; import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; @Getter @RequiredArgsConstructor @@ -33,45 +37,67 @@ class PostgresStratificationFunctions extends StratificationFunctions { private final PostgreSqlFunctionProvider functionProvider; + @Override + public ColumnDateRange ofStartAndEnd(Field start, Field end) { + return ColumnDateRange.of(functionProvider.daterange(start, end, "[)")); + } + @Override public Field absoluteIndexStartDate(ColumnDateRange dateRange) { return lower(dateRange); } @Override - public Field yearStart(ColumnDateRange dateRange) { + public Field lowerBoundYearStart(ColumnDateRange dateRange) { return castExpressionToDate(jumpToYearStart(lower(dateRange))); } @Override - public Field nextYearStart(ColumnDateRange dateRange) { + public Field upperBoundYearEnd(ColumnDateRange dateRange) { return DSL.field( "{0} + {1} {2}", Date.class, - dateTruncate(DSL.val("year"), upper(dateRange)), + dateTruncate(DSL.val("year"), inclusiveUpper(dateRange)), INTERVAL_KEYWORD, INTERVAL_MAP.get(Interval.ONE_YEAR_INTERVAL) ); } @Override - public Field yearEndQuarterAligned(ColumnDateRange dateRange) { - Field quarter = functionProvider.extract(DatePart.QUARTER, lower(dateRange)); - Field nextYearStart = nextYearStart(dateRange); - return addQuarters(nextYearStart, quarter, Offset.MINUS_ONE); + public Field upperBoundYearEndQuarterAligned(ColumnDateRange dateRange) { + Field lowerBoundQuarter = functionProvider.extract(DatePart.QUARTER, lower(dateRange)); + Field upperBound = inclusiveUpper(dateRange); + Field yearStartOfUpperBound = castExpressionToDate(jumpToYearStart(upperBound)); + Field yearEndQuarterAligned = addQuarters(yearStartOfUpperBound, lowerBoundQuarter, Offset.MINUS_ONE); + // we add +1 year to the quarter aligned end if it is less than the upper bound we want to align + return DSL.when( + yearEndQuarterAligned.lessThan(upperBound), + shiftByInterval(yearEndQuarterAligned, Interval.ONE_YEAR_INTERVAL, DSL.val(1), Offset.NONE) + ) + .otherwise(yearEndQuarterAligned); + } + + @Override + public Field lowerBoundQuarterStart(ColumnDateRange dateRange) { + return jumpToQuarterStart(lower(dateRange)); } @Override - public Field quarterStart(ColumnDateRange dateRange) { - Field quarter = functionProvider.extract(DatePart.QUARTER, lower(dateRange)); - return addQuarters(jumpToYearStart(lower(dateRange)), quarter, Offset.MINUS_ONE); + protected Field jumpToQuarterStart(Field date) { + Field quarter = functionProvider.extract(DatePart.QUARTER, date); + return addQuarters(jumpToYearStart(date), quarter, Offset.MINUS_ONE); } @Override - public Field nextQuartersStart(ColumnDateRange dateRange) { - Field yearStart = dateTruncate(DSL.val("year"), upper(dateRange)); - Field quarterEndInclusive = upper(dateRange).minus(1); - Field quarter = functionProvider.extract(DatePart.QUARTER, quarterEndInclusive); + public Field upperBoundQuarterEnd(ColumnDateRange dateRange) { + Field inclusiveEnd = inclusiveUpper(dateRange); + return jumpToNextQuarterStart(inclusiveEnd); + } + + @Override + protected Field jumpToNextQuarterStart(Field date) { + Field yearStart = dateTruncate(DSL.val("year"), date); + Field quarter = functionProvider.extract(DatePart.QUARTER, date); return addQuarters(yearStart, quarter, Offset.NONE); } @@ -85,6 +111,31 @@ public Table generateIntSeries(int from, int to) { return DSL.table("generate_series({0}, {1})", from, to); } + @Override + public Field indexSelectorField(TemporalSamplerFactory indexSelector, ColumnDateRange validityDate) { + return switch (indexSelector) { + case EARLIEST -> DSL.min(lower(validityDate)); + // upper returns the exclusive end date, we want to inclusive one, so we add -1 day + case LATEST -> DSL.max(inclusiveUpper(validityDate)); + case RANDOM -> { + // we calculate a random int which is in range of the date distance between upper and lower bound + Field dateDistanceInDays = functionProvider.dateDistance(ChronoUnit.DAYS, lower(validityDate), exclusiveUpper(validityDate)); + Field randomAmountOfDays = DSL.rand().times(dateDistanceInDays); + Field flooredAsInt = functionProvider.cast(DSL.floor(randomAmountOfDays), SQLDataType.INTEGER); + // then we add this random amount (of days) to the start date + Field randomDateInRange = functionProvider.addDays(lower(validityDate), flooredAsInt); + // finally, we handle multiple ranges by randomizing which range we use to select a random date from + yield functionProvider.random(randomDateInRange); + } + }; + } + + @Override + public Field shiftByInterval(Field startDate, Interval interval, Field amount, Offset offset) { + Field intervalExpression = INTERVAL_MAP.get(interval); + return addMultipliedInterval(startDate, intervalExpression, amount, offset); + } + @Override protected Field lower(ColumnDateRange dateRange) { checkIsSingleColumnRange(dateRange); @@ -92,7 +143,13 @@ protected Field lower(ColumnDateRange dateRange) { } @Override - protected Field upper(ColumnDateRange dateRange) { + protected Field inclusiveUpper(ColumnDateRange dateRange) { + checkIsSingleColumnRange(dateRange); + return exclusiveUpper(dateRange).minus(1); + } + + @Override + protected Field exclusiveUpper(ColumnDateRange dateRange) { checkIsSingleColumnRange(dateRange); return DSL.function("upper", Date.class, dateRange.getRange()); } @@ -100,24 +157,23 @@ protected Field upper(ColumnDateRange dateRange) { @Override protected ColumnDateRange calcRange(Field start, Interval interval) { Field intervalExpression = INTERVAL_MAP.get(interval); - return ColumnDateRange.of(functionProvider.daterange( + return ofStartAndEnd( calcStartDate(start, intervalExpression), - calcEndDate(start, intervalExpression), - "[)" - )); + calcEndDate(start, intervalExpression) + ); } private Field calcStartDate(Field start, Field intervalExpression) { Field intSeriesField = intSeriesField(); - return multiplyByInterval(start, intervalExpression, intSeriesField, Offset.MINUS_ONE); + return addMultipliedInterval(start, intervalExpression, intSeriesField, Offset.MINUS_ONE); } private Field calcEndDate(Field start, Field intervalExpression) { Field intSeriesField = intSeriesField(); - return multiplyByInterval(start, intervalExpression, intSeriesField, Offset.NONE); + return addMultipliedInterval(start, intervalExpression, intSeriesField, Offset.NONE); } - private Field multiplyByInterval(Field start, Field intervalExpression, Field amount, Offset offset) { + private Field addMultipliedInterval(Field start, Field intervalExpression, Field amount, Offset offset) { Field multiplier = amount.plus(offset.getOffset()); Field shiftedDate = DSL.field( "{0} + {1} {2} * {3}", @@ -136,7 +192,7 @@ private Field dateTruncate(Field field, Field date) { } private Field addQuarters(Field start, Field amountOfQuarters, Offset offset) { - return multiplyByInterval(start, INTERVAL_MAP.get(Interval.QUARTER_INTERVAL), amountOfQuarters, offset); + return addMultipliedInterval(start, INTERVAL_MAP.get(Interval.QUARTER_INTERVAL), amountOfQuarters, offset); } private Field jumpToYearStart(Field date) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/README.md b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/README.md index 90966677f2..d5895c9456 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/README.md +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/README.md @@ -1,42 +1,85 @@ # Form conversion - how to apply stratification in SQL? This document outlines the procedure to apply stratification within SQL in the context of the form conversion process. +The process or to be more precise, specific functions used while creating stratification tables differ from dialect +to dialect, but the overall process is the same. This document is using Postgres dialect for the SQL examples. + +# Table of Contents + +1. [Prerequisite conversion](#prerequisite-conversion) +2. [Absolute stratification](#absolute-stratification-for-absolute-forms-and-entity-date-forms) + 1. [For absolute forms](#absolute-forms) + 2. [For entity date queries](#entity-date) + 3. [Stratification tables](#stratification-tables) +3. [Relative stratification](#relative-stratification) +4. [Full stratification table](#full-stratification-table) +5. [Feature conversion](#feature-conversion) +6. [Left-join converted features](#left-join-converted-features-with-the-full-stratification-table-for-the-final-select) +7. [Full export form](#full-export-form) ## Prerequisite conversion -The prerequisite query conversion produces a CTE, which will contain the IDs of those entities relevant for the form. +The prerequisite query conversion produces a CTE, which will contain the IDs of those subjects relevant for the form. Because this could be any kind of Query, the CTE might also contain a validity date and converted Selects. Take this CTE representing a converted CQExternal as an example: **CTE:** `external` ```sql -select '1' "primary_id", - TO_DATE('2001-12-01', 'yyyy-mm-dd') "dates_start", - TO_DATE('2016-12-02', 'yyyy-mm-dd') "dates_end" +select '1' "primary_id", + daterange('2001-12-01', '2016-12-02', '[)') as "validity_date" from "DUMMY"; -- "DUMMY" is SAP HANAs built-in no-op table ``` -## Absolute stratification +## Absolute stratification (for absolute forms and entity date forms) -This example is covering the following -testcase: `src/test/resources/tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/ABS_EXPORT_FORM.test.json`. +### Absolute forms -For an absolute form, we only care for the primary ID, so we extract the primary IDs (and discard the validity date). -The `stratification_bounds` represent the absolute forms date range. They define the required complete stratification -window. For an entity date form, the `stratification_bounds` would be the intersection of an entity's validity date -and the forms date range. -We group by primary ID to keep only 1 entry per subject (a `select distinct` would do the trick too). +For an absolute form, we only care for the primary ID, so we extract the primary IDs and discard the validitiy date. For entity date queries, it's kept. The `stratification_bounds` represent the absolute forms date +range. They define the required complete stratification window. We group by primary ID (and validity date, if present) +to keep only distinct entries for each entity and discard any duplicated entries which, for example, might occur due +to a preceding secondary id query. **CTE:** `extract_ids` ```sql select "primary_id", + "validity_date", -- the validity date is only kept in case we convert an entity date query daterange('2012-06-01', '2012-09-30', '[]') as "stratification_bounds" from "external" -group by "primary_id" +group by "primary_id", "validity_date" +``` + +### Entity date + +For an entity date form, we only care for the primary ID and the validity date. We group by primary ID and validity +date to keep only distinct entries for each entity and discard any duplicated entries which, for example, might occur +due to a preceding secondary id query. + +**CTE:** `extract_ids` + +```sql +select "primary_id", + "validity_date" +from "external" +group by "primary_id", "validity_date" ``` +**CTE:** `overwrite_bounds` + +We create an additional CTE which intersects the entities validity dates with the given forms date range (if there is +one). The intersection defines the required complete stratification window. Besides this, there is no difference in the +following conversion process between absolute forms and entity date queries. + +```sql +select "primary_id", + -- the validity date is a multirange, so we unnest first + daterange('2012-06-01', '2012-09-30', '[]') * unnest("validity_date") as "stratification_bounds" +from "extract_ids" +``` + +### Stratification tables + Now, we want to create a resolution table for each resolution (`COMPLETE`, `YEAR`, `QUARTER`). **CTE:** `complete` @@ -46,7 +89,7 @@ select "primary_id", 'COMPLETE' "resolution", 1 "index", "stratification_bounds" -from "extract_ids" +from "extract_ids" -- or `overwrite_bounds` if it is an entity date query ``` A complete range shall have a `null` index, because it spans the complete range, but we set it to 1 to ensure we can @@ -150,9 +193,124 @@ in our case. The result looks like this: | 1 | YEARS | 2 | \[2013-04-01,2014-04-01\) | | 1 | YEARS | 3 | \[2014-04-01,2014-12-18\) | -**CTE:** `full_stratification` +## Relative stratification -Now, we union all the resolution tables. +Like for entity date queries, we need to extract the primary ID and the corresponding validity date for each distinct +entity. + +**CTE:** `extract_ids` + +```sql +select "primary_id", + unnest("validity_date") as "validity_date" -- unnesting is only required for dialects with multiranges +from "external" +group by "primary_id", "validity_date" +``` + +Next, we need to find the index selector date: For each validity date of an entity, we calculate either: + +- the `EARLIEST` date of the given range +- the `LATEST` date of the given range +- or a `RANDOM` date within the given range + depending on the relative forms + [index selector](../../../apiv1/query/concept/specific/temporal/TemporalSamplerFactory.java). + +**CTE:** `index_selector` + +```sql +select "primary_id", + min(lower("dates")) as "index_selector" -- example for EARLIEST +from "extract_ids" +group by "primary_id" +``` + +Using the index selector date, we can now define the index start dates from where the feature and/or outcome ranges of +the relative form start. Their exact calculation depends on the +[index placement](../../../apiv1/forms/IndexPlacement.java) of the relative form. + +For the `BEFORE` and `AFTER` placement, the positive start (outcome range) and negative start (feature range) is the +same. Only for the `NEUTRAL` placement, the start dates differ. + +We take the `BEFORE` placement with the time unit `QUARTERS` as an example: We jump to the next quarters start of the +index selector date. From these index start dates, we will start in the next step when calculating the stratification +windows of the feature and outcome range. + +**CTE:** `index_start` + +```sql +select "primary_id", + "index_selector", + (date_trunc('year', "index_selector") + + interval '3 months' * (extract(quarter from ("index_selector" + -1)) + 0))::date as "index_start_positive", + (date_trunc('year', "index_selector") + + interval '3 months' * (extract(quarter from ("index_selector" + -1)) + 0))::date as "index_start_negative" +from "index_selector" +``` + +The last step before calculating the actual stratification windows is to calculate the min and max date of the +stratification for each entity, which is basically the lower bound of the feature range and the upper bound of the +outcome range. Assuming a time count before of 6 quarters and a time count after of 2 quarters, the calculation looks +like the following: + +**CTE:** `total_bounds` + +```sql +select "primary_id", + daterange( + ("index_start_negative" + interval '3 months' * (-6 + 0))::date, + ("index_start_positive" + interval '3 months' * (2 + 0))::date, + '[)' + ) as "stratification_bounds", + "index_selector", + "index_start_positive", + "index_start_negative" +from "index_start" +``` + +We will intersect this range with the calculated ranges in the next step, because calculated ranges always span over +whole intervals (`YEARS`, `QUARTERS`), but must be ultimately bound by the complete min and max dates. In the following +example, we take a look at how the `YEARS` resolution is calculated. We still assume a time count before of 6 quarters +and a time count after of 2 quarters. This means we need to jump 2 years back for the feature range and 1 year forward +for the outcome range. Similar to absolute stratification, we create a set of date ranges by manipulating the start +dates by adding positive or negative time intervals times an index. The index is again taken from a generated integer +series. + +**CTE:** `years` + +```sql +-- feature range +select "primary_id", + 'YEARS' as "resolution", + "index", + "index_selector", + daterange( + ("index_start_negative" + interval '1 year' * ("index" + 0))::date, + ("index_start_negative" + interval '1 year' * ("index" + 1))::date, + '[)' + ) * "stratification_bounds" as "stratification_bounds" +from "total_bounds", + generate_series(-2, -1) as "index" -- -2 -> we jump 2 years back +union all +-- outcome range +select "primary_id", + 'YEARS' as "resolution", + "index", + "index_selector", + daterange( + ("index_start_positive" + interval '1 year' * ("index" + -1))::date, + ("index_start_positive" + interval '1 year' * ("index" + 0))::date, + '[)' + ) * "stratification_bounds" as "stratification_bounds" +from "total_bounds", + generate_series(1, 1) as "index" -- 1 -> we jump 1 year forward +``` + +Using the `stratification_bounds`, representing the min and max stratification date calculated in the previous step, +we intersect this range with our calculated time frame to generate stratification windows which are correctly bound. + +## Full stratification table + +Now, we union all the resolution tables to obtain the full stratification table. ```sql select "complete"."primary_id", @@ -246,3 +404,43 @@ For an absolute form, we expect the final result to contain all stratification r chosen resolutions. Because we filter all entries where stratification range and validity date do not overlap in each concept conversion's event filter step, the converted feature(s) table might not contain all stratification ranges. Thus, we left-join the table with the converted feature(s) back with the full stratification table. + +## Full export form + +When converting full export forms, we must ensure the columns in the final select query have the right order. Because +the `TableExportQuery` of an `FullExportForm` contains already the mapping from each required column to the position +in the final select, we just export each required table by selecting all the required columns. For each column, we +check if the current table contains the respective column. If so, we select the column, otherwise we just select null. +This way, we can easily union all export tables without the need to apply additional logic to obtain the correct column +order. + +```sql +with "external" as (select 1 as "primary_id"), + "extract_ids" as (select "primary_id" + from "external" + group by "primary_id") +(select "table"."pid", + "validity_date"::varchar as "validity_date", + 'table' as "source", -- name of the table we select from + "table"."second_id" as "second_id-2", + "table"."column" as "column-3", + null as "date_end-4", + null as "geburtsdatum-5", + null as "geschlecht-6" + from "table" + join "extract_ids" + on "table"."pid" = "extract_ids"."primary_id" + where "table"."column" in ('A')) + union all +(select "vers_stamm"."pid", + "vers_stamm"."validity_date"::varchar as "validity_date", + 'vers_stamm' as "source", + null as "second_id-2", + null as "column-3", + "vers_stamm"."date_end" as "date_end-4", + "vers_stamm"."geburtsdatum" as "geburtsdatum-5", + "vers_stamm"."geschlecht" as "geschlecht-6" + from "vers_stamm" + join "extract_ids" + on "vers_stamm"."pid" = "extract_ids"."primary_id") +``` diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java new file mode 100644 index 0000000000..baaa89b7f6 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/RelativeStratification.java @@ -0,0 +1,313 @@ +package com.bakdata.conquery.sql.conversion.forms; + +import static com.bakdata.conquery.sql.conversion.forms.FormConstants.INDEX_SELECTOR; +import static com.bakdata.conquery.sql.conversion.forms.FormConstants.INDEX_START_NEGATIVE; +import static com.bakdata.conquery.sql.conversion.forms.FormConstants.INDEX_START_POSITIVE; + +import java.sql.Date; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import com.bakdata.conquery.apiv1.forms.IndexPlacement; +import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; +import com.bakdata.conquery.apiv1.query.concept.specific.temporal.TemporalSamplerFactory; +import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.models.forms.managed.RelativeFormQuery; +import com.bakdata.conquery.models.forms.util.CalendarUnit; +import com.bakdata.conquery.models.forms.util.Resolution; +import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.google.common.base.Preconditions; +import lombok.RequiredArgsConstructor; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.impl.DSL; + +@RequiredArgsConstructor +class RelativeStratification { + + private final QueryStep baseStep; + private final StratificationFunctions stratificationFunctions; + private final SqlFunctionProvider functionProvider; + + public QueryStep createRelativeStratificationTable(RelativeFormQuery form) { + + // we want to create the stratification for each distinct validity date range of an entity, + // so we first need to unnest the validity date in case it is a multirange + QueryStep withUnnestedValidityDate = functionProvider.unnestValidityDate(baseStep, FormCteStep.UNNEST_DATES.getSuffix()); + + QueryStep indexSelectorStep = createIndexSelectorStep(form, withUnnestedValidityDate); + QueryStep indexStartStep = createIndexStartStep(form, indexSelectorStep); + QueryStep totalBoundsStep = createTotalBoundsStep(form, indexStartStep); + + List tables = form.getResolutionsAndAlignmentMap().stream() + .map(ExportForm.ResolutionAndAlignment::getResolution) + .map(resolution -> createResolutionTable(totalBoundsStep, resolution, form)) + .toList(); + + List predecessors = new ArrayList<>(); + predecessors.add(baseStep); + if (baseStep != withUnnestedValidityDate) { + predecessors.add(withUnnestedValidityDate); + } + predecessors.addAll(List.of(indexSelectorStep, indexStartStep, totalBoundsStep)); + + return StratificationTableFactory.unionResolutionTables(tables, predecessors); + } + + /** + * Creates {@link QueryStep} containing the date select for the corresponding {@link TemporalSamplerFactory} of the relative form. + */ + private QueryStep createIndexSelectorStep(RelativeFormQuery form, QueryStep prerequisite) { + + Selects predecessorSelects = prerequisite.getQualifiedSelects(); + ColumnDateRange validityDate = predecessorSelects.getValidityDate() + .orElseThrow(() -> new IllegalStateException("Expecting a validity date to be present")); + Field indexDate = stratificationFunctions.indexSelectorField(form.getIndexSelector(), validityDate) + .as(SharedAliases.INDEX_SELECTOR.getAlias()); + + Selects selects = Selects.builder() + .ids(predecessorSelects.getIds()) + .sqlSelect(new FieldWrapper<>(indexDate)) + .build(); + + return QueryStep.builder() + .cteName(FormCteStep.INDEX_SELECTOR.getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(prerequisite.getCteName())) + .groupBy(predecessorSelects.getIds().toFields()) + .build(); + } + + /** + * Creates {@link QueryStep} containing the start date selects ({@link FormConstants#INDEX_START_POSITIVE} and {@link FormConstants#INDEX_START_NEGATIVE}) + * from where the feature and/or outcome ranges of the relative form start. Their placement depends on the relative forms {@link IndexPlacement}. + */ + private QueryStep createIndexStartStep(RelativeFormQuery form, QueryStep indexSelectorStep) { + + List> indexStartFields = stratificationFunctions.indexStartFields(form.getIndexPlacement(), form.getTimeUnit()).stream() + .map(FieldWrapper::new) + .toList(); + + // add index start fields to qualified selects of previous step + Selects selects = indexSelectorStep.getQualifiedSelects() + .toBuilder() + .sqlSelects(indexStartFields) + .build(); + + return QueryStep.builder() + .cteName(FormCteStep.INDEX_START.getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(indexSelectorStep.getCteName())) + .build(); + } + + /** + * Creates a {@link QueryStep} containing the minimum and maximum stratification date for each entity. + */ + private QueryStep createTotalBoundsStep(RelativeFormQuery form, QueryStep indexStartStep) { + + Interval interval = getInterval(form.getTimeUnit(), Resolution.COMPLETE); + Range intRange = toGenerateSeriesBounds(form, Resolution.COMPLETE); + + Field minStratificationDate = stratificationFunctions.shiftByInterval(INDEX_START_NEGATIVE, interval, DSL.val(intRange.getMin()), Offset.NONE); + Field maxStratificationDate = stratificationFunctions.shiftByInterval(INDEX_START_POSITIVE, interval, DSL.val(intRange.getMax()), Offset.NONE); + ColumnDateRange minAndMaxStratificationDate = stratificationFunctions.ofStartAndEnd(minStratificationDate, maxStratificationDate) + .as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); + + // add min and max stratification date to qualified selects of previous step + Selects selects = indexStartStep.getQualifiedSelects() + .toBuilder() + .stratificationDate(Optional.of(minAndMaxStratificationDate)) + .build(); + + return QueryStep.builder() + .cteName(FormCteStep.TOTAL_BOUNDS.getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(indexStartStep.getCteName())) + .build(); + } + + private QueryStep createResolutionTable(QueryStep indexStartStep, Resolution resolution, RelativeFormQuery form) { + return switch (resolution) { + case COMPLETE -> createCompleteTable(indexStartStep, form); + case YEARS, QUARTERS, DAYS -> createIntervalTable(indexStartStep, resolution, form); + }; + } + + private QueryStep createCompleteTable(QueryStep totalBoundsStep, RelativeFormQuery form) { + + Selects predecessorSelects = totalBoundsStep.getQualifiedSelects(); + Interval interval = getInterval(form.getTimeUnit(), Resolution.COMPLETE); + Range intRange = toGenerateSeriesBounds(form, Resolution.COMPLETE); + + QueryStep featureTable = form.getTimeCountBefore() > 0 ? createCompleteFeatureTable(predecessorSelects, interval, intRange, totalBoundsStep) : null; + QueryStep outcomeTable = form.getTimeCountAfter() > 0 ? createCompleteOutcomeTable(predecessorSelects, interval, intRange, totalBoundsStep) : null; + + return QueryStep.createUnionStep( + Stream.concat(Stream.ofNullable(outcomeTable), Stream.ofNullable(featureTable)).toList(), + FormCteStep.COMPLETE.getSuffix(), + Collections.emptyList() + ); + } + + private QueryStep createCompleteFeatureTable(Selects predecessorSelects, Interval interval, Range intRange, QueryStep totalBoundsStep) { + Field featureIndex = DSL.field(DSL.val(-1)).as(SharedAliases.INDEX.getAlias()); + SqlIdColumns featureIds = predecessorSelects.getIds().withRelativeStratification(Resolution.COMPLETE, featureIndex, INDEX_SELECTOR); + Field rangeStart = stratificationFunctions.shiftByInterval(INDEX_START_NEGATIVE, interval, DSL.val(intRange.getMin()), Offset.NONE); + return createIntervalStep(featureIds, rangeStart, INDEX_START_NEGATIVE, Optional.empty(), totalBoundsStep); + } + + private QueryStep createCompleteOutcomeTable(Selects predecessorSelects, Interval interval, Range intRange, QueryStep totalBoundsStep) { + Field outcomeIndex = DSL.field(DSL.val(1)).as(SharedAliases.INDEX.getAlias()); + SqlIdColumns outcomeIds = predecessorSelects.getIds().withRelativeStratification(Resolution.COMPLETE, outcomeIndex, INDEX_SELECTOR); + Field rangeEnd = stratificationFunctions.shiftByInterval(INDEX_START_POSITIVE, interval, DSL.val(intRange.getMax()), Offset.NONE); + return createIntervalStep(outcomeIds, INDEX_START_POSITIVE, rangeEnd, Optional.empty(), totalBoundsStep); + } + + private QueryStep createIntervalTable(QueryStep totalBoundsStep, Resolution resolution, RelativeFormQuery form) { + + Field seriesIndex = stratificationFunctions.intSeriesField(); + Selects predecessorSelects = totalBoundsStep.getQualifiedSelects(); + SqlIdColumns ids = predecessorSelects.getIds().withRelativeStratification(resolution, seriesIndex, INDEX_SELECTOR); + Interval interval = getInterval(form.getTimeUnit(), resolution); + Range bounds = toGenerateSeriesBounds(form, resolution); + + QueryStep timeBeforeStep = createFeatureTable(totalBoundsStep, interval, seriesIndex, bounds, ids); + QueryStep timeAfterStep = createOutcomeTable(totalBoundsStep, interval, seriesIndex, bounds, ids); + + return QueryStep.createUnionStep( + List.of(timeBeforeStep, timeAfterStep), + FormCteStep.stratificationCte(resolution).getSuffix(), + Collections.emptyList() + ); + } + + private QueryStep createOutcomeTable(QueryStep totalBoundsStep, Interval interval, Field seriesIndex, Range bounds, SqlIdColumns ids) { + Field outcomeRangeStart = stratificationFunctions.shiftByInterval(INDEX_START_POSITIVE, interval, seriesIndex, Offset.MINUS_ONE); + Field outcomeRangeEnd = stratificationFunctions.shiftByInterval(INDEX_START_POSITIVE, interval, seriesIndex, Offset.NONE); + Table outcomeSeries = stratificationFunctions.generateIntSeries(1, bounds.getMax()).as(SharedAliases.INDEX.getAlias()); + return createIntervalStep(ids, outcomeRangeStart, outcomeRangeEnd, Optional.of(outcomeSeries), totalBoundsStep); + } + + private QueryStep createFeatureTable(QueryStep totalBoundsStep, Interval interval, Field seriesIndex, Range bounds, SqlIdColumns ids) { + Field featureRangeStart = stratificationFunctions.shiftByInterval(INDEX_START_NEGATIVE, interval, seriesIndex, Offset.NONE); + Field featureRangeEnd = stratificationFunctions.shiftByInterval(INDEX_START_NEGATIVE, interval, seriesIndex, Offset.ONE); + Table featureSeries = stratificationFunctions.generateIntSeries(bounds.getMin(), -1).as(SharedAliases.INDEX.getAlias()); + return createIntervalStep(ids, featureRangeStart, featureRangeEnd, Optional.of(featureSeries), totalBoundsStep); + } + + private QueryStep createIntervalStep( + SqlIdColumns ids, + Field rangeStart, + Field rangeEnd, + Optional> seriesTable, + QueryStep predecessor + ) { + Preconditions.checkArgument( + predecessor.getSelects().getStratificationDate().isPresent(), + "Expecting %s to contain a stratification date representing the min and max stratification bounds" + ); + ColumnDateRange finalRange = functionProvider.intersection( + stratificationFunctions.ofStartAndEnd(rangeStart, rangeEnd), + predecessor.getQualifiedSelects().getStratificationDate().get() + ) + .as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); + + Selects selects = Selects.builder() + .ids(ids) + .stratificationDate(Optional.of(finalRange)) + .build(); + + QueryStep.QueryStepBuilder queryStep = QueryStep.builder() + .selects(selects) + .fromTable(QueryStep.toTableLike(predecessor.getCteName())); + + seriesTable.ifPresent(queryStep::fromTable); + return queryStep.build(); + } + + /** + * Adjusts the {@link RelativeFormQuery#getTimeCountBefore()} && {@link RelativeFormQuery#getTimeCountAfter()} bounds, so they fit the SQL approach. + * Take time unit QUARTERS and Resolution YEARS as an example: If the time counts are not divisible by 4 (because 1 year == 4 quarters), we need to round + * up for each starting year. 5 Quarters mean 2 years we have to consider when creating the stratification. + */ + private static Range toGenerateSeriesBounds(RelativeFormQuery relativeForm, Resolution resolution) { + + int timeCountBefore; + int timeCountAfter; + + switch (relativeForm.getTimeUnit()) { + case QUARTERS -> { + if (resolution == Resolution.YEARS) { + timeCountBefore = divideAndRoundUp(relativeForm.getTimeCountBefore(), 4); + timeCountAfter = divideAndRoundUp(relativeForm.getTimeCountAfter(), 4); + } + else { + timeCountBefore = relativeForm.getTimeCountBefore(); + timeCountAfter = relativeForm.getTimeCountAfter(); + } + } + case DAYS -> { + switch (resolution) { + case COMPLETE, DAYS -> { + timeCountBefore = relativeForm.getTimeCountBefore(); + timeCountAfter = relativeForm.getTimeCountAfter(); + } + case YEARS -> { + timeCountBefore = divideAndRoundUp(relativeForm.getTimeCountBefore(), Interval.YEAR_AS_DAYS_INTERVAL.getAmount()); + timeCountAfter = divideAndRoundUp(relativeForm.getTimeCountAfter(), Interval.YEAR_AS_DAYS_INTERVAL.getAmount()); + } + case QUARTERS -> { + timeCountBefore = divideAndRoundUp(relativeForm.getTimeCountBefore(), Interval.NINETY_DAYS_INTERVAL.getAmount()); + timeCountAfter = divideAndRoundUp(relativeForm.getTimeCountAfter(), Interval.NINETY_DAYS_INTERVAL.getAmount()); + } + default -> throw new CombinationNotSupportedException(relativeForm.getTimeUnit(), resolution); + } + } + default -> throw new CombinationNotSupportedException(relativeForm.getTimeUnit(), resolution); + } + + return Range.of( + - timeCountBefore, + timeCountAfter + ); + } + + private static int divideAndRoundUp(int numerator, int denominator) { + if (denominator == 0) { + throw new IllegalArgumentException("Denominator cannot be zero."); + } + return (int) Math.ceil((double) numerator / denominator); + } + + /** + * @return The interval expression which will be multiplied by the {@link StratificationFunctions#intSeriesField()} and added to the + * {@link SharedAliases#INDEX_START_NEGATIVE} or {@link SharedAliases#INDEX_START_POSITIVE}. + */ + private static Interval getInterval(CalendarUnit timeUnit, Resolution resolution) { + return switch (timeUnit) { + case QUARTERS -> switch (resolution) { + case COMPLETE, QUARTERS -> Interval.QUARTER_INTERVAL; + case YEARS -> Interval.ONE_YEAR_INTERVAL; + case DAYS -> Interval.ONE_DAY_INTERVAL; + }; + case DAYS -> switch (resolution) { + case COMPLETE, DAYS -> Interval.ONE_DAY_INTERVAL; + case YEARS -> Interval.YEAR_AS_DAYS_INTERVAL; + case QUARTERS -> Interval.NINETY_DAYS_INTERVAL; + }; + default -> throw new CombinationNotSupportedException(timeUnit, resolution); + }; + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java index e17c9421f2..a997cf7dc9 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationFunctions.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.sql.conversion.forms; import static com.bakdata.conquery.sql.conversion.forms.FormConstants.DAY_ALIGNED_COUNT; +import static com.bakdata.conquery.sql.conversion.forms.FormConstants.INDEX_SELECTOR; import static com.bakdata.conquery.sql.conversion.forms.FormConstants.INDEX_START; import static com.bakdata.conquery.sql.conversion.forms.FormConstants.QUARTER_ALIGNED_COUNT; import static com.bakdata.conquery.sql.conversion.forms.FormConstants.QUARTER_END; @@ -20,7 +21,11 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import com.bakdata.conquery.ConqueryConstants; +import com.bakdata.conquery.apiv1.forms.IndexPlacement; import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; +import com.bakdata.conquery.apiv1.query.concept.specific.temporal.TemporalSamplerFactory; +import com.bakdata.conquery.models.forms.util.CalendarUnit; import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.HanaSqlFunctionProvider; @@ -28,7 +33,6 @@ import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import org.jetbrains.annotations.NotNull; import org.jooq.Condition; import org.jooq.Field; import org.jooq.Record; @@ -45,6 +49,10 @@ static StratificationFunctions create(ConversionContext context) { }; } + public ColumnDateRange ofStartAndEnd(Field start, Field end) { + return ColumnDateRange.of(start, end); // needs to be overwritten for dialects that support single-column ranges + } + protected abstract SqlFunctionProvider getFunctionProvider(); /** @@ -53,9 +61,14 @@ static StratificationFunctions create(ConversionContext context) { protected abstract Field lower(ColumnDateRange dateRange); /** - * Extract the upper bound from a given daterange. + * Extract the inclusive upper bound from a given daterange. + */ + protected abstract Field inclusiveUpper(ColumnDateRange dateRange); + + /** + * Extract the exclusive upper bound from a given daterange. */ - protected abstract Field upper(ColumnDateRange dateRange); + protected abstract Field exclusiveUpper(ColumnDateRange dateRange); /** * Calculates the start and end date based on the given start date and an interval expression. @@ -70,28 +83,38 @@ static StratificationFunctions create(ConversionContext context) { /** * Determines the start of the year based on the lower bound of the provided date range. */ - public abstract Field yearStart(ColumnDateRange dateRange); + public abstract Field lowerBoundYearStart(ColumnDateRange dateRange); /** - * Determines the start of the next year based on the upper bound of the provided date range. + * Determines the exclusive end (first day of the next year) of the upper bound of the provided date range. */ - public abstract Field nextYearStart(ColumnDateRange dateRange); + public abstract Field upperBoundYearEnd(ColumnDateRange dateRange); /** - * Determines the start of the next year based on the upper bound of the provided date range, but aligned on the quarter of the lower bound of the + * Determines the end of the upper bound of the provided date range, but aligned on the quarter of the lower bound of the * provided daterange. */ - public abstract Field yearEndQuarterAligned(ColumnDateRange dateRange); + public abstract Field upperBoundYearEndQuarterAligned(ColumnDateRange dateRange); /** * Calculates the start of the quarter using the lower bound of the provided date range. */ - public abstract Field quarterStart(ColumnDateRange dateRange); + public abstract Field lowerBoundQuarterStart(ColumnDateRange dateRange); + + /** + * Calculates the start of the quarter of the given date. + */ + protected abstract Field jumpToQuarterStart(Field date); + + /** + * Calculates the exclusive end (first day of the next quarter) of the upper bound of the provided date range. + */ + public abstract Field upperBoundQuarterEnd(ColumnDateRange dateRange); /** - * Calculates the start of next quarter of the upper bound of the provided date range. + * Calculates the start of the next quarter of the given date. */ - public abstract Field nextQuartersStart(ColumnDateRange dateRange); + protected abstract Field jumpToNextQuarterStart(Field date); /** * The int field generated by the {@link #generateIntSeries(int, int)} @@ -103,6 +126,66 @@ static StratificationFunctions create(ConversionContext context) { */ public abstract Table generateIntSeries(int start, int end); + /** + * Generates a date field representing the {@link TemporalSamplerFactory} using the given validity date range. + */ + public abstract Field indexSelectorField(TemporalSamplerFactory indexSelector, ColumnDateRange validityDate); + + /** + * Shift's a start date by an interval times an amount. The offset will we added to the amount before multiplying. + */ + public abstract Field shiftByInterval(Field startDate, Interval interval, Field amount, Offset offset); + + /** + * Generates the start and end field for the respective {@link IndexPlacement} and {@link CalendarUnit timeUnit}. + */ + public List> indexStartFields(IndexPlacement indexPlacement, CalendarUnit timeUnit) { + + Field positiveStart; + Field negativeStart; + + switch (timeUnit) { + case QUARTERS -> { + switch (indexPlacement) { + case BEFORE -> { + Field nextQuarterStart = jumpToNextQuarterStart(INDEX_SELECTOR); + positiveStart = nextQuarterStart; + negativeStart = nextQuarterStart; + } + case AFTER -> { + Field quarterStart = jumpToQuarterStart(INDEX_SELECTOR); + positiveStart = quarterStart; + negativeStart = quarterStart; + } + case NEUTRAL -> { + positiveStart = jumpToNextQuarterStart(INDEX_SELECTOR); + negativeStart = jumpToQuarterStart(INDEX_SELECTOR); + } + default -> throw new CombinationNotSupportedException(indexPlacement, timeUnit); + } + } + case DAYS -> { + switch (indexPlacement) { + case BEFORE, AFTER -> { + positiveStart = INDEX_SELECTOR; + negativeStart = INDEX_SELECTOR; + } + case NEUTRAL -> { + positiveStart = getFunctionProvider().addDays(INDEX_SELECTOR, DSL.val(1)); + negativeStart = INDEX_SELECTOR; + } + default -> throw new CombinationNotSupportedException(indexPlacement, timeUnit); + } + } + default -> throw new CombinationNotSupportedException(indexPlacement, timeUnit); + } + + return List.of( + positiveStart.as(SharedAliases.INDEX_START_POSITIVE.getAlias()), + negativeStart.as(SharedAliases.INDEX_START_NEGATIVE.getAlias()) + ); + } + /** * Calculates the count of the required resolution windows based on the provided resolution, alignment, and date range. * @@ -112,10 +195,10 @@ static StratificationFunctions create(ConversionContext context) { public Field calculateResolutionWindowCount(ExportForm.ResolutionAndAlignment resolutionAndAlignment, ColumnDateRange bounds) { SqlFunctionProvider functionProvider = getFunctionProvider(); return switch (resolutionAndAlignment.getResolution()) { - case COMPLETE -> DSL.field(DSL.val(1)); + case COMPLETE -> DSL.val(1); case YEARS -> calculateResolutionWindowForYearResolution(resolutionAndAlignment, bounds, functionProvider); case QUARTERS -> calculateResolutionWindowForQuarterResolution(resolutionAndAlignment, bounds, functionProvider); - case DAYS -> functionProvider.dateDistance(ChronoUnit.DAYS, lower(bounds), upper(bounds)) + case DAYS -> functionProvider.dateDistance(ChronoUnit.DAYS, lower(bounds), exclusiveUpper(bounds)) .as(SharedAliases.DAY_ALIGNED_COUNT.getAlias()); }; } @@ -138,14 +221,14 @@ public ColumnDateRange createStratificationRange(ExportForm.ResolutionAndAlignme } /** - * The index field for the corresponding resolution index {@link com.bakdata.conquery.ConqueryConstants#CONTEXT_INDEX_INFO}. + * The index field for the corresponding resolution index {@link ConqueryConstants#CONTEXT_INDEX_INFO}. */ - public Field index(SqlIdColumns ids, Optional validityDate) { + public Field index(SqlIdColumns ids, Optional stratificationBounds) { List> partitioningFields = Stream.concat( ids.toFields().stream(), - validityDate.stream().flatMap(columnDateRange -> columnDateRange.toFields().stream()) + stratificationBounds.stream().flatMap(columnDateRange -> columnDateRange.toFields().stream()) ) .collect(Collectors.toList()); @@ -177,7 +260,7 @@ private Field calculateResolutionWindowForQuarterResolution( case QUARTER -> functionProvider.dateDistance(ChronoUnit.MONTHS, QUARTER_START, QUARTER_END) .divide(MONTHS_PER_QUARTER) .as(SharedAliases.QUARTER_ALIGNED_COUNT.getAlias()); - case DAY -> functionProvider.dateDistance(ChronoUnit.DAYS, lower(bounds), upper(bounds)) + case DAY -> functionProvider.dateDistance(ChronoUnit.DAYS, lower(bounds), exclusiveUpper(bounds)) .plus(89) .divide(DAYS_PER_QUARTER) .as(SharedAliases.DAY_ALIGNED_COUNT.getAlias()); @@ -195,7 +278,7 @@ private Field calculateResolutionWindowForYearResolution( .as(SharedAliases.YEAR_ALIGNED_COUNT.getAlias()); case QUARTER -> functionProvider.dateDistance(ChronoUnit.YEARS, QUARTER_START, YEAR_END_QUARTER_ALIGNED) .as(SharedAliases.QUARTER_ALIGNED_COUNT.getAlias()); - case DAY -> functionProvider.dateDistance(ChronoUnit.DAYS, lower(bounds), upper(bounds)) + case DAY -> functionProvider.dateDistance(ChronoUnit.DAYS, lower(bounds), exclusiveUpper(bounds)) .plus(364) .divide(DAYS_PER_YEAR) .as(SharedAliases.DAY_ALIGNED_COUNT.getAlias()); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java index 5bbcf0c3b7..c02cdb9f45 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/forms/StratificationTableFactory.java @@ -1,31 +1,19 @@ package com.bakdata.conquery.sql.conversion.forms; -import java.sql.Date; import java.util.List; -import java.util.Optional; import java.util.stream.Stream; import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; -import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery; -import com.bakdata.conquery.models.forms.util.Resolution; -import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.models.forms.managed.RelativeFormQuery; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.Selects; -import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.google.common.base.Preconditions; +import lombok.AccessLevel; import lombok.Getter; import lombok.RequiredArgsConstructor; -import org.jooq.Condition; -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Table; -import org.jooq.impl.DSL; -import com.google.common.base.Preconditions; -@Getter +@Getter(AccessLevel.PROTECTED) @RequiredArgsConstructor public class StratificationTableFactory { @@ -42,162 +30,17 @@ public StratificationTableFactory(QueryStep baseStep, ConversionContext context) this.functionProvider = context.getSqlDialect().getFunctionProvider(); } - public QueryStep createStratificationTable(AbsoluteFormQuery form) { - - QueryStep intSeriesStep = createIntSeriesStep(); - QueryStep indexStartStep = createIndexStartStep(); - - List tables = form.getResolutionsAndAlignmentMap().stream() - .map(resolutionAndAlignment -> createResolutionTable(indexStartStep, resolutionAndAlignment)) - .toList(); - - List predecessors = List.of(getBaseStep(), intSeriesStep, indexStartStep); - return unionResolutionTables(tables, predecessors); - } - - private QueryStep createIntSeriesStep() { - - // not actually required, but Selects expect at least 1 SqlIdColumn - Field rowNumber = DSL.rowNumber().over().coerce(Object.class); - SqlIdColumns ids = new SqlIdColumns(rowNumber); - - FieldWrapper seriesIndex = new FieldWrapper<>(stratificationFunctions.intSeriesField()); - - Selects selects = Selects.builder() - .ids(ids) - .sqlSelect(seriesIndex) - .build(); - - Table seriesTable = stratificationFunctions.generateIntSeries(INDEX_START, INDEX_END) - .as(SharedAliases.SERIES_INDEX.getAlias()); - - return QueryStep.builder() - .cteName(FormCteStep.INT_SERIES.getSuffix()) - .selects(selects) - .fromTable(seriesTable) - .build(); - } - - private QueryStep createIndexStartStep() { - - Selects baseStepSelects = getBaseStep().getQualifiedSelects(); - Preconditions.checkArgument(baseStepSelects.getStratificationDate().isPresent()); - ColumnDateRange bounds = baseStepSelects.getStratificationDate().get(); - - Field indexStart = stratificationFunctions.absoluteIndexStartDate(bounds).as(SharedAliases.INDEX_START.getAlias()); - Field yearStart = stratificationFunctions.yearStart(bounds).as(SharedAliases.YEAR_START.getAlias()); - Field yearEnd = stratificationFunctions.nextYearStart(bounds).as(SharedAliases.YEAR_END.getAlias()); - Field yearEndQuarterAligned = stratificationFunctions.yearEndQuarterAligned(bounds).as(SharedAliases.YEAR_END_QUARTER_ALIGNED.getAlias()); - Field quarterStart = stratificationFunctions.quarterStart(bounds).as(SharedAliases.QUARTER_START.getAlias()); - Field quarterEnd = stratificationFunctions.nextQuartersStart(bounds).as(SharedAliases.QUARTER_END.getAlias()); - - List> startDates = Stream.of( - indexStart, - yearStart, - yearEnd, - yearEndQuarterAligned, - quarterStart, - quarterEnd - ) - .map(FieldWrapper::new) - .toList(); - - Selects selects = Selects.builder() - .ids(baseStepSelects.getIds()) - .stratificationDate(Optional.of(bounds)) - .sqlSelects(startDates) - .build(); - - return QueryStep.builder() - .cteName(FormCteStep.INDEX_START.getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(getBaseStep().getCteName())) - .build(); - } - - private QueryStep createResolutionTable(QueryStep indexStartStep, ExportForm.ResolutionAndAlignment resolutionAndAlignment) { - return switch (resolutionAndAlignment.getResolution()) { - case COMPLETE -> createCompleteTable(); - case YEARS, QUARTERS, DAYS -> createIntervalTable(indexStartStep, resolutionAndAlignment); - }; + public QueryStep createRelativeStratificationTable(RelativeFormQuery form) { + RelativeStratification relativeStratification = new RelativeStratification(baseStep, stratificationFunctions, functionProvider); + return relativeStratification.createRelativeStratificationTable(form); } - private QueryStep createCompleteTable() { - - Selects baseStepSelects = baseStep.getQualifiedSelects(); - - // complete range shall have a null index because it spans the complete range, but we set it to 1 to ensure we can join tables on index, - // because a condition involving null in a join (e.g., null = some_value or null = null) always evaluates to false - Field index = DSL.field(DSL.val(1, Integer.class)).as(SharedAliases.INDEX.getAlias()); - SqlIdColumns ids = baseStepSelects.getIds().withAbsoluteStratification(Resolution.COMPLETE, index); - - ColumnDateRange completeRange = baseStepSelects.getStratificationDate().get(); - - Selects selects = Selects.builder() - .ids(ids) - .stratificationDate(Optional.of(completeRange)) - .build(); - - return QueryStep.builder() - .cteName(FormCteStep.COMPLETE.getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(baseStep.getCteName())) - .build(); - } - - private QueryStep createIntervalTable(QueryStep indexStartStep, ExportForm.ResolutionAndAlignment resolutionAndAlignment) { - - QueryStep countsCte = createCountsCte(indexStartStep, resolutionAndAlignment); - Preconditions.checkArgument(countsCte.getSelects().getStratificationDate().isPresent()); - Selects countsCteSelects = countsCte.getQualifiedSelects(); - - ColumnDateRange stratificationRange = stratificationFunctions.createStratificationRange( - resolutionAndAlignment, - countsCteSelects.getStratificationDate().get() - ); - - Field index = stratificationFunctions.index(countsCteSelects.getIds(), Optional.empty()); - SqlIdColumns ids = countsCteSelects.getIds().withAbsoluteStratification(resolutionAndAlignment.getResolution(), index); - - Selects selects = Selects.builder() - .ids(ids) - .stratificationDate(Optional.ofNullable(stratificationRange)) - .build(); - - Condition stopOnMaxResolutionWindowCount = stratificationFunctions.stopOnMaxResolutionWindowCount(resolutionAndAlignment); - - return QueryStep.builder() - .cteName(FormCteStep.stratificationCte(resolutionAndAlignment.getResolution()).getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(countsCte.getCteName())) - .fromTable(QueryStep.toTableLike(FormCteStep.INT_SERIES.getSuffix())) - .conditions(List.of(stopOnMaxResolutionWindowCount)) - .predecessor(countsCte) - .build(); - } - - private QueryStep createCountsCte(QueryStep indexStartStep, ExportForm.ResolutionAndAlignment resolutionAndAlignment) { - - Selects indexStartSelects = indexStartStep.getQualifiedSelects(); - Preconditions.checkArgument(indexStartSelects.getStratificationDate().isPresent()); - - Field resolutionWindowCount = stratificationFunctions.calculateResolutionWindowCount( - resolutionAndAlignment, - indexStartSelects.getStratificationDate().get() - ); - - Selects selects = indexStartSelects.toBuilder() - .sqlSelect(new FieldWrapper<>(resolutionWindowCount)) - .build(); - - return QueryStep.builder() - .cteName(FormCteStep.countsCte(resolutionAndAlignment.getResolution()).getSuffix()) - .selects(selects) - .fromTable(QueryStep.toTableLike(indexStartStep.getCteName())) - .build(); + public QueryStep createAbsoluteStratificationTable(List resolutionAndAlignments) { + AbsoluteStratification absoluteStratification = new AbsoluteStratification(baseStep, stratificationFunctions); + return absoluteStratification.createStratificationTable(resolutionAndAlignments); } - private QueryStep unionResolutionTables(List unionSteps, List predecessors) { + protected static QueryStep unionResolutionTables(List unionSteps, List predecessors) { Preconditions.checkArgument(!unionSteps.isEmpty(), "Expecting at least 1 resolution table"); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java index 902e8be75f..dea4df7760 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/ColumnDateRange.java @@ -9,6 +9,7 @@ import lombok.Getter; import org.jooq.Condition; import org.jooq.Field; +import org.jooq.impl.DSL; @Getter public class ColumnDateRange implements SqlSelect { @@ -52,6 +53,11 @@ public static ColumnDateRange of(Field startColumn, Field endColumn, return new ColumnDateRange(startColumn, endColumn, alias); } + public static ColumnDateRange empty() { + Field emptyRange = DSL.field(DSL.val("{}")); + return ColumnDateRange.of(emptyRange); + } + public ColumnDateRange asValidityDateRange(String alias) { return this.as(alias + VALIDITY_DATE_COLUMN_NAME_SUFFIX); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java index bc7371bf22..7058ce04e7 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStepTransformer.java @@ -9,6 +9,8 @@ import org.jooq.Record; import org.jooq.Select; import org.jooq.SelectConditionStep; +import org.jooq.SelectHavingStep; +import org.jooq.SelectSeekStepN; import org.jooq.impl.DSL; /** @@ -32,13 +34,21 @@ public Select toSelectQuery(QueryStep queryStep) { .from(queryStep.getFromTables()) .where(queryStep.getConditions()); - List> orderByFields = queryStep.getSelects().getIds().toFields(); + // grouping + SelectHavingStep grouped = queryBase; if (queryStep.isGroupBy()) { - return queryBase.groupBy(queryStep.getGroupBy()).orderBy(orderByFields); + grouped = queryBase.groupBy(queryStep.getGroupBy()); } - else { - return queryBase.orderBy(orderByFields); + + // ordering + List> orderByFields = queryStep.getSelects().getIds().toFields(); + SelectSeekStepN ordered = grouped.orderBy(orderByFields); + + // union + if (!queryStep.isUnion()) { + return ordered; } + return unionAll(queryStep, ordered); } private List> constructPredecessorCteList(QueryStep queryStep) { @@ -60,6 +70,11 @@ private Stream> predecessorCtes(QueryStep querySte } private CommonTableExpression toCte(QueryStep queryStep) { + Select selectStep = toSelectStep(queryStep); + return DSL.name(queryStep.getCteName()).as(selectStep); + } + + private Select toSelectStep(QueryStep queryStep) { Select selectStep = this.dslContext .select(queryStep.getSelects().all()) @@ -71,18 +86,17 @@ private CommonTableExpression toCte(QueryStep queryStep) { } if (queryStep.isUnion()) { - for (QueryStep unionStep : queryStep.getUnion()) { - // we only use the union as part of the date aggregation process - the entries of the UNION tables are all unique - // thus we can use a UNION ALL because it's way faster than UNION - selectStep = selectStep.unionAll( - this.dslContext.select(unionStep.getSelects().all()) - .from(unionStep.getFromTables()) - .where(unionStep.getConditions()) - ); - } + selectStep = unionAll(queryStep, selectStep); } - return DSL.name(queryStep.getCteName()).as(selectStep); + return selectStep; + } + + private Select unionAll(QueryStep queryStep, Select base) { + for (QueryStep unionStep : queryStep.getUnion()) { + base = base.unionAll(toSelectStep(unionStep)); + } + return base; } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java index f92ea088a7..20f4a84f8b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/SqlIdColumns.java @@ -44,7 +44,7 @@ public SqlIdColumns qualify(String qualifier) { } public SqlIdColumns withAbsoluteStratification(Resolution resolution, Field index) { - Field resolutionField = DSL.field(DSL.val(resolution.toString())).as(SharedAliases.RESOLUTION.getAlias()); + Field resolutionField = DSL.val(resolution.toString()).as(SharedAliases.RESOLUTION.getAlias()); return StratificationSqlIdColumns.builder() .primaryColumn(this.primaryColumn) .secondaryId(this.secondaryId) @@ -55,7 +55,7 @@ public SqlIdColumns withAbsoluteStratification(Resolution resolution, Field index, Field eventDate) { - Field resolutionField = DSL.field(DSL.val(resolution.toString())).as(SharedAliases.RESOLUTION.getAlias()); + Field resolutionField = DSL.val(resolution.toString()).as(SharedAliases.RESOLUTION.getAlias()); return StratificationSqlIdColumns.builder() .primaryColumn(this.primaryColumn) .secondaryId(this.secondaryId) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java index 0781187db8..ba392784c8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/StratificationSqlIdColumns.java @@ -60,7 +60,7 @@ public SqlIdColumns forFinalSelect() { Field withNulledCompleteIndex = DSL.when( this.resolution.eq(DSL.val(Resolution.COMPLETE.toString().toUpperCase())), - DSL.field(DSL.val(null, Integer.class)) + DSL.val(null, Integer.class) ) .otherwise(this.index) .as(SharedAliases.INDEX.getAlias()); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java index 48b13bda9e..1964e59afb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/FlagSqlAggregator.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.sql.conversion.model.aggregator; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.stream.Collectors; @@ -96,11 +95,10 @@ public static FlagSqlAggregator create(FlagFilter flagFilter, FilterContext> rootSelects = - getRequiredColumnNames(flagFilter.getFlags(), filterContext.getValue()) - .stream() - .map(columnName -> new ExtractingSqlSelect<>(rootTable, columnName, Boolean.class)) - .collect(Collectors.toList()); + List> rootSelects = FlagCondition.getRequiredColumns(flagFilter.getFlags(), filterContext.getValue()).stream() + .map(Column::getName) + .map(columnName -> new ExtractingSqlSelect<>(rootTable, columnName, Boolean.class)) + .collect(Collectors.toList()); SqlSelects selects = SqlSelects.builder() .preprocessingSelects(rootSelects) .build(); @@ -165,14 +163,4 @@ private static Map> createRootSelectReferences( )); } - /** - * @return Columns names of a given flags map that match the selected flags of the filter value. - */ - private static List getRequiredColumnNames(Map flags, String[] selectedFlags) { - return Arrays.stream(selectedFlags) - .map(flags::get) - .map(Column::getName) - .toList(); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/CountCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/CountCondition.java index 728aafee2b..1eea111b58 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/CountCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/CountCondition.java @@ -1,7 +1,9 @@ package com.bakdata.conquery.sql.conversion.model.filter; import com.bakdata.conquery.models.common.IRange; +import com.bakdata.conquery.models.datasets.Column; import org.jooq.Field; +import org.jooq.impl.DSL; public class CountCondition extends RangeCondition { @@ -9,6 +11,13 @@ public CountCondition(Field column, IRange range) { super(column, range); } + public static CountCondition onColumn(Column column, IRange range) { + String tableName = column.getTable().getName(); + String columnName = column.getName(); + Field field = DSL.field(DSL.name(tableName, columnName), Number.class); + return new CountCondition(field, range); + } + @Override public ConditionType type() { return ConditionType.GROUP; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/DateDistanceCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/DateDistanceCondition.java index 5344330c16..6b3d4d096d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/DateDistanceCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/DateDistanceCondition.java @@ -1,7 +1,16 @@ package com.bakdata.conquery.sql.conversion.model.filter; +import java.sql.Date; +import java.time.LocalDate; +import java.time.temporal.ChronoUnit; + import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; +import com.bakdata.conquery.sql.conversion.cqelement.concept.FilterContext; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import org.jooq.Field; +import org.jooq.impl.DSL; public class DateDistanceCondition extends RangeCondition { @@ -9,6 +18,21 @@ public DateDistanceCondition(Field column, Range.LongRange range) { super(column, range); } + public static DateDistanceCondition onColumn(Column column, ChronoUnit timeUnit, FilterContext filterContext) { + + String tableName = column.getTable().getName(); + String columnName = column.getName(); + Field startDateField = DSL.field(DSL.name(tableName, columnName), Date.class); + + ConversionContext conversionContext = filterContext.getConversionContext(); + SqlFunctionProvider functionProvider = filterContext.getSqlDialect().getFunctionProvider(); + LocalDate endDate = conversionContext.getSqlDialect().getDateNowSupplier().getLocalDateNow(); + Field endDateField = functionProvider.toDateField(Date.valueOf(endDate).toString()); + + Field dateDistance = functionProvider.dateDistance(timeUnit, startDateField, endDateField); + return new DateDistanceCondition(dateDistance, filterContext.getValue()); + } + @Override public ConditionType type() { return ConditionType.EVENT; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/FlagCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/FlagCondition.java index 01ebb4aeba..f5b3e68dd3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/FlagCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/FlagCondition.java @@ -1,7 +1,10 @@ package com.bakdata.conquery.sql.conversion.model.filter; +import java.util.Arrays; import java.util.List; +import java.util.Map; +import com.bakdata.conquery.models.datasets.Column; import lombok.RequiredArgsConstructor; import org.jooq.Condition; import org.jooq.Field; @@ -12,6 +15,23 @@ public class FlagCondition implements WhereCondition { private final List> flagFields; + public static FlagCondition onColumn(Map flags, String[] selectedFlags) { + List> flagFields = getRequiredColumns(flags, selectedFlags) + .stream() + .map(column -> DSL.field(DSL.name(column.getTable().getName(), column.getName()), Boolean.class)) + .toList(); + return new FlagCondition(flagFields); + } + + /** + * @return Columns names of a given flags map that match the selected flags of the filter value. + */ + public static List getRequiredColumns(Map flags, String[] selectedFlags) { + return Arrays.stream(selectedFlags) + .map(flags::get) + .toList(); + } + @Override public Condition condition() { return flagFields.stream() diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/MultiSelectCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/MultiSelectCondition.java index 576b068d8e..e37449f024 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/MultiSelectCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/MultiSelectCondition.java @@ -2,6 +2,7 @@ import java.util.Arrays; +import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.google.common.base.Strings; import lombok.RequiredArgsConstructor; @@ -16,6 +17,13 @@ public class MultiSelectCondition implements WhereCondition { private final String[] values; private final SqlFunctionProvider functionProvider; + public static MultiSelectCondition onColumn(Column column, String[] values, SqlFunctionProvider functionProvider) { + String tableName = column.getTable().getName(); + String columnName = column.getName(); + Field field = DSL.field(DSL.name(tableName, columnName), String.class); + return new MultiSelectCondition(field, values, functionProvider); + } + @Override public WhereCondition negate() { // we want all entries that don't satisfy a condition - because in SQL a comparison with NULL equals UNKNOWN and not FALSE, diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberCondition.java index 169ead8af7..dddd91a079 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/NumberCondition.java @@ -1,7 +1,9 @@ package com.bakdata.conquery.sql.conversion.model.filter; import com.bakdata.conquery.models.common.IRange; +import com.bakdata.conquery.models.datasets.Column; import org.jooq.Field; +import org.jooq.impl.DSL; public class NumberCondition extends RangeCondition { @@ -9,6 +11,13 @@ public NumberCondition(Field column, IRange range) { + String tableName = column.getTable().getName(); + String columnName = column.getName(); + Field field = DSL.field(DSL.name(tableName, columnName), Number.class); + return new NumberCondition(field, range); + } + @Override public ConditionType type() { return ConditionType.EVENT; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/SumCondition.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/SumCondition.java index 1159b87343..ee19b8ee40 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/SumCondition.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/filter/SumCondition.java @@ -1,7 +1,11 @@ package com.bakdata.conquery.sql.conversion.model.filter; +import javax.annotation.Nullable; + import com.bakdata.conquery.models.common.IRange; +import com.bakdata.conquery.models.datasets.Column; import org.jooq.Field; +import org.jooq.impl.DSL; public class SumCondition extends RangeCondition { @@ -9,6 +13,22 @@ public SumCondition(Field column, IRange super(column, range); } + public static SumCondition onColumn(Column column, @Nullable Column subtractColumn, IRange range) { + + String tableName = column.getTable().getName(); + String columnName = column.getName(); + Field field = DSL.field(DSL.name(tableName, columnName), Number.class); + + if (subtractColumn == null) { + return new SumCondition(field, range); + } + + String subtractColumnName = subtractColumn.getName(); + String subtractTableName = subtractColumn.getTable().getName(); + Field subtractField = DSL.field(DSL.name(subtractTableName, subtractColumnName), Number.class); + return new SumCondition(field.minus(subtractField), range); + } + @Override public ConditionType type() { return ConditionType.GROUP; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java index 0612f6a995..2b0e2ecdd1 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java @@ -1,40 +1,28 @@ package com.bakdata.conquery.sql.conversion.query; -import java.time.LocalDate; import java.util.List; import java.util.Optional; -import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.Query; -import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.forms.managed.AbsoluteFormQuery; -import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.SharedAliases; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.forms.FormCteStep; +import com.bakdata.conquery.sql.conversion.forms.FormType; import com.bakdata.conquery.sql.conversion.forms.StratificationTableFactory; import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; -import com.bakdata.conquery.sql.conversion.model.ConqueryJoinType; import com.bakdata.conquery.sql.conversion.model.QueryStep; -import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; -import com.bakdata.conquery.sql.conversion.model.QueryStepTransformer; import com.bakdata.conquery.sql.conversion.model.Selects; import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; -import com.bakdata.conquery.sql.conversion.model.SqlQuery; -import lombok.RequiredArgsConstructor; -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Select; -import org.jooq.TableLike; import com.google.common.base.Preconditions; +import lombok.RequiredArgsConstructor; @RequiredArgsConstructor public class AbsoluteFormQueryConverter implements NodeConverter { - private final QueryStepTransformer queryStepTransformer; + private final FormConversionHelper formHelper; @Override public Class getConversionClass() { @@ -44,30 +32,32 @@ public Class getConversionClass() { @Override public ConversionContext convert(AbsoluteFormQuery form, ConversionContext context) { - // base population query conversion - QueryStep prerequisite = convertPrerequisite(form.getQuery(), form.getDateRange(), context); - - // creating stratification tables - StratificationTableFactory tableFactory = new StratificationTableFactory(prerequisite, context); - QueryStep stratificationTable = tableFactory.createStratificationTable(form); - - // feature conversion - ConversionContext childContext = convertFeatures(form, context, stratificationTable); - - List queriesToJoin = childContext.getQuerySteps(); - QueryStep joinedFeatures = QueryStepJoiner.joinSteps(queriesToJoin, ConqueryJoinType.OUTER_JOIN, DateAggregationAction.BLOCK, context); - return createFinalSelect(form, stratificationTable, joinedFeatures, childContext); + QueryStep convertedPrerequisite = convertPrerequisite(form, context); + StratificationTableFactory tableFactory = new StratificationTableFactory(convertedPrerequisite, context); + QueryStep stratificationTable = tableFactory.createAbsoluteStratificationTable(form.getResolutionsAndAlignmentMap()); + + return formHelper.convertForm( + FormType.ABSOLUTE, + stratificationTable, + form.getFeatures(), + form.getResultInfos(), + context + ); } - private static QueryStep convertPrerequisite(Query query, Range formDateRange, ConversionContext context) { + /** + * Converts the given {@link Query} and creates another {@link QueryStep} on top which extracts only the primary id. The form's date range is set + * as stratification range. + */ + private static QueryStep convertPrerequisite(AbsoluteFormQuery absoluteForm, ConversionContext context) { - ConversionContext withConvertedPrerequisite = context.getNodeConversions().convert(query, context); + ConversionContext withConvertedPrerequisite = context.getNodeConversions().convert(absoluteForm.getQuery(), context); Preconditions.checkArgument(withConvertedPrerequisite.getQuerySteps().size() == 1, "Base query conversion should produce exactly 1 QueryStep"); QueryStep convertedPrerequisite = withConvertedPrerequisite.getQuerySteps().get(0); ColumnDateRange bounds = context.getSqlDialect() .getFunctionProvider() - .forCDateRange(CDateRange.of(formDateRange)).as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); + .forCDateRange(CDateRange.of(absoluteForm.getDateRange())).as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); Selects prerequisiteSelects = convertedPrerequisite.getQualifiedSelects(); // we only keep the primary column for the upcoming form @@ -85,59 +75,4 @@ private static QueryStep convertPrerequisite(Query query, Range formD .build(); } - private static ConversionContext convertFeatures(AbsoluteFormQuery form, ConversionContext context, QueryStep stratificationTable) { - ConversionContext childContext = context.createChildContext().withStratificationTable(stratificationTable); - for (ConceptQuery conceptQuery : form.getFeatures().getChildQueries()) { - childContext = context.getNodeConversions().convert(conceptQuery, childContext); - } - return childContext; - } - - /** - * Left-joins the full stratification table back with the converted feature tables to keep all the resolutions. - *

- * When converting features, we filter out rows where a subjects validity date and the stratification date do not overlap. - * Thus, the pre-final step might not contain an entry for each expected stratification window. That's why we need to left-join the converted - * features with the full stratification table again. - */ - private ConversionContext createFinalSelect(AbsoluteFormQuery form, QueryStep stratificationTable, QueryStep convertedFeatures, ConversionContext context) { - - List queriesToJoin = List.of(stratificationTable, convertedFeatures); - TableLike joinedTable = QueryStepJoiner.constructJoinedTable(queriesToJoin, ConqueryJoinType.LEFT_JOIN, context); - - QueryStep finalStep = QueryStep.builder() - .cteName(null) // the final QueryStep won't be converted to a CTE - .selects(getFinalSelects(stratificationTable, convertedFeatures, context.getSqlDialect().getFunctionProvider())) - .fromTable(joinedTable) - .predecessors(queriesToJoin) - .build(); - - Select selectQuery = queryStepTransformer.toSelectQuery(finalStep); - return context.withFinalQuery(new SqlQuery(selectQuery, form.getResultInfos())); - } - - /** - * Selects the ID, resolution, index and date range from stratification table plus all explicit selects from the converted features step. - */ - private static Selects getFinalSelects(QueryStep stratificationTable, QueryStep convertedFeatures, SqlFunctionProvider functionProvider) { - - Selects preFinalSelects = convertedFeatures.getQualifiedSelects(); - - if (preFinalSelects.getStratificationDate().isEmpty() || !preFinalSelects.getIds().isWithStratification()) { - throw new IllegalStateException("Expecting the pre-final CTE to contain a stratification date, resolution and index"); - } - - Selects stratificationSelects = stratificationTable.getQualifiedSelects(); - SqlIdColumns ids = stratificationSelects.getIds().forFinalSelect(); - Field daterangeConcatenated = functionProvider.daterangeStringExpression(stratificationSelects.getStratificationDate().get()) - .as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); - - return Selects.builder() - .ids(ids) - .validityDate(Optional.empty()) - .stratificationDate(Optional.of(ColumnDateRange.of(daterangeConcatenated))) - .sqlSelects(preFinalSelects.getSqlSelects()) - .build(); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java index dec62d16c2..00c78c2c76 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java @@ -17,7 +17,6 @@ import org.jooq.Field; import org.jooq.Record; import org.jooq.Select; -import org.jooq.impl.DSL; @RequiredArgsConstructor public class ConceptQueryConverter implements NodeConverter { @@ -54,8 +53,7 @@ private Selects getFinalSelects(ConceptQuery conceptQuery, Selects preFinalSelec return preFinalSelects.blockValidityDate(); } else if (preFinalSelects.getValidityDate().isEmpty()) { - Field emptyRange = DSL.field(DSL.val("{}")); - return preFinalSelects.withValidityDate(ColumnDateRange.of(emptyRange)); + return preFinalSelects.withValidityDate(ColumnDateRange.empty()); } Field validityDateStringAggregation = functionProvider.daterangeStringAggregation(preFinalSelects.getValidityDate().get()); return preFinalSelects.withValidityDate(ColumnDateRange.of(validityDateStringAggregation)); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java new file mode 100644 index 0000000000..808c672a63 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/EntityDateQueryConverter.java @@ -0,0 +1,96 @@ +package com.bakdata.conquery.sql.conversion.query; + +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.models.forms.managed.EntityDateQuery; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.conversion.forms.FormCteStep; +import com.bakdata.conquery.sql.conversion.forms.FormType; +import com.bakdata.conquery.sql.conversion.forms.StratificationTableFactory; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.conversion.model.Selects; +import com.google.common.base.Preconditions; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class EntityDateQueryConverter implements NodeConverter { + + private final FormConversionHelper formHelper; + + @Override + public Class getConversionClass() { + return EntityDateQuery.class; + } + + @Override + public ConversionContext convert(EntityDateQuery entityDateQuery, ConversionContext context) { + + QueryStep prerequisite = formHelper.convertPrerequisite(entityDateQuery.getQuery(), context); + QueryStep withOverwrittenValidityDateBounds = overwriteBounds(prerequisite, entityDateQuery, context); + StratificationTableFactory tableFactory = new StratificationTableFactory(withOverwrittenValidityDateBounds, context); + QueryStep stratificationTable = tableFactory.createAbsoluteStratificationTable(entityDateQuery.getResolutionsAndAlignments()); + + return formHelper.convertForm( + FormType.ENTITY_DATE, + stratificationTable, + entityDateQuery.getFeatures(), + entityDateQuery.getResultInfos(), + context + ); + } + + /** + * Computes the intersection of the entity date and the entity date query's daterange. + */ + private static QueryStep overwriteBounds(QueryStep prerequisite, EntityDateQuery entityDateQuery, ConversionContext context) { + + Preconditions.checkArgument( + prerequisite.getQualifiedSelects().getValidityDate().isPresent(), + "Expecting the prerequisite step's Selects to contain a validity date" + ); + + SqlFunctionProvider functionProvider = context.getSqlDialect().getFunctionProvider(); + // we want to create a stratification for each distinct validity date range of an entity, + // so we first need to unnest the validity date in case it is a multirange + QueryStep unnestedEntityDate = functionProvider.unnestValidityDate(prerequisite, FormCteStep.UNNEST_ENTITY_DATE_CTE.getSuffix()); + Selects unnestedSelects = unnestedEntityDate.getQualifiedSelects(); + + ColumnDateRange withOverwrittenBounds; + if (entityDateQuery.getDateRange() == null) { + withOverwrittenBounds = unnestedSelects.getValidityDate().get(); + } + else { + ColumnDateRange formDateRange = functionProvider.forCDateRange(entityDateQuery.getDateRange()); + ColumnDateRange entityDate = unnestedSelects.getValidityDate().get(); + withOverwrittenBounds = functionProvider.intersection(formDateRange, entityDate) + .as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); + } + + Selects selects = Selects.builder() + .ids(unnestedSelects.getIds()) + .stratificationDate(Optional.of(withOverwrittenBounds)) + .build(); + + List predecessors; + // unnest step is optional depending on the dialect + if (unnestedEntityDate == prerequisite) { + predecessors = List.of(prerequisite); + } + else { + predecessors = List.of(prerequisite, unnestedEntityDate); + } + + return QueryStep.builder() + .cteName(FormCteStep.OVERWRITE_BOUNDS.getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(unnestedEntityDate.getCteName())) + .predecessors(predecessors) + .build(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java new file mode 100644 index 0000000000..39e3286c44 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java @@ -0,0 +1,168 @@ +package com.bakdata.conquery.sql.conversion.query; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.bakdata.conquery.apiv1.forms.FeatureGroup; +import com.bakdata.conquery.apiv1.query.ArrayConceptQuery; +import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.apiv1.query.Query; +import com.bakdata.conquery.models.query.queryplan.DateAggregationAction; +import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.conversion.forms.FormCteStep; +import com.bakdata.conquery.sql.conversion.forms.FormType; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.conversion.model.ConqueryJoinType; +import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; +import com.bakdata.conquery.sql.conversion.model.QueryStepTransformer; +import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.conversion.model.SqlQuery; +import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.google.common.base.Preconditions; +import lombok.RequiredArgsConstructor; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Select; +import org.jooq.TableLike; +import org.jooq.impl.DSL; + +@RequiredArgsConstructor +public class FormConversionHelper { + + private final QueryStepTransformer queryStepTransformer; + + /** + * Converts the given {@link Query} and creates another {@link QueryStep} on top which extracts only the primary id and the validity dates. + */ + public QueryStep convertPrerequisite(Query query, ConversionContext context) { + + ConversionContext withConvertedPrerequisite = context.getNodeConversions().convert(query, context); + Preconditions.checkArgument(withConvertedPrerequisite.getQuerySteps().size() == 1, "Base query conversion should produce exactly 1 QueryStep"); + QueryStep convertedPrerequisite = withConvertedPrerequisite.getQuerySteps().get(0); + + Selects prerequisiteSelects = convertedPrerequisite.getQualifiedSelects(); + // we keep the primary column and the validity date + Selects selects = Selects.builder() + .ids(new SqlIdColumns(prerequisiteSelects.getIds().getPrimaryColumn())) + .validityDate(prerequisiteSelects.getValidityDate()) + .build(); + + // we want to keep each primary column and the corresponding distinct validity date ranges + List> groupByFields = Stream.concat( + Stream.of(prerequisiteSelects.getIds().getPrimaryColumn()), + prerequisiteSelects.getValidityDate().stream().flatMap(validityDate -> validityDate.toFields().stream()) + ) + .collect(Collectors.toList()); + + // filter out entries with a null validity date + Condition dateNotNullCondition = prerequisiteSelects.getValidityDate().get().isNotNull(); + + return QueryStep.builder() + .cteName(FormCteStep.EXTRACT_IDS.getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(convertedPrerequisite.getCteName())) + .conditions(List.of(dateNotNullCondition)) + .groupBy(groupByFields) + .predecessors(List.of(convertedPrerequisite)) + .build(); + } + + public ConversionContext convertForm( + FormType formType, + QueryStep stratificationTable, + ArrayConceptQuery features, + List resultInfos, + ConversionContext context + ) { + // feature conversion + ConversionContext childContext = context.createChildContext().withStratificationTable(stratificationTable); + for (ConceptQuery conceptQuery : features.getChildQueries()) { + childContext = context.getNodeConversions().convert(conceptQuery, childContext); + } + + // child context contains the converted feature's QuerySteps + List queriesToJoin = childContext.getQuerySteps(); + QueryStep joinedFeatures = QueryStepJoiner.joinSteps(queriesToJoin, ConqueryJoinType.OUTER_JOIN, DateAggregationAction.BLOCK, context); + return createFinalSelect(formType, stratificationTable, joinedFeatures, resultInfos, context); + } + + /** + * Left-joins the full stratification table back with the converted feature tables to keep all the resolutions. + *

+ * When converting features, we filter out rows where a subjects validity date and the stratification date do not overlap. + * Thus, the pre-final step might not contain an entry for each expected stratification window. That's why we need to left-join the converted + * features with the full stratification table again. + */ + private ConversionContext createFinalSelect( + FormType formType, + QueryStep stratificationTable, + QueryStep convertedFeatures, + List resultInfos, + ConversionContext context + ) { + Preconditions.checkArgument( + stratificationTable.getSelects().getStratificationDate().isPresent() && convertedFeatures.getSelects().getStratificationDate().isPresent(), + "Expecting stratification table and converted features table to contain a stratification date" + ); + + List queriesToJoin = List.of(stratificationTable, convertedFeatures); + TableLike joinedTable = QueryStepJoiner.constructJoinedTable(queriesToJoin, ConqueryJoinType.LEFT_JOIN, context); + SqlFunctionProvider functionProvider = context.getSqlDialect().getFunctionProvider(); + + QueryStep finalStep = QueryStep.builder() + .cteName(null) // the final QueryStep won't be converted to a CTE + .selects(getFinalSelects(formType, stratificationTable, convertedFeatures, functionProvider)) + .fromTable(joinedTable) + .predecessors(queriesToJoin) + .build(); + + Select selectQuery = queryStepTransformer.toSelectQuery(finalStep); + return context.withFinalQuery(new SqlQuery(selectQuery, resultInfos)); + } + + /** + * Selects the ID, resolution, index and date range from stratification table plus all explicit selects from the converted features step. + * For {@link FormType#RELATIVE}, the {@link FeatureGroup} will be set too. + */ + private static Selects getFinalSelects( + FormType formType, + QueryStep stratificationTable, + QueryStep convertedFeatures, + SqlFunctionProvider functionProvider + ) { + Selects preFinalSelects = convertedFeatures.getQualifiedSelects(); + + Selects stratificationSelects = stratificationTable.getQualifiedSelects(); + SqlIdColumns ids = stratificationSelects.getIds().forFinalSelect(); + Field daterangeConcatenated = functionProvider.daterangeStringExpression(stratificationSelects.getStratificationDate().get()) + .as(SharedAliases.STRATIFICATION_BOUNDS.getAlias()); + + Selects.SelectsBuilder selects = Selects.builder() + .ids(ids) + .validityDate(Optional.empty()) + .stratificationDate(Optional.of(ColumnDateRange.of(daterangeConcatenated))); + + if (formType != FormType.RELATIVE) { + return selects.sqlSelects(preFinalSelects.getSqlSelects()).build(); + } + + // relative forms have FeatureGroup information after the stratification date and before all other selects + Field indexField = DSL.field(DSL.name(stratificationTable.getCteName(), SharedAliases.INDEX.getAlias()), Integer.class); + Field scope = DSL.when(indexField.isNull().or(indexField.lessThan(0)), DSL.val(FeatureGroup.FEATURE.toString())) + .otherwise(DSL.val(FeatureGroup.OUTCOME.toString())) + .as(SharedAliases.OBSERVATION_SCOPE.getAlias()); + + return selects.sqlSelect(new FieldWrapper<>(scope)) + .sqlSelects(preFinalSelects.getSqlSelects()) + .build(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java new file mode 100644 index 0000000000..544d1e0548 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/RelativFormQueryConverter.java @@ -0,0 +1,37 @@ +package com.bakdata.conquery.sql.conversion.query; + +import com.bakdata.conquery.models.forms.managed.RelativeFormQuery; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; +import com.bakdata.conquery.sql.conversion.forms.FormType; +import com.bakdata.conquery.sql.conversion.forms.StratificationTableFactory; +import com.bakdata.conquery.sql.conversion.model.QueryStep; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class RelativFormQueryConverter implements NodeConverter { + + private final FormConversionHelper formHelper; + + @Override + public Class getConversionClass() { + return RelativeFormQuery.class; + } + + @Override + public ConversionContext convert(RelativeFormQuery form, ConversionContext context) { + + QueryStep convertedPrerequisite = formHelper.convertPrerequisite(form.getQuery(), context); + StratificationTableFactory tableFactory = new StratificationTableFactory(convertedPrerequisite, context); + QueryStep stratificationTable = tableFactory.createRelativeStratificationTable(form); + + return formHelper.convertForm( + FormType.RELATIVE, + stratificationTable, + form.getFeatures(), + form.getResultInfos(), + context + ); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java new file mode 100644 index 0000000000..b2a7b642f2 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java @@ -0,0 +1,206 @@ +package com.bakdata.conquery.sql.conversion.query; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; + +import com.bakdata.conquery.apiv1.query.Query; +import com.bakdata.conquery.apiv1.query.TableExportQuery; +import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.SharedAliases; +import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.conversion.forms.FormCteStep; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; +import com.bakdata.conquery.sql.conversion.model.QueryStep; +import com.bakdata.conquery.sql.conversion.model.QueryStepTransformer; +import com.bakdata.conquery.sql.conversion.model.Selects; +import com.bakdata.conquery.sql.conversion.model.SqlIdColumns; +import com.bakdata.conquery.sql.conversion.model.SqlQuery; +import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; +import com.bakdata.conquery.util.TablePrimaryColumnUtil; +import com.google.common.base.Preconditions; +import com.google.common.base.Predicate; +import lombok.RequiredArgsConstructor; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Select; +import org.jooq.Table; +import org.jooq.impl.DSL; + +@RequiredArgsConstructor +public class TableExportQueryConverter implements NodeConverter { + + private final QueryStepTransformer queryStepTransformer; + + @Override + public Class getConversionClass() { + return TableExportQuery.class; + } + + @Override + public ConversionContext convert(TableExportQuery tableExportQuery, ConversionContext context) { + + QueryStep convertedPrerequisite = convertPrerequisite(tableExportQuery, context); + Map positions = tableExportQuery.getPositions(); + CDateRange dateRestriction = CDateRange.of(tableExportQuery.getDateRange()); + + List convertedTables = tableExportQuery.getTables().stream() + .flatMap(concept -> concept.getTables().stream().map(table -> convertTable( + table, + concept, + dateRestriction, + convertedPrerequisite, + positions, + context + ))) + .toList(); + + QueryStep unionedTables = QueryStep.createUnionStep( + convertedTables, + null, // no CTE name required as this step will be the final select + List.of(convertedPrerequisite) + ); + Select selectQuery = queryStepTransformer.toSelectQuery(unionedTables); + + return context.withFinalQuery(new SqlQuery(selectQuery, tableExportQuery.getResultInfos())); + } + + /** + * Converts the {@link Query} of the given {@link TableExportQuery} and creates another {@link QueryStep} on top which extracts only the primary id. + */ + private static QueryStep convertPrerequisite(TableExportQuery exportQuery, ConversionContext context) { + + ConversionContext withConvertedPrerequisite = context.getNodeConversions().convert(exportQuery.getQuery(), context); + Preconditions.checkArgument(withConvertedPrerequisite.getQuerySteps().size() == 1, "Base query conversion should produce exactly 1 QueryStep"); + QueryStep convertedPrerequisite = withConvertedPrerequisite.getQuerySteps().get(0); + + Selects prerequisiteSelects = convertedPrerequisite.getQualifiedSelects(); + Selects selects = Selects.builder() + .ids(new SqlIdColumns(prerequisiteSelects.getIds().getPrimaryColumn())) + .build(); + + return QueryStep.builder() + .cteName(FormCteStep.EXTRACT_IDS.getSuffix()) + .selects(selects) + .fromTable(QueryStep.toTableLike(convertedPrerequisite.getCteName())) + .groupBy(selects.getIds().toFields()) // group by primary column to ensure max. 1 entry per subject + .predecessors(List.of(convertedPrerequisite)) + .build(); + } + + /** + * Create a CTE selecting all positions of a {@link TableExportQuery} from the given {@link CQTable} properly ordered. For all columns, which are not + * linked to the given table, we just do a null select. + */ + private static QueryStep convertTable( + CQTable cqTable, + CQConcept concept, + CDateRange dateRestriction, + QueryStep convertedPrerequisite, + Map positions, + ConversionContext context + ) { + Field primaryColumn = TablePrimaryColumnUtil.findPrimaryColumn(cqTable.getConnector().getTable(), context.getConfig()); + SqlIdColumns ids = new SqlIdColumns(primaryColumn); + String conceptConnectorName = context.getNameGenerator().conceptConnectorName(concept, cqTable.getConnector()); + Optional validityDate = convertTablesValidityDate(cqTable, conceptConnectorName, dateRestriction, context); + + List> exportColumns = new ArrayList<>(); + exportColumns.add(createSourceInfoSelect(cqTable)); + + positions.entrySet().stream() + .sorted(Comparator.comparingInt(Map.Entry::getValue)) + .map(entry -> createColumnSelect(cqTable, entry)) + .forEach(exportColumns::add); + + Selects selects = Selects.builder() + .ids(ids) + .validityDate(validityDate) + .sqlSelects(exportColumns.stream().map(FieldWrapper::new).collect(Collectors.toList())) + .build(); + + List filters = cqTable.getFilters().stream().map(filterValue -> filterValue.convertForTableExport(ids, context)).toList(); + Table joinedTable = joinConnectorTableWithPrerequisite(cqTable, ids, convertedPrerequisite, context); + + return QueryStep.builder() + .cteName(conceptConnectorName) + .selects(selects) + .fromTable(joinedTable) + .conditions(filters) + .build(); + } + + private static Optional convertTablesValidityDate(CQTable table, String alias, CDateRange dateRestriction, ConversionContext context) { + if (table.findValidityDate() == null) { + return Optional.of(ColumnDateRange.empty()); + } + SqlFunctionProvider functionProvider = context.getSqlDialect().getFunctionProvider(); + ColumnDateRange validityDate = functionProvider.forValidityDate(table.findValidityDate(), dateRestriction); + // when exporting tables, we want the validity date as a single-column daterange string expression straightaway + Field asStringExpression = functionProvider.encloseInCurlyBraces(functionProvider.daterangeStringExpression(validityDate)); + return Optional.of(ColumnDateRange.of(asStringExpression).asValidityDateRange(alias)); + } + + private static Field createSourceInfoSelect(CQTable cqTable) { + String tableName = cqTable.getConnector().getTable().getName(); + return DSL.val(tableName).as(SharedAliases.SOURCE.getAlias()); + } + + private static Field createColumnSelect(CQTable table, Map.Entry entry) { + + Column column = entry.getKey(); + Integer columnPosition = entry.getValue(); + String columnName = "%s-%s".formatted(column.getName(), columnPosition); + + if (!isColumnOfTable(column, table)) { + return DSL.inline(null, Object.class).as(columnName); + } + return DSL.field(DSL.name(column.getTable().getName(), column.getName())) + .as(columnName); + } + + private static boolean isColumnOfTable(Column column, CQTable table) { + return columnIsConnectorColumn(column, table) + || columnIsSecondaryIdOfConnectorTable(column, table) + || columnIsConnectorTableColumn(column, table); + } + + private static boolean columnIsConnectorTableColumn(Column column, CQTable table) { + return matchesTableColumnOn(table, tableColumn -> tableColumn == column); + } + + private static boolean columnIsSecondaryIdOfConnectorTable(Column column, CQTable table) { + return column.getSecondaryId() != null && matchesTableColumnOn(table, tableColumn -> tableColumn.getSecondaryId() == column.getSecondaryId()); + } + + private static boolean matchesTableColumnOn(CQTable table, Predicate condition) { + return Arrays.stream(table.getConnector().getTable().getColumns()).anyMatch(condition); + } + + private static boolean columnIsConnectorColumn(Column column, CQTable table) { + return table.getConnector().getColumn() != null && table.getConnector().getColumn() == column; + } + + private static Table joinConnectorTableWithPrerequisite( + CQTable cqTable, + SqlIdColumns ids, + QueryStep convertedPrerequisite, + ConversionContext context + ) { + SqlFunctionProvider functionProvider = context.getSqlDialect().getFunctionProvider(); + Table connectorTable = DSL.table(DSL.name(cqTable.getConnector().getTable().getName())); + List joinOnIds = ids.join(convertedPrerequisite.getQualifiedSelects().getIds()); + return functionProvider.innerJoin(connectorTable, convertedPrerequisite, joinOnIds); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/util/SMTPAppenderFactory.java b/backend/src/main/java/com/bakdata/conquery/util/SMTPAppenderFactory.java new file mode 100644 index 0000000000..e912db7fbf --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/util/SMTPAppenderFactory.java @@ -0,0 +1,63 @@ +package com.bakdata.conquery.util; + +import java.net.URI; + +import ch.qos.logback.access.net.SMTPAppender; +import ch.qos.logback.access.spi.IAccessEvent; +import ch.qos.logback.classic.LoggerContext; +import ch.qos.logback.core.Appender; +import com.fasterxml.jackson.annotation.JsonTypeName; +import com.google.common.base.Strings; +import io.dropwizard.logging.common.AbstractAppenderFactory; +import io.dropwizard.logging.common.async.AsyncAppenderFactory; +import io.dropwizard.logging.common.filter.LevelFilterFactory; +import io.dropwizard.logging.common.layout.LayoutFactory; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@JsonTypeName("SMTP") +@Data +public class SMTPAppenderFactory extends AbstractAppenderFactory { + + private String from; + private String[] to; + private URI server; + private String subject; + + + @Override + public Appender build(LoggerContext context, String applicationName, LayoutFactory layoutFactory, LevelFilterFactory levelFilterFactory, AsyncAppenderFactory asyncAppenderFactory) { + final SMTPAppender appender = new SMTPAppender(); + appender.setName("smtp-appender"); + + appender.setFrom(getFrom()); + appender.setSmtpHost(server.getHost()); + appender.setSMTPPort(server.getPort()); + + if (!Strings.isNullOrEmpty(getServer().getUserInfo())) { + final String[] userInfo = getServer().getUserInfo().split(":"); + final String userName = userInfo[0]; + final String password = userInfo.length > 1 ? userInfo[1] : null; + + appender.setPassword(password); + appender.setUsername(userName); + } + + appender.setSubject(getSubject()); + + for (String target : getTo()) { + appender.addTo(target); + } + + appender.setContext(context); + appender.setAsynchronousSending(true); + + appender.setLayout(buildLayout(context, layoutFactory)); + appender.addFilter(levelFilterFactory.build(threshold)); + getFilterFactories().forEach(f -> appender.addFilter(f.build())); + + appender.start(); + return wrapAsync(appender, asyncAppenderFactory); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java b/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java new file mode 100644 index 0000000000..491bc04338 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java @@ -0,0 +1,17 @@ +package com.bakdata.conquery.util; + +import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.models.datasets.Table; +import org.jooq.Field; +import org.jooq.impl.DSL; + +public class TablePrimaryColumnUtil { + + public static Field findPrimaryColumn(Table table, SqlConnectorConfig sqlConfig) { + String primaryColumnName = table.getPrimaryColum() == null + ? sqlConfig.getPrimaryColumn() + : table.getPrimaryColum().getName(); + return DSL.field(DSL.name(table.getName(), primaryColumnName)); + } + +} diff --git a/backend/src/main/resources/assets/custom/js/script.js b/backend/src/main/resources/assets/custom/js/script.js index 5e12ff0b29..325d59e9c6 100644 --- a/backend/src/main/resources/assets/custom/js/script.js +++ b/backend/src/main/resources/assets/custom/js/script.js @@ -29,15 +29,32 @@ async function rest(url, options) { { method: 'get', credentials: 'same-origin', + ...options, + // Overwrite options[headers] here, BUT merge them with our headers headers: { - 'Content-Type': 'application/json' - }, - ...options + 'Content-Type': 'application/json', + // Csrf token is provided via global variable + 'X-Csrf-Token': csrf_token, + ...options?.headers, + } } ); return res; } +async function restOptionalForce(url, options) { + return rest(url, options).then((res) => { + // force button in case of 409 status + const customButton = createCustomButton('Force delete'); + customButton.onclick = () => rest(toForceURL(url), options).then((res) => { + res.ok && location.reload(); + }); + + showMessageForResponse(res, customButton); + return res; + }); +} + function getToast(type, title, text, smalltext = "", customButton) { if (!type) type = ToastTypes.INFO; @@ -148,10 +165,9 @@ function postFile(event, url) { let reader = new FileReader(); reader.onload = function () { let json = reader.result; - fetch(url, { - method: 'post', credentials: 'same-origin', body: json, headers: { - "Content-Type": "application/json" - } + rest(url, { + method: 'post', + body: json }) .then(function (response) { if (response.ok) { @@ -180,10 +196,9 @@ function postFile(event, url) { function postScriptHandler(event, jsonOut, responseTargetTextfield) { event.preventDefault(); responseTargetTextfield.innerHTML = 'waiting for response'; - fetch('/admin/script', + rest('/admin/script', { method: 'post', - credentials: 'same-origin', body: document.getElementById('script').value, headers: { 'Content-Type': 'text/plain', @@ -207,4 +222,85 @@ $("ul.nav-tabs > li > a").on("shown.bs.tab", function (e) { // on load of the page: switch to the currently selected tab var hash = window.location.hash; -$('#myTab a[href="' + hash + '"]').tab('show'); \ No newline at end of file +$('#myTab a[href="' + hash + '"]').tab('show'); + +const uploadFormMapping = { + mapping: { + name: "mapping", + uri: "internToExtern", + accept: "*.mapping.json", + }, + table: { + name: "table_schema", + uri: "tables", + accept: "*.table.json" + }, + concept: { + name: "concept_schema", + uri: "concepts", + accept: "*.concept.json", + }, + structure: { + name: "structure_schema", + uri: "structure", + accept: "structure.json", + }, +}; + +function updateDatasetUploadForm(select, datasetId) { + const data = uploadFormMapping[select.value]; + const fileInput = $(select).next(); + fileInput.value = ""; + fileInput.attr("accept", data.accept); + fileInput.attr("name", data.name); + $(select) + .parent() + .attr( + "onsubmit", + `postFile(event, '/admin/datasets/${datasetId}/${data.uri}')` + ); +} + +function createDataset(event) { + event.preventDefault(); + rest( + '/admin/datasets', + { + method: 'post', + body: JSON.stringify({ + name: document.getElementById('entity_id').value, + label: document.getElementById('entity_name').value + }) + }).then(function (res) { + if (res.ok) + location.reload(); + else + showMessageForResponse(res); + }); +} + + + +function deleteDataset(event, datasetId) { + event.preventDefault(); + rest( + `/admin/datasets/${datasetId}`, + { + method: 'delete', + }).then(function (res) { + if (res.ok) + location.reload(); + else + showMessageForResponse(res); + }); +} + +function shutdown(event) { + event.preventDefault(); + rest( + '/tasks/shutdown', + { + method: 'post' + } + ); +} \ No newline at end of file diff --git a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/dataset.html.ftl b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/dataset.html.ftl index 51b1aa8e22..65d07c7f6d 100644 --- a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/dataset.html.ftl +++ b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/dataset.html.ftl @@ -27,8 +27,6 @@ <#macro idMapping>Here <@layout.layout> - - <@breadcrumbs.breadcrumbs @@ -51,7 +49,7 @@ - + @@ -32,43 +32,4 @@ - \ No newline at end of file diff --git a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/group.html.ftl b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/group.html.ftl index 402ffd0494..611ea50a5c 100644 --- a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/group.html.ftl +++ b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/group.html.ftl @@ -84,23 +84,20 @@ diff --git a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl index 64db072311..17981be3f3 100644 --- a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl +++ b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl @@ -7,17 +7,16 @@ function cancelJob(jobId) { event.preventDefault(); - fetch( + rest( "/admin/jobs/" + jobId + "/cancel", { method: "post", - credentials: "same-origin" } ); } function getJobs() { - return fetch("/admin/jobs") + return rest("/admin/jobs") .then((res) => res.json()) .then((entries) => { const origins = {}; diff --git a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/queries.html.ftl b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/queries.html.ftl index 5bac320b71..c73c81cd19 100644 --- a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/queries.html.ftl +++ b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/queries.html.ftl @@ -1,159 +1,130 @@ <#import "templates/template.html.ftl" as layout> -<@layout.layout> + <@layout.layout> -

Queries

@@ -230,7 +200,7 @@
@@ -244,7 +214,7 @@
@@ -258,7 +228,7 @@
@@ -272,7 +242,7 @@
@@ -283,5 +253,4 @@
- - \ No newline at end of file + \ No newline at end of file diff --git a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/scripts/dataset.js b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/scripts/dataset.js deleted file mode 100644 index 6cbb9c846e..0000000000 --- a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/scripts/dataset.js +++ /dev/null @@ -1,45 +0,0 @@ -const uploadFormMapping = { - mapping: { - name: "mapping", - uri: "internToExtern", - accept: "*.mapping.json", - }, - table: { name: "table_schema", uri: "tables", accept: "*.table.json" }, - concept: { - name: "concept_schema", - uri: "concepts", - accept: "*.concept.json", - }, - structure: { - name: "structure_schema", - uri: "structure", - accept: "structure.json", - }, -}; - -function updateDatasetUploadForm(select) { - const data = uploadFormMapping[select.value]; - const fileInput = $(select).next(); - fileInput.value = ""; - fileInput.attr("accept", data.accept); - fileInput.attr("name", data.name); - $(select) - .parent() - .attr( - "onsubmit", - "postFile(event, '/admin/datasets/${c.ds.id}/" + data.uri + "')" - ); -} - -async function restOptionalForce(url, options) { - return rest(url, options).then((res) => { - // force button in case of 409 status - const customButton = createCustomButton('Force delete'); - customButton.onclick = () => rest(toForceURL(url), options).then((res) => { - res.ok && location.reload(); - }); - - showMessageForResponse(res, customButton); - return res; - }); -} \ No newline at end of file diff --git a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/authEntityOverview.html.ftl b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/authEntityOverview.html.ftl index 371a846874..dd62fa7206 100644 --- a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/authEntityOverview.html.ftl +++ b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/authEntityOverview.html.ftl @@ -46,11 +46,9 @@ + + + + ${title} <#nested /> diff --git a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/groupHandler.html.ftl b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/groupHandler.html.ftl index 17731552c4..4b89d2367a 100644 --- a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/groupHandler.html.ftl +++ b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/groupHandler.html.ftl @@ -40,22 +40,19 @@