From 8b7a72c2995bcaa24448278f8c2898525f666ad0 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 8 Apr 2024 21:57:24 +0200 Subject: [PATCH 01/23] wip update version info format Signed-off-by: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> --- .../apiv1/frontend/FrontendConfiguration.java | 9 +++---- .../apiv1/frontend/VersionContainer.java | 14 ++++++++++ .../external/form/ExternalFormBackendApi.java | 15 +++++------ .../io/external/form/FormBackendVersion.java | 5 ++++ .../models/config/FormBackendConfig.java | 18 ++++++------- .../resources/api/ConfigResource.java | 5 ++-- .../bakdata/conquery/util/VersionInfo.java | 26 +++++++++++++++---- .../tests/ExternalFormBackendTest.java | 11 ++++---- .../conquery/models/SerializationTests.java | 14 ++++++++++ 9 files changed, 81 insertions(+), 36 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/frontend/VersionContainer.java diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendConfiguration.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendConfiguration.java index 9afa35d5be..1a75ccf769 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendConfiguration.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendConfiguration.java @@ -2,7 +2,7 @@ import java.net.URL; import java.time.LocalDate; -import java.util.Map; +import java.util.List; import com.bakdata.conquery.models.config.FrontendConfig; import com.bakdata.conquery.models.config.IdColumnConfig; @@ -10,17 +10,14 @@ /** * API Response for the dynamic configuration of the frontend * - * @param version backend version - * @param formBackendVersions mapping of form backend ids to their versions (version can be null) + * @param versions mapping of backend and form backend ids to their versions (version can be null) * @param currency currency representation * @param queryUpload identifier specific column configuration for the query upload * @param manualUrl url to a user manual * @param contactEmail typical a mailto-url */ public record FrontendConfiguration( - String version, - - Map formBackendVersions, + List versions, FrontendConfig.CurrencyConfig currency, IdColumnConfig queryUpload, URL manualUrl, diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/VersionContainer.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/VersionContainer.java new file mode 100644 index 0000000000..ba4cbfb17f --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/VersionContainer.java @@ -0,0 +1,14 @@ +package com.bakdata.conquery.apiv1.frontend; + +import java.util.Date; + +import com.fasterxml.jackson.annotation.JsonFormat; +import lombok.NonNull; + +public record VersionContainer( + @NonNull String name, + String version, + @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss") + Date buildTime +) { +} diff --git a/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java b/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java index d27a11481a..7e8ced87cc 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java +++ b/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java @@ -7,14 +7,6 @@ import java.util.UUID; import java.util.function.Function; -import jakarta.ws.rs.client.Client; -import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.client.Invocation; -import jakarta.ws.rs.client.WebTarget; -import jakarta.ws.rs.core.GenericType; -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.Response; - import com.bakdata.conquery.apiv1.forms.ExternalForm; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; @@ -24,6 +16,13 @@ import com.codahale.metrics.health.HealthCheck; import com.fasterxml.jackson.databind.node.ObjectNode; import io.dropwizard.health.check.http.HttpHealthCheck; +import jakarta.ws.rs.client.Client; +import jakarta.ws.rs.client.Entity; +import jakarta.ws.rs.client.Invocation; +import jakarta.ws.rs.client.WebTarget; +import jakarta.ws.rs.core.GenericType; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.Response; import lombok.extern.slf4j.Slf4j; @Slf4j diff --git a/backend/src/main/java/com/bakdata/conquery/io/external/form/FormBackendVersion.java b/backend/src/main/java/com/bakdata/conquery/io/external/form/FormBackendVersion.java index 93a9588819..0ac93b80c7 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/external/form/FormBackendVersion.java +++ b/backend/src/main/java/com/bakdata/conquery/io/external/form/FormBackendVersion.java @@ -1,8 +1,13 @@ package com.bakdata.conquery.io.external.form; +import java.util.Date; + +import com.fasterxml.jackson.annotation.JsonFormat; import lombok.Data; @Data public class FormBackendVersion { private String version; + @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss") + private Date buildTime; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java index 69e22c8c86..b6412eebb9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java @@ -6,13 +6,8 @@ import java.util.HashSet; import java.util.Set; -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; -import jakarta.validation.constraints.Pattern; -import jakarta.ws.rs.client.Client; - import com.bakdata.conquery.apiv1.forms.ExternalForm; +import com.bakdata.conquery.apiv1.frontend.VersionContainer; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.cps.CPSTypeIdResolver; @@ -33,6 +28,11 @@ import com.google.common.collect.ImmutableCollection; import io.dropwizard.client.JerseyClientBuilder; import io.dropwizard.jersey.jackson.JacksonMessageBodyProvider; +import jakarta.validation.Valid; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.Pattern; +import jakarta.ws.rs.client.Client; import lombok.Data; import lombok.extern.slf4j.Slf4j; @@ -113,7 +113,7 @@ private void updateVersion(ExternalFormBackendApi externalApi) { try { final String version = externalApi.getVersion(); - final String oldVersion = VersionInfo.INSTANCE.setFormBackendVersion(getId(), version); + final VersionContainer oldVersion = VersionInfo.INSTANCE.setFormBackendVersion(new VersionContainer(getId(), version, null)); if (!version.equals(oldVersion)) { log.info("Form Backend '{}' version update: {} -> {}", getId(), oldVersion, version); } @@ -122,8 +122,8 @@ private void updateVersion(ExternalFormBackendApi externalApi) { log.warn("Unable to retrieve version from form backend '{}'. Enable trace logging for more info", getId(), (Exception) (log.isTraceEnabled() ? e : null)); - // Set place holder - VersionInfo.INSTANCE.setFormBackendVersion(getId(), "no-version-available"); + + VersionInfo.INSTANCE.setFormBackendVersion(new VersionContainer(getId(), null, null)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java index b0a00a0586..c641784154 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java @@ -33,9 +33,10 @@ public FrontendConfiguration getFrontendConfig() { .toList()); final FrontendConfig frontendConfig = config.getFrontend(); + + return new FrontendConfiguration( - VersionInfo.INSTANCE.getProjectVersion(), - VersionInfo.INSTANCE.getFormBackendVersions(), + VersionInfo.INSTANCE.getVersions(), frontendConfig.getCurrency(), idColumns, frontendConfig.getManualUrl(), diff --git a/backend/src/main/java/com/bakdata/conquery/util/VersionInfo.java b/backend/src/main/java/com/bakdata/conquery/util/VersionInfo.java index ae60711cf0..8df1f966f9 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/VersionInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/util/VersionInfo.java @@ -2,10 +2,13 @@ import java.io.BufferedReader; import java.time.ZonedDateTime; -import java.util.HashMap; +import java.util.ArrayList; +import java.util.List; import java.util.Map; import java.util.Properties; +import java.util.TreeMap; +import com.bakdata.conquery.apiv1.frontend.VersionContainer; import com.github.powerlibraries.io.In; import lombok.Getter; import lombok.ToString; @@ -21,8 +24,12 @@ public class VersionInfo { private ZonedDateTime buildTime; private String projectVersion; - // Form backend id -> version - private final Map formBackendVersions = new HashMap<>(); + /** + * Form backend id -> version + * + * @implNote using {@link TreeMap} to have a stable key order + */ + private final Map formBackendVersions = new TreeMap<>(); private VersionInfo() { try { @@ -45,7 +52,16 @@ private VersionInfo() { } } - public String setFormBackendVersion(String formBackendId, String version) { - return formBackendVersions.put(formBackendId, version); + public List getVersions() { + List versions = new ArrayList<>(); + + versions.add(new VersionContainer("Backend", projectVersion, null)); + versions.addAll(formBackendVersions.values()); + + return versions; + } + + public VersionContainer setFormBackendVersion(VersionContainer version) { + return formBackendVersions.put(version.name(), version); } } \ No newline at end of file diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java index be99b88a00..fb80d58642 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java @@ -9,14 +9,11 @@ import java.nio.file.Path; import java.util.Collections; import java.util.List; -import java.util.Map; - -import jakarta.ws.rs.core.MediaType; -import jakarta.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.apiv1.frontend.FrontendConfiguration; +import com.bakdata.conquery.apiv1.frontend.VersionContainer; import com.bakdata.conquery.integration.common.IntegrationUtils; import com.bakdata.conquery.io.result.ExternalResult; import com.bakdata.conquery.models.auth.entities.User; @@ -34,6 +31,8 @@ import com.bakdata.conquery.resources.hierarchies.HierarchyHelper; import com.bakdata.conquery.util.support.StandaloneSupport; import com.bakdata.conquery.util.support.TestConquery; +import jakarta.ws.rs.core.MediaType; +import jakarta.ws.rs.core.UriBuilder; import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; @@ -79,9 +78,9 @@ public void execute(String name, TestConquery testConquery) throws Exception { frontendConfiguration = support.getClient().target(frontendConfigURI).request(MediaType.APPLICATION_JSON_TYPE).get().readEntity(FrontendConfiguration.class); - assertThat(frontendConfiguration.formBackendVersions()) + assertThat(frontendConfiguration.versions()) .describedAs("Checking health of form backend") - .containsExactlyEntriesOf(Map.of(FORM_BACKEND_ID, "3.2.1-ge966c285")); // example value from OpenAPI Spec + .contains(new VersionContainer(FORM_BACKEND_ID, "3.2.1-ge966c285", null)); // example value from OpenAPI Spec log.info("Send an external form"); final User testUser = support.getTestUser(); diff --git a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java index 7df0c58cff..95d3e655b9 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java @@ -4,10 +4,12 @@ import static org.assertj.core.api.Assertions.assertThat; import java.io.IOException; +import java.time.Instant; import java.time.LocalDate; import java.util.Arrays; import java.util.BitSet; import java.util.Collections; +import java.util.Date; import java.util.List; import java.util.Locale; import java.util.Map; @@ -29,6 +31,7 @@ import com.bakdata.conquery.apiv1.query.concept.specific.CQOr; import com.bakdata.conquery.io.AbstractSerializationTest; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.io.external.form.FormBackendVersion; import com.bakdata.conquery.io.jackson.Injectable; import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.jackson.serializer.SerializationTestUtil; @@ -833,4 +836,15 @@ public void arrayObject2Int() throws JSONException, IOException { } + @Test + public void formBackendVersion() throws JSONException, IOException { + final FormBackendVersion version = new FormBackendVersion(); + version.setVersion("3.45.45-g85ut85u43t8"); + version.setBuildTime(Date.from(Instant.ofEpochSecond(1712554025))); //new Date(2024, Calendar.APRIL, 8, 21, 17, 44)); + + SerializationTestUtil.forType(FormBackendVersion.class) + .objectMappers(getApiMapper(), getManagerInternalMapper()) + .test(version); + } + } From 878062cc45f03f9399186a37555743071cc29907 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 9 Apr 2024 13:48:06 +0200 Subject: [PATCH 02/23] passthrough buildTime from form backends Signed-off-by: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> --- .../conquery/apiv1/frontend/VersionContainer.java | 6 ++---- .../io/external/form/ExternalFormBackendApi.java | 4 ++-- .../io/external/form/FormBackendVersion.java | 13 +++++-------- .../conquery/models/config/FormBackendConfig.java | 11 +++++++---- .../conquery/external/openapi-form-backend.yaml | 4 ++++ .../integration/tests/ExternalFormBackendTest.java | 3 ++- .../bakdata/conquery/models/SerializationTests.java | 7 ++----- 7 files changed, 24 insertions(+), 24 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/VersionContainer.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/VersionContainer.java index ba4cbfb17f..e2739ec9ce 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/VersionContainer.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/VersionContainer.java @@ -1,14 +1,12 @@ package com.bakdata.conquery.apiv1.frontend; -import java.util.Date; +import java.time.ZonedDateTime; -import com.fasterxml.jackson.annotation.JsonFormat; import lombok.NonNull; public record VersionContainer( @NonNull String name, String version, - @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss") - Date buildTime + ZonedDateTime buildTime ) { } diff --git a/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java b/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java index 7e8ced87cc..6d47ab247d 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java +++ b/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java @@ -128,8 +128,8 @@ public HealthCheck createHealthCheck() { return new HttpHealthCheck(getHealthTarget.getUri().toString(), client); } - public String getVersion() { - return getVersionTarget.request(MediaType.APPLICATION_JSON_TYPE).get(FormBackendVersion.class).getVersion(); + public FormBackendVersion getVersion() { + return getVersionTarget.request(MediaType.APPLICATION_JSON_TYPE).get(FormBackendVersion.class); } public ExternalTaskState cancelTask(UUID taskId) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/external/form/FormBackendVersion.java b/backend/src/main/java/com/bakdata/conquery/io/external/form/FormBackendVersion.java index 0ac93b80c7..c661159eff 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/external/form/FormBackendVersion.java +++ b/backend/src/main/java/com/bakdata/conquery/io/external/form/FormBackendVersion.java @@ -1,13 +1,10 @@ package com.bakdata.conquery.io.external.form; -import java.util.Date; +import java.time.ZonedDateTime; -import com.fasterxml.jackson.annotation.JsonFormat; -import lombok.Data; -@Data -public class FormBackendVersion { - private String version; - @JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss") - private Date buildTime; +public record FormBackendVersion( + String version, + ZonedDateTime buildTime +) { } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java index b6412eebb9..143f0fadf4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java @@ -13,6 +13,7 @@ import com.bakdata.conquery.io.cps.CPSTypeIdResolver; import com.bakdata.conquery.io.external.form.ExternalFormBackendApi; import com.bakdata.conquery.io.external.form.ExternalFormMixin; +import com.bakdata.conquery.io.external.form.FormBackendVersion; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.config.auth.AuthenticationClientFilterProvider; @@ -112,10 +113,12 @@ public void initialize(ManagerNode managerNode) { private void updateVersion(ExternalFormBackendApi externalApi) { try { - final String version = externalApi.getVersion(); - final VersionContainer oldVersion = VersionInfo.INSTANCE.setFormBackendVersion(new VersionContainer(getId(), version, null)); - if (!version.equals(oldVersion)) { - log.info("Form Backend '{}' version update: {} -> {}", getId(), oldVersion, version); + final FormBackendVersion versionInfo = externalApi.getVersion(); + final VersionContainer + oldVersion = + VersionInfo.INSTANCE.setFormBackendVersion(new VersionContainer(getId(), versionInfo.version(), versionInfo.buildTime())); + if (!versionInfo.equals(oldVersion)) { + log.info("Form Backend '{}' versionInfo update: {} -> {}", getId(), oldVersion, versionInfo); } } catch (Exception e) { diff --git a/backend/src/main/resources/com/bakdata/conquery/external/openapi-form-backend.yaml b/backend/src/main/resources/com/bakdata/conquery/external/openapi-form-backend.yaml index 25d2e581a8..7b68a722ae 100644 --- a/backend/src/main/resources/com/bakdata/conquery/external/openapi-form-backend.yaml +++ b/backend/src/main/resources/com/bakdata/conquery/external/openapi-form-backend.yaml @@ -321,6 +321,10 @@ components: version: type: string example: "3.2.1-ge966c285" + buildTime: + type: string + format: date-time + example: "2007-08-31T16:47:00+00:00" securitySchemes: ApiKeyAuth: type: apiKey diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java index fb80d58642..e6cacdd25c 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java @@ -7,6 +7,7 @@ import java.io.File; import java.net.URI; import java.nio.file.Path; +import java.time.ZonedDateTime; import java.util.Collections; import java.util.List; @@ -80,7 +81,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { assertThat(frontendConfiguration.versions()) .describedAs("Checking health of form backend") - .contains(new VersionContainer(FORM_BACKEND_ID, "3.2.1-ge966c285", null)); // example value from OpenAPI Spec + .contains(new VersionContainer(FORM_BACKEND_ID, "3.2.1-ge966c285", ZonedDateTime.parse("2007-08-31T16:47:00+00:00"))); // example value from OpenAPI Spec log.info("Send an external form"); final User testUser = support.getTestUser(); diff --git a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java index 95d3e655b9..2ae22e13bf 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/models/SerializationTests.java @@ -4,12 +4,11 @@ import static org.assertj.core.api.Assertions.assertThat; import java.io.IOException; -import java.time.Instant; import java.time.LocalDate; +import java.time.ZonedDateTime; import java.util.Arrays; import java.util.BitSet; import java.util.Collections; -import java.util.Date; import java.util.List; import java.util.Locale; import java.util.Map; @@ -838,9 +837,7 @@ public void arrayObject2Int() throws JSONException, IOException { @Test public void formBackendVersion() throws JSONException, IOException { - final FormBackendVersion version = new FormBackendVersion(); - version.setVersion("3.45.45-g85ut85u43t8"); - version.setBuildTime(Date.from(Instant.ofEpochSecond(1712554025))); //new Date(2024, Calendar.APRIL, 8, 21, 17, 44)); + final FormBackendVersion version = new FormBackendVersion("3.45.45-g85ut85u43t8", ZonedDateTime.parse("2007-12-03T10:15:30+00:00")); SerializationTestUtil.forType(FormBackendVersion.class) .objectMappers(getApiMapper(), getManagerInternalMapper()) From bc03b16ff9d32c556a158f923f65914a8b443ce6 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 9 Apr 2024 14:02:49 +0200 Subject: [PATCH 03/23] update openapi specs Signed-off-by: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> --- .../external/openapi-form-backend.yaml | 2 +- openapi.yaml | 26 ++++++++++++++++--- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/backend/src/main/resources/com/bakdata/conquery/external/openapi-form-backend.yaml b/backend/src/main/resources/com/bakdata/conquery/external/openapi-form-backend.yaml index 7b68a722ae..43d43a5e86 100644 --- a/backend/src/main/resources/com/bakdata/conquery/external/openapi-form-backend.yaml +++ b/backend/src/main/resources/com/bakdata/conquery/external/openapi-form-backend.yaml @@ -1,7 +1,7 @@ openapi: 3.0.0 info: title: Form Backend - version: 1.0.1 + version: 1.0.2 description: | API for generic external form backends in [Conquery](https://github.com/ingef/conquery). diff --git a/openapi.yaml b/openapi.yaml index 4fa06a6fe1..3d228bbc94 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1,6 +1,6 @@ openapi: 3.0.0 info: - version: 0.0.0 + version: "0.0.1" title: Conquery API description: Conquery API license: @@ -381,13 +381,31 @@ components: pattern: "[.,]" decimalScale: type: string - - FrontendConfig: + VersionContainer: type: object properties: + name: + type: string + example: backend + description: Name of the versioned component version: - description: Revision of the backend. type: string + example: "1.2.3-g983u4r84u" + description: Version id of the component + buildTime: + type: string + format: date-time + example: "2007-08-31T16:47:00+00:00" + required: + - name + FrontendConfig: + type: object + properties: + versions: + description: Revisions of backend components such as the backend and form-backends + type: array + items: + $ref: "#/components/schemas/VersionContainer" manualUrl: description: URL to this instances manual if available. type: string From 71db6bbbc1e07b58d86f60f8f424a36bb7a3ac34 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 7 May 2024 13:39:35 +0200 Subject: [PATCH 04/23] fix not respecting money format --- frontend/src/js/preview/tableUtils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/js/preview/tableUtils.ts b/frontend/src/js/preview/tableUtils.ts index a61d73e4e9..2ee65d8888 100644 --- a/frontend/src/js/preview/tableUtils.ts +++ b/frontend/src/js/preview/tableUtils.ts @@ -71,7 +71,7 @@ export function useCustomTableRenderers(queryData: GetQueryResponseDoneT) { } else if (cellType === "MONEY") { return (value) => { if (value && !isNaN(value as unknown as number)) { - return currencyFormatter.format(value as unknown as number); + return currencyFormatter.format((value as unknown as number) / 100); // MONEY is sent as cent } return ""; }; From a6208283623becab30425e7edc55c978cb100297 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 8 May 2024 15:28:43 +0200 Subject: [PATCH 05/23] create a new user from validated jwt Signed-off-by: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> --- .../auth/oidc/JwtPkceVerifyingRealm.java | 36 +++++++++++++++++-- .../conquery/models/index/IndexService.java | 2 +- .../auth/oidc/JwtPkceVerifyingRealmTest.java | 35 ++++++++++++++++++ 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealm.java b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealm.java index e42b311976..67f374b872 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealm.java @@ -6,6 +6,8 @@ import java.util.Optional; import java.util.Set; import java.util.function.Supplier; +import java.util.stream.Collectors; +import java.util.stream.Stream; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.ConqueryAuthenticationInfo; @@ -21,11 +23,11 @@ import lombok.Data; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; +import org.apache.logging.log4j.util.Strings; import org.apache.shiro.authc.AuthenticationException; import org.apache.shiro.authc.AuthenticationToken; import org.apache.shiro.authc.BearerToken; import org.apache.shiro.authc.IncorrectCredentialsException; -import org.apache.shiro.authc.UnknownAccountException; import org.apache.shiro.authc.pam.UnsupportedTokenException; import org.apache.shiro.realm.AuthenticatingRealm; import org.keycloak.TokenVerifier; @@ -152,13 +154,41 @@ public ConqueryAuthenticationInfo doGetAuthenticationInfo(AuthenticationToken to user = storage.getUser(userId); if (user != null) { - log.trace("Successfully mapped subject {} using user id {}", subject, userId); + log.trace("Successfully mapped subject {} using user id {}", accessToken.getSubject(), userId); handleRoleClaims(accessToken, user); return new ConqueryAuthenticationInfo(user, token, this, true, idpConfiguration.logoutEndpoint()); } } - throw new UnknownAccountException("The user id was unknown: " + subject); + // Create a new user if none could be found + final User newUser = createUser(accessToken); + + return new ConqueryAuthenticationInfo(newUser, token, this, true, idpConfiguration.logoutEndpoint()); + } + + /** + * Creates a new user from values in the access token + * + * @param accessToken + * @return + */ + private User createUser(AccessToken accessToken) { + String + userLabel = + Stream.of(accessToken.getGivenName(), accessToken.getMiddleName(), accessToken.getFamilyName()) + .filter(Strings::isNotBlank) + .collect(Collectors.joining(" ")); + if (Strings.isBlank(userLabel)) { + userLabel = accessToken.getPreferredUsername(); + } + + final User user = new User(accessToken.getSubject(), userLabel, storage); + user.updateStorage(); + handleRoleClaims(accessToken, user); + + log.info("Created a new user from a valid JWT: {}", user); + + return user; } private void handleRoleClaims(AccessToken accessToken, User user) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java b/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java index 518266bca3..4da2c358e2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java @@ -2,7 +2,6 @@ import java.io.IOException; import java.io.InputStream; -import java.time.Duration; import java.util.List; import java.util.Map; import java.util.Set; @@ -64,6 +63,7 @@ public Index load(@NotNull IndexKey key) throws Exception { // Iterate records for (Record row : records) { + // There can be multiple templates and multiple external values, hence the right side is a map final Pair> pair = computeInternalExternal(key, csvParser, row); if (pair == null) { continue; diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealmTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealmTest.java index 1c39b1ac37..c3f2e379ec 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealmTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealmTest.java @@ -30,6 +30,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.keycloak.common.VerificationException; +import org.keycloak.representations.IDToken; @TestInstance(TestInstance.Lifecycle.PER_CLASS) class JwtPkceVerifyingRealmTest { @@ -125,6 +126,40 @@ void verifyTokenAndAddRole() { assertThat(expected.getRoles()).contains(role.getId()); } + @Test + void verifyTokenAndAddRoleNewUser() { + + // Setup the expected user id + User expected = new User("Test", "New Tester", STORAGE); + Role role = new Role("admin", "admin", STORAGE); + + STORAGE.updateRole(role); + + Date issueDate = new Date(); + Date expDate = DateUtils.addMinutes(issueDate, 1); + String token = JWT.create() + .withClaim(IDToken.GIVEN_NAME, "New") + .withClaim(IDToken.FAMILY_NAME, "Tester") + .withIssuer(HTTP_REALM_URL) + .withAudience(AUDIENCE) + .withSubject(expected.getName()) + .withIssuedAt(issueDate) + .withExpiresAt(expDate) + .withClaim("groups", "conquery") + .withClaim("resource_access", Map.of(AUDIENCE, Map.of("roles", List.of("admin", "unknown")))) // See structure of AccessToken.Access + .withIssuedAt(issueDate) + .withExpiresAt(expDate) + .withKeyId(KEY_ID) + .withJWTId(UUID.randomUUID().toString()) + .sign(Algorithm.RSA256(PUBLIC_KEY, PRIVATE_KEY)); + + BearerToken accessToken = new BearerToken(token); + + assertThat(REALM.doGetAuthenticationInfo(accessToken).getPrincipals().getPrimaryPrincipal().getId()).isEqualTo(expected.getId()); + assertThat(STORAGE.getUser(expected.getId()).getRoles()).contains(role.getId()); + assertThat(STORAGE.getUser(expected.getId()).getLabel()).isEqualTo(expected.getLabel()); + } + @Test void verifyTokenInLeeway() { From ee91faccfc08f0cf1b6f1b65d19f484c8fbede53 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 8 May 2024 15:35:07 +0200 Subject: [PATCH 06/23] fix storage cleanup Signed-off-by: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> --- .../models/auth/oidc/JwtPkceVerifyingRealmTest.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealmTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealmTest.java index c3f2e379ec..8e093e2a25 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealmTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealmTest.java @@ -27,6 +27,7 @@ import org.apache.shiro.authc.BearerToken; import org.apache.shiro.authc.pam.UnsupportedTokenException; import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.keycloak.common.VerificationException; @@ -67,6 +68,11 @@ static void setup() throws NoSuchAlgorithmException { ); } + @BeforeEach + void cleanUpBefore() { + STORAGE.clear(); + } + @Test void verifyToken() { @@ -130,7 +136,7 @@ void verifyTokenAndAddRole() { void verifyTokenAndAddRoleNewUser() { // Setup the expected user id - User expected = new User("Test", "New Tester", STORAGE); + User expected = new User("new_user", "New User", STORAGE); Role role = new Role("admin", "admin", STORAGE); STORAGE.updateRole(role); @@ -139,7 +145,7 @@ void verifyTokenAndAddRoleNewUser() { Date expDate = DateUtils.addMinutes(issueDate, 1); String token = JWT.create() .withClaim(IDToken.GIVEN_NAME, "New") - .withClaim(IDToken.FAMILY_NAME, "Tester") + .withClaim(IDToken.FAMILY_NAME, "User") .withIssuer(HTTP_REALM_URL) .withAudience(AUDIENCE) .withSubject(expected.getName()) From f42d8a5cf05a3cba46eb58027c0578405223aaf0 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 8 May 2024 17:20:35 +0200 Subject: [PATCH 07/23] add better username fallbacks and validate created user Signed-off-by: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> --- .../auth/oidc/JwtPkceVerifyingRealm.java | 33 ++++++++++--------- .../auth/JwtPkceVerifyingRealmFactory.java | 3 +- .../auth/oidc/JwtPkceVerifyingRealmTest.java | 4 ++- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealm.java b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealm.java index 67f374b872..1c1aa59fe7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealm.java @@ -6,8 +6,6 @@ import java.util.Optional; import java.util.Set; import java.util.function.Supplier; -import java.util.stream.Collectors; -import java.util.stream.Stream; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.ConqueryAuthenticationInfo; @@ -16,14 +14,16 @@ import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.util.SkippingCredentialsMatcher; import com.bakdata.conquery.models.config.auth.JwtPkceVerifyingRealmFactory; +import com.bakdata.conquery.models.exceptions.ValidatorHelper; import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; import com.bakdata.conquery.models.identifiable.ids.specific.UserId; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import jakarta.validation.Validator; import lombok.Data; import lombok.NonNull; import lombok.extern.slf4j.Slf4j; -import org.apache.logging.log4j.util.Strings; +import org.apache.commons.lang3.ObjectUtils; import org.apache.shiro.authc.AuthenticationException; import org.apache.shiro.authc.AuthenticationToken; import org.apache.shiro.authc.BearerToken; @@ -51,6 +51,7 @@ public class JwtPkceVerifyingRealm extends AuthenticatingRealm implements Conque private final List alternativeIdClaims; private final ActiveWithLeewayVerifier activeVerifier; private final MetaStorage storage; + private final Validator validator; /** * Used in handleRoleClaims as size-limited set, with LRU characteristics. @@ -63,12 +64,21 @@ public class JwtPkceVerifyingRealm extends AuthenticatingRealm implements Conque Supplier> idpConfigurationSupplier; - public JwtPkceVerifyingRealm(@NonNull Supplier> idpConfigurationSupplier, @NonNull String allowedAudience, List> additionalTokenChecks, List alternativeIdClaims, MetaStorage storage, int tokenLeeway) { + public JwtPkceVerifyingRealm( + @NonNull Supplier> idpConfigurationSupplier, + @NonNull String allowedAudience, + List> additionalTokenChecks, + List alternativeIdClaims, + MetaStorage storage, + int tokenLeeway, + Validator validator + ) { this.storage = storage; this.idpConfigurationSupplier = idpConfigurationSupplier; this.allowedAudience = new String[]{allowedAudience}; this.alternativeIdClaims = alternativeIdClaims; this.tokenChecks = additionalTokenChecks.toArray(TokenVerifier.Predicate[]::new); + this.validator = validator; setCredentialsMatcher(SkippingCredentialsMatcher.INSTANCE); setAuthenticationTokenClass(TOKEN_CLASS); activeVerifier = new ActiveWithLeewayVerifier(tokenLeeway); @@ -168,21 +178,14 @@ public ConqueryAuthenticationInfo doGetAuthenticationInfo(AuthenticationToken to /** * Creates a new user from values in the access token - * - * @param accessToken - * @return */ private User createUser(AccessToken accessToken) { - String - userLabel = - Stream.of(accessToken.getGivenName(), accessToken.getMiddleName(), accessToken.getFamilyName()) - .filter(Strings::isNotBlank) - .collect(Collectors.joining(" ")); - if (Strings.isBlank(userLabel)) { - userLabel = accessToken.getPreferredUsername(); - } + String userLabel = ObjectUtils.firstNonNull(accessToken.getName(), accessToken.getPreferredUsername(), accessToken.getSubject()); final User user = new User(accessToken.getSubject(), userLabel, storage); + + ValidatorHelper.failOnError(log, getValidator().validate(user)); + user.updateStorage(); handleRoleClaims(accessToken, user); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/auth/JwtPkceVerifyingRealmFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/auth/JwtPkceVerifyingRealmFactory.java index 82fac62d01..ad01456c10 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/auth/JwtPkceVerifyingRealmFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/auth/JwtPkceVerifyingRealmFactory.java @@ -45,7 +45,6 @@ import io.dropwizard.core.setup.Environment; import io.dropwizard.jersey.DropwizardResourceConfig; import io.dropwizard.validation.ValidationMethod; -import jakarta.inject.Named; import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotEmpty; import jakarta.validation.constraints.NotNull; @@ -175,7 +174,7 @@ public ConqueryAuthenticationRealm createRealm(Environment environment, Conquery RedirectingAuthFilter.registerAuthAttemptChecker(jerseyAdminUi, this::checkAndRedeemAuthzCode, "jwt-authz-redeemer"); RedirectingAuthFilter.registerAuthAttemptChecker(jerseyAdminUi, this::checkAndRedeemRefreshToken, "jwt-refresh-redeemer"); - return new JwtPkceVerifyingRealm(idpConfigurationSupplier, client, additionalVerifiers, alternativeIdClaims, authorizationController.getStorage(), tokenLeeway); + return new JwtPkceVerifyingRealm(idpConfigurationSupplier, client, additionalVerifiers, alternativeIdClaims, authorizationController.getStorage(), tokenLeeway, environment.getValidator()); } @Data diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealmTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealmTest.java index 8e093e2a25..41e57bcbc5 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealmTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealmTest.java @@ -23,6 +23,7 @@ import com.bakdata.conquery.models.config.auth.JwtPkceVerifyingRealmFactory; import com.bakdata.conquery.models.identifiable.ids.specific.UserId; import com.bakdata.conquery.util.NonPersistentStoreFactory; +import io.dropwizard.validation.BaseValidator; import org.apache.commons.lang3.time.DateUtils; import org.apache.shiro.authc.BearerToken; import org.apache.shiro.authc.pam.UnsupportedTokenException; @@ -64,7 +65,8 @@ static void setup() throws NoSuchAlgorithmException { List.of(JwtPkceVerifyingRealmFactory.ScriptedTokenChecker.create("t.getOtherClaims().get(\"groups\").equals(\"conquery\")")), List.of(ALTERNATIVE_ID_CLAIM), STORAGE, - TOKEN_LEEWAY + TOKEN_LEEWAY, + BaseValidator.newValidator() ); } From 34a87ed5ad0e82a0a952dd6ffd4018b4008d1ddd Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 8 May 2024 17:27:43 +0200 Subject: [PATCH 08/23] fix username claim in test Signed-off-by: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> --- .../conquery/models/auth/oidc/JwtPkceVerifyingRealmTest.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealmTest.java b/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealmTest.java index 41e57bcbc5..a86999d04a 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealmTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/auth/oidc/JwtPkceVerifyingRealmTest.java @@ -146,8 +146,7 @@ void verifyTokenAndAddRoleNewUser() { Date issueDate = new Date(); Date expDate = DateUtils.addMinutes(issueDate, 1); String token = JWT.create() - .withClaim(IDToken.GIVEN_NAME, "New") - .withClaim(IDToken.FAMILY_NAME, "User") + .withClaim(IDToken.NAME, "New User") .withIssuer(HTTP_REALM_URL) .withAudience(AUDIENCE) .withSubject(expected.getName()) From b71551b525a9a5119b16af17bc4fe604e87b72e5 Mon Sep 17 00:00:00 2001 From: Jonas Arnhold Date: Mon, 13 May 2024 10:25:18 +0200 Subject: [PATCH 09/23] Fix feature conversion of AND/OR nodes (#3431) --- .../conversion/cqelement/CQAndConverter.java | 29 ++++++--- .../cqelement/CQNegationConverter.java | 2 +- .../conversion/cqelement/CQOrConverter.java | 29 ++++++--- .../cqelement/ConversionContext.java | 8 +++ .../sql/conversion/model/NameGenerator.java | 4 -- .../sql/conversion/model/QueryStep.java | 7 +++ .../model/aggregator/ExistsSqlAggregator.java | 5 +- .../model/select/ExistsSqlSelect.java | 22 +++++++ .../query/AbsoluteFormQueryConverter.java | 2 +- .../query/FormConversionHelper.java | 2 +- .../query/TableExportQueryConverter.java | 2 +- .../MULTIPLE_FEATURES_AND_CONNECTOR.test.json | 63 +++++++++++++++++++ .../MULTIPLE_FEATURES_AND_CONNECTOR.json | 63 +++++++++++++++++++ .../expected_and_connector.csv | 7 +++ 14 files changed, 219 insertions(+), 26 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSqlSelect.java create mode 100644 backend/src/test/resources/tests/form/MULTIPLE_FEATURES_AND_CONNECTOR.test.json create mode 100644 backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/MULTIPLE_FEATURES_AND_CONNECTOR.json create mode 100644 backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/expected_and_connector.csv diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java index fbc47a811b..1df7f0d738 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java @@ -5,6 +5,7 @@ import com.bakdata.conquery.sql.conversion.model.ConqueryJoinType; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; +import com.bakdata.conquery.sql.conversion.model.select.ExistsSqlSelect; public class CQAndConverter implements NodeConverter { @@ -15,16 +16,28 @@ public Class getConversionClass() { @Override public ConversionContext convert(CQAnd andNode, ConversionContext context) { + + QueryStep joined; if (andNode.getChildren().size() == 1) { - return context.getNodeConversions().convert(andNode.getChildren().get(0), context); + ConversionContext withConvertedChild = context.getNodeConversions().convert(andNode.getChildren().get(0), context); + joined = withConvertedChild.getLastConvertedStep(); + } + else { + joined = QueryStepJoiner.joinChildren( + andNode.getChildren(), + context, + ConqueryJoinType.INNER_JOIN, + andNode.getDateAction() + ); } - QueryStep joined = QueryStepJoiner.joinChildren( - andNode.getChildren(), - context, - ConqueryJoinType.INNER_JOIN, - andNode.getDateAction() - ); - return context.withQueryStep(joined); + + if (andNode.getCreateExists().isEmpty()) { + return context.withQueryStep(joined); + } + + String joinedNodeName = joined.getCteName(); + ExistsSqlSelect existsSqlSelect = new ExistsSqlSelect(joinedNodeName); + return context.withQueryStep(joined.addSqlSelect(existsSqlSelect)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java index 8206694d45..e184f5894b 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java @@ -25,7 +25,7 @@ public ConversionContext convert(CQNegation negationNode, ConversionContext cont converted.getQuerySteps().size() == 1, "As we convert only 1 child CQElement, their should be only a single query step." ); - QueryStep queryStep = converted.getQuerySteps().get(0); + QueryStep queryStep = converted.getLastConvertedStep(); if (negationNode.getDateAction() != DateAggregationAction.NEGATE) { QueryStep withBlockedValidityDate = queryStep.toBuilder() diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java index 46e14a522c..9f889e6e86 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java @@ -5,6 +5,7 @@ import com.bakdata.conquery.sql.conversion.model.ConqueryJoinType; import com.bakdata.conquery.sql.conversion.model.QueryStep; import com.bakdata.conquery.sql.conversion.model.QueryStepJoiner; +import com.bakdata.conquery.sql.conversion.model.select.ExistsSqlSelect; public class CQOrConverter implements NodeConverter { @@ -15,16 +16,28 @@ public Class getConversionClass() { @Override public ConversionContext convert(CQOr orNode, ConversionContext context) { + + QueryStep joined; if (orNode.getChildren().size() == 1) { - return context.getNodeConversions().convert(orNode.getChildren().get(0), context); + ConversionContext withConvertedChild = context.getNodeConversions().convert(orNode.getChildren().get(0), context); + joined = withConvertedChild.getLastConvertedStep(); + } + else { + joined = QueryStepJoiner.joinChildren( + orNode.getChildren(), + context, + ConqueryJoinType.OUTER_JOIN, + orNode.getDateAction() + ); } - QueryStep joined = QueryStepJoiner.joinChildren( - orNode.getChildren(), - context, - ConqueryJoinType.OUTER_JOIN, - orNode.getDateAction() - ); - return context.withQueryStep(joined); + + if (orNode.getCreateExists().isEmpty()) { + return context.withQueryStep(joined); + } + + String joinedNodeName = joined.getCteName(); + ExistsSqlSelect existsSqlSelect = new ExistsSqlSelect(joinedNodeName); + return context.withQueryStep(joined.addSqlSelect(existsSqlSelect)); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java index aa4bf1041f..ad10c04235 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java @@ -84,4 +84,12 @@ public ConversionContext createChildContext() { public ConversionContext getConversionContext() { return this; } + + /** + * Get the last query {@link QueryStep} that has been added to this context query steps. + */ + public QueryStep getLastConvertedStep() { + return this.querySteps.get(this.querySteps.size() - 1); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java index c4865b38ef..af6923cbcf 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/NameGenerator.java @@ -47,10 +47,6 @@ public String cteStepName(CteStep cteStep, String nodeLabel) { return ensureValidLength(cteStep.cteName(nodeLabel)); } - public String cteStepName(String cteStep, String nodeLabel) { - return ensureValidLength("%s-%s".formatted(nodeLabel, cteStep)); - } - public String selectName(Labeled selectOrFilter) { int selectCount = this.selectCountMap.merge(selectOrFilter.getName(), 1, Integer::sum); String name = lowerAndReplaceWhitespace(selectOrFilter.getName()); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java index bb55739819..c6ea7da616 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/QueryStep.java @@ -3,6 +3,8 @@ import java.util.Collections; import java.util.List; +import com.bakdata.conquery.sql.conversion.model.select.ExistsSqlSelect; +import com.bakdata.conquery.sql.conversion.model.select.SqlSelect; import lombok.Builder; import lombok.Singular; import lombok.Value; @@ -54,6 +56,11 @@ public static TableLike toTableLike(String fromTableName) { return DSL.table(DSL.name(fromTableName)); } + public QueryStep addSqlSelect(SqlSelect sqlSelect) { + Selects withAdditionalSelect = this.selects.toBuilder().sqlSelect(sqlSelect).build(); + return this.toBuilder().selects(withAdditionalSelect).build(); + } + /** * @return All selects re-mapped to a qualifier, which is the cteName of this QueryStep. */ diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/ExistsSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/ExistsSqlAggregator.java index b6a9e88087..d377fb7ecf 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/ExistsSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/ExistsSqlAggregator.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.sql.conversion.model.aggregator; import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.ExistsSelect; +import com.bakdata.conquery.sql.conversion.model.select.ExistsSqlSelect; import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; import com.bakdata.conquery.sql.conversion.model.select.FieldWrapper; @@ -16,9 +17,9 @@ public class ExistsSqlAggregator implements SqlAggregator { WhereClauses whereClauses; private ExistsSqlAggregator(String alias) { - FieldWrapper existsSelect = new UniversalSqlSelect<>(DSL.field("1", Integer.class).as(alias)); + ExistsSqlSelect existsSqlSelect = new ExistsSqlSelect(alias); this.sqlSelects = SqlSelects.builder() - .finalSelect(existsSelect) + .finalSelect(existsSqlSelect) .build(); this.whereClauses = WhereClauses.empty(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSqlSelect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSqlSelect.java new file mode 100644 index 0000000000..f8bdbc6a3b --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/select/ExistsSqlSelect.java @@ -0,0 +1,22 @@ +package com.bakdata.conquery.sql.conversion.model.select; + +import java.util.Collections; +import java.util.List; + +import org.jooq.Field; +import org.jooq.impl.DSL; + +public class ExistsSqlSelect extends UniversalSqlSelect { + + private static final Field EXISTS = DSL.val(1); + + public ExistsSqlSelect(String alias) { + super(EXISTS.as(alias)); + } + + @Override + public List requiredColumns() { + return Collections.emptyList(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java index 2b0e2ecdd1..189a1748b6 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/AbsoluteFormQueryConverter.java @@ -53,7 +53,7 @@ private static QueryStep convertPrerequisite(AbsoluteFormQuery absoluteForm, Con ConversionContext withConvertedPrerequisite = context.getNodeConversions().convert(absoluteForm.getQuery(), context); Preconditions.checkArgument(withConvertedPrerequisite.getQuerySteps().size() == 1, "Base query conversion should produce exactly 1 QueryStep"); - QueryStep convertedPrerequisite = withConvertedPrerequisite.getQuerySteps().get(0); + QueryStep convertedPrerequisite = withConvertedPrerequisite.getLastConvertedStep(); ColumnDateRange bounds = context.getSqlDialect() .getFunctionProvider() diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java index 39e3286c44..af31541d95 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/FormConversionHelper.java @@ -46,7 +46,7 @@ public QueryStep convertPrerequisite(Query query, ConversionContext context) { ConversionContext withConvertedPrerequisite = context.getNodeConversions().convert(query, context); Preconditions.checkArgument(withConvertedPrerequisite.getQuerySteps().size() == 1, "Base query conversion should produce exactly 1 QueryStep"); - QueryStep convertedPrerequisite = withConvertedPrerequisite.getQuerySteps().get(0); + QueryStep convertedPrerequisite = withConvertedPrerequisite.getLastConvertedStep(); Selects prerequisiteSelects = convertedPrerequisite.getQualifiedSelects(); // we keep the primary column and the validity date diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java index b2a7b642f2..a576737695 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/TableExportQueryConverter.java @@ -82,7 +82,7 @@ private static QueryStep convertPrerequisite(TableExportQuery exportQuery, Conve ConversionContext withConvertedPrerequisite = context.getNodeConversions().convert(exportQuery.getQuery(), context); Preconditions.checkArgument(withConvertedPrerequisite.getQuerySteps().size() == 1, "Base query conversion should produce exactly 1 QueryStep"); - QueryStep convertedPrerequisite = withConvertedPrerequisite.getQuerySteps().get(0); + QueryStep convertedPrerequisite = withConvertedPrerequisite.getLastConvertedStep(); Selects prerequisiteSelects = convertedPrerequisite.getQualifiedSelects(); Selects selects = Selects.builder() diff --git a/backend/src/test/resources/tests/form/MULTIPLE_FEATURES_AND_CONNECTOR.test.json b/backend/src/test/resources/tests/form/MULTIPLE_FEATURES_AND_CONNECTOR.test.json new file mode 100644 index 0000000000..d949c3e938 --- /dev/null +++ b/backend/src/test/resources/tests/form/MULTIPLE_FEATURES_AND_CONNECTOR.test.json @@ -0,0 +1,63 @@ +{ + "type": "FORM_TEST", + "label": "ABS-EXPORT-FORM with multiple features and AND connector", + "expectedCsv": { + "results": "tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/expected_and_connector.csv" + }, + "form": { + "type": "EXPORT_FORM", + "queryGroup": "00000000-0000-0000-0000-000000000001", + "resolution": "QUARTERS", + "alsoCreateCoarserSubdivisions": true, + "features": [ + { + "type": "AND", + "children": [ + { + "ids": [ + "alter" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "alter.alter", + "filters": [] + } + ] + }, + { + "ids": [ + "geschlecht" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "geschlecht.geschlecht", + "filters": [] + } + ] + } + ] + } + ], + "timeMode": { + "value": "ABSOLUTE", + "dateRange": { + "min": "2012-01-16", + "max": "2012-12-17" + } + } + }, + "concepts": [ + "/tests/form/shared/alter.concept.json", + "/tests/form/shared/geschlecht.concept.json" + ], + "content": { + "tables": [ + "/tests/form/shared/vers_stamm.table.json" + ], + "previousQueryResults": [ + "tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/query_results_1.csv" + ] + } +} diff --git a/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/MULTIPLE_FEATURES_AND_CONNECTOR.json b/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/MULTIPLE_FEATURES_AND_CONNECTOR.json new file mode 100644 index 0000000000..ca5f139e9f --- /dev/null +++ b/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/MULTIPLE_FEATURES_AND_CONNECTOR.json @@ -0,0 +1,63 @@ +{ + "type": "FORM_TEST", + "label": "ABS-EXPORT-FORM with multiple features and AND connector", + "expectedCsv": { + "results": "tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/expected_and_connector.csv" + }, + "form": { + "type": "EXPORT_FORM", + "queryGroup": "00000000-0000-0000-0000-000000000001", + "resolution": "QUARTERS", + "alsoCreateCoarserSubdivisions": true, + "features": [ + { + "type": "AND", + "children": [ + { + "ids": [ + "alter" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "alter.alter", + "filters": [] + } + ] + }, + { + "ids": [ + "geschlecht" + ], + "type": "CONCEPT", + "tables": [ + { + "id": "geschlecht.geschlecht", + "filters": [] + } + ] + } + ] + } + ], + "timeMode": { + "value": "ABSOLUTE", + "dateRange": { + "min": "2012-01-16", + "max": "2012-12-17" + } + } + }, + "concepts": [ + "/shared/alter.concept.json", + "/shared/geschlecht.concept.json" + ], + "content": { + "tables": [ + "/shared/vers_stamm.table.json" + ], + "previousQueryResults": [ + "tests/form/EXPORT_FORM/ABSOLUT/SIMPLE/query_results_1.csv" + ] + } +} diff --git a/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/expected_and_connector.csv b/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/expected_and_connector.csv new file mode 100644 index 0000000000..b6502df1ef --- /dev/null +++ b/backend/src/test/resources/tests/sql/form/ABSOLUT/MULTIPLE_FEATURES/expected_and_connector.csv @@ -0,0 +1,7 @@ +result,resolution,index,date_range,Alter and Geschlecht +1,complete,,2012-01-16/2012-12-17,1 +1,year,1,2012-01-16/2012-12-17,1 +1,quarter,1,2012-01-16/2012-03-31,1 +1,quarter,2,2012-04-01/2012-06-30,1 +1,quarter,3,2012-07-01/2012-09-30,1 +1,quarter,4,2012-10-01/2012-12-17,1 From a4456bc7bd84d7a96436f6e2c870393f902c3b4b Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 13 May 2024 11:28:21 +0200 Subject: [PATCH 10/23] Run npm audit fix --- frontend/package-lock.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 8e86b137b3..4e5a1879be 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -8167,9 +8167,10 @@ "license": "MIT" }, "node_modules/ejs": { - "version": "3.1.9", + "version": "3.1.10", + "resolved": "https://registry.npmjs.org/ejs/-/ejs-3.1.10.tgz", + "integrity": "sha512-UeJmFfOrAQS8OJWPZ4qtgHyWExa088/MtK5UEyoJGFH67cDEXkZSviOiKRCZ4Xij0zxI3JECgYs3oKx+AizQBA==", "dev": true, - "license": "Apache-2.0", "dependencies": { "jake": "^10.8.5" }, From 9de698aeb67495ae9274bb822436b0becfdbd132 Mon Sep 17 00:00:00 2001 From: Jonas Arnhold Date: Mon, 13 May 2024 12:32:51 +0200 Subject: [PATCH 11/23] Fix duration sum for infinity dates (#3430) --- .../EventDurationSumSqlAggregator.java | 37 ++++++++++++++----- .../model/aggregator/SqlAggregator.java | 2 + .../sql/selects/sum/duration_sum/content.csv | 3 ++ .../sql/selects/sum/duration_sum/expected.csv | 1 + 4 files changed, 34 insertions(+), 9 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/EventDurationSumSqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/EventDurationSumSqlAggregator.java index ea363e3a39..62409f4850 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/EventDurationSumSqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/EventDurationSumSqlAggregator.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.sql.conversion.model.aggregator; import java.math.BigDecimal; +import java.sql.Date; import java.time.temporal.ChronoUnit; import com.bakdata.conquery.models.datasets.concepts.select.concept.specific.EventDurationSumSelect; @@ -14,6 +15,8 @@ import com.bakdata.conquery.sql.conversion.model.select.SelectContext; import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; import lombok.Value; +import org.jooq.Condition; +import org.jooq.Field; import org.jooq.impl.DSL; @Value @@ -28,29 +31,45 @@ private EventDurationSumSqlAggregator( ConceptConversionTables tables, SqlFunctionProvider functionProvider ) { - ColumnDateRange qualified = validityDate.qualify(tables.getPredecessor(ConceptCteStep.INTERVAL_PACKING_SELECTS)); - ColumnDateRange asDualColumn = functionProvider.toDualColumn(qualified); - FieldWrapper durationSum = new FieldWrapper<>( - DSL.sum(functionProvider.dateDistance(ChronoUnit.DAYS, asDualColumn.getStart(), asDualColumn.getEnd())) - .as(alias) - ); + Field durationSum = DSL.sum( + DSL.when(containsInfinityDate(validityDate, functionProvider), DSL.val(null, Integer.class)) + .otherwise(functionProvider.dateDistance(ChronoUnit.DAYS, validityDate.getStart(), validityDate.getEnd())) + ) + .as(alias); - ExtractingSqlSelect finalSelect = durationSum.qualify(tables.getLastPredecessor()); + FieldWrapper durationSumWrapper = new FieldWrapper<>(durationSum); + ExtractingSqlSelect finalSelect = durationSumWrapper.qualify(tables.getLastPredecessor()); this.sqlSelects = SqlSelects.builder() - .intervalPackingSelect(durationSum) + .intervalPackingSelect(durationSumWrapper) .finalSelect(finalSelect) .build(); this.whereClauses = WhereClauses.builder().build(); } public static EventDurationSumSqlAggregator create(EventDurationSumSelect eventDurationSumSelect, SelectContext selectContext) { + + ColumnDateRange validityDate = selectContext.getValidityDate().orElseThrow( + () -> new IllegalStateException("Can't convert a EventDurationSum select without a validity date") + ); + return new EventDurationSumSqlAggregator( selectContext.getNameGenerator().selectName(eventDurationSumSelect), - selectContext.getValidityDate().orElseThrow(() -> new IllegalStateException("Can't convert a EventDurationSum select without a validity date")), + prepareValidityDate(validityDate, selectContext), selectContext.getTables(), selectContext.getConversionContext().getSqlDialect().getFunctionProvider() ); } + private static ColumnDateRange prepareValidityDate(ColumnDateRange validityDate, SelectContext selectContext) { + ColumnDateRange qualified = validityDate.qualify(selectContext.getTables().getPredecessor(ConceptCteStep.INTERVAL_PACKING_SELECTS)); + return selectContext.getSqlDialect().getFunctionProvider().toDualColumn(qualified); + } + + private static Condition containsInfinityDate(ColumnDateRange validityDate, SqlFunctionProvider functionProvider) { + Field negativeInfinity = functionProvider.toDateField(functionProvider.getMinDateExpression()); + Field positiveInfinity = functionProvider.toDateField(functionProvider.getMaxDateExpression()); + return validityDate.getStart().eq(negativeInfinity).or(validityDate.getEnd().eq(positiveInfinity)); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SqlAggregator.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SqlAggregator.java index f79ed962a5..81fb9d6c67 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SqlAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/model/aggregator/SqlAggregator.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.sql.conversion.model.aggregator; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.conversion.model.ColumnDateRange; import com.bakdata.conquery.sql.conversion.model.filter.SqlFilters; import com.bakdata.conquery.sql.conversion.model.filter.WhereClauses; import com.bakdata.conquery.sql.conversion.model.select.SqlSelects; diff --git a/backend/src/test/resources/tests/sql/selects/sum/duration_sum/content.csv b/backend/src/test/resources/tests/sql/selects/sum/duration_sum/content.csv index dda9eaa270..f70ad54470 100644 --- a/backend/src/test/resources/tests/sql/selects/sum/duration_sum/content.csv +++ b/backend/src/test/resources/tests/sql/selects/sum/duration_sum/content.csv @@ -20,3 +20,6 @@ pid,datum,column 5,2012-01-03,"B" 5,2012-01-04,"B" 5,2012-01-05,"B" + +6,,"A" +6,,"B" diff --git a/backend/src/test/resources/tests/sql/selects/sum/duration_sum/expected.csv b/backend/src/test/resources/tests/sql/selects/sum/duration_sum/expected.csv index 403d444e82..9d58b233f6 100644 --- a/backend/src/test/resources/tests/sql/selects/sum/duration_sum/expected.csv +++ b/backend/src/test/resources/tests/sql/selects/sum/duration_sum/expected.csv @@ -3,3 +3,4 @@ result,dates,tree a event_duration_sum 3,{2012-01-01/2012-01-02},1 4,{2012-01-01/2012-01-02},2 5,{2012-01-01/2012-01-05},4 +6,{-∞/+∞}, From 6699171e7f92ded1e5e0a203ae7a4862e5203dd0 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 13 May 2024 12:38:22 +0200 Subject: [PATCH 12/23] Adapt frontend to new versions api --- frontend/src/js/api/types.ts | 7 ++- frontend/src/js/app/About.tsx | 71 +++++++++++++----------------- frontend/src/js/startup/reducer.ts | 3 +- 3 files changed, 37 insertions(+), 44 deletions(-) diff --git a/frontend/src/js/api/types.ts b/frontend/src/js/api/types.ts index a37f3061dd..12503f728b 100644 --- a/frontend/src/js/api/types.ts +++ b/frontend/src/js/api/types.ts @@ -292,8 +292,11 @@ export interface QueryUploadConfigT { } export interface GetFrontendConfigResponseT { - version: string; - formBackendVersions: Record; + versions: { + name: string; + version: string | null; // probably shouldn't be nullable at all + buildTime: string | null; // should be optional, not null + }[]; currency: CurrencyConfigT; queryUpload: QueryUploadConfigT; manualUrl?: string; diff --git a/frontend/src/js/app/About.tsx b/frontend/src/js/app/About.tsx index 4e4a7ae2c3..def9194095 100644 --- a/frontend/src/js/app/About.tsx +++ b/frontend/src/js/app/About.tsx @@ -1,6 +1,6 @@ -import styled from "@emotion/styled"; import { faCopy } from "@fortawesome/free-regular-svg-icons"; import { + Fragment, ReactNode, createContext, memo, @@ -14,6 +14,7 @@ import { useSelector } from "react-redux"; import IconButton from "../button/IconButton"; import Modal from "../modal/Modal"; +import { GetFrontendConfigResponseT } from "../api/types"; import { StateT } from "./reducers"; const initialState = { @@ -37,39 +38,18 @@ export const useAbout = () => { return useContext(Context); }; -const Grid = styled("div")` - display: grid; - grid-template-columns: auto 1fr; - margin-bottom: 20px; - gap: 5px 20px; -`; - -const Version = styled("code")` - font-size: 16px; - font-weight: bold; -`; - const useVersion = () => { - const backendGitDescribe = useSelector( - (state) => state.startup.config.version, - ); - - // TODO: GET THIS TO WORK WHEN BUILDING INSIDE A DOCKER CONTAINER - // const frontendGitCommit = preval` - // const { execSync } = require('child_process'); - // module.exports = execSync('git rev-parse --short HEAD').toString(); - // `; - // const frontendGitTag = preval` - // const { execSync } = require('child_process'); - // module.exports = execSync('git describe --all --exact-match \`git rev-parse HEAD\`').toString(); - // `; + const backendVersions = useSelector< + StateT, + GetFrontendConfigResponseT["versions"] + >((state) => state.startup.config.versions); // THIS IS GETTING STATICALLY REPLACED USING "VITE DEFINE" const frontendTimestamp = `__BUILD_TIMESTAMP__`.replace(/"/g, ""); const frontendGitDescribe = `__BUILD_GIT_DESCRIBE__`.replace(/"/g, ""); return { - backendGitDescribe, + backendVersions, frontendTimestamp, frontendGitDescribe, }; @@ -78,12 +58,14 @@ const useVersion = () => { export const About = memo(() => { const { isOpen, setOpen } = useAbout(); const toggleOpen = useCallback(() => setOpen((open) => !open), [setOpen]); - const { backendGitDescribe, frontendTimestamp, frontendGitDescribe } = + const { backendVersions, frontendTimestamp, frontendGitDescribe } = useVersion(); const copyVersionToClipboard = () => { navigator.clipboard.writeText( - `BE: ${backendGitDescribe} FE: ${frontendGitDescribe}`, + `${backendVersions + .map(({ name, version }) => `${name}: ${version}`) + .join(" ")} Frontend: ${frontendGitDescribe}`, ); setOpen(false); }; @@ -94,17 +76,26 @@ export const About = memo(() => { return ( setOpen(false)}> - -
Backend
- {backendGitDescribe} -
Frontend
- - {frontendGitDescribe} – {frontendTimestamp} - - - - Copy version info - +
+
+ {backendVersions.map((version) => ( + +
{version.name}
+ + {version.version || "-"} + {version.buildTime && ` – ${version.buildTime}`} + + + ))} +
Frontend
+ + {frontendGitDescribe} – {frontendTimestamp} + +
+ + Copy version info + +
); }); diff --git a/frontend/src/js/startup/reducer.ts b/frontend/src/js/startup/reducer.ts index bd7d13f2ef..fad6374b0b 100644 --- a/frontend/src/js/startup/reducer.ts +++ b/frontend/src/js/startup/reducer.ts @@ -15,11 +15,10 @@ const initialState: StartupStateT = { loading: false, error: null, config: { - version: "No version loaded", + versions: [], queryUpload: { ids: [], }, - formBackendVersions: {}, currency: { unit: "€", thousandSeparator: ".", From 25a5f8349b1d30fe965e68981de0edbe95a1f198 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 13 May 2024 13:35:50 +0200 Subject: [PATCH 13/23] Fix query upload --- .../FormConceptCopyModal.tsx | 6 +- frontend/src/js/headings/Headings.tsx | 17 +- frontend/src/js/modal/Modal.tsx | 23 +-- frontend/src/js/pane/TabNavigation.tsx | 27 ++- frontend/src/js/preview/DiagramModal.tsx | 2 +- .../previous-queries/list/AddFolderModal.tsx | 2 +- .../js/previous-queries/list/ProjectItem.tsx | 28 ++- .../upload/CSVColumnPicker.tsx | 166 ++++++++---------- .../upload/UploadQueryResults.tsx | 9 +- .../upload/UploadQueryResultsModal.tsx | 1 - .../UploadConceptListModal.tsx | 6 +- 11 files changed, 108 insertions(+), 179 deletions(-) diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptCopyModal.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptCopyModal.tsx index bc219477b1..95d7036481 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptCopyModal.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptCopyModal.tsx @@ -147,11 +147,7 @@ const FormConceptCopyModal = ({ } return ( - + theme.font.md}; - color: ${({ theme }) => theme.col.black}; -`; +export const Heading3 = tw("h3")`text-lg font-normal text-gray-800`; -export const Heading4 = styled("h4")` - line-height: 1.2; - font-weight: 400; - font-size: ${({ theme }) => theme.font.sm}; - color: ${({ theme }) => theme.col.gray}; - text-transform: uppercase; -`; +export const Heading4 = tw("h4")`text-sm font-normal text-gray-500 uppercase`; diff --git a/frontend/src/js/modal/Modal.tsx b/frontend/src/js/modal/Modal.tsx index 1b62439a87..57e1d2210e 100644 --- a/frontend/src/js/modal/Modal.tsx +++ b/frontend/src/js/modal/Modal.tsx @@ -1,12 +1,11 @@ import styled from "@emotion/styled"; -import { faTimes } from "@fortawesome/free-solid-svg-icons"; import { ReactNode, useRef } from "react"; import { useHotkeys } from "react-hotkeys-hook"; import { useTranslation } from "react-i18next"; import { TransparentButton } from "../button/TransparentButton"; import { useClickOutside } from "../common/helpers/useClickOutside"; -import FaIcon from "../icon/FaIcon"; +import { Heading3 } from "../headings/Headings"; import WithTooltip from "../tooltip/WithTooltip"; const Root = styled("div")` @@ -43,12 +42,6 @@ const TopRow = styled("div")` align-items: flex-start; `; -const Headline = styled("h3")` - margin: 0 10px 15px 0; - font-size: ${({ theme }) => theme.font.md}; - color: ${({ theme }) => theme.col.blueGrayDark}; -`; - const Subtitle = styled(`p`)` margin: -15px 0 20px; max-width: 600px; @@ -74,8 +67,7 @@ const ModalContent = ({ ); }; -// A modal with three ways to close it -// - a button +// A modal with two ways to close it // - click outside // - press esc const Modal = ({ @@ -84,7 +76,6 @@ const Modal = ({ headline, subtitle, doneButton, - closeIcon, scrollable, onClose, }: { @@ -93,7 +84,6 @@ const Modal = ({ headline?: ReactNode; subtitle?: ReactNode; doneButton?: boolean; - closeIcon?: boolean; scrollable?: boolean; onClose: () => void; }) => { @@ -105,14 +95,7 @@ const Modal = ({ - {headline} - {closeIcon && ( - - - - - - )} + {headline} {doneButton && ( diff --git a/frontend/src/js/pane/TabNavigation.tsx b/frontend/src/js/pane/TabNavigation.tsx index 6d2535a254..b5f37d0aba 100644 --- a/frontend/src/js/pane/TabNavigation.tsx +++ b/frontend/src/js/pane/TabNavigation.tsx @@ -1,6 +1,5 @@ import styled from "@emotion/styled"; import { faSpinner } from "@fortawesome/free-solid-svg-icons"; -import { FC } from "react"; import tw from "tailwind-styled-components"; import FaIcon from "../icon/FaIcon"; @@ -15,7 +14,7 @@ const Root = styled("div")` align-items: flex-start; `; -const Headline = tw("h2")<{ active: boolean }>` +const Headline = tw("h2")<{ $active: boolean }>` text-sm mb-0 mt-[6px] @@ -30,10 +29,10 @@ const Headline = tw("h2")<{ active: boolean }>` tracking-wider border-b-[3px] - ${({ active }) => - active ? "text-primary-500" : "text-gray-500 hover:text-black"}; - ${({ active }) => - active + ${({ $active }) => + $active ? "text-primary-500" : "text-gray-500 hover:text-black"}; + ${({ $active }) => + $active ? "border-primary-500" : "border-transparent hover:border-primary-200"}; @@ -54,18 +53,16 @@ export interface TabNavigationTab { loading?: boolean; } -interface PropsT { - onClickTab: (tab: string) => void; - activeTab: string | null; - tabs: TabNavigationTab[]; - dataTestId: string; -} - -const TabNavigation: FC = ({ +const TabNavigation = ({ tabs, activeTab, onClickTab, dataTestId, +}: { + onClickTab: (tab: string) => void; + activeTab: string | null; + tabs: TabNavigationTab[]; + dataTestId: string; }) => { function createClickHandler(key: string) { return () => { @@ -82,7 +79,7 @@ const TabNavigation: FC = ({ {label} diff --git a/frontend/src/js/preview/DiagramModal.tsx b/frontend/src/js/preview/DiagramModal.tsx index 51451c7051..6f4d3b8439 100644 --- a/frontend/src/js/preview/DiagramModal.tsx +++ b/frontend/src/js/preview/DiagramModal.tsx @@ -38,7 +38,7 @@ export default function DiagramModal({ useHotkeys("esc", () => onClose()); return ( - onClose()}> + {previewStatsIsBarStats(statistic) && diff --git a/frontend/src/js/previous-queries/list/AddFolderModal.tsx b/frontend/src/js/previous-queries/list/AddFolderModal.tsx index f4815542b6..57d8c269b0 100644 --- a/frontend/src/js/previous-queries/list/AddFolderModal.tsx +++ b/frontend/src/js/previous-queries/list/AddFolderModal.tsx @@ -34,7 +34,7 @@ const AddFolderModal = ({ onClose, onSubmit, isValidName }: Props) => { const [folderName, setFolderName] = useState(""); return ( - +

{t("addFolderModal.description")}

- {!isFormConfig(item) && item.resultUrls.length > 0 ? ( - +
+ {!isFormConfig(item) && item.resultUrls.length > 0 ? ( - {!item.containsDates && ( - - - - )} - - ) : ( - {topLeftLabel} - )} + ) : ( + {topLeftLabel} + )} + {!isFormConfig(item) && !item.containsDates && ( + + + + )} +
{executedAt} diff --git a/frontend/src/js/previous-queries/upload/CSVColumnPicker.tsx b/frontend/src/js/previous-queries/upload/CSVColumnPicker.tsx index 44d608d998..6be5c05fc4 100644 --- a/frontend/src/js/previous-queries/upload/CSVColumnPicker.tsx +++ b/frontend/src/js/previous-queries/upload/CSVColumnPicker.tsx @@ -12,6 +12,7 @@ import { saveAs } from "file-saver"; import { FC, useEffect, useState } from "react"; import { useTranslation } from "react-i18next"; +import tw from "tailwind-styled-components"; import { QueryUploadConfigT, UploadQueryResponseT } from "../../api/types"; import IconButton from "../../button/IconButton"; import PrimaryButton from "../../button/PrimaryButton"; @@ -30,31 +31,12 @@ const Row = styled("div")` margin-bottom: 15px; `; -const Grow = styled("div")` - display: flex; - align-items: center; -`; - -const HorizontalScrollContainer = styled("div")` - overflow-x: auto; - width: 100%; -`; -const Table = styled("table")` - margin: 10px 0; - border: 1px solid #ccc; - text-align: left; - width: 100%; - padding: 10px; - box-shadow: 0 0 5px 0 rgba(0, 0, 0, 0.1); - table-layout: fixed; -`; - -const Td = styled("td")` - font-size: ${({ theme }) => theme.font.xs}; - overflow: hidden; - text-overflow: ellipsis; - white-space: nowrap; - width: 150px; +const Td = tw("td")` + text-xs + text-ellipsis + overflow-hidden + whitespace-nowrap + min-w-[150px] `; const Th = styled("th")` font-size: ${({ theme }) => theme.font.xs}; @@ -62,16 +44,6 @@ const Th = styled("th")` width: 150px; `; -const FileName = styled("code")` - display: block; - margin: 0 15px 0 0; -`; - -const BoldFileName = styled(FileName)` - margin-bottom: 3px; - font-weight: 700; -`; - const Padded = styled("span")` padding: 0 6px; `; @@ -276,15 +248,15 @@ const CSVColumnPicker: FC = ({ return (
- -
- {file.name} - {csv.length} Zeilen +
+
+ {file.name} + {csv.length} Zeilen
- +
{csv.length > 0 && ( = ({ /> )} - - - - {csvLoading && ( - - - - )} - {csv.length > 0 && - csv.slice(0, 1).map((row, j) => ( - - {row.map((cell, i) => ( - - ))} +
+
+
{t("csvColumnPicker.loading")}
- o.value === csvHeader[i], - ) || SELECT_OPTIONS[0] - } - onChange={(value) => { - if (value) { - setCSVHeader([ - ...csvHeader.slice(0, i), - value.value as UploadColumnType, - ...csvHeader.slice(i + 1), - ]); - } - }} - /> - {cell} -
+ + {csvLoading && ( + + - ))} - - - {csv.length > 0 && - csv.slice(1, 6).map((row, j) => ( - - {row.map((cell, i) => ( - + {row.map((cell, i) => ( + + ))} + + ))} + + + {csv.length > 0 && + csv.slice(1, 6).map((row, j) => ( + + {row.map((cell, i) => ( + + ))} + + ))} + {csv.length > 6 && ( + + {new Array(csv[0].length).fill(null).map((_, j) => ( + ))} - ))} - {csv.length > 6 && ( - - {new Array(csv[0].length).fill(null).map((_, j) => ( - - ))} - - )} - -
{t("csvColumnPicker.loading")}
- {cell} + )} + {csv.length > 0 && + csv.slice(0, 1).map((row, j) => ( +
+ o.value === csvHeader[i], + ) || SELECT_OPTIONS[0] + } + onChange={(value) => { + if (value) { + setCSVHeader([ + ...csvHeader.slice(0, i), + value.value as UploadColumnType, + ...csvHeader.slice(i + 1), + ]); + } + }} + /> + {cell} +
+ {cell} +
+ ...
- ... -
- + )} + + +
+
{uploadResult && ( diff --git a/frontend/src/js/previous-queries/upload/UploadQueryResults.tsx b/frontend/src/js/previous-queries/upload/UploadQueryResults.tsx index 3c8638bc48..7a244cfaae 100644 --- a/frontend/src/js/previous-queries/upload/UploadQueryResults.tsx +++ b/frontend/src/js/previous-queries/upload/UploadQueryResults.tsx @@ -24,12 +24,13 @@ const SxIconButton = styled(IconButton)` padding: 9px 6px; `; -interface PropsT { +const UploadQueryResults = ({ + className, + datasetId, +}: { className?: string; datasetId: DatasetT["id"] | null; -} - -const UploadQueryResults = ({ className, datasetId }: PropsT) => { +}) => { const { t } = useTranslation(); const [isModalOpen, setIsModalOpen] = useState(false); diff --git a/frontend/src/js/previous-queries/upload/UploadQueryResultsModal.tsx b/frontend/src/js/previous-queries/upload/UploadQueryResultsModal.tsx index 26d58aafd4..a9db707f97 100644 --- a/frontend/src/js/previous-queries/upload/UploadQueryResultsModal.tsx +++ b/frontend/src/js/previous-queries/upload/UploadQueryResultsModal.tsx @@ -63,7 +63,6 @@ const UploadQueryResultsModal: FC = ({ return ( diff --git a/frontend/src/js/upload-concept-list-modal/UploadConceptListModal.tsx b/frontend/src/js/upload-concept-list-modal/UploadConceptListModal.tsx index 8efe0745db..c481ddf105 100644 --- a/frontend/src/js/upload-concept-list-modal/UploadConceptListModal.tsx +++ b/frontend/src/js/upload-concept-list-modal/UploadConceptListModal.tsx @@ -509,11 +509,7 @@ const UploadConceptListModal = ({ ); return ( - + Date: Mon, 13 May 2024 16:07:44 +0200 Subject: [PATCH 14/23] Fine-tune a disclosure's open state after appending and re-trigger validation --- .../form/fields/DisclosureListField.tsx | 57 ++++++++++++++++--- .../external-forms/form/fields/TabsField.tsx | 7 ++- 2 files changed, 53 insertions(+), 11 deletions(-) diff --git a/frontend/src/js/external-forms/form/fields/DisclosureListField.tsx b/frontend/src/js/external-forms/form/fields/DisclosureListField.tsx index d8b60729da..591ff3c3bd 100644 --- a/frontend/src/js/external-forms/form/fields/DisclosureListField.tsx +++ b/frontend/src/js/external-forms/form/fields/DisclosureListField.tsx @@ -11,6 +11,7 @@ import tw from "tailwind-styled-components"; import IconButton from "../../../button/IconButton"; import { TransparentButton } from "../../../button/TransparentButton"; import { exists } from "../../../common/helpers/exists"; +import { usePrevious } from "../../../common/helpers/usePrevious"; import FaIcon from "../../../icon/FaIcon"; import InfoTooltip from "../../../tooltip/InfoTooltip"; import { DisclosureListField as DisclosureListFieldT } from "../../config-types"; @@ -142,11 +143,6 @@ export const DisclosureListField = ({ name: field.name, }); - const { isOpen, toggleOpen } = useOpenState({ - onlyOneOpenAtATime: field.onlyOneOpenAtATime, - defaultOpen: field.defaultOpen ? fields[0]?.id : undefined, - }); - useEffect( function applyDefaultValue() { if ( @@ -165,6 +161,51 @@ export const DisclosureListField = ({ [fields.length, replace, defaultValue, commonProps], ); + const prevFieldsLength = usePrevious(fields.length); + + const { isOpen, toggleOpen } = useOpenState({ + onlyOneOpenAtATime: field.onlyOneOpenAtATime, + defaultOpen: field.defaultOpen ? fields[0]?.id : undefined, + }); + + useEffect( + function openFirstFieldIfNecessary() { + if (prevFieldsLength === 0 && fields.length > 0 && field.defaultOpen) { + const id = fields[0]?.id; + if (id && !isOpen[id]) { + toggleOpen(id); + } + } + }, + [prevFieldsLength, fields, toggleOpen, isOpen, field.defaultOpen], + ); + + useEffect( + function openLastFieldAfterAppending() { + if ( + exists(prevFieldsLength) && + prevFieldsLength > 0 && + prevFieldsLength < fields.length && + field.defaultOpen + ) { + commonProps.trigger(); + + const id = fields[fields.length - 1]?.id; + if (id && !isOpen[id]) { + toggleOpen(id); + } + } + }, + [ + prevFieldsLength, + fields, + isOpen, + toggleOpen, + field.defaultOpen, + commonProps, + ], + ); + if (field.fields.length === 0) return null; const { locale } = commonProps; @@ -187,7 +228,7 @@ export const DisclosureListField = ({ + onClick={() => { append( Object.fromEntries( field.fields.filter(isFormFieldWithValue).map((f) => [ @@ -199,8 +240,8 @@ export const DisclosureListField = ({ }), ]), ), - ) - } + ); + }} > {field.createNewLabel ? field.createNewLabel[locale] : undefined} diff --git a/frontend/src/js/external-forms/form/fields/TabsField.tsx b/frontend/src/js/external-forms/form/fields/TabsField.tsx index 48b6d64002..d2328e4d31 100644 --- a/frontend/src/js/external-forms/form/fields/TabsField.tsx +++ b/frontend/src/js/external-forms/form/fields/TabsField.tsx @@ -45,9 +45,10 @@ export const TabsField = ({ <> - commonProps.setValue(field.name, tab, setValueConfig) - } + onSelectTab={(tab) => { + commonProps.setValue(field.name, tab, setValueConfig); + commonProps.trigger(); + }} options={field.tabs.map((tab) => ({ label: () => tab.title[commonProps.locale] || "", value: tab.name, From 91ee25790ced4908577d2f554d3469701574ce27 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 14 May 2024 11:51:12 +0200 Subject: [PATCH 15/23] Handle internal server error on upload --- frontend/src/js/app/App.tsx | 2 +- frontend/src/js/dataset/actions.ts | 3 +- .../js/entity-history/LoadHistoryDropzone.tsx | 3 +- frontend/src/js/entity-history/actions.ts | 3 +- frontend/src/js/entity-history/saveAndLoad.ts | 3 +- .../js/external-forms/FormConfigLoader.tsx | 5 +- frontend/src/js/preview/actions.ts | 3 +- .../src/js/previous-queries/list/actions.ts | 17 +++-- .../list/useDeleteProjectItemFolder.tsx | 3 +- .../upload/UploadQueryResults.tsx | 9 +-- .../src/js/snack-message/SnackMessage.tsx | 68 +++++++------------ frontend/src/js/snack-message/actions.ts | 4 +- frontend/src/js/snack-message/reducer.ts | 8 +-- 13 files changed, 50 insertions(+), 81 deletions(-) diff --git a/frontend/src/js/app/App.tsx b/frontend/src/js/app/App.tsx index 8cd6bc8eeb..210407764d 100644 --- a/frontend/src/js/app/App.tsx +++ b/frontend/src/js/app/App.tsx @@ -4,9 +4,9 @@ import { useEffect, useState } from "react"; import { useIsCacheEnabled } from "../common/feature-flags/useIsCacheEnabled"; import { clearIndexedDBCache } from "../common/helpers/indexedDBCache"; import Header from "../header/Header"; -import SnackMessage from "../snack-message/SnackMessage"; import { useStartup } from "../startup/useStartup"; +import { SnackMessage } from "../snack-message/SnackMessage"; import { About } from "./About"; import Content from "./Content"; diff --git a/frontend/src/js/dataset/actions.ts b/frontend/src/js/dataset/actions.ts index caa4aee6f3..f0f8b27f77 100644 --- a/frontend/src/js/dataset/actions.ts +++ b/frontend/src/js/dataset/actions.ts @@ -16,7 +16,6 @@ import { import { useLoadQueries } from "../previous-queries/list/actions"; import { queryResultReset } from "../query-runner/actions"; import { setMessage } from "../snack-message/actions"; -import { SnackMessageType } from "../snack-message/reducer"; import { clearQuery, loadSavedQuery } from "../standard-query-editor/actions"; import type { StandardQueryStateT } from "../standard-query-editor/queryReducer"; @@ -65,7 +64,7 @@ export const useLoadDatasets = () => { dispatch( setMessage({ message: t("datasetSelector.error"), - type: SnackMessageType.ERROR, + type: "error", }), ); dispatch(loadDatasets.failure(e as Error)); diff --git a/frontend/src/js/entity-history/LoadHistoryDropzone.tsx b/frontend/src/js/entity-history/LoadHistoryDropzone.tsx index a894f206b7..72adca30e3 100644 --- a/frontend/src/js/entity-history/LoadHistoryDropzone.tsx +++ b/frontend/src/js/entity-history/LoadHistoryDropzone.tsx @@ -5,7 +5,6 @@ import { useDispatch } from "react-redux"; import type { SelectOptionT } from "../api/types"; import { parseCSV } from "../file/csv"; import { setMessage } from "../snack-message/actions"; -import { SnackMessageType } from "../snack-message/reducer"; import DropzoneWithFileInput, { DragItemFile, } from "../ui-components/DropzoneWithFileInput"; @@ -46,7 +45,7 @@ export const LoadHistoryDropzone = ({ dispatch( setMessage({ message: t("history.load.error"), - type: SnackMessageType.ERROR, + type: "error", }), ); return; diff --git a/frontend/src/js/entity-history/actions.ts b/frontend/src/js/entity-history/actions.ts index e3041d96dc..57c525d262 100644 --- a/frontend/src/js/entity-history/actions.ts +++ b/frontend/src/js/entity-history/actions.ts @@ -28,7 +28,6 @@ import { exists } from "../common/helpers/exists"; import { useDatasetId } from "../dataset/selectors"; import { loadCSV, parseCSVWithHeaderToObj } from "../file/csv"; import { setMessage } from "../snack-message/actions"; -import { SnackMessageType } from "../snack-message/reducer"; import { EntityEvent, EntityId } from "./reducer"; import { isDateColumn, isSourceColumn } from "./timeline/util"; @@ -302,7 +301,7 @@ export function useUpdateHistorySession() { dispatch( setMessage({ message: t("history.error"), - type: SnackMessageType.ERROR, + type: "error", }), ); } diff --git a/frontend/src/js/entity-history/saveAndLoad.ts b/frontend/src/js/entity-history/saveAndLoad.ts index b13979435b..b1b4d6e6af 100644 --- a/frontend/src/js/entity-history/saveAndLoad.ts +++ b/frontend/src/js/entity-history/saveAndLoad.ts @@ -5,7 +5,6 @@ import { useDispatch } from "react-redux"; import { downloadBlob } from "../common/helpers/downloadBlob"; import { toCSV } from "../file/csv"; import { setMessage } from "../snack-message/actions"; -import { SnackMessageType } from "../snack-message/reducer"; import { EntityIdsStatus } from "./History"; import { LoadingPayload } from "./LoadHistoryDropzone"; @@ -90,7 +89,7 @@ export const useLoadHistory = ({ dispatch( setMessage({ message: t("history.load.error"), - type: SnackMessageType.ERROR, + type: "error", }), ); return; diff --git a/frontend/src/js/external-forms/FormConfigLoader.tsx b/frontend/src/js/external-forms/FormConfigLoader.tsx index af438d777c..7e2726c798 100644 --- a/frontend/src/js/external-forms/FormConfigLoader.tsx +++ b/frontend/src/js/external-forms/FormConfigLoader.tsx @@ -11,7 +11,6 @@ import { DNDType } from "../common/constants/dndTypes"; import { Language, useActiveLang } from "../localization/useActiveLang"; import type { FormConfigT } from "../previous-queries/list/reducer"; import { setMessage } from "../snack-message/actions"; -import { SnackMessageType } from "../snack-message/reducer"; import Dropzone from "../ui-components/Dropzone"; import { setExternalForm } from "./actions"; @@ -151,7 +150,7 @@ const FormConfigLoader: FC = ({ message: t("formConfig.loadSuccess", { label: formConfigToLoadNext.label, }), - type: SnackMessageType.SUCCESS, + type: "success", }), ); }, @@ -179,7 +178,7 @@ const FormConfigLoader: FC = ({ dispatch( setMessage({ message: t("formConfig.loadError"), - type: SnackMessageType.ERROR, + type: "error", }), ); } diff --git a/frontend/src/js/preview/actions.ts b/frontend/src/js/preview/actions.ts index 905013e0dc..60d7b63f9f 100644 --- a/frontend/src/js/preview/actions.ts +++ b/frontend/src/js/preview/actions.ts @@ -7,7 +7,6 @@ import { GetQueryResponseT, PreviewStatisticsResponse } from "../api/types"; import { StateT } from "../app/reducers"; import { ErrorObject } from "../common/actions/genericActions"; import { setMessage } from "../snack-message/actions"; -import { SnackMessageType } from "../snack-message/reducer"; import { PreviewStateT } from "./reducer"; export type PreviewActions = ActionType< @@ -106,7 +105,7 @@ export function useLoadPreviewData() { dispatch( setMessage({ message: t("preview.loadingError"), - type: SnackMessageType.ERROR, + type: "error", }), ); dispatch(loadPreview.failure({})); diff --git a/frontend/src/js/previous-queries/list/actions.ts b/frontend/src/js/previous-queries/list/actions.ts index 3a38e65a51..f09ec0d105 100644 --- a/frontend/src/js/previous-queries/list/actions.ts +++ b/frontend/src/js/previous-queries/list/actions.ts @@ -21,7 +21,6 @@ import { } from "../../api/types"; import { useDatasetId } from "../../dataset/selectors"; import { setMessage } from "../../snack-message/actions"; -import { SnackMessageType } from "../../snack-message/reducer"; import type { FormConfigT, PreviousQueryT } from "./reducer"; @@ -59,7 +58,7 @@ export const useLoadQueries = () => { dispatch( setMessage({ message: t("previousQueries.error"), - type: SnackMessageType.ERROR, + type: "error", }), ); } @@ -97,7 +96,7 @@ export const useLoadQuery = () => { dispatch( setMessage({ message: t("previousQuery.loadError"), - type: SnackMessageType.ERROR, + type: "error", }), ); } @@ -142,7 +141,7 @@ export const useUpdateQuery = () => { dispatch( setMessage({ message: errorMessage, - type: SnackMessageType.ERROR, + type: "error", }), ); } @@ -172,7 +171,7 @@ export const useRemoveQuery = () => { dispatch( setMessage({ message: t("previousQuery.deleteError"), - type: SnackMessageType.ERROR, + type: "error", }), ); } @@ -218,7 +217,7 @@ export const useLoadFormConfigs = () => { dispatch( setMessage({ message: t("formConfigs.error"), - type: SnackMessageType.ERROR, + type: "error", }), ); } @@ -257,7 +256,7 @@ export const useLoadFormConfig = () => { dispatch( setMessage({ message: t("formConfig.loadError"), - type: SnackMessageType.ERROR, + type: "error", }), ); } @@ -297,7 +296,7 @@ export const useUpdateFormConfig = () => { dispatch( setMessage({ message: errorMessage, - type: SnackMessageType.ERROR, + type: "error", }), ); } @@ -328,7 +327,7 @@ export const useRemoveFormConfig = () => { dispatch( setMessage({ message: t("formConfig.deleteError"), - type: SnackMessageType.ERROR, + type: "error", }), ); } diff --git a/frontend/src/js/previous-queries/list/useDeleteProjectItemFolder.tsx b/frontend/src/js/previous-queries/list/useDeleteProjectItemFolder.tsx index a9186fe375..01c00aa435 100644 --- a/frontend/src/js/previous-queries/list/useDeleteProjectItemFolder.tsx +++ b/frontend/src/js/previous-queries/list/useDeleteProjectItemFolder.tsx @@ -4,7 +4,6 @@ import { useDispatch, useSelector } from "react-redux"; import type { StateT } from "../../app/reducers"; import { useDatasetId } from "../../dataset/selectors"; import { setMessage } from "../../snack-message/actions"; -import { SnackMessageType } from "../../snack-message/reducer"; import { removeFolder, @@ -74,7 +73,7 @@ export const useDeleteProjectItemFolder = () => { dispatch( setMessage({ message: t("previousQuery.retagError"), - type: SnackMessageType.ERROR, + type: "error", }), ); return Promise.reject(); diff --git a/frontend/src/js/previous-queries/upload/UploadQueryResults.tsx b/frontend/src/js/previous-queries/upload/UploadQueryResults.tsx index 7a244cfaae..1dff47ceb9 100644 --- a/frontend/src/js/previous-queries/upload/UploadQueryResults.tsx +++ b/frontend/src/js/previous-queries/upload/UploadQueryResults.tsx @@ -13,7 +13,6 @@ import type { import type { StateT } from "../../app/reducers"; import IconButton from "../../button/IconButton"; import { setMessage } from "../../snack-message/actions"; -import { SnackMessageType } from "../../snack-message/reducer"; import WithTooltip from "../../tooltip/WithTooltip"; import { useLoadQueries } from "../list/actions"; @@ -59,18 +58,20 @@ const UploadQueryResults = ({ setLoading(true); const result = await postQueryUpload(datasetId, query); - setUploadResult(result); loadQueries(datasetId); } catch (e) { - if ((e as { status?: number }).status === 400) { + if ( + (e as { status?: number }).status === 400 && + "resolved" in (e as object) + ) { setUploadResult(e as UploadQueryResponseT); } else { dispatch( setMessage({ message: t("uploadQueryResultsModal.uploadFailed"), - type: SnackMessageType.ERROR, + type: "error", }), ); } diff --git a/frontend/src/js/snack-message/SnackMessage.tsx b/frontend/src/js/snack-message/SnackMessage.tsx index 6a72212649..5b87292c20 100644 --- a/frontend/src/js/snack-message/SnackMessage.tsx +++ b/frontend/src/js/snack-message/SnackMessage.tsx @@ -1,54 +1,36 @@ -import styled from "@emotion/styled"; import { faTimes } from "@fortawesome/free-solid-svg-icons"; -import { FC, memo, useRef } from "react"; +import { memo, useRef } from "react"; import { useDispatch, useSelector } from "react-redux"; import type { StateT } from "../app/reducers"; import { useClickOutside } from "../common/helpers/useClickOutside"; import FaIcon from "../icon/FaIcon"; +import tw from "tailwind-styled-components"; import { resetMessage as resetMessageAction } from "./actions"; -import { SnackMessageStateT, SnackMessageType } from "./reducer"; - -const snackMessageTypeToColor: Record = { - [SnackMessageType.ERROR]: "rgba(0, 0, 0, 0.75)", - [SnackMessageType.SUCCESS]: "rgba(12, 100, 39, 0.9)", // #0C6427 - [SnackMessageType.DEFAULT]: "rgba(0, 0, 0, 0.75)", -}; - -const Root = styled("div")<{ type: SnackMessageType }>` - position: fixed; - z-index: 10; - bottom: 20px; - right: 20px; - background-color: ${({ type }) => - snackMessageTypeToColor[type ?? SnackMessageType.DEFAULT]}; - color: white; - display: flex; - flex-direction: row; - align-items: flex-start; - max-width: 500px; - border-radius: 5px; +import { SnackMessageStateT } from "./reducer"; + +const Root = tw("div")<{ $success?: boolean }>` + fixed + z-10 + bottom-5 + right-5 + text-white + flex items-start + max-w-[500px] + rounded-lg + ${({ $success }) => + $success ? "bg-primary-500 bg-opacity-90" : "bg-black bg-opacity-75"} `; -const Relative = styled("div")` - position: relative; - padding: 12px 40px 12px 20px; +const ClearZone = tw("div")` + absolute top-3 right-4 + z-[11] + cursor-pointer + opacity-80 hover:opacity-100 `; -const ClearZone = styled("div")` - position: absolute; - top: 12px; - right: 18px; - z-index: 11; - cursor: pointer; - opacity: 0.8; - &:hover { - opacity: 1; - } -`; - -const SnackMessage: FC = memo(function SnackMessageComponent() { +export const SnackMessage = memo(function SnackMessageComponent() { const ref = useRef(null); const { message, type } = useSelector( (state) => state.snackMessage, @@ -65,17 +47,15 @@ const SnackMessage: FC = memo(function SnackMessageComponent() { return (
{message && ( - - + +
- +
)}
); }); - -export default SnackMessage; diff --git a/frontend/src/js/snack-message/actions.ts b/frontend/src/js/snack-message/actions.ts index 942eb2d456..ea52c42579 100644 --- a/frontend/src/js/snack-message/actions.ts +++ b/frontend/src/js/snack-message/actions.ts @@ -1,6 +1,6 @@ import { ActionType, createAction } from "typesafe-actions"; -import { SnackMessageType } from "./reducer"; +import { SnackMessageStateT } from "./reducer"; export type SnackMessageActions = ActionType< typeof setMessage | typeof resetMessage @@ -8,7 +8,7 @@ export type SnackMessageActions = ActionType< export const setMessage = createAction("snack-message/SET")<{ message: string | null; - type: SnackMessageType; + type: SnackMessageStateT["type"]; }>(); export const resetMessage = createAction("snack-message/RESET")(); diff --git a/frontend/src/js/snack-message/reducer.ts b/frontend/src/js/snack-message/reducer.ts index 844253f6a6..3e276c4e46 100644 --- a/frontend/src/js/snack-message/reducer.ts +++ b/frontend/src/js/snack-message/reducer.ts @@ -4,11 +4,7 @@ import { Action } from "../app/actions"; import { resetMessage, setMessage } from "./actions"; -export enum SnackMessageType { - ERROR = "error", - SUCCESS = "success", - DEFAULT = "default", -} +type SnackMessageType = "error" | "success" | "default"; export interface SnackMessageStateT { message: string | null; type: SnackMessageType; @@ -16,7 +12,7 @@ export interface SnackMessageStateT { const initialState: SnackMessageStateT = { message: null, - type: SnackMessageType.DEFAULT, + type: "default", }; function reducer( From 67735f565a4d3b5ed18270cccded6050d2499c5c Mon Sep 17 00:00:00 2001 From: Marco Korinth Date: Tue, 14 May 2024 11:53:25 +0200 Subject: [PATCH 16/23] feat: change diagram fontfamily to Roboto --- frontend/src/js/preview/Diagram.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/js/preview/Diagram.tsx b/frontend/src/js/preview/Diagram.tsx index 78ee345de7..83eb01f31d 100644 --- a/frontend/src/js/preview/Diagram.tsx +++ b/frontend/src/js/preview/Diagram.tsx @@ -123,6 +123,7 @@ export default function Diagram({ font: { weight: "normal", size: 14, + family: "Roboto", }, position: "bottom", text: stat.label, @@ -151,6 +152,7 @@ export default function Diagram({ font: { weight: "normal", size: 14, + family: "Roboto", }, }; From cfa4f9115b4abe4002aedd7ec0e178b80043c286 Mon Sep 17 00:00:00 2001 From: Jonas Arnhold Date: Tue, 14 May 2024 16:36:01 +0200 Subject: [PATCH 17/23] Fix reused query conversion (#3429) --- .../sql/conversion/dialect/SqlDialect.java | 2 + .../query/CQReusedQueryConverter.java | 25 +++++ .../sql/programmatic/SqlReusedQueryTest.java | 92 ------------------- .../integration/tests/ReusedQueryTest.java | 22 +++-- .../SECONDARY_IDS_MIXED.test.json | 24 +++-- .../query/SECONDARY_ID_MIXED/content.csv | 14 +-- .../query/SECONDARY_ID_MIXED/content2.csv | 14 +-- 7 files changed, 70 insertions(+), 123 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/query/CQReusedQueryConverter.java delete mode 100644 backend/src/test/java/com/bakdata/conquery/integration/sql/programmatic/SqlReusedQueryTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java index 4f17b904ea..733112ac18 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java @@ -16,6 +16,7 @@ import com.bakdata.conquery.sql.conversion.cqelement.concept.CQConceptConverter; import com.bakdata.conquery.sql.conversion.model.QueryStepTransformer; import com.bakdata.conquery.sql.conversion.query.AbsoluteFormQueryConverter; +import com.bakdata.conquery.sql.conversion.query.CQReusedQueryConverter; import com.bakdata.conquery.sql.conversion.query.ConceptQueryConverter; import com.bakdata.conquery.sql.conversion.query.EntityDateQueryConverter; import com.bakdata.conquery.sql.conversion.query.FormConversionHelper; @@ -63,6 +64,7 @@ default List> getDefaultNodeConverters() { new CQNegationConverter(), new CQConceptConverter(), new CQExternalConverter(), + new CQReusedQueryConverter(), new ConceptQueryConverter(queryStepTransformer), new SecondaryIdQueryConverter(), new AbsoluteFormQueryConverter(formConversionUtil), diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/CQReusedQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/CQReusedQueryConverter.java new file mode 100644 index 0000000000..963354c0a6 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/CQReusedQueryConverter.java @@ -0,0 +1,25 @@ +package com.bakdata.conquery.sql.conversion.query; + +import com.bakdata.conquery.apiv1.query.CQElement; +import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; + +public class CQReusedQueryConverter implements NodeConverter { + + @Override + public Class getConversionClass() { + return CQReusedQuery.class; + } + + @Override + public ConversionContext convert(CQReusedQuery reusedQuery, ConversionContext context) { + CQElement reusableComponents = reusedQuery.getResolvedQuery().getReusableComponents(); + if (!reusedQuery.isExcludeFromSecondaryId()) { + return context.getNodeConversions().convert(reusableComponents, context); + } + ConversionContext withExcludedSecondaryId = context.withSecondaryIdDescription(null); + return context.getNodeConversions().convert(reusableComponents, withExcludedSecondaryId); + } + +} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/programmatic/SqlReusedQueryTest.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/programmatic/SqlReusedQueryTest.java deleted file mode 100644 index f9230b6f00..0000000000 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/programmatic/SqlReusedQueryTest.java +++ /dev/null @@ -1,92 +0,0 @@ -package com.bakdata.conquery.integration.sql.programmatic; - -import static org.assertj.core.api.Assertions.assertThat; - -import java.net.URI; -import java.util.Map; -import java.util.Set; - -import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; -import com.bakdata.conquery.apiv1.query.ConceptQuery; -import com.bakdata.conquery.apiv1.query.Query; -import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; -import com.bakdata.conquery.integration.common.IntegrationUtils; -import com.bakdata.conquery.integration.json.JsonIntegrationTest; -import com.bakdata.conquery.integration.json.QueryTest; -import com.bakdata.conquery.integration.tests.ProgrammaticIntegrationTest; -import com.bakdata.conquery.integration.tests.ReusedQueryTest; -import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.models.datasets.Dataset; -import com.bakdata.conquery.models.execution.ExecutionState; -import com.bakdata.conquery.models.execution.ManagedExecution; -import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; -import com.bakdata.conquery.resources.ResourceConstants; -import com.bakdata.conquery.resources.api.QueryResource; -import com.bakdata.conquery.resources.hierarchies.HierarchyHelper; -import com.bakdata.conquery.util.support.StandaloneSupport; -import com.bakdata.conquery.util.support.TestConquery; -import com.github.powerlibraries.io.In; -import jakarta.ws.rs.client.Entity; -import jakarta.ws.rs.core.MediaType; -import lombok.extern.slf4j.Slf4j; - -/** - * This test duplicates parts of the {@link com.bakdata.conquery.integration.tests.ReusedQueryTest}, which itself is not SQL-ready yet, because it uses - * secondary ID's in its test. As soon as the SQL connector supports secondary ID's, this test can be removed. - */ -@Slf4j -public class SqlReusedQueryTest implements ProgrammaticIntegrationTest { - - @Override - public Set forModes() { - return Set.of(StandaloneSupport.Mode.SQL); - } - - @Override - public void execute(String name, TestConquery testConquery) throws Exception { - - final StandaloneSupport conquery = testConquery.getSupport(name); - - final String testJson = In.resource("/tests/sql/filter/count_distinct/count_distinct.json").withUTF8().readAll(); - - final Dataset dataset = conquery.getDataset(); - final QueryTest test = JsonIntegrationTest.readJson(dataset, testJson); - - ReusedQueryTest.importManually(conquery, test); - - final Query query = IntegrationUtils.parseQuery(conquery, test.getRawQuery()); - final ManagedExecutionId id = IntegrationUtils.assertQueryResult(conquery, query, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); - - assertThat(id).isNotNull(); - - final MetaStorage metaStorage = conquery.getMetaStorage(); - final ManagedExecution execution = metaStorage.getExecution(id); - - // Normal reuse - { - - final ConceptQuery reused = new ConceptQuery(new CQReusedQuery(execution.getId())); - - IntegrationUtils.assertQueryResult(conquery, reused, 2L, ExecutionState.DONE, conquery.getTestUser(), 201); - } - - // Reuse by API - { - final URI reexecuteUri = - HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), QueryResource.class, "reexecute") - .buildFromMap(Map.of( - ResourceConstants.DATASET, conquery.getDataset().getName(), - ResourceConstants.QUERY, execution.getId().toString() - )); - - final FullExecutionStatus status = conquery.getClient().target(reexecuteUri) - .request(MediaType.APPLICATION_JSON) - .post(Entity.entity(null, MediaType.APPLICATION_JSON_TYPE)) - .readEntity(FullExecutionStatus.class); - - assertThat(status.getStatus()).isIn(ExecutionState.RUNNING, ExecutionState.DONE); - - } - } - -} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java index d437492a8f..bef52ead96 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java @@ -52,6 +52,11 @@ @Slf4j public class ReusedQueryTest implements ProgrammaticIntegrationTest { + @Override + public Set forModes() { + return Set.of(StandaloneSupport.Mode.WORKER, StandaloneSupport.Mode.SQL); + } + @Override public void execute(String name, TestConquery testConquery) throws Exception { @@ -66,7 +71,13 @@ public void execute(String name, TestConquery testConquery) throws Exception { final QueryTest test = (QueryTest) JsonIntegrationTest.readJson(dataset, testJson); // Manually import data, so we can do our own work. - importManually(conquery, test); + ValidatorHelper.failOnError(log, conquery.getValidator().validate(test)); + TestDataImporter testImporter = conquery.getTestImporter(); + + testImporter.importSecondaryIds(conquery, test.getContent().getSecondaryIds()); + testImporter.importTables(conquery, test.getContent().getTables(), test.getContent().isAutoConcept()); + testImporter.importConcepts(conquery, test.getRawConcepts()); + testImporter.importTableContents(conquery, test.getContent().getTables()); final SecondaryIdQuery query = (SecondaryIdQuery) IntegrationUtils.parseQuery(conquery, test.getRawQuery()); @@ -230,13 +241,4 @@ public void execute(String name, TestConquery testConquery) throws Exception { } } - public static void importManually(StandaloneSupport conquery, QueryTest test) throws Exception { - ValidatorHelper.failOnError(log, conquery.getValidator().validate(test)); - TestDataImporter testImporter = conquery.getTestImporter(); - - testImporter.importSecondaryIds(conquery, test.getContent().getSecondaryIds()); - testImporter.importTables(conquery, test.getContent().getTables(), test.getContent().isAutoConcept()); - testImporter.importConcepts(conquery, test.getRawConcepts()); - testImporter.importTableContents(conquery, test.getContent().getTables()); - } } diff --git a/backend/src/test/resources/tests/query/SECONDARY_ID_MIXED/SECONDARY_IDS_MIXED.test.json b/backend/src/test/resources/tests/query/SECONDARY_ID_MIXED/SECONDARY_IDS_MIXED.test.json index 5703bffb12..7e2c2190b8 100644 --- a/backend/src/test/resources/tests/query/SECONDARY_ID_MIXED/SECONDARY_IDS_MIXED.test.json +++ b/backend/src/test/resources/tests/query/SECONDARY_ID_MIXED/SECONDARY_IDS_MIXED.test.json @@ -51,7 +51,8 @@ "table": "table", "validityDates": { "label": "datum", - "column": "table.datum" + "startColumn": "table.datum_start", + "endColumn": "table.datum_end" }, "filters": { "label": "filter", @@ -65,7 +66,8 @@ "table": "table2", "validityDates": { "label": "datum", - "column": "table2.datum" + "startColumn": "table2.datum_start", + "endColumn": "table2.datum_end" }, "filters": { "label": "filter", @@ -105,8 +107,12 @@ "type": "REAL" }, { - "name": "datum", - "type": "DATE_RANGE" + "name": "datum_start", + "type": "DATE" + }, + { + "name": "datum_end", + "type": "DATE" }, { "name": "ignored", @@ -132,11 +138,15 @@ "type": "REAL" }, { - "name": "datum", - "type": "DATE_RANGE" + "name": "datum_start", + "type": "DATE" + }, + { + "name": "datum_end", + "type": "DATE" } ] } ] } -} \ No newline at end of file +} diff --git a/backend/src/test/resources/tests/query/SECONDARY_ID_MIXED/content.csv b/backend/src/test/resources/tests/query/SECONDARY_ID_MIXED/content.csv index 882b7b8cf5..62918f9a9b 100644 --- a/backend/src/test/resources/tests/query/SECONDARY_ID_MIXED/content.csv +++ b/backend/src/test/resources/tests/query/SECONDARY_ID_MIXED/content.csv @@ -1,7 +1,7 @@ -pid,sid,value,datum,ignored -a,f_a1,1,"2014-06-30/2015-06-30","a" -a,f_a1,1,"2016-06-30/2016-06-30","a" -a,f_a2,1,"2014-06-30/2015-06-30","a" -a,,1,"2010-06-30/2010-06-30","a" -a,f_a3,1.01,"2014-06-30/2015-06-30","a" -b,f_b1,1,"2015-02-03/2015-06-30","a" \ No newline at end of file +pid,sid,value,datum_start,datum_end,ignored +a,f_a1,1,2014-06-30,2015-06-30,"a" +a,f_a1,1,2016-06-30,2016-06-30,"a" +a,f_a2,1,2014-06-30,2015-06-30,"a" +a,,1,2010-06-30,2010-06-30,"a" +a,f_a3,1.01,2014-06-30,2015-06-30,"a" +b,f_b1,1,2015-02-03,2015-06-30,"a" diff --git a/backend/src/test/resources/tests/query/SECONDARY_ID_MIXED/content2.csv b/backend/src/test/resources/tests/query/SECONDARY_ID_MIXED/content2.csv index 341cac8290..fb989445e9 100644 --- a/backend/src/test/resources/tests/query/SECONDARY_ID_MIXED/content2.csv +++ b/backend/src/test/resources/tests/query/SECONDARY_ID_MIXED/content2.csv @@ -1,7 +1,7 @@ -pid,sid,value,datum -a,f_a4,1,"2024-06-30/2025-06-30" -a,f_a4,1,"2026-06-30/2026-06-30" -a,f_a4,1,"2024-06-30/2025-06-30" -a,,13,"2020-06-30/2020-06-30" -a,f_a5,1.01,"2024-06-30/2025-06-30" -b,f_b6,1,"2025-02-03/2025-06-30" \ No newline at end of file +pid,sid,value,datum_start,datum_end +a,f_a4,1,2024-06-30,2025-06-30 +a,f_a4,1,2026-06-30,2026-06-30 +a,f_a4,1,2024-06-30,2025-06-30 +a,,13,2020-06-30,2020-06-30 +a,f_a5,1.01,2024-06-30,2025-06-30 +b,f_b6,1,2025-02-03,2025-06-30 From ce8e694b85db2b11714caa5fcf37ffa81dfc12d0 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 14 May 2024 18:56:10 +0200 Subject: [PATCH 18/23] Try and run validation after tab switch only after timeout --- frontend/src/js/external-forms/form/fields/TabsField.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/js/external-forms/form/fields/TabsField.tsx b/frontend/src/js/external-forms/form/fields/TabsField.tsx index d2328e4d31..e4de298db0 100644 --- a/frontend/src/js/external-forms/form/fields/TabsField.tsx +++ b/frontend/src/js/external-forms/form/fields/TabsField.tsx @@ -47,7 +47,9 @@ export const TabsField = ({ selectedTab={fieldProps.value as string} onSelectTab={(tab) => { commonProps.setValue(field.name, tab, setValueConfig); - commonProps.trigger(); + setTimeout(() => { + commonProps.trigger(); + }, 100); }} options={field.tabs.map((tab) => ({ label: () => tab.title[commonProps.locale] || "", From 3d4b59dbe1e2130b6f3409d061ec4eb14de16f09 Mon Sep 17 00:00:00 2001 From: Jonas Arnhold Date: Fri, 17 May 2024 12:42:44 +0200 Subject: [PATCH 19/23] Make SQL executions async (#3442) --- .../forms/managed/ManagedInternalForm.java | 5 +- .../sql/conquery/SqlExecutionManager.java | 48 ++++++++++++++----- 2 files changed, 38 insertions(+), 15 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java index 9bd4581ae7..237a3d4c2e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedInternalForm.java @@ -21,7 +21,6 @@ import com.bakdata.conquery.models.identifiable.IdMap; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.messages.namespaces.WorkerMessage; -import com.bakdata.conquery.models.messages.namespaces.specific.CancelQuery; import com.bakdata.conquery.models.messages.namespaces.specific.ExecuteForm; import com.bakdata.conquery.models.query.ColumnDescriptor; import com.bakdata.conquery.models.query.ManagedQuery; @@ -30,7 +29,6 @@ import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.FormShardResult; -import com.bakdata.conquery.models.worker.DistributedNamespace; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.OptBoolean; @@ -135,8 +133,7 @@ protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject @Override public void cancel() { - log.debug("Sending cancel message to all workers."); - ((DistributedNamespace) getNamespace()).getWorkerHandler().sendToAll(new CancelQuery(getId())); + subQueries.values().forEach(ManagedQuery::cancel); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java index fde5116efb..a0c1616396 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java @@ -1,12 +1,17 @@ package com.bakdata.conquery.sql.conquery; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.execution.InternalExecution; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.forms.managed.ManagedInternalForm; +import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.worker.Namespace; @@ -22,29 +27,27 @@ public class SqlExecutionManager extends ExecutionManager { private final SqlExecutionService executionService; private final SqlConverter converter; + private final Map> runningExecutions; public SqlExecutionManager(final SqlContext context, SqlExecutionService sqlExecutionService, MetaStorage storage) { super(storage); - executionService = sqlExecutionService; - converter = new SqlConverter(context.getSqlDialect(), context.getConfig()); + this.executionService = sqlExecutionService; + this.converter = new SqlConverter(context.getSqlDialect(), context.getConfig()); + this.runningExecutions = new HashMap<>(); } @Override protected void doExecute(Namespace namespace, InternalExecution execution) { - // todo(tm): Non-blocking execution if (execution instanceof ManagedQuery managedQuery) { - SqlQuery sqlQuery = converter.convert(managedQuery.getQuery()); - SqlExecutionResult result = executionService.execute(sqlQuery); - addResult(managedQuery, result); - managedQuery.setLastResultCount(((long) result.getRowCount())); - managedQuery.finish(ExecutionState.DONE); + CompletableFuture sqlQueryExecution = executeAsync(managedQuery); + runningExecutions.put(managedQuery.getId(), sqlQueryExecution); return; } if (execution instanceof ManagedInternalForm managedForm) { - managedForm.getSubQueries().values().forEach(subQuery -> doExecute(namespace, subQuery)); - managedForm.finish(ExecutionState.DONE); + CompletableFuture.allOf(managedForm.getSubQueries().values().stream().map(this::executeAsync).toArray(CompletableFuture[]::new)) + .thenRun(() -> managedForm.finish(ExecutionState.DONE)); return; } @@ -53,7 +56,30 @@ protected void doExecute(Namespace namespace, InternalExecution execution) { @Override public void cancelQuery(Dataset dataset, ManagedExecution query) { - // unsupported for now + + CompletableFuture sqlQueryExecution = runningExecutions.remove(query.getId()); + + // already finished/canceled + if (sqlQueryExecution == null) { + return; + } + + if (!sqlQueryExecution.isCancelled()) { + sqlQueryExecution.cancel(true); + } + + query.cancel(); + } + + private CompletableFuture executeAsync(ManagedQuery managedQuery) { + SqlQuery sqlQuery = converter.convert(managedQuery.getQuery()); + return CompletableFuture.supplyAsync(() -> executionService.execute(sqlQuery)) + .thenAccept(result -> { + addResult(managedQuery, result); + managedQuery.setLastResultCount(((long) result.getRowCount())); + managedQuery.finish(ExecutionState.DONE); + runningExecutions.remove(managedQuery.getId()); + }); } } From 897ba54e4397bc084646fab396362d054baf892b Mon Sep 17 00:00:00 2001 From: Jonas Arnhold Date: Fri, 17 May 2024 15:45:51 +0200 Subject: [PATCH 20/23] Implement support for multiple DB tenants in SQL mode (#3444) --- .../mode/local/LocalManagerProvider.java | 37 +++------- .../mode/local/LocalNamespaceHandler.java | 34 +++++++-- .../models/config/DatabaseConfig.java | 23 ++++++ .../models/config/SqlConnectorConfig.java | 22 +++--- .../models/worker/LocalNamespace.java | 40 +++++++++-- .../conquery/sql/DSLContextWrapper.java | 29 ++++++++ .../conquery/sql/DslContextFactory.java | 16 +++-- .../com/bakdata/conquery/sql/SqlContext.java | 11 --- .../sql/conquery/SqlExecutionManager.java | 5 +- .../sql/conversion/NodeConversions.java | 9 +-- .../conquery/sql/conversion/SqlConverter.java | 7 +- .../cqelement/ConversionContext.java | 4 +- .../conversion/dialect/HanaSqlDialect.java | 13 +--- .../conversion/dialect/PostgreSqlDialect.java | 13 +--- .../sql/conversion/dialect/SqlDialect.java | 8 +-- .../conversion/dialect/SqlDialectFactory.java | 14 ++++ .../conquery/util/TablePrimaryColumnUtil.java | 4 +- .../com/bakdata/conquery/util/io/Cloner.java | 11 +-- .../integration/IntegrationTests.java | 9 +-- .../integration/sql/CsvTableImporter.java | 10 +-- .../integration/sql/SqlStandaloneCommand.java | 18 +---- .../sql/dialect/HanaSqlIntegrationTests.java | 72 +++++++++---------- .../dialect/PostgreSqlIntegrationTests.java | 50 ++++++------- .../sql/dialect/TestContextProvider.java | 10 +-- .../sql/dialect/TestSqlConnectorConfig.java | 23 ++++++ .../sql/dialect/TestSqlDialectFactory.java | 16 +++++ 26 files changed, 310 insertions(+), 198 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConfig.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/DSLContextWrapper.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/SqlContext.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialectFactory.java create mode 100644 backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlConnectorConfig.java create mode 100644 backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlDialectFactory.java diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java index 91588c3bf5..a6d5f63277 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java @@ -10,39 +10,30 @@ import com.bakdata.conquery.mode.ManagerProvider; import com.bakdata.conquery.mode.NamespaceHandler; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.LocalNamespace; import com.bakdata.conquery.models.worker.ShardNodeInformation; -import com.bakdata.conquery.sql.DslContextFactory; -import com.bakdata.conquery.sql.SqlContext; -import com.bakdata.conquery.sql.conversion.dialect.HanaSqlDialect; -import com.bakdata.conquery.sql.conversion.dialect.PostgreSqlDialect; -import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; -import com.bakdata.conquery.sql.execution.ResultSetProcessorFactory; -import com.bakdata.conquery.sql.execution.SqlExecutionService; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialectFactory; import io.dropwizard.core.setup.Environment; -import org.jooq.DSLContext; public class LocalManagerProvider implements ManagerProvider { private static final Supplier> EMPTY_NODE_PROVIDER = Collections::emptyList; - public DelegateManager provideManager(ConqueryConfig config, Environment environment) { + private final SqlDialectFactory dialectFactory; - InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, environment.getValidator()); + public LocalManagerProvider() { + this.dialectFactory = new SqlDialectFactory(); + } - SqlConnectorConfig sqlConnectorConfig = config.getSqlConnectorConfig(); - DSLContext dslContext = DslContextFactory.create(sqlConnectorConfig); - SqlDialect sqlDialect = createSqlDialect(sqlConnectorConfig, dslContext); - SqlContext sqlContext = new SqlContext(sqlConnectorConfig, sqlDialect); + public LocalManagerProvider(SqlDialectFactory dialectFactory) { + this.dialectFactory = dialectFactory; + } - SqlExecutionService sqlExecutionService = new SqlExecutionService( - sqlDialect.getDSLContext(), - ResultSetProcessorFactory.create(sqlDialect) - ); + public DelegateManager provideManager(ConqueryConfig config, Environment environment) { - NamespaceHandler namespaceHandler = new LocalNamespaceHandler(config, creator, sqlContext, sqlExecutionService); + InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, environment.getValidator()); + NamespaceHandler namespaceHandler = new LocalNamespaceHandler(config, creator, dialectFactory); DatasetRegistry datasetRegistry = ManagerProvider.createLocalDatasetRegistry(namespaceHandler, config, creator); creator.init(datasetRegistry); @@ -59,10 +50,4 @@ public DelegateManager provideManager(ConqueryConfig config, Env ); } - protected SqlDialect createSqlDialect(SqlConnectorConfig sqlConnectorConfig, DSLContext dslContext) { - return switch (sqlConnectorConfig.getDialect()) { - case POSTGRESQL -> new PostgreSqlDialect(dslContext); - case HANA -> new HanaSqlDialect(dslContext); - }; - } } diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java index ce79a76c6c..52909c50ef 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java @@ -6,33 +6,57 @@ import com.bakdata.conquery.mode.NamespaceHandler; import com.bakdata.conquery.mode.NamespaceSetupData; import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.config.DatabaseConfig; +import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.index.IndexService; import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.models.worker.LocalNamespace; -import com.bakdata.conquery.sql.SqlContext; +import com.bakdata.conquery.sql.DSLContextWrapper; +import com.bakdata.conquery.sql.DslContextFactory; import com.bakdata.conquery.sql.conquery.SqlExecutionManager; +import com.bakdata.conquery.sql.conversion.SqlConverter; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialectFactory; +import com.bakdata.conquery.sql.execution.ResultSetProcessorFactory; +import com.bakdata.conquery.sql.execution.SqlExecutionResult; import com.bakdata.conquery.sql.execution.SqlExecutionService; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.jooq.DSLContext; @RequiredArgsConstructor +@Slf4j public class LocalNamespaceHandler implements NamespaceHandler { private final ConqueryConfig config; private final InternalObjectMapperCreator mapperCreator; - private final SqlContext sqlContext; - private final SqlExecutionService sqlExecutionService; + private final SqlDialectFactory dialectFactory; @Override public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaStorage metaStorage, IndexService indexService) { + NamespaceSetupData namespaceData = NamespaceHandler.createNamespaceSetup(namespaceStorage, config, mapperCreator, indexService); - ExecutionManager executionManager = new SqlExecutionManager(sqlContext, sqlExecutionService, metaStorage); + + SqlConnectorConfig sqlConnectorConfig = config.getSqlConnectorConfig(); + DatabaseConfig databaseConfig = sqlConnectorConfig.getDatabaseConfig(namespaceStorage.getDataset()); + + DSLContextWrapper dslContextWrapper = DslContextFactory.create(databaseConfig, sqlConnectorConfig); + DSLContext dslContext = dslContextWrapper.getDslContext(); + SqlDialect sqlDialect = dialectFactory.createSqlDialect(databaseConfig.getDialect()); + + SqlConverter sqlConverter = new SqlConverter(sqlDialect, dslContext, databaseConfig); + SqlExecutionService sqlExecutionService = new SqlExecutionService(dslContext, ResultSetProcessorFactory.create(sqlDialect)); + ExecutionManager executionManager = new SqlExecutionManager(sqlConverter, sqlExecutionService, metaStorage); + SqlStorageHandler sqlStorageHandler = new SqlStorageHandler(sqlExecutionService); + return new LocalNamespace( namespaceData.getPreprocessMapper(), namespaceData.getCommunicationMapper(), namespaceStorage, executionManager, - sqlExecutionService, + dslContextWrapper, + sqlStorageHandler, namespaceData.getJobManager(), namespaceData.getFilterSearch(), namespaceData.getIndexService(), diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConfig.java new file mode 100644 index 0000000000..b42aa9bf0e --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/config/DatabaseConfig.java @@ -0,0 +1,23 @@ +package com.bakdata.conquery.models.config; + +import lombok.Builder; +import lombok.Data; + +@Data +@Builder +public class DatabaseConfig { + + private static final String DEFAULT_PRIMARY_COLUMN = "pid"; + + private Dialect dialect; + + private String databaseUsername; + + private String databasePassword; + + private String jdbcConnectionUrl; + + @Builder.Default + private String primaryColumn = DEFAULT_PRIMARY_COLUMN; + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java index 7393a58f01..2a65aa5ca6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java @@ -1,8 +1,13 @@ package com.bakdata.conquery.models.config; +import java.util.Map; + +import com.bakdata.conquery.models.datasets.Dataset; +import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Data; +import lombok.Getter; import lombok.NoArgsConstructor; @Data @@ -11,22 +16,21 @@ @AllArgsConstructor public class SqlConnectorConfig { - public static final String DEFAULT_PRIMARY_COLUMN = "pid"; - boolean enabled; - private Dialect dialect; - /** * Determines if generated SQL should be formatted. */ private boolean withPrettyPrinting; - private String databaseUsername; - - private String databasePassword; + /** + * Keys must match the name of existing {@link Dataset}s. + */ + @Getter(AccessLevel.PRIVATE) + private Map databaseConfigs; - private String jdbcConnectionUrl; + public DatabaseConfig getDatabaseConfig(Dataset dataset) { + return databaseConfigs.get(dataset.getName()); + } - private String primaryColumn = DEFAULT_PRIMARY_COLUMN; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java index 5663baedc5..e5a2c8f5aa 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.worker; +import java.io.IOException; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -13,31 +14,34 @@ import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.models.query.FilterSearch; -import com.bakdata.conquery.sql.execution.SqlExecutionService; +import com.bakdata.conquery.sql.DSLContextWrapper; +import com.bakdata.conquery.sql.execution.SqlExecutionResult; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.Getter; +import lombok.extern.slf4j.Slf4j; @Getter +@Slf4j public class LocalNamespace extends Namespace { - private final SqlExecutionService sqlExecutionService; - + private final DSLContextWrapper dslContextWrapper; private final SqlStorageHandler storageHandler; public LocalNamespace( ObjectMapper preprocessMapper, ObjectMapper communicationMapper, NamespaceStorage storage, - ExecutionManager executionManager, - SqlExecutionService sqlExecutionService, + ExecutionManager executionManager, + DSLContextWrapper dslContextWrapper, + SqlStorageHandler storageHandler, JobManager jobManager, FilterSearch filterSearch, IndexService indexService, List injectables ) { super(preprocessMapper, communicationMapper, storage, executionManager, jobManager, filterSearch, indexService, injectables); - this.sqlExecutionService = sqlExecutionService; - this.storageHandler = new SqlStorageHandler(sqlExecutionService); + this.dslContextWrapper = dslContextWrapper; + this.storageHandler = storageHandler; } @Override @@ -52,4 +56,26 @@ void registerColumnValuesInSearch(Set columns) { getFilterSearch().registerValues(column, stringStream.collect(Collectors.toSet())); } } + + @Override + public void close() { + closeDslContextWrapper(); + super.close(); + } + + @Override + public void remove() { + closeDslContextWrapper(); + super.remove(); + } + + private void closeDslContextWrapper() { + try { + dslContextWrapper.close(); + } + catch (IOException e) { + log.warn("Could not close namespace's {} DSLContext/Datasource directly", getDataset().getId(), e); + } + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/DSLContextWrapper.java b/backend/src/main/java/com/bakdata/conquery/sql/DSLContextWrapper.java new file mode 100644 index 0000000000..cf031678ad --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/DSLContextWrapper.java @@ -0,0 +1,29 @@ +package com.bakdata.conquery.sql; + +import java.io.Closeable; +import java.io.IOException; + +import com.zaxxer.hikari.HikariDataSource; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import org.jooq.DSLContext; + +/** + * Provides access to a configured {@link DSLContext} and enables closing the underlying connection pool. + */ +@RequiredArgsConstructor +public class DSLContextWrapper implements Closeable { + + @Getter + private final DSLContext dslContext; + + private final HikariDataSource dataSource; + + @Override + public void close() throws IOException { + // Hikari opens a connection pool under the hood which we won't be able to close after passing it to the DSLContext. + // That's why we keep the HikariDataSource reference. + dataSource.close(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java index 980aa597bb..b7ea1916d3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.sql; -import javax.sql.DataSource; - +import com.bakdata.conquery.models.config.DatabaseConfig; import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; @@ -12,25 +11,28 @@ public class DslContextFactory { - public static DSLContext create(SqlConnectorConfig config) { + public static DSLContextWrapper create(DatabaseConfig config, SqlConnectorConfig connectorConfig) { + HikariConfig hikariConfig = new HikariConfig(); hikariConfig.setJdbcUrl(config.getJdbcConnectionUrl()); hikariConfig.setUsername(config.getDatabaseUsername()); hikariConfig.setPassword(config.getDatabasePassword()); - DataSource dataSource = new HikariDataSource(hikariConfig); + HikariDataSource hikariDataSource = new HikariDataSource(hikariConfig); Settings settings = new Settings() - .withRenderFormatted(config.isWithPrettyPrinting()) + .withRenderFormatted(connectorConfig.isWithPrettyPrinting()) // enforces all identifiers to be quoted if not explicitly unquoted via DSL.unquotedName() // to prevent any lowercase/uppercase SQL dialect specific identifier naming issues .withRenderQuotedNames(RenderQuotedNames.EXPLICIT_DEFAULT_QUOTED); - return DSL.using( - dataSource, + DSLContext dslContext = DSL.using( + hikariDataSource, config.getDialect().getJooqDialect(), settings ); + + return new DSLContextWrapper(dslContext, hikariDataSource); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/SqlContext.java b/backend/src/main/java/com/bakdata/conquery/sql/SqlContext.java deleted file mode 100644 index bca5ca6f2e..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/SqlContext.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.bakdata.conquery.sql; - -import com.bakdata.conquery.models.config.SqlConnectorConfig; -import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; -import lombok.Value; - -@Value -public class SqlContext { - SqlConnectorConfig config; - SqlDialect sqlDialect; -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java index a0c1616396..5fb2e663ad 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java @@ -15,7 +15,6 @@ import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.worker.Namespace; -import com.bakdata.conquery.sql.SqlContext; import com.bakdata.conquery.sql.conversion.SqlConverter; import com.bakdata.conquery.sql.conversion.model.SqlQuery; import com.bakdata.conquery.sql.execution.SqlExecutionResult; @@ -29,10 +28,10 @@ public class SqlExecutionManager extends ExecutionManager { private final SqlConverter converter; private final Map> runningExecutions; - public SqlExecutionManager(final SqlContext context, SqlExecutionService sqlExecutionService, MetaStorage storage) { + public SqlExecutionManager(SqlConverter sqlConverter, SqlExecutionService sqlExecutionService, MetaStorage storage) { super(storage); + this.converter = sqlConverter; this.executionService = sqlExecutionService; - this.converter = new SqlConverter(context.getSqlDialect(), context.getConfig()); this.runningExecutions = new HashMap<>(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java index 81b936e558..769fe973ed 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConversions.java @@ -1,11 +1,12 @@ package com.bakdata.conquery.sql.conversion; import com.bakdata.conquery.apiv1.query.QueryDescription; -import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.models.config.DatabaseConfig; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; import com.bakdata.conquery.sql.conversion.model.NameGenerator; +import org.jooq.DSLContext; /** * Entry point for converting {@link QueryDescription} to an SQL query. @@ -13,10 +14,10 @@ public class NodeConversions extends Conversions { private final SqlDialect dialect; - private final SqlConnectorConfig config; + private final DatabaseConfig config; - public NodeConversions(SqlDialect dialect, SqlConnectorConfig config) { - super(dialect.getNodeConverters()); + public NodeConversions(SqlDialect dialect, DSLContext dslContext, DatabaseConfig config) { + super(dialect.getNodeConverters(dslContext)); this.dialect = dialect; this.config = config; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java index 261195c7a7..af2a6fa681 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java @@ -1,17 +1,18 @@ package com.bakdata.conquery.sql.conversion; import com.bakdata.conquery.apiv1.query.QueryDescription; -import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.models.config.DatabaseConfig; import com.bakdata.conquery.sql.conversion.cqelement.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; import com.bakdata.conquery.sql.conversion.model.SqlQuery; +import org.jooq.DSLContext; public class SqlConverter { private final NodeConversions nodeConversions; - public SqlConverter(SqlDialect dialect, SqlConnectorConfig config) { - this.nodeConversions = new NodeConversions(dialect, config); + public SqlConverter(SqlDialect dialect, DSLContext dslContext, DatabaseConfig config) { + this.nodeConversions = new NodeConversions(dialect, dslContext, config); } public SqlQuery convert(QueryDescription queryDescription) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java index ad10c04235..f55754e70f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConversionContext.java @@ -7,7 +7,7 @@ import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; import com.bakdata.conquery.apiv1.query.concept.specific.CQDateRestriction; import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.models.config.DatabaseConfig; import com.bakdata.conquery.models.datasets.SecondaryIdDescription; import com.bakdata.conquery.sql.conversion.Context; import com.bakdata.conquery.sql.conversion.NodeConversions; @@ -25,7 +25,7 @@ @Builder(toBuilder = true) public class ConversionContext implements Context { - SqlConnectorConfig config; + DatabaseConfig config; NodeConversions nodeConversions; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlDialect.java index 17e54c5d84..72228209df 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlDialect.java @@ -16,29 +16,22 @@ public class HanaSqlDialect implements SqlDialect { private final IntervalPacker hanaIntervalPacker; private final SqlDateAggregator hanaSqlDateAggregator; private final DefaultSqlCDateSetParser defaultNotationParser; - private final DSLContext dslContext; - public HanaSqlDialect(DSLContext dslContext) { - this.dslContext = dslContext; + public HanaSqlDialect() { this.hanaSqlFunctionProvider = new HanaSqlFunctionProvider(); this.hanaIntervalPacker = new AnsiSqlIntervalPacker(); this.hanaSqlDateAggregator = new AnsiSqlDateAggregator(this.hanaIntervalPacker); this.defaultNotationParser = new DefaultSqlCDateSetParser(); } - @Override - public DSLContext getDSLContext() { - return this.dslContext; - } - @Override public SqlCDateSetParser getCDateSetParser() { return this.defaultNotationParser; } @Override - public List> getNodeConverters() { - return getDefaultNodeConverters(); + public List> getNodeConverters(DSLContext dslContext) { + return getDefaultNodeConverters(dslContext); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlDialect.java index 014308bbe2..b5f385343a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlDialect.java @@ -16,21 +16,14 @@ public class PostgreSqlDialect implements SqlDialect { private final IntervalPacker postgresqlIntervalPacker; private final SqlDateAggregator postgresqlDateAggregator; private final DefaultSqlCDateSetParser defaultNotationParser; - private final DSLContext dslContext; - public PostgreSqlDialect(DSLContext dslContext) { - this.dslContext = dslContext; + public PostgreSqlDialect() { this.postgresqlFunctionProvider = new PostgreSqlFunctionProvider(); this.postgresqlIntervalPacker = new PostgreSqlIntervalPacker(this.postgresqlFunctionProvider); this.postgresqlDateAggregator = new PostgreSqlDateAggregator(this.postgresqlFunctionProvider); this.defaultNotationParser = new DefaultSqlCDateSetParser(); } - @Override - public DSLContext getDSLContext() { - return this.dslContext; - } - @Override public SqlCDateSetParser getCDateSetParser() { return this.defaultNotationParser; @@ -42,8 +35,8 @@ public boolean supportsSingleColumnRanges() { } @Override - public List> getNodeConverters() { - return getDefaultNodeConverters(); + public List> getNodeConverters(DSLContext dslContext) { + return getDefaultNodeConverters(dslContext); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java index 733112ac18..d201bdf720 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java @@ -38,9 +38,7 @@ public interface SqlDialect { SqlDateAggregator getDateAggregator(); - List> getNodeConverters(); - - DSLContext getDSLContext(); + List> getNodeConverters(DSLContext context); SqlCDateSetParser getCDateSetParser(); @@ -52,9 +50,9 @@ default boolean supportsSingleColumnRanges() { return false; } - default List> getDefaultNodeConverters() { + default List> getDefaultNodeConverters(DSLContext dslContext) { - QueryStepTransformer queryStepTransformer = new QueryStepTransformer(getDSLContext()); + QueryStepTransformer queryStepTransformer = new QueryStepTransformer(dslContext); FormConversionHelper formConversionUtil = new FormConversionHelper(queryStepTransformer); return List.of( diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialectFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialectFactory.java new file mode 100644 index 0000000000..3480926432 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialectFactory.java @@ -0,0 +1,14 @@ +package com.bakdata.conquery.sql.conversion.dialect; + +import com.bakdata.conquery.models.config.Dialect; + +public class SqlDialectFactory { + + public SqlDialect createSqlDialect(Dialect dialect) { + return switch (dialect) { + case POSTGRESQL -> new PostgreSqlDialect(); + case HANA -> new HanaSqlDialect(); + }; + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java b/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java index 491bc04338..8ed9cafaf9 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/util/TablePrimaryColumnUtil.java @@ -1,13 +1,13 @@ package com.bakdata.conquery.util; -import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.models.config.DatabaseConfig; import com.bakdata.conquery.models.datasets.Table; import org.jooq.Field; import org.jooq.impl.DSL; public class TablePrimaryColumnUtil { - public static Field findPrimaryColumn(Table table, SqlConnectorConfig sqlConfig) { + public static Field findPrimaryColumn(Table table, DatabaseConfig sqlConfig) { String primaryColumnName = table.getPrimaryColum() == null ? sqlConfig.getPrimaryColumn() : table.getPrimaryColum().getName(); diff --git a/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java b/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java index 30829dc1c7..9c217e30d3 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java +++ b/backend/src/main/java/com/bakdata/conquery/util/io/Cloner.java @@ -25,6 +25,7 @@ public static ConqueryConfig clone(ConqueryConfig config, Map, T> i ConqueryConfig.class ); clone.setLoggingFactory(config.getLoggingFactory()); + clone.setSqlConnectorConfig(config.getSqlConnectorConfig()); return clone; } catch (IOException e) { @@ -35,11 +36,11 @@ public static ConqueryConfig clone(ConqueryConfig config, Map, T> i public static T clone(T element, Injectable injectable, Class valueType) { try { return injectable - .injectIntoNew(Jackson.BINARY_MAPPER) - .readValue( - Jackson.BINARY_MAPPER.writeValueAsBytes(element), - valueType - ); + .injectIntoNew(Jackson.BINARY_MAPPER) + .readValue( + Jackson.BINARY_MAPPER.writeValueAsBytes(element), + valueType + ); } catch (IOException e) { throw new IllegalStateException("Failed to clone the CQElement " + element, e); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java index 5034162860..d48f99401c 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java @@ -23,13 +23,14 @@ import com.bakdata.conquery.integration.json.JsonIntegrationTest; import com.bakdata.conquery.integration.json.TestDataImporter; import com.bakdata.conquery.integration.json.WorkerTestDataImporter; +import com.bakdata.conquery.integration.sql.dialect.TestSqlConnectorConfig; import com.bakdata.conquery.integration.tests.ProgrammaticIntegrationTest; import com.bakdata.conquery.io.cps.CPSTypeIdResolver; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.config.DatabaseConfig; import com.bakdata.conquery.models.config.Dialect; -import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.util.support.ConfigOverride; import com.bakdata.conquery.util.support.StandaloneSupport; import com.bakdata.conquery.util.support.TestConquery; @@ -132,18 +133,18 @@ public Stream programmaticTests(TestDataImporter testImporter, Stan } @SneakyThrows - public Stream sqlProgrammaticTests(SqlConnectorConfig sqlConfig, TestDataImporter testDataImporter) { + public Stream sqlProgrammaticTests(DatabaseConfig databaseConfig, TestSqlConnectorConfig sqlConfig, TestDataImporter testDataImporter) { this.config.setSqlConnectorConfig(sqlConfig); return programmaticTests(testDataImporter, StandaloneSupport.Mode.SQL); } @SneakyThrows - public List sqlQueryTests(SqlConnectorConfig sqlConfig, TestDataImporter testDataImporter) { + public List sqlQueryTests(DatabaseConfig databaseConfig, TestSqlConnectorConfig sqlConfig, TestDataImporter testDataImporter) { this.config.setSqlConnectorConfig(sqlConfig); final String testRoot = Objects.requireNonNullElse(System.getenv(TestTags.SQL_BACKEND_TEST_DIRECTORY_ENVIRONMENT_VARIABLE), defaultTestRoot); ResourceTree tree = scanForResources(testRoot, SQL_TEST_PATTERN); - return collectTestTree(tree, testRoot, testDataImporter, sqlConfig.getDialect()); + return collectTestTree(tree, testRoot, testDataImporter, databaseConfig.getDialect()); } private List collectTestTree(ResourceTree tree, String testRoot, TestDataImporter testImporter, Dialect sqlDialect) { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java index 046887dbae..010d6ee112 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java @@ -17,7 +17,7 @@ import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.config.CSVConfig; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.models.config.DatabaseConfig; import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.preproc.parser.specific.DateRangeParser; import com.google.common.base.Strings; @@ -43,14 +43,14 @@ public class CsvTableImporter { private final DateRangeParser dateRangeParser; private final CsvParser csvReader; private final TestSqlDialect testSqlDialect; - private final SqlConnectorConfig sqlConnectorConfig; + private final DatabaseConfig databaseConfig; - public CsvTableImporter(DSLContext dslContext, TestSqlDialect testSqlDialect, SqlConnectorConfig sqlConnectorConfig) { + public CsvTableImporter(DSLContext dslContext, TestSqlDialect testSqlDialect, DatabaseConfig databaseConfig) { this.dslContext = dslContext; this.dateRangeParser = new DateRangeParser(new ConqueryConfig()); this.csvReader = new CSVConfig().withSkipHeader(true).createParser(); this.testSqlDialect = testSqlDialect; - this.sqlConnectorConfig = sqlConnectorConfig; + this.databaseConfig = databaseConfig; } /** @@ -121,7 +121,7 @@ private Field createField(RequiredColumn requiredColumn) { }; // Set all columns except 'pid' to nullable, important for ClickHouse compatibility - if (!requiredColumn.getName().equals(sqlConnectorConfig.getPrimaryColumn())) { + if (!requiredColumn.getName().equals(databaseConfig.getPrimaryColumn())) { dataType = dataType.nullable(true); } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlStandaloneCommand.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlStandaloneCommand.java index 37c74be44b..6504e804a2 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlStandaloneCommand.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlStandaloneCommand.java @@ -7,14 +7,11 @@ import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.commands.ShardNode; import com.bakdata.conquery.commands.StandaloneCommand; -import com.bakdata.conquery.integration.sql.dialect.HanaSqlIntegrationTests; -import com.bakdata.conquery.integration.sql.dialect.PostgreSqlIntegrationTests; +import com.bakdata.conquery.integration.sql.dialect.TestSqlDialectFactory; import com.bakdata.conquery.mode.DelegateManager; import com.bakdata.conquery.mode.local.LocalManagerProvider; import com.bakdata.conquery.models.config.ConqueryConfig; -import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.models.worker.LocalNamespace; -import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; import com.bakdata.conquery.util.io.ConqueryMDC; import io.dropwizard.core.cli.ServerCommand; import io.dropwizard.core.setup.Bootstrap; @@ -22,7 +19,6 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import net.sourceforge.argparse4j.inf.Namespace; -import org.jooq.DSLContext; @Slf4j @Getter @@ -42,7 +38,7 @@ public SqlStandaloneCommand(Conquery conquery) { public void startStandalone(Environment environment, Namespace namespace, ConqueryConfig config) throws Exception { ConqueryMDC.setLocation("ManagerNode"); log.debug("Starting ManagerNode"); - this.manager = new TestLocalManagerProvider().provideManager(config, environment); + this.manager = new LocalManagerProvider(new TestSqlDialectFactory()).provideManager(config, environment); this.conquery.setManagerNode(managerNode); this.conquery.run(manager); // starts the Jersey Server @@ -74,14 +70,6 @@ public void run(Bootstrap bootstrap, Namespace namespace, Conque startStandalone(environment, namespace, configuration); } - private static class TestLocalManagerProvider extends LocalManagerProvider { - @Override - protected SqlDialect createSqlDialect(SqlConnectorConfig sqlConnectorConfig, DSLContext dslContext) { - return switch (sqlConnectorConfig.getDialect()) { - case POSTGRESQL -> new PostgreSqlIntegrationTests.TestPostgreSqlDialect(dslContext); - case HANA -> new HanaSqlIntegrationTests.TestHanaDialect(dslContext); - }; - } - } + } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java index 5ca44ad7d6..9ec5ea4d3e 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java @@ -21,8 +21,9 @@ import com.bakdata.conquery.integration.json.TestDataImporter; import com.bakdata.conquery.integration.sql.CsvTableImporter; import com.bakdata.conquery.integration.sql.testcontainer.hana.HanaContainer; +import com.bakdata.conquery.models.config.DatabaseConfig; import com.bakdata.conquery.models.config.Dialect; -import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.sql.DSLContextWrapper; import com.bakdata.conquery.sql.DslContextFactory; import com.bakdata.conquery.sql.conversion.dialect.HanaSqlDialect; import com.bakdata.conquery.sql.conversion.supplier.DateNowSupplier; @@ -50,6 +51,7 @@ public class HanaSqlIntegrationTests extends IntegrationTests { private final static DockerImageName HANA_IMAGE = DockerImageName.parse("saplabs/hanaexpress:2.00.072.00.20231123.1"); private static final Path TMP_HANA_MOUNT_DIR = Paths.get("/tmp/data/hana"); private static boolean useLocalHanaDb = true; + private static DSLContextWrapper dslContextWrapper; static { final String USE_LOCAL_HANA_DB = System.getenv("USE_LOCAL_HANA_DB"); @@ -72,14 +74,15 @@ public Stream sqlBackendTests() { log.info("Running HANA tests with %s.".formatted(provider.getClass().getSimpleName())); - DSLContext dslContext = provider.getDslContext(); - SqlConnectorConfig config = provider.getSqlConnectorConfig(); - TestHanaDialect testHanaDialect = new TestHanaDialect(dslContext); - TestDataImporter testDataImporter = new SqlTestDataImporter(new CsvTableImporter(dslContext, testHanaDialect, config)); + dslContextWrapper = provider.getDslContextWrapper(); + DatabaseConfig databaseConfig = provider.getDatabaseConfig(); + TestSqlConnectorConfig config = provider.getSqlConnectorConfig(); + TestHanaDialect testHanaDialect = new TestHanaDialect(); + TestDataImporter testDataImporter = new SqlTestDataImporter(new CsvTableImporter(dslContextWrapper.getDslContext(), testHanaDialect, databaseConfig)); return Stream.concat( - super.sqlProgrammaticTests(config, testDataImporter), - super.sqlQueryTests(config, testDataImporter).stream() + super.sqlProgrammaticTests(databaseConfig, config, testDataImporter), + super.sqlQueryTests(databaseConfig, config, testDataImporter).stream() ); } @@ -102,6 +105,9 @@ public static void prepareTmpHanaDir() { @SneakyThrows @AfterAll public static void tearDownClass() { + + dslContextWrapper.close(); + if (!Files.exists(TMP_HANA_MOUNT_DIR)) { return; } @@ -116,11 +122,6 @@ public static class TestHanaDialect extends HanaSqlDialect implements TestSqlDia public static final MockDateNowSupplier DATE_NOW_SUPPLIER = new MockDateNowSupplier(); - - public TestHanaDialect(DSLContext dslContext) { - super(dslContext); - } - @Override public DateNowSupplier getDateNowSupplier() { return DATE_NOW_SUPPLIER; @@ -157,8 +158,9 @@ public String createDropTableStatement(Table table, DSLContext dslContex @Getter private static class HanaTestcontainerContextProvider implements TestContextProvider { - private final DSLContext dslContext; - private final SqlConnectorConfig sqlConnectorConfig; + private final DSLContextWrapper dslContextWrapper; + private final DatabaseConfig databaseConfig; + private final TestSqlConnectorConfig sqlConnectorConfig; @Container private final HanaContainer hanaContainer; @@ -167,17 +169,14 @@ public HanaTestcontainerContextProvider() { this.hanaContainer = new HanaContainer<>(HANA_IMAGE) .withFileSystemBind(TMP_HANA_MOUNT_DIR.toString(), "/home/secrets"); this.hanaContainer.start(); - - this.sqlConnectorConfig = SqlConnectorConfig.builder() - .enabled(true) - .dialect(Dialect.HANA) - .withPrettyPrinting(true) - .jdbcConnectionUrl(hanaContainer.getJdbcUrl()) - .databaseUsername(hanaContainer.getUsername()) - .databasePassword(hanaContainer.getPassword()) - .primaryColumn("pid") - .build(); - this.dslContext = DslContextFactory.create(sqlConnectorConfig); + this.databaseConfig = DatabaseConfig.builder() + .dialect(Dialect.HANA) + .jdbcConnectionUrl(hanaContainer.getJdbcUrl()) + .databaseUsername(hanaContainer.getUsername()) + .databasePassword(hanaContainer.getPassword()) + .build(); + this.sqlConnectorConfig = new TestSqlConnectorConfig(databaseConfig); + this.dslContextWrapper = DslContextFactory.create(this.databaseConfig, sqlConnectorConfig); } } @@ -190,20 +189,19 @@ private static class RemoteHanaContextProvider implements TestContextProvider { private final static String CONNECTION_URL = "jdbc:sap://%s:%s/databaseName=HXE&encrypt=true&validateCertificate=false".formatted(HOST, PORT); private final static String USERNAME = System.getenv("CONQUERY_SQL_USER"); private final static String PASSWORD = System.getenv("CONQUERY_SQL_PASSWORD"); - private final DSLContext dslContext; - private final SqlConnectorConfig sqlConnectorConfig; + private final DSLContextWrapper dslContextWrapper; + private final DatabaseConfig databaseConfig; + private final TestSqlConnectorConfig sqlConnectorConfig; public RemoteHanaContextProvider() { - this.sqlConnectorConfig = SqlConnectorConfig.builder() - .enabled(true) - .dialect(Dialect.HANA) - .withPrettyPrinting(true) - .jdbcConnectionUrl(CONNECTION_URL) - .databaseUsername(USERNAME) - .databasePassword(PASSWORD) - .primaryColumn("pid") - .build(); - this.dslContext = DslContextFactory.create(sqlConnectorConfig); + this.databaseConfig = DatabaseConfig.builder() + .dialect(Dialect.HANA) + .jdbcConnectionUrl(CONNECTION_URL) + .databaseUsername(USERNAME) + .databasePassword(PASSWORD) + .build(); + this.sqlConnectorConfig = new TestSqlConnectorConfig(databaseConfig); + this.dslContextWrapper = DslContextFactory.create(databaseConfig, sqlConnectorConfig); } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java index 6e7dfaa61e..3a31b73d7c 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.integration.sql.dialect; +import java.io.IOException; import java.util.stream.Stream; import com.bakdata.conquery.TestTags; @@ -7,10 +8,11 @@ import com.bakdata.conquery.integration.IntegrationTests; import com.bakdata.conquery.integration.json.SqlTestDataImporter; import com.bakdata.conquery.integration.sql.CsvTableImporter; +import com.bakdata.conquery.models.config.DatabaseConfig; import com.bakdata.conquery.models.config.Dialect; -import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.i18n.I18n; +import com.bakdata.conquery.sql.DSLContextWrapper; import com.bakdata.conquery.sql.DslContextFactory; import com.bakdata.conquery.sql.conversion.dialect.PostgreSqlDialect; import com.bakdata.conquery.sql.conversion.model.SqlQuery; @@ -20,7 +22,7 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.assertj.core.api.Assertions; -import org.jooq.DSLContext; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.DynamicNode; import org.junit.jupiter.api.Tag; @@ -39,8 +41,10 @@ public class PostgreSqlIntegrationTests extends IntegrationTests { private static final String DATABASE_NAME = "test"; private static final String USERNAME = "user"; private static final String PASSWORD = "pass"; - private static DSLContext dslContext; - private static SqlConnectorConfig sqlConfig; + + private static DSLContextWrapper dslContextWrapper; + private static DatabaseConfig databaseConfig; + private static TestSqlConnectorConfig sqlConfig; private static TestSqlDialect testSqlDialect; private static SqlTestDataImporter testDataImporter; @@ -58,18 +62,21 @@ public PostgreSqlIntegrationTests() { @BeforeAll static void before() { POSTGRESQL_CONTAINER.start(); - sqlConfig = SqlConnectorConfig.builder() - .enabled(true) - .dialect(Dialect.POSTGRESQL) - .jdbcConnectionUrl(POSTGRESQL_CONTAINER.getJdbcUrl()) - .databaseUsername(USERNAME) - .databasePassword(PASSWORD) - .withPrettyPrinting(true) - .primaryColumn("pid") - .build(); - dslContext = DslContextFactory.create(sqlConfig); - testSqlDialect = new TestPostgreSqlDialect(dslContext); - testDataImporter = new SqlTestDataImporter(new CsvTableImporter(dslContext, testSqlDialect, sqlConfig)); + databaseConfig = DatabaseConfig.builder() + .dialect(Dialect.POSTGRESQL) + .jdbcConnectionUrl(POSTGRESQL_CONTAINER.getJdbcUrl()) + .databaseUsername(USERNAME) + .databasePassword(PASSWORD) + .build(); + sqlConfig = new TestSqlConnectorConfig(databaseConfig); + dslContextWrapper = DslContextFactory.create(databaseConfig, sqlConfig); + testSqlDialect = new TestPostgreSqlDialect(); + testDataImporter = new SqlTestDataImporter(new CsvTableImporter(dslContextWrapper.getDslContext(), testSqlDialect, databaseConfig)); + } + + @AfterAll + static void after() throws IOException { + dslContextWrapper.close(); } @Test @@ -77,7 +84,7 @@ static void before() { public void shouldThrowException() { // This can be removed as soon as we switch to a full integration test including the REST API I18n.init(); - SqlExecutionService executionService = new SqlExecutionService(dslContext, ResultSetProcessorFactory.create(testSqlDialect)); + SqlExecutionService executionService = new SqlExecutionService(dslContextWrapper.getDslContext(), ResultSetProcessorFactory.create(testSqlDialect)); SqlQuery validQuery = new TestSqlQuery("SELECT 1"); Assertions.assertThatNoException().isThrownBy(() -> executionService.execute(validQuery)); @@ -92,8 +99,8 @@ public void shouldThrowException() { @Tag(TestTags.INTEGRATION_SQL_BACKEND) public Stream sqlBackendTests() { return Stream.concat( - super.sqlProgrammaticTests(sqlConfig, testDataImporter), - super.sqlQueryTests(sqlConfig, testDataImporter).stream() + super.sqlProgrammaticTests(databaseConfig, sqlConfig, testDataImporter), + super.sqlQueryTests(databaseConfig, sqlConfig, testDataImporter).stream() ); } @@ -101,11 +108,6 @@ public static class TestPostgreSqlDialect extends PostgreSqlDialect implements T public static final MockDateNowSupplier DATE_NOW_SUPPLIER = new MockDateNowSupplier(); - - public TestPostgreSqlDialect(DSLContext dslContext) { - super(dslContext); - } - @Override public DateNowSupplier getDateNowSupplier() { return DATE_NOW_SUPPLIER; diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestContextProvider.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestContextProvider.java index c111f83d8b..d5cf8eb140 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestContextProvider.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestContextProvider.java @@ -1,12 +1,14 @@ package com.bakdata.conquery.integration.sql.dialect; -import com.bakdata.conquery.models.config.SqlConnectorConfig; -import org.jooq.DSLContext; +import com.bakdata.conquery.models.config.DatabaseConfig; +import com.bakdata.conquery.sql.DSLContextWrapper; public interface TestContextProvider { - SqlConnectorConfig getSqlConnectorConfig(); + DatabaseConfig getDatabaseConfig(); - DSLContext getDslContext(); + TestSqlConnectorConfig getSqlConnectorConfig(); + + DSLContextWrapper getDslContextWrapper(); } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlConnectorConfig.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlConnectorConfig.java new file mode 100644 index 0000000000..ec5dccddc8 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlConnectorConfig.java @@ -0,0 +1,23 @@ +package com.bakdata.conquery.integration.sql.dialect; + +import java.util.Map; + +import com.bakdata.conquery.models.config.DatabaseConfig; +import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.models.datasets.Dataset; + +public class TestSqlConnectorConfig extends SqlConnectorConfig { + + private final DatabaseConfig databaseConfig; + + public TestSqlConnectorConfig(DatabaseConfig databaseConfig) { + super(true, true, Map.of()); + this.databaseConfig = databaseConfig; + } + + @Override + public DatabaseConfig getDatabaseConfig(Dataset dataset) { + return databaseConfig; + } + +} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlDialectFactory.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlDialectFactory.java new file mode 100644 index 0000000000..cadd8cb746 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestSqlDialectFactory.java @@ -0,0 +1,16 @@ +package com.bakdata.conquery.integration.sql.dialect; + +import com.bakdata.conquery.models.config.Dialect; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialectFactory; + +public class TestSqlDialectFactory extends SqlDialectFactory { + + @Override + public SqlDialect createSqlDialect(Dialect dialect) { + return switch (dialect) { + case POSTGRESQL -> new PostgreSqlIntegrationTests.TestPostgreSqlDialect(); + case HANA -> new HanaSqlIntegrationTests.TestHanaDialect(); + }; + } +} From a11d7ab74c4430161be4fa77111d36ee7449a98c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 04:11:34 +0000 Subject: [PATCH 21/23] Bump com.nimbusds:oauth2-oidc-sdk from 9.1 to 11.12 Bumps [com.nimbusds:oauth2-oidc-sdk](https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions) from 9.1 to 11.12. - [Changelog](https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions/src/master/CHANGELOG.txt) - [Commits](https://bitbucket.org/connect2id/oauth-2.0-sdk-with-openid-connect-extensions/branches/compare/11.12..9.1) --- updated-dependencies: - dependency-name: com.nimbusds:oauth2-oidc-sdk dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- backend/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pom.xml b/backend/pom.xml index 88e0aafdcb..7354901368 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -303,7 +303,7 @@ com.nimbusds oauth2-oidc-sdk - 9.1 + 11.12 org.keycloak From 93cb179f050aa5d5ef8f6f9e7b1c23295beb86bb Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Fri, 24 May 2024 15:38:12 +0200 Subject: [PATCH 22/23] return description for StringStoreString Signed-off-by: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> --- .../bakdata/conquery/io/storage/NamespacedStorage.java | 10 ---------- .../events/stores/primitive/StringStoreString.java | 4 ++-- .../models/events/stores/root/ColumnStore.java | 2 +- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java index 800aae6fa5..dbfa3e68d1 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java @@ -3,7 +3,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.Set; import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore; import com.bakdata.conquery.models.config.StoreFactory; @@ -15,7 +14,6 @@ import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.concepts.Connector; import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; -import com.bakdata.conquery.models.exceptions.ValidatorHelper; import com.bakdata.conquery.models.identifiable.CentralRegistry; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; import com.bakdata.conquery.models.identifiable.ids.specific.ImportId; @@ -23,7 +21,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.TableId; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; -import jakarta.validation.ConstraintViolation; import jakarta.validation.Validator; import lombok.Getter; import lombok.SneakyThrows; @@ -123,13 +120,6 @@ private void decorateConceptStore(IdentifiableStore> store) { concept.initElements(); - if (log.isTraceEnabled()) { - // Validating concepts is quite slow, so we only validate when requested. - final Set>> violations = validator.validate(concept); - - ValidatorHelper.failOnError(log, violations); - } - concept.getSelects().forEach(centralRegistry::register); for (Connector connector : concept.getConnectors()) { centralRegistry.register(connector); diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java index bca5009799..58524e069a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/primitive/StringStoreString.java @@ -67,8 +67,8 @@ public int getLines() { } @Override - public T createDescription() { - return null; + public StringStoreString createDescription() { + return ColumnStore.emptyCopy(this); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java index 3e1bb36ca9..93db593245 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/stores/root/ColumnStore.java @@ -105,7 +105,7 @@ default long estimateTypeSizeBytes() { @JsonIgnore T createDescription(); - public static T emptyCopy(T store) { + static T emptyCopy(T store) { return store.select(new int[0], new int[0]); } From ed704b6b1d568bc06867fec01278baeea9d33fd0 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Fri, 24 May 2024 15:46:24 +0200 Subject: [PATCH 23/23] remove validator from contructor Signed-off-by: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> --- .../com/bakdata/conquery/io/storage/NamespaceStorage.java | 5 ++--- .../com/bakdata/conquery/io/storage/NamespacedStorage.java | 7 +------ .../com/bakdata/conquery/io/storage/WorkerStorage.java | 2 +- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java index 899d591b46..f84b4dc544 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespaceStorage.java @@ -4,8 +4,6 @@ import java.util.Objects; import java.util.OptionalInt; -import jakarta.validation.Validator; - import com.bakdata.conquery.io.storage.xodus.stores.CachedStore; import com.bakdata.conquery.io.storage.xodus.stores.SingletonStore; import com.bakdata.conquery.models.config.StoreFactory; @@ -19,6 +17,7 @@ import com.bakdata.conquery.models.worker.WorkerToBucketsMap; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; +import jakarta.validation.Validator; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -34,7 +33,7 @@ public class NamespaceStorage extends NamespacedStorage { protected CachedStore entity2Bucket; public NamespaceStorage(StoreFactory storageFactory, String pathName, Validator validator) { - super(storageFactory, pathName, validator); + super(storageFactory, pathName); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java index dbfa3e68d1..a4a4c07d54 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/NamespacedStorage.java @@ -21,7 +21,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.TableId; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; -import jakarta.validation.Validator; import lombok.Getter; import lombok.SneakyThrows; import lombok.ToString; @@ -45,19 +44,15 @@ public abstract class NamespacedStorage extends ConqueryStorage { @Getter private final StoreFactory storageFactory; - @Getter - private final Validator validator; - protected SingletonStore dataset; protected IdentifiableStore secondaryIds; protected IdentifiableStore tables; protected IdentifiableStore imports; protected IdentifiableStore> concepts; - public NamespacedStorage(StoreFactory storageFactory, String pathName, Validator validator) { + public NamespacedStorage(StoreFactory storageFactory, String pathName) { this.pathName = pathName; this.storageFactory = storageFactory; - this.validator = validator; } public void openStores(ObjectMapper objectMapper) { diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java index 10927513b2..adfe9b2841 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/WorkerStorage.java @@ -26,7 +26,7 @@ public class WorkerStorage extends NamespacedStorage { private IdentifiableStore cBlocks; public WorkerStorage(StoreFactory storageFactory, Validator validator, String pathName) { - super(storageFactory, pathName, validator); + super(storageFactory, pathName); } @Override