From e1388096d20d7c89814accc60648a122e04d7949 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Mon, 30 Jan 2023 18:01:53 +0100 Subject: [PATCH 001/679] capsule result urls in api object --- .../java/com/bakdata/conquery/Constants.java | 6 +++--- .../conquery/apiv1/QueryProcessor.java | 21 +++++++++++-------- .../{ => execution}/ExecutionStatus.java | 15 +++++++------ .../{ => execution}/FullExecutionStatus.java | 13 ++++++------ .../OverviewExecutionStatus.java | 2 +- .../conquery/apiv1/execution/ResultAsset.java | 6 ++++++ .../apiv1/query/TableExportQuery.java | 2 +- .../ResultRender/ResultRendererProvider.java | 4 ++-- .../models/config/ArrowResultProvider.java | 8 +++---- .../models/config/CsvResultProvider.java | 6 +++--- .../models/config/ExcelResultProvider.java | 5 +++-- .../models/config/ParquetResultProvider.java | 8 +++---- .../models/execution/ManagedExecution.java | 6 +++--- .../models/forms/managed/ManagedForm.java | 2 +- .../conquery/models/query/ManagedQuery.java | 4 ++-- .../query/preview/EntityPreviewExecution.java | 2 +- .../query/preview/EntityPreviewStatus.java | 2 +- .../resources/admin/rest/AdminResource.java | 2 +- .../resources/api/DatasetQueryResource.java | 4 ++-- .../conquery/resources/api/QueryResource.java | 2 +- .../api/StoredQueriesProcessorTest.java | 12 ++++------- .../integration/DownloadLinkGeneration.java | 6 ++++-- .../integration/common/IntegrationUtils.java | 4 ++-- .../integration/tests/EntityExportTest.java | 2 ++ .../integration/tests/ReusedQueryTest.java | 3 +-- 25 files changed, 77 insertions(+), 70 deletions(-) rename backend/src/main/java/com/bakdata/conquery/apiv1/{ => execution}/ExecutionStatus.java (92%) rename backend/src/main/java/com/bakdata/conquery/apiv1/{ => execution}/FullExecutionStatus.java (97%) rename backend/src/main/java/com/bakdata/conquery/apiv1/{ => execution}/OverviewExecutionStatus.java (83%) create mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/execution/ResultAsset.java diff --git a/autodoc/src/main/java/com/bakdata/conquery/Constants.java b/autodoc/src/main/java/com/bakdata/conquery/Constants.java index 36c3fe1fe4..005ad4a0b4 100644 --- a/autodoc/src/main/java/com/bakdata/conquery/Constants.java +++ b/autodoc/src/main/java/com/bakdata/conquery/Constants.java @@ -16,13 +16,13 @@ import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; -import com.bakdata.conquery.apiv1.ExecutionStatus; import com.bakdata.conquery.apiv1.FilterTemplate; -import com.bakdata.conquery.apiv1.FullExecutionStatus; import com.bakdata.conquery.apiv1.IdLabel; import com.bakdata.conquery.apiv1.KeyValue; import com.bakdata.conquery.apiv1.MetaDataPatch; -import com.bakdata.conquery.apiv1.OverviewExecutionStatus; +import com.bakdata.conquery.apiv1.execution.ExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.OverviewExecutionStatus; import com.bakdata.conquery.apiv1.frontend.FrontendRoot; import com.bakdata.conquery.apiv1.frontend.FrontendValue; import com.bakdata.conquery.apiv1.query.CQElement; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index e6394d3543..fa226a35d8 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -2,7 +2,6 @@ import static com.bakdata.conquery.models.auth.AuthorizationHelper.buildDatasetAbilityMap; -import java.net.URL; import java.time.LocalDate; import java.util.Collection; import java.util.List; @@ -21,6 +20,10 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.UriBuilder; +import com.bakdata.conquery.apiv1.execution.ExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.OverviewExecutionStatus; +import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.ExternalUpload; @@ -191,7 +194,7 @@ private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId } - public Stream getAllQueries(Dataset dataset, HttpServletRequest req, Subject subject, boolean allProviders) { + public Stream getAllQueries(Dataset dataset, HttpServletRequest req, Subject subject, boolean allProviders) { Collection> allQueries = storage.getAllExecutions(); return getQueriesFiltered(dataset, RequestAwareUriBuilder.fromRequest(req), subject, allQueries, allProviders); @@ -212,7 +215,7 @@ public Stream getQueriesFiltered(Dataset datasetId, UriBuilder .map(mq -> { OverviewExecutionStatus status = mq.buildStatusOverview(uriBuilder.clone(), subject); if (mq.isReadyToDownload(datasetAbilities)) { - status.setResultUrls(getDownloadUrls(config.getResultProviders(), mq, uriBuilder, allProviders)); + status.setResultUrls(getResultAssets(config.getResultProviders(), mq, uriBuilder, allProviders)); } return status; }); @@ -229,7 +232,7 @@ public Stream getQueriesFiltered(Dataset datasetId, UriBuilder * @param allProviders If true, forces {@link ResultRendererProvider} to return an URL if possible. * @return The modified status */ - public static List getDownloadUrls(List renderer, ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { + public static List getResultAssets(List renderer, ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { return renderer.stream() .map(r -> r.generateResultURLs(exec, uriBuilder.clone(), allProviders)) @@ -332,15 +335,15 @@ public void deleteQuery(Subject subject, ManagedExecution execution) { storage.removeExecution(execution.getId()); } - public FullExecutionStatus getQueryFullStatus(ManagedExecution query, Subject subject, UriBuilder url, Boolean allProviders) { + public com.bakdata.conquery.apiv1.execution.FullExecutionStatus getQueryFullStatus(ManagedExecution query, Subject subject, UriBuilder url, Boolean allProviders) { query.initExecutable(datasetRegistry, config); Map> datasetAbilities = buildDatasetAbilityMap(subject, datasetRegistry); - final FullExecutionStatus status = query.buildStatusFull(storage, subject, datasetRegistry, config); + final com.bakdata.conquery.apiv1.execution.FullExecutionStatus status = query.buildStatusFull(storage, subject, datasetRegistry, config); if (query.isReadyToDownload(datasetAbilities)) { - status.setResultUrls(getDownloadUrls(config.getResultProviders(), query, url, allProviders)); + status.setResultUrls(getResultAssets(config.getResultProviders(), query, url, allProviders)); } return status; } @@ -389,7 +392,7 @@ public ExternalUploadResult uploadEntities(Subject subject, Dataset dataset, Ext /** * Create and submit {@link EntityPreviewForm} for subject on to extract sources for entity, and extract some additional infos to be used as infocard. */ - public FullExecutionStatus getSingleEntityExport(Subject subject, UriBuilder uriBuilder, String idKind, String entity, List sources, Dataset dataset, Range dateRange) { + public com.bakdata.conquery.apiv1.execution.FullExecutionStatus getSingleEntityExport(Subject subject, UriBuilder uriBuilder, String idKind, String entity, List sources, Dataset dataset, Range dateRange) { EntityPreviewForm form = EntityPreviewForm.create(entity, idKind, dateRange, sources, datasetRegistry.get(dataset.getId()).getPreviewConfig().getSelects()); @@ -411,7 +414,7 @@ public FullExecutionStatus getSingleEntityExport(Subject subject, UriBuilder uri FullExecutionStatus status = execution.buildStatusFull(storage, subject, datasetRegistry, config); - status.setResultUrls(getDownloadUrls(config.getResultProviders(), execution, uriBuilder, false)); + status.setResultUrls(getResultAssets(config.getResultProviders(), execution, uriBuilder, false)); return status; } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/ExecutionStatus.java similarity index 92% rename from backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java rename to backend/src/main/java/com/bakdata/conquery/apiv1/execution/ExecutionStatus.java index a9c674b11e..4aec080f22 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/ExecutionStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/ExecutionStatus.java @@ -1,4 +1,9 @@ -package com.bakdata.conquery.apiv1; +package com.bakdata.conquery.apiv1.execution; + +import java.time.LocalDateTime; +import java.time.ZonedDateTime; +import java.util.Collections; +import java.util.List; import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; @@ -10,12 +15,6 @@ import lombok.ToString; import lombok.experimental.FieldNameConstants; -import java.net.URL; -import java.time.LocalDateTime; -import java.time.ZonedDateTime; -import java.util.Collections; -import java.util.List; - @NoArgsConstructor @ToString @Data @@ -49,7 +48,7 @@ public abstract class ExecutionStatus { /** * The urls under from which the result of the execution can be downloaded as soon as it finished successfully. */ - private List resultUrls = Collections.emptyList(); + private List resultUrls = Collections.emptyList(); } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/FullExecutionStatus.java b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/FullExecutionStatus.java similarity index 97% rename from backend/src/main/java/com/bakdata/conquery/apiv1/FullExecutionStatus.java rename to backend/src/main/java/com/bakdata/conquery/apiv1/execution/FullExecutionStatus.java index 61c5d4a917..17c058884d 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/FullExecutionStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/FullExecutionStatus.java @@ -1,4 +1,10 @@ -package com.bakdata.conquery.apiv1; +package com.bakdata.conquery.apiv1.execution; + +import java.util.Collection; +import java.util.List; +import java.util.Set; + +import javax.annotation.Nullable; import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.io.jackson.serializer.NsIdRefCollection; @@ -11,11 +17,6 @@ import lombok.NoArgsConstructor; import lombok.experimental.FieldNameConstants; -import javax.annotation.Nullable; -import java.util.Collection; -import java.util.List; -import java.util.Set; - /** * This status holds extensive information about the query description and meta data that is computational heavy * and can produce a larger payload to requests. diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/OverviewExecutionStatus.java b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/OverviewExecutionStatus.java similarity index 83% rename from backend/src/main/java/com/bakdata/conquery/apiv1/OverviewExecutionStatus.java rename to backend/src/main/java/com/bakdata/conquery/apiv1/execution/OverviewExecutionStatus.java index 3e06d0e74c..f68c83ec96 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/OverviewExecutionStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/OverviewExecutionStatus.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.apiv1; +package com.bakdata.conquery.apiv1.execution; import lombok.NoArgsConstructor; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/execution/ResultAsset.java b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/ResultAsset.java new file mode 100644 index 0000000000..1457bc1dc6 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/ResultAsset.java @@ -0,0 +1,6 @@ +package com.bakdata.conquery.apiv1.execution; + +import java.net.URL; + +public record ResultAsset(String label, URL url) { +} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java index aa190fe549..6d85e72edf 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java @@ -19,7 +19,7 @@ import javax.validation.constraints.NotNull; import com.bakdata.conquery.ConqueryConstants; -import com.bakdata.conquery.apiv1.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.io.cps.CPSType; diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/ResultRender/ResultRendererProvider.java b/backend/src/main/java/com/bakdata/conquery/io/result/ResultRender/ResultRendererProvider.java index df1714e6f5..2dc64b25e4 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/ResultRender/ResultRendererProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/ResultRender/ResultRendererProvider.java @@ -1,10 +1,10 @@ package com.bakdata.conquery.io.result.ResultRender; -import java.net.URL; import java.util.Collection; import javax.ws.rs.core.UriBuilder; +import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.models.execution.ManagedExecution; @@ -24,7 +24,7 @@ public interface ResultRendererProvider { * @param allProviders A flag that should override internal "hide-this-url" flags. * @return An Optional with the url or an empty optional. */ - Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders); + Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders); void registerResultResource(DropwizardResourceConfig environment, ManagerNode manager); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ArrowResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ArrowResultProvider.java index 2e828c6edb..40317e9847 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ArrowResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ArrowResultProvider.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.models.config; import java.net.MalformedURLException; -import java.net.URL; import java.util.Collection; import java.util.Collections; import java.util.List; @@ -10,6 +9,7 @@ import javax.validation.constraints.NotNull; import javax.ws.rs.core.UriBuilder; +import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider; @@ -33,7 +33,7 @@ public class ArrowResultProvider implements ResultRendererProvider { @Override @SneakyThrows(MalformedURLException.class) - public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { + public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { if (!(exec instanceof SingleTableResult)) { return Collections.emptyList(); } @@ -43,8 +43,8 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuilder u } return List.of( - ResultArrowResource.getFileDownloadURL(uriBuilder.clone(), (ManagedExecution & SingleTableResult) exec), - ResultArrowResource.getStreamDownloadURL(uriBuilder.clone(), (ManagedExecution & SingleTableResult) exec) + new ResultAsset("Arrow File", ResultArrowResource.getFileDownloadURL(uriBuilder.clone(), (ManagedExecution & SingleTableResult) exec)), + new ResultAsset("Arrow Stream", ResultArrowResource.getStreamDownloadURL(uriBuilder.clone(), (ManagedExecution & SingleTableResult) exec)) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/CsvResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/CsvResultProvider.java index ee2edfa3ea..3c7a4ddc3d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/CsvResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/CsvResultProvider.java @@ -1,13 +1,13 @@ package com.bakdata.conquery.models.config; import java.net.MalformedURLException; -import java.net.URL; import java.util.Collection; import java.util.Collections; import java.util.List; import javax.ws.rs.core.UriBuilder; +import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider; @@ -30,7 +30,7 @@ public class CsvResultProvider implements ResultRendererProvider { private boolean hidden = false; @SneakyThrows(MalformedURLException.class) - public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { + public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { if (!(exec instanceof SingleTableResult)) { return Collections.emptyList(); } @@ -39,7 +39,7 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuilder u return Collections.emptyList(); } - return List.of(ResultCsvResource.getDownloadURL(uriBuilder, (ManagedExecution & SingleTableResult) exec)); + return List.of(new ResultAsset("CSV", ResultCsvResource.getDownloadURL(uriBuilder, (ManagedExecution & SingleTableResult) exec))); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java index 139d08196a..706097198d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ExcelResultProvider.java @@ -11,6 +11,7 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.UriBuilder; +import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider; @@ -45,7 +46,7 @@ public class ExcelResultProvider implements ResultRendererProvider { @Override @SneakyThrows(MalformedURLException.class) - public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { + public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { // We only support/produce xlsx files with one sheet for now if (!(exec instanceof SingleTableResult singleExecution)) { log.trace("Execution result is not a single table"); @@ -82,7 +83,7 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuilder u final URL resultUrl = ResultExcelResource.getDownloadURL(uriBuilder, (ManagedExecution & SingleTableResult) exec); log.trace("Generated URL: {}", resultUrl); - return List.of(resultUrl); + return List.of(new ResultAsset("XLSX", resultUrl)); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ParquetResultProvider.java b/backend/src/main/java/com/bakdata/conquery/models/config/ParquetResultProvider.java index 68bfb1c210..c32d728bdf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ParquetResultProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ParquetResultProvider.java @@ -1,21 +1,19 @@ package com.bakdata.conquery.models.config; import java.net.MalformedURLException; -import java.net.URL; import java.util.Collection; import java.util.Collections; import java.util.List; import javax.ws.rs.core.UriBuilder; +import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.result.ResultRender.ResultRendererProvider; -import com.bakdata.conquery.io.result.arrow.ResultArrowProcessor; import com.bakdata.conquery.io.result.parquet.ResultParquetProcessor; import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.query.SingleTableResult; -import com.bakdata.conquery.resources.api.ResultArrowResource; import com.bakdata.conquery.resources.api.ResultParquetResource; import io.dropwizard.jersey.DropwizardResourceConfig; import lombok.Data; @@ -30,7 +28,7 @@ public class ParquetResultProvider implements ResultRendererProvider { @Override @SneakyThrows(MalformedURLException.class) - public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { + public Collection generateResultURLs(ManagedExecution exec, UriBuilder uriBuilder, boolean allProviders) { if (!(exec instanceof SingleTableResult)) { return Collections.emptyList(); } @@ -40,7 +38,7 @@ public Collection generateResultURLs(ManagedExecution exec, UriBuilder u } return List.of( - ResultParquetResource.getDownloadURL(uriBuilder.clone(), (ManagedExecution & SingleTableResult) exec) + new ResultAsset("PARQUET", ResultParquetResource.getDownloadURL(uriBuilder.clone(), (ManagedExecution & SingleTableResult) exec)) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java index 71e097357c..bc1c9385ad 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java @@ -17,9 +17,9 @@ import javax.validation.constraints.NotNull; import javax.ws.rs.core.UriBuilder; -import com.bakdata.conquery.apiv1.ExecutionStatus; -import com.bakdata.conquery.apiv1.FullExecutionStatus; -import com.bakdata.conquery.apiv1.OverviewExecutionStatus; +import com.bakdata.conquery.apiv1.execution.ExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.OverviewExecutionStatus; import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.io.jackson.serializer.MetaIdRef; diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedForm.java b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedForm.java index 2ac8379861..dc9dece1db 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedForm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/managed/ManagedForm.java @@ -6,7 +6,7 @@ import java.util.function.Consumer; import java.util.stream.Collectors; -import com.bakdata.conquery.apiv1.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.apiv1.forms.Form; import com.bakdata.conquery.apiv1.forms.FormConfigAPI; import com.bakdata.conquery.apiv1.query.QueryDescription; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java index 376fc37996..df8ff022d3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java @@ -13,8 +13,8 @@ import java.util.stream.Stream; import c10n.C10N; -import com.bakdata.conquery.apiv1.ExecutionStatus; -import com.bakdata.conquery.apiv1.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.ExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java index b5fdfc280f..72ef84d85c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java @@ -6,7 +6,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import com.bakdata.conquery.apiv1.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.entities.Subject; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewStatus.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewStatus.java index 64b4de088c..26a664b6b9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewStatus.java @@ -3,7 +3,7 @@ import java.util.List; import java.util.Set; -import com.bakdata.conquery.apiv1.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.models.query.ColumnDescriptor; import com.bakdata.conquery.models.types.SemanticType; import lombok.Data; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java index eec99abc41..1049cd2757 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java @@ -23,7 +23,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.UriBuilder; -import com.bakdata.conquery.apiv1.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.io.jersey.ExtraMimeTypes; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.entities.Subject; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetQueryResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetQueryResource.java index 7997c6ac5f..1d84b57ed2 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetQueryResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/DatasetQueryResource.java @@ -22,10 +22,10 @@ import javax.ws.rs.core.UriBuilder; import com.bakdata.conquery.apiv1.AdditionalMediaTypes; -import com.bakdata.conquery.apiv1.ExecutionStatus; -import com.bakdata.conquery.apiv1.FullExecutionStatus; import com.bakdata.conquery.apiv1.QueryProcessor; import com.bakdata.conquery.apiv1.RequestAwareUriBuilder; +import com.bakdata.conquery.apiv1.execution.ExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.apiv1.query.ExternalUpload; import com.bakdata.conquery.apiv1.query.ExternalUploadResult; import com.bakdata.conquery.apiv1.query.QueryDescription; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java index 210cd63543..be0756a98f 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/QueryResource.java @@ -19,10 +19,10 @@ import javax.ws.rs.core.Context; import com.bakdata.conquery.apiv1.AdditionalMediaTypes; -import com.bakdata.conquery.apiv1.FullExecutionStatus; import com.bakdata.conquery.apiv1.MetaDataPatch; import com.bakdata.conquery.apiv1.QueryProcessor; import com.bakdata.conquery.apiv1.RequestAwareUriBuilder; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.models.auth.entities.Subject; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.execution.ManagedExecution; diff --git a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java index 0b123abfcd..78097cf627 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java @@ -3,7 +3,6 @@ import static com.bakdata.conquery.models.execution.ExecutionState.*; import static org.assertj.core.api.Assertions.assertThat; -import java.net.URL; import java.time.LocalDateTime; import java.time.ZoneId; import java.util.ArrayList; @@ -14,9 +13,10 @@ import javax.ws.rs.core.UriBuilder; -import com.bakdata.conquery.apiv1.ExecutionStatus; -import com.bakdata.conquery.apiv1.OverviewExecutionStatus; import com.bakdata.conquery.apiv1.QueryProcessor; +import com.bakdata.conquery.apiv1.execution.ExecutionStatus; +import com.bakdata.conquery.apiv1.execution.OverviewExecutionStatus; +import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.apiv1.forms.export_form.ExportForm; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.apiv1.query.ConceptQuery; @@ -48,10 +48,6 @@ import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.worker.DatasetRegistry; -import com.bakdata.conquery.resources.api.ResultArrowResource; -import com.bakdata.conquery.resources.api.ResultCsvResource; -import com.bakdata.conquery.resources.api.ResultExcelResource; -import com.bakdata.conquery.resources.api.ResultParquetResource; import com.bakdata.conquery.util.NonPersistentStoreFactory; import com.google.common.collect.ImmutableList; import lombok.SneakyThrows; @@ -233,7 +229,7 @@ public List getResultInfos() { status.setNumberOfResults(resultCount); status.setSecondaryId(secondaryId); // This is probably not interesting on the overview (only if there is an filter for the search) if(state.equals(DONE)) { - List resultUrls = new ArrayList<>(); + List resultUrls = new ArrayList<>(); resultUrls.addAll(EXCEL_RESULT_PROVIDER.generateResultURLs(execMock, URI_BUILDER.clone(), true)); resultUrls.addAll(CSV_RESULT_PROVIDER.generateResultURLs(execMock, URI_BUILDER.clone(), true)); resultUrls.addAll(ARROW_RESULT_PROVIDER.generateResultURLs(execMock, URI_BUILDER.clone(), true)); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java b/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java index cd4395227e..7f8e28b585 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java @@ -5,7 +5,8 @@ import java.net.URL; import java.util.Set; -import com.bakdata.conquery.apiv1.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.integration.common.IntegrationUtils; import com.bakdata.conquery.integration.json.JsonIntegrationTest; import com.bakdata.conquery.integration.json.QueryTest; @@ -67,7 +68,8 @@ public void execute(StandaloneSupport conquery) throws Exception { FullExecutionStatus status = IntegrationUtils.getExecutionStatus(conquery, exec.getId(), user, 200); // This Url is missing the `/api` path part, because we use the standard UriBuilder here - assertThat(status.getResultUrls()).contains(new URL(String.format("%s/result/%s.csv", conquery.defaultApiURIBuilder().toString(), exec.getId()))); + assertThat(status.getResultUrls()).contains(new ResultAsset("CSV", new URL(String.format("%s/result/%s.csv", conquery.defaultApiURIBuilder() + .toString(), exec.getId())))); } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java b/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java index 571b9ccdd8..f765d107bc 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java @@ -10,8 +10,8 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; -import com.bakdata.conquery.apiv1.ExecutionStatus; -import com.bakdata.conquery.apiv1.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.ExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.integration.json.ConqueryTestSpec; import com.bakdata.conquery.io.storage.MetaStorage; diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/EntityExportTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/EntityExportTest.java index 53847259bc..c52a2264c6 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/EntityExportTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/EntityExportTest.java @@ -19,6 +19,7 @@ import javax.ws.rs.core.Response; import com.bakdata.conquery.apiv1.AdditionalMediaTypes; +import com.bakdata.conquery.apiv1.execution.ResultAsset; import com.bakdata.conquery.integration.common.LoadingUtil; import com.bakdata.conquery.integration.common.RequiredData; import com.bakdata.conquery.integration.json.JsonIntegrationTest; @@ -174,6 +175,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { final Optional csvUrl = result.getResultUrls().stream() + .map(ResultAsset::url) .filter(url -> url.getFile().endsWith(".csv")) .findFirst(); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java index b02e64ab2d..70d34e9dc2 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ReusedQueryTest.java @@ -12,7 +12,7 @@ import javax.ws.rs.client.Entity; import javax.ws.rs.core.MediaType; -import com.bakdata.conquery.apiv1.FullExecutionStatus; +import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.SecondaryIdQuery; import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; @@ -22,7 +22,6 @@ import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; import com.bakdata.conquery.integration.common.IntegrationUtils; import com.bakdata.conquery.integration.common.LoadingUtil; -import com.bakdata.conquery.integration.common.RequiredData; import com.bakdata.conquery.integration.json.JsonIntegrationTest; import com.bakdata.conquery.integration.json.QueryTest; import com.bakdata.conquery.io.storage.MetaStorage; From 5061e825a304fb0768b8efd3218862055d97917e Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 31 Jan 2023 18:57:38 +0100 Subject: [PATCH 002/679] adds result provider to path so that an external provider would not collide with providers of a similar name --- .../com/bakdata/conquery/resources/api/ResultArrowResource.java | 2 +- .../com/bakdata/conquery/resources/api/ResultCsvResource.java | 2 +- .../com/bakdata/conquery/resources/api/ResultExcelResource.java | 2 +- .../bakdata/conquery/resources/api/ResultParquetResource.java | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowResource.java index 310cce1e5e..f5338f28af 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultArrowResource.java @@ -29,7 +29,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -@Path("result/") +@Path("result/arrow") @RequiredArgsConstructor(onConstructor_ = {@Inject}) public class ResultArrowResource { diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java index 86e645be86..30d7b5fd2e 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java @@ -30,7 +30,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -@Path("result/") +@Path("result/csv") @RequiredArgsConstructor(onConstructor_ = {@Inject}) public class ResultCsvResource { diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExcelResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExcelResource.java index ab087cd662..801545b7ff 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExcelResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultExcelResource.java @@ -29,7 +29,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -@Path("result/") +@Path("result/xlsx") @RequiredArgsConstructor(onConstructor_ = {@Inject}) public class ResultExcelResource { diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultParquetResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultParquetResource.java index b2a7610957..822c7c5320 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultParquetResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultParquetResource.java @@ -30,7 +30,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -@Path("result/") +@Path("result/parquet") @RequiredArgsConstructor(onConstructor_ = {@Inject}) public class ResultParquetResource { From ac48e18901e3020210cc186e6becd3dfa40d3a46 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 31 Jan 2023 21:47:22 +0100 Subject: [PATCH 003/679] fix tests --- .../conquery/io/result/ResultUtil.java | 3 +- .../io/result/arrow/ResultArrowProcessor.java | 4 +- .../io/result/csv/ResultCsvProcessor.java | 4 +- .../io/result/excel/ResultExcelProcessor.java | 4 +- .../parquet/ResultParquetProcessor.java | 4 +- .../integration/DownloadLinkGeneration.java | 4 +- .../tests/endpoints/apiEndpointInfo.json | 48 +++++++++---------- 7 files changed, 36 insertions(+), 35 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java b/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java index 47c80e2ba9..cff45a3070 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java @@ -77,7 +77,8 @@ public static void checkSingleTableResult(ManagedExecution exec) { } - public static void authorizeExecutable(Subject subject, ManagedExecution exec, Dataset dataset) { + public static void authorizeExecutable(Subject subject, ManagedExecution exec) { + final Dataset dataset = exec.getDataset(); subject.authorize(dataset, Ability.READ); subject.authorize(dataset, Ability.DOWNLOAD); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java index 61a32303d0..26fe56bca2 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java @@ -99,9 +99,9 @@ public static & SingleTableResult> Response getAr final Dataset dataset = exec.getDataset(); - log.info("Downloading results for {} on dataset {}", exec, dataset); + log.info("Downloading results for {}", exec.getId()); - ResultUtil.authorizeExecutable(subject, exec, dataset); + ResultUtil.authorizeExecutable(subject, exec); if (!(exec instanceof ManagedQuery || (exec instanceof ManagedForm && ((ManagedForm) exec).getSubQueries().size() == 1))) { return Response.status(HttpStatus.SC_UNPROCESSABLE_ENTITY, "Execution result is not a single Table").build(); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java index 0c6b7c2375..ab2ccaa790 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java @@ -45,9 +45,9 @@ public & SingleTableResult> Response createResult final Namespace namespace = datasetRegistry.get(dataset.getId()); ConqueryMDC.setLocation(subject.getName()); - log.info("Downloading results for {} on dataset {}", exec, dataset); + log.info("Downloading results for {}", exec.getId()); - ResultUtil.authorizeExecutable(subject, exec, dataset); + ResultUtil.authorizeExecutable(subject, exec); // Check if subject is permitted to download on all datasets that were referenced by the query authorizeDownloadDatasets(subject, exec); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java index f96e906683..f6b34821d6 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java @@ -44,9 +44,9 @@ public & SingleTableResult> Response createResult final Dataset dataset = exec.getDataset(); - log.info("Downloading results for {} on dataset {}", exec, dataset); + log.info("Downloading results for {}", exec.getId()); - ResultUtil.authorizeExecutable(subject, exec, dataset); + ResultUtil.authorizeExecutable(subject, exec); ResultUtil.checkSingleTableResult(exec); subject.authorize(dataset, Ability.DOWNLOAD); diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java index bee9a72e10..7a9e25a067 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java @@ -41,9 +41,9 @@ public Response createResultFile(Subject subject, ManagedExecution exec, bool final Dataset dataset = exec.getDataset(); - log.info("Downloading results for {} on dataset {}", exec, dataset); + log.info("Downloading results for {}", exec.getId()); - ResultUtil.authorizeExecutable(subject, exec, dataset); + ResultUtil.authorizeExecutable(subject, exec); ResultUtil.checkSingleTableResult(exec); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java b/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java index 7f8e28b585..51a6080b77 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/DownloadLinkGeneration.java @@ -68,8 +68,8 @@ public void execute(StandaloneSupport conquery) throws Exception { FullExecutionStatus status = IntegrationUtils.getExecutionStatus(conquery, exec.getId(), user, 200); // This Url is missing the `/api` path part, because we use the standard UriBuilder here - assertThat(status.getResultUrls()).contains(new ResultAsset("CSV", new URL(String.format("%s/result/%s.csv", conquery.defaultApiURIBuilder() - .toString(), exec.getId())))); + assertThat(status.getResultUrls()).contains(new ResultAsset("CSV", new URL(String.format("%s/result/csv/%s.csv", conquery.defaultApiURIBuilder() + .toString(), exec.getId())))); } } diff --git a/backend/src/test/resources/tests/endpoints/apiEndpointInfo.json b/backend/src/test/resources/tests/endpoints/apiEndpointInfo.json index 18984d7fe3..7439ccb6eb 100644 --- a/backend/src/test/resources/tests/endpoints/apiEndpointInfo.json +++ b/backend/src/test/resources/tests/endpoints/apiEndpointInfo.json @@ -45,30 +45,30 @@ "clazz": "QueryResource" }, { - "method": "GET", - "path": "/result/{query}.csv", - "clazz": "ResultCsvResource" - }, - { - "method": "GET", - "path": "/result/{query}.xlsx", - "clazz": "ResultExcelResource" - }, - { - "method": "GET", - "path": "/result/{query}.arrs", - "clazz": "ResultArrowResource" - }, - { - "method": "GET", - "path": "/result/{query}.arrf", - "clazz": "ResultArrowResource" - }, - { - "method": "GET", - "path": "/result/{query}.parquet", - "clazz": "ResultParquetResource" - }, + "method": "GET", + "path": "/result/csv/{query}.csv", + "clazz": "ResultCsvResource" + }, + { + "method": "GET", + "path": "/result/xlsx/{query}.xlsx", + "clazz": "ResultExcelResource" + }, + { + "method": "GET", + "path": "/result/arrow/{query}.arrs", + "clazz": "ResultArrowResource" + }, + { + "method": "GET", + "path": "/result/arrow/{query}.arrf", + "clazz": "ResultArrowResource" + }, + { + "method": "GET", + "path": "/result/parquet/{query}.parquet", + "clazz": "ResultParquetResource" + }, { "method": "GET", "path": "/datasets/{dataset}/queries", From 185f05b8d95ccf3bd7112695c2cbce4ecd79780f Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 1 Feb 2023 15:26:20 +0100 Subject: [PATCH 004/679] generalize result name normalization --- .../java/com/bakdata/conquery/io/result/ResultUtil.java | 8 ++++---- .../conquery/io/result/arrow/ResultArrowProcessor.java | 2 +- .../conquery/io/result/csv/ResultCsvProcessor.java | 8 +++++++- .../conquery/io/result/excel/ResultExcelProcessor.java | 3 ++- .../io/result/parquet/ResultParquetProcessor.java | 4 ++-- .../com/bakdata/conquery/resources/ResourceConstants.java | 7 ++++++- .../resources/admin/rest/AuthOverviewResource.java | 2 +- .../bakdata/conquery/resources/api/ResultCsvResource.java | 5 +++-- .../main/java/com/bakdata/conquery/util/io/FileUtil.java | 4 ++-- .../com/bakdata/conquery/io/result/ResultNameTest.java | 4 ++-- 10 files changed, 30 insertions(+), 17 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java b/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java index cff45a3070..e8e0a7b421 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/ResultUtil.java @@ -35,12 +35,12 @@ String getHeaderValue() { } - public static Response makeResponseWithFileName(Response.ResponseBuilder response, String label, String fileExtension, MediaType mediaType, ContentDispositionOption disposition) { + public static Response makeResponseWithFileName(Response.ResponseBuilder response, String filename, MediaType mediaType, ContentDispositionOption disposition) { response.header(HttpHeaders.CONTENT_TYPE, mediaType); - if (!(Strings.isNullOrEmpty(label) || label.isBlank())) { - // Set filename from label if the label was set, otherwise the browser will name the file according to the request path + if (!(Strings.isNullOrEmpty(filename) || filename.isBlank())) { + // Set filename from filename if the filename was set, otherwise the browser will name the file according to the request path response.header("Content-Disposition", String.format( - "%s; filename=\"%s\"", disposition.getHeaderValue(), FileUtil.makeSafeFileName(label, fileExtension))); + "%s; filename=\"%s\"", disposition.getHeaderValue(), FileUtil.makeSafeFileName(filename))); } return response.build(); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java index 26fe56bca2..baf6757aca 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/arrow/ResultArrowProcessor.java @@ -142,7 +142,7 @@ public static & SingleTableResult> Response getAr } }; - return makeResponseWithFileName(Response.ok(out), exec.getLabelWithoutAutoLabelSuffix(), fileExtension, mediaType, ResultUtil.ContentDispositionOption.ATTACHMENT); + return makeResponseWithFileName(Response.ok(out), String.join(".", exec.getLabelWithoutAutoLabelSuffix(), fileExtension), mediaType, ResultUtil.ContentDispositionOption.ATTACHMENT); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java index ab2ccaa790..7fbcd6a40f 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/csv/ResultCsvProcessor.java @@ -25,6 +25,7 @@ import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.resources.ResourceConstants; import com.bakdata.conquery.util.io.ConqueryMDC; import com.bakdata.conquery.util.io.IdColumnUtil; import lombok.RequiredArgsConstructor; @@ -74,7 +75,12 @@ public & SingleTableResult> Response createResult } }; - return makeResponseWithFileName(Response.ok(out), exec.getLabelWithoutAutoLabelSuffix(), "csv", new MediaType("text", "csv", charset.toString()), ResultUtil.ContentDispositionOption.ATTACHMENT); + return makeResponseWithFileName( + Response.ok(out), + String.join(".", exec.getLabelWithoutAutoLabelSuffix(), ResourceConstants.FILE_EXTENTION_CSV), + new MediaType("text", "csv", charset.toString()), + ResultUtil.ContentDispositionOption.ATTACHMENT + ); } } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java index f6b34821d6..52987fc54a 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/excel/ResultExcelProcessor.java @@ -22,6 +22,7 @@ import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.resources.ResourceConstants; import com.bakdata.conquery.util.io.ConqueryMDC; import com.bakdata.conquery.util.io.IdColumnUtil; import lombok.RequiredArgsConstructor; @@ -63,7 +64,7 @@ public & SingleTableResult> Response createResult log.trace("FINISHED downloading {}", exec.getId()); }; - return makeResponseWithFileName(Response.ok(out), exec.getLabelWithoutAutoLabelSuffix(), "xlsx", MEDIA_TYPE, ResultUtil.ContentDispositionOption.ATTACHMENT); + return makeResponseWithFileName(Response.ok(out), String.join(".", exec.getLabelWithoutAutoLabelSuffix(), ResourceConstants.FILE_EXTENTION_XLSX), MEDIA_TYPE, ResultUtil.ContentDispositionOption.ATTACHMENT); } diff --git a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java index 7a9e25a067..28f17efeb4 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/io/result/parquet/ResultParquetProcessor.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.io.result.parquet; import static com.bakdata.conquery.io.result.ResultUtil.makeResponseWithFileName; -import static com.bakdata.conquery.resources.ResourceConstants.FILE_EXTENTION_PARQUET; import java.util.Locale; @@ -21,6 +20,7 @@ import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.resources.ResourceConstants; import com.bakdata.conquery.resources.api.ResultParquetResource; import com.bakdata.conquery.util.io.ConqueryMDC; import com.bakdata.conquery.util.io.IdColumnUtil; @@ -74,6 +74,6 @@ public Response createResultFile(Subject subject, ManagedExecution exec, bool }; - return makeResponseWithFileName(Response.ok(out), exec.getLabelWithoutAutoLabelSuffix(), FILE_EXTENTION_PARQUET, PARQUET_MEDIA_TYPE, ResultUtil.ContentDispositionOption.ATTACHMENT); + return makeResponseWithFileName(Response.ok(out), String.join(".", exec.getLabelWithoutAutoLabelSuffix(), ResourceConstants.FILE_EXTENTION_PARQUET), PARQUET_MEDIA_TYPE, ResultUtil.ContentDispositionOption.ATTACHMENT); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/ResourceConstants.java b/backend/src/main/java/com/bakdata/conquery/resources/ResourceConstants.java index 27b99c6d60..267d22e181 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/ResourceConstants.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/ResourceConstants.java @@ -35,7 +35,12 @@ public class ResourceConstants { public static final String FILE_EXTENTION_ARROW_FILE = "arrf"; public static final String FILE_EXTENTION_ARROW_STREAM = "arrs"; public static final String FILE_EXTENTION_PARQUET = "parquet"; - + + public static final String FILE_EXTENTION_CSV = "csv"; + + public static final String FILE_EXTENTION_XLSX = "xlsx"; + + /** * Method to generate a data-model of this class's static members so that they * are accessible from within a freemarker template. diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AuthOverviewResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AuthOverviewResource.java index 0e57feb2ae..6d94aac4c8 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AuthOverviewResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AuthOverviewResource.java @@ -39,7 +39,7 @@ public Response getPermissionOverviewAsCSV() { public Response getPermissionOverviewAsCSV(@PathParam(GROUP_ID) Group group) { return Response .ok(processor.getPermissionOverviewAsCSV(group)) - .header("Content-Disposition", String.format("attachment; filename=\"authOverview_%s.csv\"", FileUtil.makeSafeFileName(group.getName(), "csv"))) + .header("Content-Disposition", String.format("attachment; filename=\"authOverview_%s.csv\"", FileUtil.makeSafeFileName(group.getName()))) .build(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java index 30d7b5fd2e..5a652c5485 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ResultCsvResource.java @@ -9,6 +9,7 @@ import java.util.Optional; import javax.inject.Inject; +import javax.ws.rs.DefaultValue; import javax.ws.rs.GET; import javax.ws.rs.HeaderParam; import javax.ws.rs.Path; @@ -54,11 +55,11 @@ public & SingleTableResult> Response getAsCsv( @PathParam(QUERY) ManagedExecution execution, @HeaderParam(HttpHeaders.USER_AGENT) String userAgent, @QueryParam("charset") String queryCharset, - @QueryParam("pretty") Optional pretty) { + @QueryParam("pretty") @DefaultValue("true") boolean pretty) { checkSingleTableResult(execution); log.info("Result for {} download on dataset {} by subject {} ({}).", execution, execution.getDataset().getId(), subject.getId(), subject.getName()); - return processor.createResult(subject, (E) execution, pretty.orElse(Boolean.TRUE), determineCharset(userAgent, queryCharset)); + return processor.createResult(subject, (E) execution, pretty, determineCharset(userAgent, queryCharset)); } } diff --git a/backend/src/main/java/com/bakdata/conquery/util/io/FileUtil.java b/backend/src/main/java/com/bakdata/conquery/util/io/FileUtil.java index 940e31fae5..2a97bbb271 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/io/FileUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/util/io/FileUtil.java @@ -26,8 +26,8 @@ public class FileUtil { public static final Pattern SAVE_FILENAME_REPLACEMENT_MATCHER = Pattern.compile("[^a-zA-Z0-9äÄöÖüÜß .\\-]"); - public static String makeSafeFileName(String label, String fileExtension) { - return SAVE_FILENAME_REPLACEMENT_MATCHER.matcher(label + "." + fileExtension).replaceAll("_"); + public static String makeSafeFileName(String label) { + return SAVE_FILENAME_REPLACEMENT_MATCHER.matcher(label).replaceAll("_"); } public void deleteRecursive(Path path) throws IOException { diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/ResultNameTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/ResultNameTest.java index abc2aed9cc..dae43745ed 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/ResultNameTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/ResultNameTest.java @@ -12,7 +12,7 @@ public class ResultNameTest { @Test public void resultNameOk(){ final String label = "azAZ19 äü-ÄÜ"; - String fileName = FileUtil.makeSafeFileName(label, FILE_EXTENSION); + String fileName = FileUtil.makeSafeFileName(label); assertThat(fileName).isEqualTo(label + "." +FILE_EXTENSION); } @@ -20,7 +20,7 @@ public void resultNameOk(){ public void resultNameModified(){ final String label = "()§ $ \\ \" "; - String fileName = FileUtil.makeSafeFileName(label, FILE_EXTENSION); + String fileName = FileUtil.makeSafeFileName(label); assertThat(fileName).isEqualTo("___ _ _ _ ." + FILE_EXTENSION); } } From 7f6a9ba5fe377a9ac4756b4f8706d216c19a4336 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 1 Feb 2023 15:42:03 +0100 Subject: [PATCH 005/679] adds NoSuchElementException->404 mapper --- .../bakdata/conquery/io/jersey/RESTServer.java | 2 ++ .../io/jetty/NoSuchElementExceptionMapper.java | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 backend/src/main/java/com/bakdata/conquery/io/jetty/NoSuchElementExceptionMapper.java diff --git a/backend/src/main/java/com/bakdata/conquery/io/jersey/RESTServer.java b/backend/src/main/java/com/bakdata/conquery/io/jersey/RESTServer.java index b81085af3e..535d33c116 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jersey/RESTServer.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jersey/RESTServer.java @@ -7,6 +7,7 @@ import com.bakdata.conquery.io.jetty.ConqueryErrorExecptionMapper; import com.bakdata.conquery.io.jetty.ConqueryJsonExceptionMapper; import com.bakdata.conquery.io.jetty.JsonValidationExceptionMapper; +import com.bakdata.conquery.io.jetty.NoSuchElementExceptionMapper; import com.bakdata.conquery.models.auth.entities.Subject; import com.bakdata.conquery.models.auth.web.AuthenticationExceptionMapper; import com.bakdata.conquery.models.auth.web.AuthorizationExceptionMapper; @@ -32,6 +33,7 @@ public static void configure(ConqueryConfig config, ResourceConfig jersey) { jersey.register(new AuthenticationExceptionMapper()); jersey.register(new AuthorizationExceptionMapper()); jersey.register(JsonValidationExceptionMapper.class); + jersey.register(NoSuchElementExceptionMapper.class); // default Dropwizard's exception mappers jersey.register(new ConqueryErrorExecptionMapper()); jersey.register(ConqueryJsonExceptionMapper.class); diff --git a/backend/src/main/java/com/bakdata/conquery/io/jetty/NoSuchElementExceptionMapper.java b/backend/src/main/java/com/bakdata/conquery/io/jetty/NoSuchElementExceptionMapper.java new file mode 100644 index 0000000000..531786b0c5 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/io/jetty/NoSuchElementExceptionMapper.java @@ -0,0 +1,18 @@ +package com.bakdata.conquery.io.jetty; + +import java.util.NoSuchElementException; + +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.ws.rs.ext.ExceptionMapper; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class NoSuchElementExceptionMapper implements ExceptionMapper { + @Override + public Response toResponse(NoSuchElementException exception) { + log.trace("Mapping exception:", exception); + return Response.status(Response.Status.NOT_FOUND).type(MediaType.APPLICATION_JSON_TYPE).entity(exception.getMessage()).build(); + } +} From cbd8b21fd98e078641774a2ac980a67fcda7f8b1 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 1 Feb 2023 16:05:36 +0100 Subject: [PATCH 006/679] fix ResultNameTest --- .../bakdata/conquery/io/result/ResultNameTest.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/src/test/java/com/bakdata/conquery/io/result/ResultNameTest.java b/backend/src/test/java/com/bakdata/conquery/io/result/ResultNameTest.java index dae43745ed..04131c1288 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/result/ResultNameTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/result/ResultNameTest.java @@ -1,26 +1,26 @@ package com.bakdata.conquery.io.result; +import static org.assertj.core.api.Assertions.assertThat; + import com.bakdata.conquery.util.io.FileUtil; import org.junit.jupiter.api.Test; -import static org.assertj.core.api.Assertions.assertThat; - public class ResultNameTest { public static final String FILE_EXTENSION = "test"; @Test - public void resultNameOk(){ + public void resultNameOk() { final String label = "azAZ19 äü-ÄÜ"; - String fileName = FileUtil.makeSafeFileName(label); - assertThat(fileName).isEqualTo(label + "." +FILE_EXTENSION); + String fileName = FileUtil.makeSafeFileName(label + "." + FILE_EXTENSION); + assertThat(fileName).isEqualTo(label + "." + FILE_EXTENSION); } @Test - public void resultNameModified(){ + public void resultNameModified() { final String label = "()§ $ \\ \" "; - String fileName = FileUtil.makeSafeFileName(label); + String fileName = FileUtil.makeSafeFileName(label + "." + FILE_EXTENSION); assertThat(fileName).isEqualTo("___ _ _ _ ." + FILE_EXTENSION); } } From 01cd6cc6ccef60aa1751fb33678c641fd03ac083 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 1 Feb 2023 16:35:44 +0100 Subject: [PATCH 007/679] adapt openapi.yaml --- .../bakdata/conquery/apiv1/QueryProcessor.java | 8 ++++---- openapi.yaml | 16 +++++++++++++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index fa226a35d8..8bbd6a88a6 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -194,7 +194,7 @@ private ManagedExecution tryReuse(QueryDescription query, ManagedExecutionId } - public Stream getAllQueries(Dataset dataset, HttpServletRequest req, Subject subject, boolean allProviders) { + public Stream getAllQueries(Dataset dataset, HttpServletRequest req, Subject subject, boolean allProviders) { Collection> allQueries = storage.getAllExecutions(); return getQueriesFiltered(dataset, RequestAwareUriBuilder.fromRequest(req), subject, allQueries, allProviders); @@ -335,12 +335,12 @@ public void deleteQuery(Subject subject, ManagedExecution execution) { storage.removeExecution(execution.getId()); } - public com.bakdata.conquery.apiv1.execution.FullExecutionStatus getQueryFullStatus(ManagedExecution query, Subject subject, UriBuilder url, Boolean allProviders) { + public FullExecutionStatus getQueryFullStatus(ManagedExecution query, Subject subject, UriBuilder url, Boolean allProviders) { query.initExecutable(datasetRegistry, config); Map> datasetAbilities = buildDatasetAbilityMap(subject, datasetRegistry); - final com.bakdata.conquery.apiv1.execution.FullExecutionStatus status = query.buildStatusFull(storage, subject, datasetRegistry, config); + final FullExecutionStatus status = query.buildStatusFull(storage, subject, datasetRegistry, config); if (query.isReadyToDownload(datasetAbilities)) { status.setResultUrls(getResultAssets(config.getResultProviders(), query, url, allProviders)); @@ -392,7 +392,7 @@ public ExternalUploadResult uploadEntities(Subject subject, Dataset dataset, Ext /** * Create and submit {@link EntityPreviewForm} for subject on to extract sources for entity, and extract some additional infos to be used as infocard. */ - public com.bakdata.conquery.apiv1.execution.FullExecutionStatus getSingleEntityExport(Subject subject, UriBuilder uriBuilder, String idKind, String entity, List sources, Dataset dataset, Range dateRange) { + public FullExecutionStatus getSingleEntityExport(Subject subject, UriBuilder uriBuilder, String idKind, String entity, List sources, Dataset dataset, Range dateRange) { EntityPreviewForm form = EntityPreviewForm.create(entity, idKind, dateRange, sources, datasetRegistry.get(dataset.getId()).getPreviewConfig().getSelects()); diff --git a/openapi.yaml b/openapi.yaml index 59b480048b..2d8d505421 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -295,9 +295,19 @@ components: description: Available result urls of the query. type: array items: - type: string #TODO url? - format: url - + $ref: "#/components/schemas/result-asset" + result-asset: + type: object + properties: + label: + type: string + minLength: 1 + url: + type: string + format: url + required: + - label + - url labeled: type: object properties: From 1afd0346d5c5886e72a4e3559b127519c706a427 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Wed, 8 Feb 2023 12:14:46 +0100 Subject: [PATCH 008/679] switch to labels, switch internal type, mock api --- frontend/mock-api/index.js | 22 +++++++++++-- frontend/mock-api/stored-queries/25.json | 15 ++++++++- frontend/src/js/api/types.ts | 9 +++-- frontend/src/js/button/DownloadButton.tsx | 33 ++++++++++++++++--- frontend/src/js/button/PreviewButton.tsx | 6 ++-- .../js/button/QueryResultHistoryButton.tsx | 6 ++-- frontend/src/js/entity-history/History.tsx | 4 +-- frontend/src/js/entity-history/actions.ts | 11 ++++--- frontend/src/js/entity-history/reducer.ts | 11 ++++--- frontend/src/js/preview/actions.ts | 8 ++--- frontend/src/js/preview/reducer.ts | 4 +-- .../src/js/previous-queries/list/reducer.ts | 4 +-- .../DownloadResultsDropdownButton.tsx | 19 ++++++----- frontend/src/js/query-runner/QueryResults.tsx | 6 ++-- frontend/src/js/query-runner/reducer.ts | 3 +- 15 files changed, 111 insertions(+), 50 deletions(-) diff --git a/frontend/mock-api/index.js b/frontend/mock-api/index.js index afb4aeb211..63193c2fbf 100644 --- a/frontend/mock-api/index.js +++ b/frontend/mock-api/index.js @@ -151,8 +151,14 @@ module.exports = function (app, port) { status: "DONE", numberOfResults: 5, resultUrls: [ - `/api/results/results.xlsx`, - `/api/results/results.csv`, + { + label: "XLSX", + url: "http://localhost:8080/api/result/xlsx/mimic-iii-demo.51cd95fd-90b2-4573-aab5-11846126427b.xlsx", + }, + { + label: "CSV", + url: "http://localhost:8080/api/result/csv/mimic-iii-demo.51cd95fd-90b2-4573-aab5-11846126427b.csv", + }, ], columnDescriptions: [ { @@ -258,7 +264,17 @@ module.exports = function (app, port) { shared: Math.random() < 0.8, resultUrls: notExecuted ? [] - : [`/api/results/results.xlsx`, `/api/results/results.csv`], + : [ + { + label: "XLSX", + url: "http://localhost:8080/api/result/xlsx/mimic-iii-demo.51cd95fd-90b2-4573-aab5-11846126427b.xlsx", + }, + { + label: "CSV", + url: "http://localhost:8080/api/result/csv/mimic-iii-demo.51cd95fd-90b2-4573-aab5-11846126427b.csv", + }, + ], + ownerName: "System", ...(Math.random() > 0.2 ? { queryType: "CONCEPT_QUERY" } diff --git a/frontend/mock-api/stored-queries/25.json b/frontend/mock-api/stored-queries/25.json index fe34fef9fd..19503436fc 100644 --- a/frontend/mock-api/stored-queries/25.json +++ b/frontend/mock-api/stored-queries/25.json @@ -2,7 +2,20 @@ "id": 25, "status": "DONE", "numberOfResults": 5, - "resultUrls": ["/api/results/1234-123123123-123123-1231.csv"], + "resultUrls": [ + { + "label": "Alle Dateien", + "url": "http://localhost:8080/api/results/all.zip" + }, + { + "label": "Bericht.pdf", + "url": "http://localhost:8080/api/results/Bericht.pdf" + }, + { + "label": "Statistik.xlsx", + "url": "http://localhost:8080/api/results/Statistik.xlsx" + } + ], "createdAt": "2016-12-02T14:19:09Z", "version": 0, "tags": ["Fun"], diff --git a/frontend/src/js/api/types.ts b/frontend/src/js/api/types.ts index 311659bd91..7d013f5087 100644 --- a/frontend/src/js/api/types.ts +++ b/frontend/src/js/api/types.ts @@ -390,12 +390,17 @@ export interface ColumnDescription { userConceptLabel: string | null; } +export interface ResultUrlsWithLabel { + label: string; + url: string; +} + // TODO: This actually returns GETQueryResponseT => a lot of unused fields export interface GetQueryResponseDoneT { status: "DONE" | "NEW"; // NEW might mean canceled (query not (yet) executed) label: string; numberOfResults: number | null; - resultUrls: string[]; + resultUrls: ResultUrlsWithLabel[]; columnDescriptions: ColumnDescription[] | null; queryType: "CONCEPT_QUERY" | "SECONDARY_ID_QUERY"; requiredTime: number; // In ms, unused at the moment @@ -529,7 +534,7 @@ export interface EntityInfo { } export type GetEntityHistoryResponse = { - resultUrls: string[]; + resultUrls: ResultUrlsWithLabel[]; columnDescriptions: ColumnDescription[]; infos: EntityInfo[]; }; diff --git a/frontend/src/js/button/DownloadButton.tsx b/frontend/src/js/button/DownloadButton.tsx index 00173e566d..c4b136d274 100644 --- a/frontend/src/js/button/DownloadButton.tsx +++ b/frontend/src/js/button/DownloadButton.tsx @@ -1,5 +1,7 @@ import styled from "@emotion/styled"; import { ReactNode, useContext, forwardRef } from "react"; +import { ResultUrlsWithLabel } from "../api/types"; +import { IconName } from "@fortawesome/fontawesome-svg-core"; import { AuthTokenContext } from "../authorization/AuthTokenProvider"; @@ -13,8 +15,31 @@ const Link = styled("a")` line-height: 1; `; +const fileTypeToIcon: Record = { + "ZIP": "file-archive", + "XLSX": "file-excel", + "PDF": "file-pdf", + "CSV": "file-csv", +} +function getFileIcon(label:string): IconName { + // Editor Requests + if(label in fileTypeToIcon) { + return fileTypeToIcon[label]; + } + + // Forms + if(label.includes(".")){ + const ext = label.split(".").pop(); + if(!ext) return "file-download"; + if(ext in fileTypeToIcon){ + return fileTypeToIcon[ext]; + } + } + return "file-download"; +} + interface Props extends Omit { - url: string; + url: ResultUrlsWithLabel; className?: string; children?: ReactNode; onClick?: () => void; @@ -24,15 +49,13 @@ const DownloadButton = forwardRef( ({ url, className, children, onClick, ...restProps }, ref) => { const { authToken } = useContext(AuthTokenContext); - const href = `${url}?access_token=${encodeURIComponent( + const href = `${url.url}?access_token=${encodeURIComponent( authToken, )}&charset=ISO_8859_1`; - const icon = "download"; - return ( - + {children} diff --git a/frontend/src/js/button/PreviewButton.tsx b/frontend/src/js/button/PreviewButton.tsx index 09144a9dec..196c37ba50 100644 --- a/frontend/src/js/button/PreviewButton.tsx +++ b/frontend/src/js/button/PreviewButton.tsx @@ -3,7 +3,7 @@ import { FC } from "react"; import { useTranslation } from "react-i18next"; import { useDispatch } from "react-redux"; -import type { ColumnDescription } from "../api/types"; +import type { ColumnDescription, ResultUrlsWithLabel } from "../api/types"; import { useGetAuthorizedUrl } from "../authorization/useAuthorizedUrl"; import { openPreview, useLoadPreviewData } from "../preview/actions"; @@ -16,7 +16,7 @@ const Button = styled(TransparentButton)` interface PropsT { columns: ColumnDescription[]; - url: string; + url: ResultUrlsWithLabel; } const PreviewButton: FC = ({ url, columns, ...restProps }) => { @@ -29,7 +29,7 @@ const PreviewButton: FC = ({ url, columns, ...restProps }) => { return ( - -
-
- ${text.trim()} -
-
- `; - if (customButton) { - toast.querySelector('.toast-body').appendChild(customButton); - } - return toast; -} - - - -function showToastMessage(type, title, text, smalltext = "", customButton) { - let toastContainer = document.getElementById("toast-container"); - let toast = getToast(type, title, text, smalltext, customButton); - toastContainer.appendChild(toast); - $(toast).toast({ delay: 10000 }); - $(toast).toast('show'); -} - -async function showMessageForResponse(response, customButton) { - if(response){ - try { - let body = await response.json(); - if(!response.ok){ - showToastMessage( - ToastTypes.ERROR, - "Error", - "The send request came back with the following error: " + JSON.stringify(body), - "Status " + response.status, - response.status == 409 ? customButton : undefined - ); - } - } catch (e) { - // ignore, because some responses don't have a body - } - } -} - - -function loginClickHandler(){ - event.preventDefault(); - fetch( - '/auth', - { - method: 'POST', - headers: {'Content-Type': 'application/json'}, - body: JSON.stringify({ - user: document.getElementById("inputEmail").value, - password: document.getElementById("inputPassword").value - }) - }) - .then((response) => { - if(!response.ok) { - throw new Error("Error fetching token"); - } - return response.json(); - }) - .then( (json) => { - window.location = '${c}?access_token='+json.access_token; + ); + return res; + } + + function getToast(type, title, text, smalltext = "", customButton) { + if (!type) type = ToastTypes.INFO; + + let toast = document.createElement("div"); + toast.classList.add("toast"); + toast.setAttribute("role", "alert"); + toast.setAttribute("aria-live", "assertive"); + toast.setAttribute("aria-atomic", "true"); + toast.setAttribute("style", "width: 500px; max-width: none"); + toast.setAttribute('data-test-id', 'toast'); + toast.innerHTML = ` +
+ ${title} + ${smalltext ? '' + smalltext + '' : ''} + +
+
+
+ ${text.trim()} +
+
+ `; + if (customButton) { + toast.querySelector('.toast-body').appendChild(customButton); } - ) - .catch(function(error) { - var p = document.createElement('p'); - p.appendChild( - document.createTextNode('Error: ' + error.message) - ); - document.body.insertBefore(p, myImage); - }); -} - -function logout() { - event.preventDefault(); - rest('/${ctx.staticUriElem.ADMIN_SERVLET_PATH}/logout') - .then(function () { location.reload() }); -} - -function postFile(event, url) { - event.preventDefault(); - - let inputs = event.target.getElementsByClassName("restparam"); - if (inputs.length != 1) { - console.log('Unexpected number of inputs in ' + inputs); + return toast; } - - var file; - - for (var i = 0; i < inputs[0].files.length; i++) { - let file = inputs[0].files[i]; - let reader = new FileReader(); - reader.onload = function () { - let json = reader.result; - fetch(url, { - method: 'post', credentials: 'same-origin', body: json, headers: { - "Content-Type": "application/json" + + + + function showToastMessage(type, title, text, smalltext = "", customButton) { + let toastContainer = document.getElementById("toast-container"); + let toast = getToast(type, title, text, smalltext, customButton); + toastContainer.appendChild(toast); + $(toast).toast({ delay: 10000 }); + $(toast).toast('show'); + } + + async function showMessageForResponse(response, customButton) { + if(response){ + try { + let body = await response.json(); + if(!response.ok){ + showToastMessage( + ToastTypes.ERROR, + "Error", + "The send request came back with the following error: " + JSON.stringify(body), + "Status " + response.status, + response.status == 409 ? customButton : undefined + ); + } + } catch (e) { + // ignore, because some responses don't have a body } - }) - .then(function (response) { - if (response.ok) { - setTimeout(() => location.reload(), 2000); - showToastMessage(ToastTypes.SUCCESS, "Success", "The file has been posted successfully"); - } else { - // force button in case of 409 status - let customButton; - // only apply for concept uploads - if (toForceURL(url).pathname.includes('/concepts')) { - customButton = createCustomButton('Replace file'); - customButton.onclick = () => postFile(event, toForceURL(url)); - } - showMessageForResponse(response, customButton); + } + } + + + function loginClickHandler(){ + event.preventDefault(); + fetch( + '/auth', + { + method: 'POST', + headers: {'Content-Type': 'application/json'}, + body: JSON.stringify({ + user: document.getElementById("inputEmail").value, + password: document.getElementById("inputPassword").value + }) + }) + .then((response) => { + if(!response.ok) { + throw new Error("Error fetching token"); } + return response.json(); }) - .catch(function (error) { - showToastMessage(ToastTypes.ERROR, "Error", "There has been a problem with posting a file: " + error.message); - console.log('There has been a problem with posting a file', error.message); + .then( (json) => { + window.location = '${c}?access_token='+json.access_token; + } + ) + .catch(function(error) { + var p = document.createElement('p'); + p.appendChild( + document.createTextNode('Error: ' + error.message) + ); + document.body.insertBefore(p, myImage); + }); + } + + function logout() { + event.preventDefault(); + rest('/${ctx.staticUriElem.ADMIN_SERVLET_PATH}/logout') + .then(function () { location.reload() }); + } + + function postFile(event, url) { + event.preventDefault(); + + let inputs = event.target.getElementsByClassName("restparam"); + if (inputs.length != 1) { + console.log('Unexpected number of inputs in ' + inputs); + } + + var file; + + for (var i = 0; i < inputs[0].files.length; i++) { + let file = inputs[0].files[i]; + let reader = new FileReader(); + reader.onload = function () { + let json = reader.result; + fetch(url, { + method: 'post', credentials: 'same-origin', body: json, headers: { + "Content-Type": "application/json" + } + }) + .then(function (response) { + if (response.ok) { + setTimeout(() => location.reload(), 2000); + showToastMessage(ToastTypes.SUCCESS, "Success", "The file has been posted successfully"); + } else { + // force button in case of 409 status + let customButton; + // only apply for concept uploads + if (toForceURL(url).pathname.includes('/concepts')) { + customButton = createCustomButton('Replace file'); + customButton.onclick = () => postFile(event, toForceURL(url)); + } + showMessageForResponse(response, customButton); + } + }) + .catch(function (error) { + showToastMessage(ToastTypes.ERROR, "Error", "There has been a problem with posting a file: " + error.message); + console.log('There has been a problem with posting a file', error.message); + }); + }; + reader.readAsText(file); + } + } + + + // Get to recent tab: https://stackoverflow.com/a/19015027 + $('#myTab a').click(function (e) { + e.preventDefault(); + $(this).tab('show'); }); - }; - reader.readAsText(file); - } -} - - -// Get to recent tab: https://stackoverflow.com/a/19015027 -$('#myTab a').click(function (e) { - e.preventDefault(); - $(this).tab('show'); -}); - -// store the currently selected tab in the hash value -$("ul.nav-tabs > li > a").on("shown.bs.tab", function (e) { - var id = $(e.target).attr("href").substr(1); - window.location.hash = id; -}); - -// on load of the page: switch to the currently selected tab -var hash = window.location.hash; -$('#myTab a[href="' + hash + '"]').tab('show'); \ No newline at end of file + + // store the currently selected tab in the hash value + $("ul.nav-tabs > li > a").on("shown.bs.tab", function (e) { + var id = $(e.target).attr("href").substr(1); + window.location.hash = id; + }); + + // on load of the page: switch to the currently selected tab + var hash = window.location.hash; + $('#myTab a[href="' + hash + '"]').tab('show'); \ No newline at end of file diff --git a/cypress/integration/backend-admin-ui/test_2_dataset.js b/cypress/integration/backend-admin-ui/test_2_dataset.js index ad3e6e09e7..ae20ec1eb6 100644 --- a/cypress/integration/backend-admin-ui/test_2_dataset.js +++ b/cypress/integration/backend-admin-ui/test_2_dataset.js @@ -36,13 +36,12 @@ context("Admin UI Single Dataset", () => { }); describe("Can upload test table and concept", () => { - before(() => { visitAdminUI(`datasets/${testDSID}`); }); + beforeEach(() => { visitAdminUI(`datasets/${testDSID}`); }); it("Can upload test table", () => { cy.get('[data-test-id="upload-select"]').select('Table JSON'); cy.get('[data-test-id="upload-input"]').selectFile('./cypress/support/test_data/all_types.table.json'); cy.get('[data-test-id="upload-btn"]').click(); - cy.wait(3000).reload(); }); it("Is new table visible", () => { @@ -53,16 +52,16 @@ context("Admin UI Single Dataset", () => { cy.get('[data-test-id="upload-select"]').select('Concept JSON'); cy.get('[data-test-id="upload-input"]').selectFile('./cypress/support/test_data/all_types.concept.json'); cy.get('[data-test-id="upload-btn"]').click(); - cy.wait(3000).reload(); }); it("Can replace test concept", () => { + cy.intercept('/admin/datasets/*/concepts*').as('apiCall'); cy.get('[data-test-id="upload-select"]').select('Concept JSON'); cy.get('[data-test-id="upload-input"]').selectFile('./cypress/support/test_data/all_types.concept.json'); cy.get('[data-test-id="upload-btn"]').click(); - cy.wait(1000); + cy.wait('@apiCall'); cy.get(`[data-test-id="toast-custom-button"]`).click(); - cy.wait(1000); + cy.wait('@apiCall'); cy.get('[data-test-id="toast"]').contains('The file has been posted successfully'); }); @@ -72,7 +71,7 @@ context("Admin UI Single Dataset", () => { }); describe("Can delete test concept and table", () => { - beforeEach(() => visitAdminUI(`datasets/${testDSID}`)); + beforeEach(() => visitAdminUI(`datasets/${testDSID}`)); it("Can delete test concept", () => { cy.get('[data-test-id="accordion-Concepts"]').click(); @@ -95,7 +94,6 @@ context("Admin UI Single Dataset", () => { cy.get('[data-test-id="upload-select"]').select('Concept JSON'); cy.get('[data-test-id="upload-input"]').selectFile('./cypress/support/test_data/all_types.concept.json'); cy.get('[data-test-id="upload-btn"]').click(); - cy.wait(1000).reload(); }); it("Can force delete test table", () => { From 6b5ec257affa2cde746e0600531e82d7357ef865 Mon Sep 17 00:00:00 2001 From: Marco Korinth <103998475+MarcoKorinth@users.noreply.github.com> Date: Wed, 5 Apr 2023 13:32:46 +0200 Subject: [PATCH 191/679] Update cypress/integration/backend-admin-ui/test_2_dataset.js Co-authored-by: MT <12283268+thoniTUB@users.noreply.github.com> --- cypress/integration/backend-admin-ui/test_2_dataset.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cypress/integration/backend-admin-ui/test_2_dataset.js b/cypress/integration/backend-admin-ui/test_2_dataset.js index 83dd704596..83e442e84e 100644 --- a/cypress/integration/backend-admin-ui/test_2_dataset.js +++ b/cypress/integration/backend-admin-ui/test_2_dataset.js @@ -48,7 +48,7 @@ context("Admin UI Single Dataset", () => { cy.get('[data-test-id="accordion-Tables"]').contains('td', `table`); }); - it("Can upload concept table", () => { + it("Can upload concept", () => { cy.get('[data-test-id="upload-select"]').select('Concept JSON'); cy.get('[data-test-id="upload-input"]').selectFile('./cypress/support/test_data/all_types.concept.json'); cy.get('[data-test-id="upload-btn"]').click(); From 99313efb00f9433c55880ae4ef57eb84598cfb1c Mon Sep 17 00:00:00 2001 From: Marco Korinth Date: Wed, 5 Apr 2023 13:51:30 +0200 Subject: [PATCH 192/679] refactored dataset cypress test --- .../backend-admin-ui/test_2_dataset.js | 29 +++++++++---------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/cypress/integration/backend-admin-ui/test_2_dataset.js b/cypress/integration/backend-admin-ui/test_2_dataset.js index 83e442e84e..84983db798 100644 --- a/cypress/integration/backend-admin-ui/test_2_dataset.js +++ b/cypress/integration/backend-admin-ui/test_2_dataset.js @@ -12,9 +12,6 @@ context("Admin UI Single Dataset", () => { cy.get('[data-test-id="entity-name"]').type(testDSLabel); cy.get('[data-test-id="entity-id"]').type(testDSID); cy.get('[data-test-id="create-dataset-btn"]').click().as('createDataset'); - }); - - it("Can see the new dataset", () => { cy.contains(testDSID); }); }); @@ -42,16 +39,16 @@ context("Admin UI Single Dataset", () => { cy.get('[data-test-id="upload-select"]').select('Table JSON'); cy.get('[data-test-id="upload-input"]').selectFile('./cypress/support/test_data/all_types.table.json'); cy.get('[data-test-id="upload-btn"]').click(); - }); - - it("Is new table visible", () => { - cy.get('[data-test-id="accordion-Tables"]').contains('td', `table`); + cy.reload(); + cy.get('[data-test-id="accordion-Tables"]').contains('td', `table`); }); it("Can upload concept", () => { cy.get('[data-test-id="upload-select"]').select('Concept JSON'); cy.get('[data-test-id="upload-input"]').selectFile('./cypress/support/test_data/all_types.concept.json'); cy.get('[data-test-id="upload-btn"]').click(); + cy.reload(); + cy.get('[data-test-id="accordion-Concepts"]').contains('td', `concept1`); }); it("Can replace test concept", () => { @@ -63,9 +60,7 @@ context("Admin UI Single Dataset", () => { cy.get(`[data-test-id="toast-custom-button"]`).click(); cy.wait('@apiCall'); cy.get('[data-test-id="toast"]').contains('The file has been posted successfully'); - }); - - it("Is new concept visible", () => { + cy.reload(); cy.get('[data-test-id="accordion-Concepts"]').contains('td', `concept1`); }); }); @@ -76,8 +71,11 @@ context("Admin UI Single Dataset", () => { it("Can use page components", () => { cy.contains('Table table'); cy.get('[data-test-id="accordion-Tags"]').click(); + cy.hash().should('eq', '#Tags'); cy.get('[data-test-id="accordion-Concepts"]').click(); + cy.hash().should('eq', '#Concepts'); cy.get('[data-test-id="accordion-Columns"]').click(); + cy.hash().should('eq', '#Columns'); }); }); @@ -87,7 +85,9 @@ context("Admin UI Single Dataset", () => { it("Can use page components", () => { cy.contains('Concept Concept1'); cy.get('[data-test-id="accordion-Selects"]').first().click(); + cy.hash().should('eq', '#Selects'); cy.get('[data-test-id="accordion-Connectors"]').click(); + cy.hash().should('eq', '#Connectors'); }); }); @@ -153,18 +153,17 @@ context("Admin UI Single Dataset", () => { cy.get(`[data-test-id="delete-btn-table-${testDSID}.table"]`).should('not.exist'); }); - it("Reupload test data", () => { - // upload table + it("Can force delete test table", () => { + // reupload table and concept cy.get('[data-test-id="upload-select"]').select('Table JSON'); cy.get('[data-test-id="upload-input"]').selectFile('./cypress/support/test_data/all_types.table.json'); cy.get('[data-test-id="upload-btn"]').click(); - // upload concept + cy.reload(); cy.get('[data-test-id="upload-select"]').select('Concept JSON'); cy.get('[data-test-id="upload-input"]').selectFile('./cypress/support/test_data/all_types.concept.json'); cy.get('[data-test-id="upload-btn"]').click(); - }); + cy.reload(); - it("Can force delete test table", () => { cy.get('[data-test-id="accordion-Tables"]').click(); cy.get(`[data-test-id="delete-btn-table-${testDSID}.table"]`).click({force: true}); cy.get(`[data-test-id="toast-custom-button"]`).click({force: true}); From 48bce1a0fa6204debf1fbe81ceb76cd93ef22a67 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Fri, 31 Mar 2023 13:24:18 +0200 Subject: [PATCH 193/679] wip: created doc --- docs/authentication.md | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 docs/authentication.md diff --git a/docs/authentication.md b/docs/authentication.md new file mode 100644 index 0000000000..1d78ee2ed7 --- /dev/null +++ b/docs/authentication.md @@ -0,0 +1,2 @@ +# Authentication +Conquery uses Shiro for authentication of users. \ No newline at end of file From 4bc830a275161efcad4fc63eb837142b23d5b34a Mon Sep 17 00:00:00 2001 From: Marco Korinth Date: Wed, 5 Apr 2023 14:04:02 +0200 Subject: [PATCH 194/679] updated test_2_dataset.js --- cypress/integration/backend-admin-ui/test_2_dataset.js | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cypress/integration/backend-admin-ui/test_2_dataset.js b/cypress/integration/backend-admin-ui/test_2_dataset.js index 84983db798..48cf0ecabd 100644 --- a/cypress/integration/backend-admin-ui/test_2_dataset.js +++ b/cypress/integration/backend-admin-ui/test_2_dataset.js @@ -35,23 +35,25 @@ context("Admin UI Single Dataset", () => { describe("Can upload test table and concept", () => { beforeEach(() => { visitAdminUI(`datasets/${testDSID}`); }); - it("Can upload test table", () => { + it("Can upload table", () => { + cy.intercept('/admin/datasets/*/tables').as('apiCall'); cy.get('[data-test-id="upload-select"]').select('Table JSON'); cy.get('[data-test-id="upload-input"]').selectFile('./cypress/support/test_data/all_types.table.json'); cy.get('[data-test-id="upload-btn"]').click(); - cy.reload(); + cy.wait('@apiCall').reload(); cy.get('[data-test-id="accordion-Tables"]').contains('td', `table`); }); it("Can upload concept", () => { + cy.intercept('/admin/datasets/*/concepts').as('apiCall'); cy.get('[data-test-id="upload-select"]').select('Concept JSON'); cy.get('[data-test-id="upload-input"]').selectFile('./cypress/support/test_data/all_types.concept.json'); cy.get('[data-test-id="upload-btn"]').click(); - cy.reload(); + cy.wait('@apiCall').reload(); cy.get('[data-test-id="accordion-Concepts"]').contains('td', `concept1`); }); - it("Can replace test concept", () => { + it("Can replace concept", () => { cy.intercept('/admin/datasets/*/concepts*').as('apiCall'); cy.get('[data-test-id="upload-select"]').select('Concept JSON'); cy.get('[data-test-id="upload-input"]').selectFile('./cypress/support/test_data/all_types.concept.json'); From b3deb4b6fd477848b4f3887f61b1e5e182c02e8c Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Wed, 5 Apr 2023 14:30:51 +0200 Subject: [PATCH 195/679] =?UTF-8?q?add=20space=20before=20=E2=82=AC=20Symb?= =?UTF-8?q?ol?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/bakdata/conquery/models/config/FrontendConfig.java | 2 +- frontend/mock-api/config.json | 2 +- frontend/src/js/startup/reducer.ts | 2 +- frontend/src/js/ui-components/CurrencyInput.tsx | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index 52030609eb..1d11526c02 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -60,7 +60,7 @@ public class FrontendConfig { @Data public static class CurrencyConfig { - private String suffix = "€"; + private String suffix = " €"; private String thousandSeparator = "."; private String decimalSeparator = ","; private int decimalScale = 2; diff --git a/frontend/mock-api/config.json b/frontend/mock-api/config.json index b2bd865248..04acc163ce 100644 --- a/frontend/mock-api/config.json +++ b/frontend/mock-api/config.json @@ -3,7 +3,7 @@ "production": true, "currency": { "factor": 1, - "suffix": "€", + "suffix": " €", "thousandSeparator": ".", "decimalSeparator": ",", "decimalScale": 2 diff --git a/frontend/src/js/startup/reducer.ts b/frontend/src/js/startup/reducer.ts index 07c61c5f18..812d664438 100644 --- a/frontend/src/js/startup/reducer.ts +++ b/frontend/src/js/startup/reducer.ts @@ -20,7 +20,7 @@ const initialState: StartupStateT = { ids: [], }, currency: { - suffix: "€", + suffix: " €", thousandSeparator: ".", decimalSeparator: ",", decimalScale: 2, diff --git a/frontend/src/js/ui-components/CurrencyInput.tsx b/frontend/src/js/ui-components/CurrencyInput.tsx index dcb44f92d7..38b7d4c17e 100644 --- a/frontend/src/js/ui-components/CurrencyInput.tsx +++ b/frontend/src/js/ui-components/CurrencyInput.tsx @@ -48,7 +48,7 @@ const CurrencyInput: FC = ({ setNumberFormatValue(""); } }, [value]); - + console.log("Number Format Value: ", numberFormatValue) return ( Date: Wed, 5 Apr 2023 14:33:29 +0200 Subject: [PATCH 196/679] format --- frontend/src/js/ui-components/CurrencyInput.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/src/js/ui-components/CurrencyInput.tsx b/frontend/src/js/ui-components/CurrencyInput.tsx index 38b7d4c17e..3fa51f7556 100644 --- a/frontend/src/js/ui-components/CurrencyInput.tsx +++ b/frontend/src/js/ui-components/CurrencyInput.tsx @@ -48,7 +48,6 @@ const CurrencyInput: FC = ({ setNumberFormatValue(""); } }, [value]); - console.log("Number Format Value: ", numberFormatValue) return ( Date: Wed, 5 Apr 2023 14:43:13 +0200 Subject: [PATCH 197/679] add highlighting of hovered concepts, allow adding duplicate concepts --- frontend/src/js/query-node-editor/ConceptDropzone.tsx | 4 ++-- .../src/js/small-tab-navigation/HoverNavigatable.tsx | 10 +++++++--- frontend/src/js/standard-query-editor/QueryNode.tsx | 6 ++---- frontend/src/js/standard-query-editor/queryReducer.ts | 8 +++++++- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/frontend/src/js/query-node-editor/ConceptDropzone.tsx b/frontend/src/js/query-node-editor/ConceptDropzone.tsx index 6669ca0585..d05e5ece5c 100644 --- a/frontend/src/js/query-node-editor/ConceptDropzone.tsx +++ b/frontend/src/js/query-node-editor/ConceptDropzone.tsx @@ -30,8 +30,8 @@ const ConceptDropzone: FC = ({ node, onDropConcept }) => { const conceptId = (item as DragItemConceptTreeNode).ids[0]; return ( - (item as DragItemConceptTreeNode).tree === node.tree && - !node.ids.some((id) => id === conceptId) + (item as DragItemConceptTreeNode).tree === node.tree + // !node.ids.some((id) => id === conceptId) ); }} > diff --git a/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx b/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx index 7bc0a2d4f9..c6454025d5 100644 --- a/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx +++ b/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx @@ -13,14 +13,16 @@ interface PropsT { item: PossibleDroppableObject, monitor: DropTargetMonitor, ) => boolean; + highlightDroppable?: boolean; } const Root = styled("div")<{ isOver?: boolean; isDroppable?: boolean; + highlightDroppable?: boolean; }>` - background-color: ${({ theme, isOver, isDroppable }) => - isOver && isDroppable ? `${theme.col.grayVeryLight}` : "inherit"}; + background-color: ${({ theme, isDroppable, highlightDroppable, isOver }) => + isOver&& isDroppable ? `${theme.col.grayLight}` : highlightDroppable && isDroppable ? `${theme.col.grayVeryLight}` : "inherit"}; position: relative; border-radius: ${({ theme }) => theme.borderRadius}; display: inline-flex; @@ -34,6 +36,7 @@ export const HoverNavigatable = ({ children, className, canDrop, + highlightDroppable }: PropsT) => { const [timeoutVar, setTimeoutVar] = useState(null); @@ -52,7 +55,7 @@ export const HoverNavigatable = ({ setTimeout(() => { setTimeoutVar(null); if (monitor.isOver()) { - triggerNavigate(); + triggerNavigate(); } }, TIME_UNTIL_NAVIGATE), ); @@ -70,6 +73,7 @@ export const HoverNavigatable = ({ isOver={isOver} isDroppable={isDroppable} className={className} + highlightDroppable={highlightDroppable} > {children} diff --git a/frontend/src/js/standard-query-editor/QueryNode.tsx b/frontend/src/js/standard-query-editor/QueryNode.tsx index 0f725b6fba..86c01b1fb4 100644 --- a/frontend/src/js/standard-query-editor/QueryNode.tsx +++ b/frontend/src/js/standard-query-editor/QueryNode.tsx @@ -204,11 +204,9 @@ const QueryNode = ({ ) { return false; } - const conceptId = item.ids[0]; - const itemAlreadyInNode = node.ids.includes(conceptId); - const itemHasConceptRoot = item.tree === node.tree; - return itemHasConceptRoot && !itemAlreadyInNode; + return item.tree === node.tree; }} + highlightDroppable={true} > { diff --git a/frontend/src/js/standard-query-editor/queryReducer.ts b/frontend/src/js/standard-query-editor/queryReducer.ts index 45f5bf4fd2..0adbcfa6be 100644 --- a/frontend/src/js/standard-query-editor/queryReducer.ts +++ b/frontend/src/js/standard-query-editor/queryReducer.ts @@ -688,8 +688,14 @@ const onAddConceptToNode = ( if (!nodeIsConceptQueryNode(node)) return state; + let ids = concept.ids; + + node.ids.forEach(nodeId => { + if(!ids.includes(nodeId)) ids.push(nodeId); + }); + return setElementProperties(state, andIdx, orIdx, { - ids: [...concept.ids, ...node.ids], + ids: ids, }); }; From 18e73a5b62ec7e01e07fa85ef851b5259a900065 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Wed, 5 Apr 2023 15:03:45 +0200 Subject: [PATCH 198/679] update workflow triggers and check names Signed-off-by: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 5 ++++- .github/workflows/prepare-merge-release.yml | 2 +- .github/workflows/reintegrate-master.yml | 2 +- .github/workflows/run_autodoc.yml | 2 +- .github/workflows/test_backend.yml | 8 +++++++- .github/workflows/test_cypress.yml | 8 +++++++- .github/workflows/test_frontend.yml | 8 +++++++- 7 files changed, 28 insertions(+), 7 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 5d8fe3542c..44c6ced0c1 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -13,7 +13,10 @@ name: "CodeQL" on: push: - branches: [ develop, master, release/* ] + branches: + - develop + - master + - release pull_request: # The branches below must be a subset of the branches above branches: [ develop ] diff --git a/.github/workflows/prepare-merge-release.yml b/.github/workflows/prepare-merge-release.yml index 70e74cf111..2265bda627 100644 --- a/.github/workflows/prepare-merge-release.yml +++ b/.github/workflows/prepare-merge-release.yml @@ -6,7 +6,7 @@ on: - release jobs: - bump-test: + merge-release: runs-on: ubuntu-latest timeout-minutes: 3 steps: diff --git a/.github/workflows/reintegrate-master.yml b/.github/workflows/reintegrate-master.yml index 10ae75eec9..b6896e4c00 100644 --- a/.github/workflows/reintegrate-master.yml +++ b/.github/workflows/reintegrate-master.yml @@ -6,7 +6,7 @@ on: - master jobs: - bump-test: + reintegrate-master: runs-on: ubuntu-latest timeout-minutes: 3 steps: diff --git a/.github/workflows/run_autodoc.yml b/.github/workflows/run_autodoc.yml index 2eb49230e7..ba1ce0abb9 100644 --- a/.github/workflows/run_autodoc.yml +++ b/.github/workflows/run_autodoc.yml @@ -13,7 +13,7 @@ on: - "backend/src/main/java/com/bakdata/conquery/apiv1/**" jobs: - generate_auto_doc: + generate-auto-doc: runs-on: ubuntu-latest timeout-minutes: 10 steps: diff --git a/.github/workflows/test_backend.yml b/.github/workflows/test_backend.yml index f976ebd6f4..00a5fa85d3 100644 --- a/.github/workflows/test_backend.yml +++ b/.github/workflows/test_backend.yml @@ -1,12 +1,18 @@ name: "Test Backend" on: + push: + branches: + # Always run on protected branches + - master + - develop + - release pull_request: paths: - "backend/**" jobs: - run_tests: + test: runs-on: ubuntu-latest timeout-minutes: 10 steps: diff --git a/.github/workflows/test_cypress.yml b/.github/workflows/test_cypress.yml index 947d1b9c9e..e552a5c9cd 100644 --- a/.github/workflows/test_cypress.yml +++ b/.github/workflows/test_cypress.yml @@ -1,12 +1,18 @@ name: "End-To-End Tests" on: + push: + branches: + # Always run on protected branches + - master + - develop + - release pull_request: paths: - "**" jobs: - end-to-end-test: + test: runs-on: ubuntu-latest timeout-minutes: 10 steps: diff --git a/.github/workflows/test_frontend.yml b/.github/workflows/test_frontend.yml index b23cbb734f..f68a285161 100644 --- a/.github/workflows/test_frontend.yml +++ b/.github/workflows/test_frontend.yml @@ -1,12 +1,18 @@ name: "Test Frontend" on: + push: + branches: + # Always run on protected branches + - master + - develop + - release pull_request: paths: - "frontend/**" jobs: - format-lint-unittest: + test: runs-on: ubuntu-latest timeout-minutes: 6 steps: From a9b7e0060fbc82c26a1a1bc44ff6dbb261a52bb4 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 5 Apr 2023 15:24:40 +0200 Subject: [PATCH 199/679] code review changes --- .../query/preview/EntityPreviewExecution.java | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java index 5dcc7bfe58..bdd4626442 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java @@ -154,7 +154,7 @@ private List toChronoInfos(PreviewConfi .collect(Collectors.toMap(PreviewConfig.InfoCardSelect::select, Function.identity())); // Group lines by year and quarter. - final Function> lineTransformer = createLineTransformer(query.getResultInfos(), select2desc, printSettings); + final Function> lineTransformer = createLineToMapTransformer(query.getResultInfos(), select2desc, printSettings); final List yearEntries = createYearEntries(entityResult, lineTransformer); final Object[] completeResult = getCompleteLine(entityResult); @@ -220,6 +220,9 @@ private static Map> getQuarterLines(EntityResult final Map> quarterLines = new HashMap<>(); for (Object[] line : entityResult.listResultLines()) { + if (Resolution.valueOf((String) line[AbsoluteFormQuery.RESOLUTION_INDEX]) != Resolution.QUARTERS) { + continue; + } // Since we know the dates are always aligned we need to only respect their starts. final LocalDate date = CDate.toLocalDate(((List) line[AbsoluteFormQuery.TIME_INDEX]).get(0)); @@ -227,9 +230,7 @@ private static Map> getQuarterLines(EntityResult final int year = date.getYear(); final int quarter = QuarterUtils.getQuarter(date); - if (Resolution.valueOf((String) line[AbsoluteFormQuery.RESOLUTION_INDEX]) == Resolution.QUARTERS) { - quarterLines.computeIfAbsent(year, (ignored) -> new HashMap<>(4)).put(quarter, line); - } + quarterLines.computeIfAbsent(year, (ignored) -> new HashMap<>(4)).put(quarter, line); } return quarterLines; @@ -246,15 +247,16 @@ private static Map getYearLines(EntityResult entityResult) { for (Object[] line : entityResult.listResultLines()) { + if (Resolution.valueOf((String) line[AbsoluteFormQuery.RESOLUTION_INDEX]) != Resolution.YEARS) { + continue; + } + // Since we know the dates are always aligned we need to only respect their starts. final LocalDate date = CDate.toLocalDate(((List) line[AbsoluteFormQuery.TIME_INDEX]).get(0)); final int year = date.getYear(); - final int quarter = QuarterUtils.getQuarter(date); - if (Resolution.valueOf((String) line[AbsoluteFormQuery.RESOLUTION_INDEX]) == Resolution.YEARS) { - yearLines.put(year, line); - } + yearLines.put(year, line); } return yearLines; @@ -264,7 +266,7 @@ private static Map getYearLines(EntityResult entityResult) { * Creates a transformer printing lines, transformed into a Map of label->value. * Null values are omitted. */ - private static Function> createLineTransformer(List resultInfos, Map select2desc, PrintSettings printSettings) { + private static Function> createLineToMapTransformer(List resultInfos, Map select2desc, PrintSettings printSettings) { final int size = resultInfos.size(); From 4afea9498e85bacc34f79f54770b279e13f1f2d1 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 6 Apr 2023 12:48:44 +0200 Subject: [PATCH 200/679] adds documentation to configure oauth2 Signed-off-by: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> --- docs/authentication.md | 77 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 75 insertions(+), 2 deletions(-) diff --git a/docs/authentication.md b/docs/authentication.md index 1d78ee2ed7..fac74797a7 100644 --- a/docs/authentication.md +++ b/docs/authentication.md @@ -1,2 +1,75 @@ -# Authentication -Conquery uses Shiro for authentication of users. \ No newline at end of file +# Configure Authentication with OAuth2/OpenId Connect + +We will discuss how to configure authentication with Keycloak, an open-source identity and access management tool, in +Conquery. +Using this configuration, Conquery's frontend acts as a public client, while the backend validates the provided tokens +offline. + +### Prerequisites: + +Before proceeding with the configuration process, ensure that you have the following prerequisites: + +Conquery is installed and running on your system. +The Keycloak server is installed and running on your system. +You have administrative access to both Conquery and Keycloak. + +### Configuration + +To configure authentication with Keycloak in the Conquery project, follow these steps: + +**Step 1: Create a Keycloak Realm (optional)** +The first step is to create a new Keycloak realm dedicated to Conquery. To create a new realm, follow these steps: + +- Login to the Keycloak Administration Console using your administrative credentials. +- Click on the "Add Realm" button on the left sidebar. +- Enter a name for the new realm (e.g., Conquery). +- Click on the "Create" button. + +Finally, you can retrieve the well-known url to the new realm which will be later referenced by `` +. + +**Step 2: Configure Keycloak Client** + +The next step is to configure a Keycloak client for Conquery. To do this, follow these steps: + +- In the Keycloak Administration Console, navigate to the Conquery realm. +- Click on the "Clients" tab in the left sidebar and then click on the "Create" button. +- Enter a name for the client (e.g., conquery-frontend). +- Set the "Client Protocol" to "openid-connect". +- Set the "Access Type" to "public". +- Set the "Valid Redirect URIs" to the Conquery frontend URL (e.g., http://localhost:8000). +- Click on the "Save" button. + +**Step 3: Configure Conquery** + +The final step is to configure Conquery to use Keycloak for authentication. To do this, follow these steps: + +- Open the Conquery configuration file (e.g., config.json) in a text editor to configure the backend. + Add the following lines to the configuration file: + ```json + "authenticationRealms": [ + { + "type" : "JWT_PKCE_REALM", + "wellKnownEndpoint" : "", + "client" : "" + } + ] + ``` + Replace the `` and `` with the names you used in Steps 1 and 2. + +- Save the changes to the configuration file and restart the Conquery server. +- Assuming you are running the frontend in the provided docker container, you need to set the following environment + variables, when running the container: + ```bash + docker run \ + --env REACT_APP_API_URL= \ + --env REACT_APP_IDP_REALM= \ + --env REACT_APP_IDP_CLIENT_ID= \ + --env REACT_APP_DISABLE_LOGIN=true \ + --env REACT_APP_IDP_URL= \ + --env REACT_APP_IDP_ENABLE=true \ + --publish 8000:80 \ + ghcr.io/ingef/conquery-frontend + ``` + +By following these steps, you can ensure that your Conquery instance is secure and protected from unauthorized access. \ No newline at end of file From 417f3d8a843936e3562579c0bf3bacce0afef68c Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 6 Apr 2023 15:39:28 +0200 Subject: [PATCH 201/679] adds reference to configuration to README Signed-off-by: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> --- README.md | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index b28416c6ae..276d2309e4 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,9 @@ # Conquery *fast & efficient analysis* -[![Last Release](https://img.shields.io/github/release-date/bakdata/conquery.svg?logo=github)](https://github.com/bakdata/conquery/releases/latest) -![Code Size](https://img.shields.io/github/languages/code-size/bakdata/conquery.svg) -[![License](https://img.shields.io/github/license/bakdata/conquery.svg)](https://github.com/bakdata/conquery/blob/develop/LICENSE) - +[![Last Release](https://img.shields.io/github/release-date/ingef/conquery.svg?logo=github)](https://github.com/ingef/conquery/releases/latest) +![Code Size](https://img.shields.io/github/languages/code-size/ingef/conquery.svg) +[![License](https://img.shields.io/github/license/ingef/conquery.svg)](https://github.com/ingef/conquery/blob/develop/LICENSE) ![conquery Screenshot](images/screenshot-v4.png) @@ -32,10 +31,11 @@ Check the README in `/frontend` for details. ### Frontend + Backend #### Steps + To test frontend and backend together you can start the setup that is used for end-to-end tests. First build the backend using `conquery/scripts/build_backend_version.sh` or download a JAR from -the [release page](https://github.com/bakdata/conquery/releases) and place it in `conquery/executable/target/`. +the [release page](https://github.com/ingef/conquery/releases) and place it in `conquery/executable/target/`. Build the frontend by running: @@ -46,7 +46,9 @@ yarn yarn build ``` -You can then run `conquery/scripts/run_e2e_all.sh` to start frontend and backend, and also load the test data required by cypress end-to-end test or you can run `conquery/scripts/run_e2e_backend.sh` and `conquery/scripts/run_e2e_frontend.sh` separately without loading any data. +You can then run `conquery/scripts/run_e2e_all.sh` to start frontend and backend, and also load the test data required +by cypress end-to-end test or you can run `conquery/scripts/run_e2e_backend.sh` +and `conquery/scripts/run_e2e_frontend.sh` separately without loading any data. After that, you can visit http://localhost:8081/admin-ui and explore the Admin Panel. @@ -54,6 +56,26 @@ The frontend is accessible at http://localhost:8000 as the default "superuser" i development authentication, you can switch users by passing another users "UserId" as the access token in the query string when accessing the frontend, e.g.: http://localhost:8000/?access_token=user.user2. +## Configuration + +The configuration options for the backend are based on Java classes which reside +under [this package](https://github.com/ingef/conquery/tree/develop/backend/src/main/java/com/bakdata/conquery/models/config) +. +Usually you provide configuration with in a JSON file that is referenced by the start command: + +```bash +java -jar conquery.jar standalone config.json +``` + +This `config.json` represents +configuration [root class](https://github.com/ingef/conquery/blob/develop/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java) +. + +We continuously improve and extend the documentation on these classes and will provide dedicated articles on specific +configurations, such as: + +- [Configure Authentication with OAuth2/OpenId Connect](./docs/authentication.md) + ## Development ### Testing @@ -69,7 +91,8 @@ To run the end-to-end test locally: 5. Then run `yarn cypress open` to start cypress 6. Then chose a test suite and start it. -For further informations on this and other tests, please refer to the corresponding [CI configuration](https://github.com/bakdata/conquery/tree/develop/.github/workflows). +For further informations on this and other tests, please refer to the +corresponding [CI configuration](https://github.com/ingef/conquery/tree/develop/.github/workflows). ### Data Integration From 9054f8d7f3e3429716ff14ded838387fa69cb450 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 6 Apr 2023 17:31:16 +0200 Subject: [PATCH 202/679] try fix codeql Signed-off-by: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 44c6ced0c1..79d9243fee 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -62,18 +62,20 @@ jobs: # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild + if: matrix.language != 'java' uses: github/codeql-action/autobuild@v2 - # ℹ️ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl + # ℹ️ Command-line programs to run using the OS shell. + # 📚 https://git.io/JvXDl - # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language + # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines + # and modify them (or add more) to build your code if your project + # uses a compiled language - #- run: | - # make bootstrap - # make release + - name: Build + if: matrix.language != 'java' + run: | + ./scripts/build_backend_version.sh - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 From fe0d4c7609783613f602168c1c97a9e1189c0462 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 6 Apr 2023 17:34:11 +0200 Subject: [PATCH 203/679] fix condition Signed-off-by: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 79d9243fee..44647f0f03 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -73,7 +73,7 @@ jobs: # uses a compiled language - name: Build - if: matrix.language != 'java' + if: matrix.language == 'java' run: | ./scripts/build_backend_version.sh From d9fefa5faa555f370ad9f4704c73002653b432fd Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Thu, 6 Apr 2023 17:37:14 +0200 Subject: [PATCH 204/679] adds version placeholder Signed-off-by: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> --- .github/workflows/codeql-analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 44647f0f03..570eb425f6 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -75,7 +75,7 @@ jobs: - name: Build if: matrix.language == 'java' run: | - ./scripts/build_backend_version.sh + ./scripts/build_backend_version.sh "version_placeholder" - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v2 From b26d426374b7976b2085d405bc9d82fa96940d31 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 10 Apr 2023 14:22:43 +0200 Subject: [PATCH 205/679] Transform fieldnames in the right places --- .../js/external-forms/FormConfigLoader.tsx | 4 +- frontend/src/js/external-forms/FormsTab.tsx | 34 +++++++----- frontend/src/js/external-forms/form/Field.tsx | 55 +++++++++++++++---- frontend/src/js/external-forms/helper.ts | 19 ++++++- .../src/js/external-forms/stateSelectors.ts | 30 +++++++--- .../js/external-forms/transformQueryToApi.ts | 32 +++++++---- 6 files changed, 125 insertions(+), 49 deletions(-) diff --git a/frontend/src/js/external-forms/FormConfigLoader.tsx b/frontend/src/js/external-forms/FormConfigLoader.tsx index 73db78deff..e4f4378d8b 100644 --- a/frontend/src/js/external-forms/FormConfigLoader.tsx +++ b/frontend/src/js/external-forms/FormConfigLoader.tsx @@ -17,7 +17,7 @@ import Dropzone from "../ui-components/Dropzone"; import { setExternalForm } from "./actions"; import type { Form, FormField } from "./config-types"; import type { FormConceptGroupT } from "./form-concept-group/formConceptGroupState"; -import { collectAllFormFields } from "./helper"; +import { collectAllFormFields, getUniqueFieldname } from "./helper"; import { selectActiveFormType, selectFormConfig } from "./stateSelectors"; import type { DragItemFormConfig } from "./types"; @@ -135,7 +135,7 @@ const FormConfigLoader: FC = ({ }); // -------------------------- - setValue(fieldname, fieldValue, { + setValue(getUniqueFieldname(formConfig.type, fieldname), fieldValue, { shouldValidate: true, shouldDirty: true, shouldTouch: true, diff --git a/frontend/src/js/external-forms/FormsTab.tsx b/frontend/src/js/external-forms/FormsTab.tsx index 0522b261cd..fd54111a03 100644 --- a/frontend/src/js/external-forms/FormsTab.tsx +++ b/frontend/src/js/external-forms/FormsTab.tsx @@ -14,7 +14,11 @@ import FormsQueryRunner from "./FormsQueryRunner"; import { loadFormsSuccess, setExternalForm } from "./actions"; import type { Field, Form, Tabs } from "./config-types"; import type { DynamicFormValues } from "./form/Form"; -import { collectAllFormFields, getInitialValue } from "./helper"; +import { + collectAllFormFields, + getInitialValue, + getUniqueFieldname, +} from "./helper"; import { selectFormConfig } from "./stateSelectors"; const useLoadForms = ({ datasetId }: { datasetId: DatasetT["id"] | null }) => { @@ -69,20 +73,20 @@ const useInitializeForm = () => { const datasetOptions = useDatasetOptions(); - const defaultValues = useMemo( - () => - Object.fromEntries( - allFields.map((field) => { - const initialValue = getInitialValue(field, { - availableDatasets: datasetOptions, - activeLang, - }); - - return [field.name, initialValue]; - }), - ), - [allFields, datasetOptions, activeLang], - ); + const defaultValues = useMemo(() => { + if (!config) return {}; + + return Object.fromEntries( + allFields.map((field) => { + const initialValue = getInitialValue(field, { + availableDatasets: datasetOptions, + activeLang, + }); + + return [getUniqueFieldname(config.type, field), initialValue]; + }), + ); + }, [allFields, datasetOptions, activeLang, config]); const methods = useForm({ defaultValues, diff --git a/frontend/src/js/external-forms/form/Field.tsx b/frontend/src/js/external-forms/form/Field.tsx index 73150c3ed8..bafd2a8911 100644 --- a/frontend/src/js/external-forms/form/Field.tsx +++ b/frontend/src/js/external-forms/form/Field.tsx @@ -32,6 +32,7 @@ import FormTabNavigation from "../form-tab-navigation/FormTabNavigation"; import { getFieldKey, getInitialValue, + getUniqueFieldname, isFormField, isOptionalField, } from "../helper"; @@ -66,6 +67,7 @@ const BOTTOM_MARGIN = 7; type Props = T & { children: (props: ControllerRenderProps) => ReactNode; control: Control; + formType: string; formField: FieldT | Tabs; defaultValue?: any; noContainer?: boolean; @@ -81,6 +83,7 @@ const FieldContainer = styled("div")<{ noLabel?: boolean }>` const ConnectedField = ({ children, control, + formType, formField, defaultValue, noContainer, @@ -89,7 +92,7 @@ const ConnectedField = ({ }: Props) => { const { t } = useTranslation(); const { field } = useController({ - name: formField.name, + name: getUniqueFieldname(formType, formField), defaultValue, control, rules: { @@ -154,11 +157,17 @@ const Field = ({ field, ...commonProps }: PropsT) => { const { formType, optional, locale, availableDatasets, setValue, control } = commonProps; const { t } = useTranslation(); + const defaultValue = isFormField(field) && field.type !== "GROUP" ? getInitialValue(field, { availableDatasets, activeLang: locale }) : null; + const uniqueFieldname = + isFormField(field) && field.type !== "GROUP" + ? getUniqueFieldname(formType, field) + : ""; // To avoid null checks. We won't use this value for non-form-fields and GROUP fields. + switch (field.type) { case "HEADLINE": return ( @@ -175,6 +184,7 @@ const Field = ({ field, ...commonProps }: PropsT) => { case "STRING": return ( { } fullWidth={field.style ? field.style.fullWidth : false} value={fieldProps.value as string} - onChange={(value) => setValue(field.name, value, setValueConfig)} + onChange={(value) => + setValue(uniqueFieldname, value, setValueConfig) + } tooltip={field.tooltip ? field.tooltip[locale] : undefined} optional={optional} /> @@ -199,6 +211,7 @@ const Field = ({ field, ...commonProps }: PropsT) => { case "TEXTAREA": return ( { value={fieldProps.value as string} onChange={(value) => { console.log(value); - setValue(field.name, value, setValueConfig); + setValue(uniqueFieldname, value, setValueConfig); }} tooltip={field.tooltip ? field.tooltip[locale] : undefined} optional={optional} @@ -225,6 +238,7 @@ const Field = ({ field, ...commonProps }: PropsT) => { case "NUMBER": return ( { (field.placeholder && field.placeholder[locale]) || "" } value={fieldProps.value as number | null} - onChange={(value) => setValue(field.name, value, setValueConfig)} + onChange={(value) => + setValue(uniqueFieldname, value, setValueConfig) + } inputProps={{ step: field.step || "1", pattern: field.pattern, @@ -254,6 +270,7 @@ const Field = ({ field, ...commonProps }: PropsT) => { case "DATE_RANGE": return ( { optional={optional} value={fieldProps.value as DateStringMinMax} onChange={(value) => - setValue(field.name, value, setValueConfig) + setValue(uniqueFieldname, value, setValueConfig) } /> ); @@ -277,6 +294,7 @@ const Field = ({ field, ...commonProps }: PropsT) => { case "RESULT_GROUP": return ( { tooltip={field.tooltip ? field.tooltip[locale] : undefined} optional={optional} value={fieldProps.value as DragItemQuery} - onChange={(value) => setValue(field.name, value, setValueConfig)} + onChange={(value) => + setValue(uniqueFieldname, value, setValueConfig) + } /> )} @@ -296,6 +316,7 @@ const Field = ({ field, ...commonProps }: PropsT) => { case "CHECKBOX": return ( { {({ ref, ...fieldProps }) => ( setValue(field.name, value, setValueConfig)} + onChange={(value) => + setValue(uniqueFieldname, value, setValueConfig) + } label={field.label[locale] || ""} infoTooltip={field.tooltip ? field.tooltip[locale] : undefined} /> @@ -319,6 +342,7 @@ const Field = ({ field, ...commonProps }: PropsT) => { return ( { tooltip={field.tooltip ? field.tooltip[locale] : undefined} optional={optional} value={fieldProps.value as SelectOptionT | null} - onChange={(value) => setValue(field.name, value, setValueConfig)} + onChange={(value) => + setValue(uniqueFieldname, value, setValueConfig) + } /> )} @@ -344,6 +370,7 @@ const Field = ({ field, ...commonProps }: PropsT) => { return ( { optional={optional} value={fieldProps.value as SelectOptionT | null} onChange={(value) => - setValue(field.name, value, setValueConfig) + setValue(uniqueFieldname, value, setValueConfig) } /> ); @@ -399,6 +426,7 @@ const Field = ({ field, ...commonProps }: PropsT) => { case "TABS": return ( { - setValue(field.name, tab, setValueConfig) + setValue(uniqueFieldname, tab, setValueConfig) } options={field.tabs.map((tab) => ({ label: () => tab.title[locale] || "", @@ -449,15 +477,18 @@ const Field = ({ field, ...commonProps }: PropsT) => { case "CONCEPT_LIST": return ( {({ ref, ...fieldProps }) => ( setValue(field.name, value, setValueConfig)} + onChange={(value) => + setValue(uniqueFieldname, value, setValueConfig) + } label={field.label[locale] || ""} tooltip={field.tooltip ? field.tooltip[locale] : undefined} conceptDropzoneText={ diff --git a/frontend/src/js/external-forms/helper.ts b/frontend/src/js/external-forms/helper.ts index 465b2e6f68..25d32eb0e1 100644 --- a/frontend/src/js/external-forms/helper.ts +++ b/frontend/src/js/external-forms/helper.ts @@ -1,10 +1,27 @@ import type { SelectOptionT } from "../api/types"; import type { Language } from "../localization/useActiveLang"; -import type { FormField, GeneralField, Group } from "./config-types"; +import type { + Field, + FormField, + GeneralField, + Group, + Tabs, +} from "./config-types"; const nonFormFieldTypes = new Set(["HEADLINE", "DESCRIPTION"]); +// Different forms may have fields with the same name. +// We want to remember values of fields of form A when switching to form B, +// so users may come back to form A and see their previous values. +// So in order to avoid field name clashes, we need unique field names +export const getUniqueFieldname = (formType: string, field: Field | Tabs) => { + return `${formType}--${field.name}`; +}; +export const getRawFieldname = (uniqueFieldname: string) => { + return uniqueFieldname.split("--").at(-1); +}; + export const getFieldKey = ( formType: string, field: GeneralField, diff --git a/frontend/src/js/external-forms/stateSelectors.ts b/frontend/src/js/external-forms/stateSelectors.ts index 5662170457..6dbc62d20c 100644 --- a/frontend/src/js/external-forms/stateSelectors.ts +++ b/frontend/src/js/external-forms/stateSelectors.ts @@ -7,6 +7,7 @@ import { useActiveLang } from "../localization/useActiveLang"; import { ConceptListField, Form, GeneralField } from "./config-types"; import type { FormConceptGroupT } from "./form-concept-group/formConceptGroupState"; +import { getUniqueFieldname } from "./helper"; export const selectAvailableForms = (state: StateT) => state.externalForms ? state.externalForms.availableForms : {}; @@ -31,22 +32,23 @@ export const selectRunningQuery = (state: StateT) => { }; function getVisibleConceptListFields( - config: { fields: GeneralField[] }, + config: Form, + fields: GeneralField[], values: Record, ): ConceptListField[] { - return config.fields + return fields .flatMap((field) => { switch (field.type) { case "GROUP": return field.fields; case "TABS": - const activeTabName = values[field.name]; + const activeTabName = values[getUniqueFieldname(config.type, field)]; const activeTab = field.tabs.find( (tab) => tab.name === activeTabName, ); return activeTab - ? getVisibleConceptListFields(activeTab, values) + ? getVisibleConceptListFields(config, activeTab.fields, values) : []; default: return [field]; @@ -65,13 +67,16 @@ export const useVisibleConceptListFields = () => { if (!config) return []; - return getVisibleConceptListFields(config, values); + return getVisibleConceptListFields(config, config.fields, values); }; export const useAllowExtendedCopying = ( targetFieldname: string, visibleConceptListFields: ConceptListField[], ) => { + const config = useSelector((state) => + selectFormConfig(state), + ); const values = useWatch({}); const otherConceptListFields = visibleConceptListFields.filter( (field) => field.name !== targetFieldname, @@ -80,11 +85,18 @@ export const useAllowExtendedCopying = ( // Need to have min 2 fields to copy from one to another if (otherConceptListFields.length < 1) return false; - const fieldHasFilledConcept = (field: ConceptListField) => - !!values[field.name] && - values[field.name].some((value: FormConceptGroupT) => - value.concepts.some(exists), + const fieldHasFilledConcept = (field: ConceptListField) => { + if (!config) return false; + + const uniqueFieldname = getUniqueFieldname(config.type, field); + + return ( + !!values[uniqueFieldname] && + values[uniqueFieldname].some((value: FormConceptGroupT) => + value.concepts.some(exists), + ) ); + }; return otherConceptListFields.some(fieldHasFilledConcept); }; diff --git a/frontend/src/js/external-forms/transformQueryToApi.ts b/frontend/src/js/external-forms/transformQueryToApi.ts index 423b90ca8d..b3022f49c7 100644 --- a/frontend/src/js/external-forms/transformQueryToApi.ts +++ b/frontend/src/js/external-forms/transformQueryToApi.ts @@ -7,7 +7,12 @@ import type { DragItemQuery } from "../standard-query-editor/types"; import type { Form, GeneralField } from "./config-types"; import type { FormConceptGroupT } from "./form-concept-group/formConceptGroupState"; import type { DynamicFormValues } from "./form/Form"; -import { collectAllFormFields, isFormField } from "./helper"; +import { + collectAllFormFields, + getRawFieldname, + getUniqueFieldname, + isFormField, +} from "./helper"; function transformElementGroupsToApi(elementGroups: FormConceptGroupT[]) { const elementGroupsWithAtLeastOneElement = elementGroups @@ -30,15 +35,17 @@ function transformElementGroupsToApi(elementGroups: FormConceptGroupT[]) { } function transformFieldToApiEntries( + configType: string, fieldConfig: GeneralField, formValues: DynamicFormValues, ): [string, any][] { if (!isFormField(fieldConfig)) { return []; } - const formValue = - fieldConfig.type === "GROUP" ? null : formValues[fieldConfig.name]; + fieldConfig.type === "GROUP" + ? null + : formValues[getUniqueFieldname(configType, fieldConfig)]; switch (fieldConfig.type) { case "CHECKBOX": @@ -79,7 +86,7 @@ function transformFieldToApiEntries( ]; case "GROUP": return fieldConfig.fields.flatMap((f) => - transformFieldToApiEntries(f, formValues), + transformFieldToApiEntries(configType, f, formValues), ); case "TABS": const selectedTab = fieldConfig.tabs.find( @@ -98,7 +105,7 @@ function transformFieldToApiEntries( { value: formValue, // Only include field values from the selected tab - ...transformFieldsToApi(selectedTab.fields, formValues), + ...transformFieldsToApi(configType, selectedTab.fields, formValues), }, ], ]; @@ -106,11 +113,14 @@ function transformFieldToApiEntries( } function transformFieldsToApi( + configType: string, fields: GeneralField[], formValues: DynamicFormValues, ): DynamicFormValues { return Object.fromEntries( - fields.flatMap((field) => transformFieldToApiEntries(field, formValues)), + fields.flatMap((field) => + transformFieldToApiEntries(configType, field, formValues), + ), ); } @@ -120,15 +130,17 @@ const transformQueryToApi = ( ) => { const formFields = collectAllFormFields(formConfig.fields); const formSpecificValuesToSave = Object.fromEntries( - Object.entries(formValues).filter(([k]) => - formFields.some((f) => f.type !== "GROUP" && f.name === k), - ), + Object.entries(formValues) + .filter(([k]) => + formFields.some((f) => f.type !== "GROUP" && f.name === k), + ) + .map(([k, v]) => [getRawFieldname(k), v]), ); return { type: formConfig.type, values: formSpecificValuesToSave, - ...transformFieldsToApi(formConfig.fields, formValues), + ...transformFieldsToApi(formConfig.type, formConfig.fields, formValues), }; }; From fb943ab662cfa053c607561a75afe4d28dd34a3c Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 10 Apr 2023 17:06:27 +0200 Subject: [PATCH 206/679] Fix copying and values saving --- frontend/src/js/entity-history/saveAndLoad.ts | 1 - .../src/js/external-forms/FormConfigLoader.tsx | 6 +++--- .../form-concept-group/FormConceptCopyModal.tsx | 17 +++++++++++++++-- .../form-concept-group/FormConceptGroup.tsx | 1 + frontend/src/js/external-forms/form/Field.tsx | 1 - .../src/js/external-forms/stateSelectors.ts | 6 ++++-- .../js/external-forms/transformQueryToApi.ts | 5 ++++- 7 files changed, 27 insertions(+), 10 deletions(-) diff --git a/frontend/src/js/entity-history/saveAndLoad.ts b/frontend/src/js/entity-history/saveAndLoad.ts index 5e98add4ef..b13979435b 100644 --- a/frontend/src/js/entity-history/saveAndLoad.ts +++ b/frontend/src/js/entity-history/saveAndLoad.ts @@ -18,7 +18,6 @@ export const saveHistory = ({ entityIds: EntityId[]; entityIdsStatus: EntityIdsStatus; }) => { - console.log(entityIdsStatus); const usedStatuses = Object.values(entityIdsStatus).reduce( (longest, el) => (longest.length > el.length ? longest : el), [], diff --git a/frontend/src/js/external-forms/FormConfigLoader.tsx b/frontend/src/js/external-forms/FormConfigLoader.tsx index e4f4378d8b..195366591a 100644 --- a/frontend/src/js/external-forms/FormConfigLoader.tsx +++ b/frontend/src/js/external-forms/FormConfigLoader.tsx @@ -15,7 +15,7 @@ import { SnackMessageType } from "../snack-message/reducer"; import Dropzone from "../ui-components/Dropzone"; import { setExternalForm } from "./actions"; -import type { Form, FormField } from "./config-types"; +import type { Field, Form, FormField, Tabs } from "./config-types"; import type { FormConceptGroupT } from "./form-concept-group/formConceptGroupState"; import { collectAllFormFields, getUniqueFieldname } from "./helper"; import { selectActiveFormType, selectFormConfig } from "./stateSelectors"; @@ -124,7 +124,7 @@ const FormConfigLoader: FC = ({ // from string, e.g. 'next' // to SelectValueT, e.g. { value: 'next', label: 'Next' } const field = collectAllFormFields(formConfig.fields).find( - (f) => f.type !== "GROUP" && f.name === fieldname, + (f): f is Field | Tabs => f.type !== "GROUP" && f.name === fieldname, ); if (!field) continue; @@ -135,7 +135,7 @@ const FormConfigLoader: FC = ({ }); // -------------------------- - setValue(getUniqueFieldname(formConfig.type, fieldname), fieldValue, { + setValue(getUniqueFieldname(formConfig.type, field), fieldValue, { shouldValidate: true, shouldDirty: true, shouldTouch: true, diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptCopyModal.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptCopyModal.tsx index ad76d7486b..60c7b27895 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptCopyModal.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptCopyModal.tsx @@ -11,6 +11,7 @@ import { useActiveLang } from "../../localization/useActiveLang"; import Modal from "../../modal/Modal"; import InputCheckbox from "../../ui-components/InputCheckbox"; import InputSelect from "../../ui-components/InputSelect/InputSelect"; +import { getUniqueFieldname } from "../helper"; import { useVisibleConceptListFields } from "../stateSelectors"; import type { FormConceptGroupT } from "./formConceptGroupState"; @@ -39,12 +40,14 @@ const SxInputCheckbox = styled(InputCheckbox)` `; interface PropsT { + formType: string; targetFieldname: string; onAccept: (selectedNodes: FormConceptGroupT[]) => void; onClose: () => void; } const FormConceptCopyModal = ({ + formType, targetFieldname, onAccept, onClose, @@ -56,10 +59,20 @@ const FormConceptCopyModal = ({ const visibleConceptListFields = useVisibleConceptListFields(); const conceptListFieldOptions = visibleConceptListFields - .filter((field) => field.name !== targetFieldname) + .filter((field) => { + const uniqueFieldname = getUniqueFieldname(formType, field); + const isAnotherField = uniqueFieldname !== targetFieldname; + const hasValues = + formValues[uniqueFieldname] && + formValues[uniqueFieldname].some((value: FormConceptGroupT) => + value.concepts.some(exists), + ); + + return isAnotherField && hasValues; + }) .map((field) => ({ label: field.label[activeLang] || "-", - value: field.name, + value: getUniqueFieldname(formType, field), })); // Since the modal is only rendered when there exists more than one concept list field diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 72a44648e3..a072c9527e 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -364,6 +364,7 @@ const FormConceptGroup = (props: Props) => { /> {isCopyModalOpen && ( setIsCopyModalOpen(false)} diff --git a/frontend/src/js/external-forms/form/Field.tsx b/frontend/src/js/external-forms/form/Field.tsx index bafd2a8911..322d023aa4 100644 --- a/frontend/src/js/external-forms/form/Field.tsx +++ b/frontend/src/js/external-forms/form/Field.tsx @@ -226,7 +226,6 @@ const Field = ({ field, ...commonProps }: PropsT) => { rows={field.style?.rows ?? 4} value={fieldProps.value as string} onChange={(value) => { - console.log(value); setValue(uniqueFieldname, value, setValueConfig); }} tooltip={field.tooltip ? field.tooltip[locale] : undefined} diff --git a/frontend/src/js/external-forms/stateSelectors.ts b/frontend/src/js/external-forms/stateSelectors.ts index 6dbc62d20c..78a9eb5d56 100644 --- a/frontend/src/js/external-forms/stateSelectors.ts +++ b/frontend/src/js/external-forms/stateSelectors.ts @@ -78,8 +78,10 @@ export const useAllowExtendedCopying = ( selectFormConfig(state), ); const values = useWatch({}); - const otherConceptListFields = visibleConceptListFields.filter( - (field) => field.name !== targetFieldname, + const otherConceptListFields = visibleConceptListFields.filter((field) => + !config + ? false + : getUniqueFieldname(config.type, field) !== targetFieldname, ); // Need to have min 2 fields to copy from one to another diff --git a/frontend/src/js/external-forms/transformQueryToApi.ts b/frontend/src/js/external-forms/transformQueryToApi.ts index b3022f49c7..1cef12d5f1 100644 --- a/frontend/src/js/external-forms/transformQueryToApi.ts +++ b/frontend/src/js/external-forms/transformQueryToApi.ts @@ -132,7 +132,10 @@ const transformQueryToApi = ( const formSpecificValuesToSave = Object.fromEntries( Object.entries(formValues) .filter(([k]) => - formFields.some((f) => f.type !== "GROUP" && f.name === k), + formFields.some( + (f) => + f.type !== "GROUP" && getUniqueFieldname(formConfig.type, f) === k, + ), ) .map(([k, v]) => [getRawFieldname(k), v]), ); From 145bfa72e7dc5f407240ceb78baf99106f4f8a10 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 11 Apr 2023 10:16:35 +0200 Subject: [PATCH 207/679] Add a small explainer why we don't reset --- frontend/src/js/external-forms/FormsNavigation.tsx | 3 +++ 1 file changed, 3 insertions(+) diff --git a/frontend/src/js/external-forms/FormsNavigation.tsx b/frontend/src/js/external-forms/FormsNavigation.tsx index f552b5c31c..92fc676ece 100644 --- a/frontend/src/js/external-forms/FormsNavigation.tsx +++ b/frontend/src/js/external-forms/FormsNavigation.tsx @@ -90,6 +90,9 @@ const FormsNavigation = ({ reset }: Props) => { onChange={(value) => { if (value) { onChangeToForm(value.value as string); + // we intentionally only change the form + // but we don't reset field state, + // so values are kept when switching forms } }} /> From 1b7c139d4801444786ff5bfc82762ed51590cdad Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 11 Apr 2023 12:39:42 +0200 Subject: [PATCH 208/679] Fixes ExternalNode failing when no dates are provided --- .../queryplan/specific/ExternalNode.java | 15 ++-- .../SIMPLE_CQEXTERNAL_QUERY.test.json | 79 +++++++++++++++++++ .../query/CQEXTERNAL_ONLY_IDS/content.csv | 7 ++ .../query/CQEXTERNAL_ONLY_IDS/expected.csv | 3 + 4 files changed, 98 insertions(+), 6 deletions(-) create mode 100644 backend/src/test/resources/tests/query/CQEXTERNAL_ONLY_IDS/SIMPLE_CQEXTERNAL_QUERY.test.json create mode 100644 backend/src/test/resources/tests/query/CQEXTERNAL_ONLY_IDS/content.csv create mode 100644 backend/src/test/resources/tests/query/CQEXTERNAL_ONLY_IDS/expected.csv diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java index 2ce2a83cdf..9780a429fd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java @@ -35,9 +35,8 @@ public class ExternalNode extends QPNode { private final Map> extraData; private final String[] extraColumns; - - private CDateSet contained; private final Map> extraAggregators; + private CDateSet contained; @ToString.Include public Set getEntities() { @@ -72,7 +71,7 @@ public void init(Entity entity, QueryExecutionContext context) { public void nextTable(QueryExecutionContext ctx, Table currentTable) { super.nextTable(ctx, currentTable); - if (table.equals(currentTable) && contained != null){ + if (table.equals(currentTable) && contained != null) { dateUnion.addAll(contained); dateUnion.retainAll(ctx.getDateRestriction()); } @@ -85,10 +84,14 @@ public void acceptEvent(Bucket bucket, int event) { @Override public boolean isContained() { + if (contained == null) { + // Entity was not in the selected set. + return false; + } + /* - If the intersection 'dateUnion' is not empty its contained. Otherwise - it is only contained, if the initial date set 'contained' was also empty, - which means that no date context was provided anyway. + If the intersection 'dateUnion' is not empty its contained. + Unless the initial dateset 'contained' was also empty, which means that no date context was provided anyway. */ return !dateUnion.isEmpty() || contained.isEmpty(); } diff --git a/backend/src/test/resources/tests/query/CQEXTERNAL_ONLY_IDS/SIMPLE_CQEXTERNAL_QUERY.test.json b/backend/src/test/resources/tests/query/CQEXTERNAL_ONLY_IDS/SIMPLE_CQEXTERNAL_QUERY.test.json new file mode 100644 index 0000000000..3c591d8ca3 --- /dev/null +++ b/backend/src/test/resources/tests/query/CQEXTERNAL_ONLY_IDS/SIMPLE_CQEXTERNAL_QUERY.test.json @@ -0,0 +1,79 @@ +{ + "type": "QUERY_TEST", + "label": "CQExternal Extra Data Test", + "expectedCsv": "tests/query/CQEXTERNAL_ONLY_IDS/expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "type": "NEGATION", + "child": { + "type": "EXTERNAL", + "format": [ + "ID" + ], + "values": [ + [ + "result" + ], + [ + 1 + ] + ] + } + }, + { + "type": "DATE_RESTRICTION", + "dateRange": { + "max": "2010-12-31" + }, + "child": { + "type": "CONCEPT", + "ids": [ + "test_tree" + ], + "tables": [ + { + "id": "test_tree.connector" + } + ] + } + } + ] + } + }, + "concepts": [ + { + "name": "test_tree", + "type": "TREE", + "connectors": { + "name": "connector", + "table": "test_table", + "validityDates": { + "label": "datum", + "column": "test_table.datum" + } + } + } + ], + "content": { + "tables": [ + { + "csv": "tests/query/CQEXTERNAL_ONLY_IDS/content.csv", + "name": "test_table", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/query/CQEXTERNAL_ONLY_IDS/content.csv b/backend/src/test/resources/tests/query/CQEXTERNAL_ONLY_IDS/content.csv new file mode 100644 index 0000000000..38f68cb1b3 --- /dev/null +++ b/backend/src/test/resources/tests/query/CQEXTERNAL_ONLY_IDS/content.csv @@ -0,0 +1,7 @@ +pid,datum +1,2010-01-01 +2,2010-01-01 +3,2010-01-01 +4,2011-01-01 +5,2011-01-01 +6,2011-01-01 diff --git a/backend/src/test/resources/tests/query/CQEXTERNAL_ONLY_IDS/expected.csv b/backend/src/test/resources/tests/query/CQEXTERNAL_ONLY_IDS/expected.csv new file mode 100644 index 0000000000..08cd9c91da --- /dev/null +++ b/backend/src/test/resources/tests/query/CQEXTERNAL_ONLY_IDS/expected.csv @@ -0,0 +1,3 @@ +result,dates +2,{2010-01-01/2010-01-01} +3,{2010-01-01/2010-01-01} \ No newline at end of file From 2d21e609a6e18bbae9378eb24fc2b32cd6ec0c0f Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 11 Apr 2023 13:03:38 +0200 Subject: [PATCH 209/679] Iterate form tabs and h1 headline styles --- .../form-components/Headline.tsx | 38 ++++++++- .../form-tab-navigation/FormTabNavigation.tsx | 4 +- frontend/src/js/external-forms/form/Field.tsx | 19 ++++- frontend/src/js/external-forms/form/Form.tsx | 4 +- frontend/src/js/external-forms/helper.ts | 16 ++++ .../SmallTabNavigation.tsx | 21 ++--- .../SmallTabNavigationButton.tsx | 85 +++++++++++-------- 7 files changed, 133 insertions(+), 54 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/Headline.tsx b/frontend/src/js/external-forms/form-components/Headline.tsx index 8d283babb2..3a15f5468e 100644 --- a/frontend/src/js/external-forms/form-components/Headline.tsx +++ b/frontend/src/js/external-forms/form-components/Headline.tsx @@ -1,3 +1,4 @@ +import { css } from "@emotion/react"; import styled from "@emotion/styled"; import { Headline as HeadlineField } from "../config-types"; @@ -16,19 +17,48 @@ export const getHeadlineFieldAs = (headline: HeadlineField) => { }; export const Headline = styled("h3")<{ size?: "h1" | "h2" | "h3" }>` + display: flex; + align-items: center; + gap: 10px; font-size: ${({ theme, size }) => size === "h3" ? theme.font.sm : size === "h2" ? theme.font.md : theme.font.lg}; + line-height: 1; color: ${({ theme }) => theme.col.black}; - margin-top: ${({ size }) => - size === "h3" ? "14px" : size === "h2" ? "10px" : "20px"}; - margin-bottom: 0; - margin-left: 10px; font-weight: ${({ size }) => (size === "h3" ? "700" : "400")}; + &:first-child { margin-top: 0; } + + position: relative; + + ${({ size }) => + (!size || size === "h1") && + css` + margin: 20px 0 5px; + margin-left: 0; + `}; + + ${({ size }) => + (size === "h2" || size === "h3") && + css` + border-left: 0; + padding-left: 0; + margin: 10px 0 3px; + margin-left: 10px; + `}; +`; + +export const HeadlineIndex = styled("span")` + padding: 0 10px; + font-size: ${({ theme }) => theme.font.lg}; + border-right: 3px solid ${({ theme }) => theme.col.grayMediumLight}; + display: flex; + align-items: center; + justify-content: center; + color: ${({ theme }) => theme.col.grayMediumLight}; `; diff --git a/frontend/src/js/external-forms/form-tab-navigation/FormTabNavigation.tsx b/frontend/src/js/external-forms/form-tab-navigation/FormTabNavigation.tsx index 262a33306f..b0457e856f 100644 --- a/frontend/src/js/external-forms/form-tab-navigation/FormTabNavigation.tsx +++ b/frontend/src/js/external-forms/form-tab-navigation/FormTabNavigation.tsx @@ -4,14 +4,14 @@ import type { ComponentProps } from "react"; import SmallTabNavigation from "../../small-tab-navigation/SmallTabNavigation"; const SxSmallTabNavigation = styled(SmallTabNavigation)` - padding-top: 8px; + padding-top: 3px; padding-left: 10px; `; const FormTavNavigation = ( props: ComponentProps, ) => { - return ; + return ; }; export default FormTavNavigation; diff --git a/frontend/src/js/external-forms/form/Field.tsx b/frontend/src/js/external-forms/form/Field.tsx index 9c6a1526a9..0d5e787cfe 100644 --- a/frontend/src/js/external-forms/form/Field.tsx +++ b/frontend/src/js/external-forms/form/Field.tsx @@ -24,7 +24,11 @@ import { InputTextarea } from "../../ui-components/InputTextarea/InputTextarea"; import ToggleButton from "../../ui-components/ToggleButton"; import type { Field as FieldT, GeneralField, Tabs } from "../config-types"; import { Description } from "../form-components/Description"; -import { getHeadlineFieldAs, Headline } from "../form-components/Headline"; +import { + getHeadlineFieldAs, + Headline, + HeadlineIndex, +} from "../form-components/Headline"; import FormConceptGroup from "../form-concept-group/FormConceptGroup"; import type { FormConceptGroupT } from "../form-concept-group/formConceptGroupState"; import FormQueryDropzone from "../form-query-dropzone/FormQueryDropzone"; @@ -133,6 +137,7 @@ const NestedFields = styled("div")` interface PropsT { formType: string; + h1Index?: number; field: GeneralField; locale: Language; availableDatasets: SelectOptionT[]; @@ -150,8 +155,15 @@ const setValueConfig = { const Field = ({ field, ...commonProps }: PropsT) => { const datasetId = useDatasetId(); - const { formType, optional, locale, availableDatasets, setValue, control } = - commonProps; + const { + formType, + h1Index, + optional, + locale, + availableDatasets, + setValue, + control, + } = commonProps; const { t } = useTranslation(); const defaultValue = isFormField(field) && field.type !== "GROUP" @@ -162,6 +174,7 @@ const Field = ({ field, ...commonProps }: PropsT) => { case "HEADLINE": return ( + {exists(h1Index) && {h1Index + 1}} {field.label[locale]} ); diff --git a/frontend/src/js/external-forms/form/Form.tsx b/frontend/src/js/external-forms/form/Form.tsx index 65d3a79e65..28be70f607 100644 --- a/frontend/src/js/external-forms/form/Form.tsx +++ b/frontend/src/js/external-forms/form/Form.tsx @@ -6,7 +6,7 @@ import type { SelectOptionT } from "../../api/types"; import { useActiveLang } from "../../localization/useActiveLang"; import FormHeader from "../FormHeader"; import type { Form as FormType } from "../config-types"; -import { getFieldKey, isOptionalField } from "../helper"; +import { getFieldKey, getH1Index, isOptionalField } from "../helper"; import Field from "./Field"; @@ -45,11 +45,13 @@ const Form = memo(({ config, datasetOptions, methods }: Props) => { {config.fields.map((field, i) => { const key = getFieldKey(config.type, field, i); const optional = isOptionalField(field); + const h1Index = getH1Index(config.fields, field); return ( { + if ( + field.type !== "HEADLINE" || + !field.style?.size || + field.style.size !== "h1" + ) { + return; + } + + const h1Fields = fields.filter( + (f) => f.type === "HEADLINE" && f.style?.size === "h1", + ); + + return h1Fields.indexOf(field); +}; + export const isOptionalField = (field: GeneralField) => { return ( isFormField(field) && diff --git a/frontend/src/js/small-tab-navigation/SmallTabNavigation.tsx b/frontend/src/js/small-tab-navigation/SmallTabNavigation.tsx index 5fbc532e7a..a5e870de93 100644 --- a/frontend/src/js/small-tab-navigation/SmallTabNavigation.tsx +++ b/frontend/src/js/small-tab-navigation/SmallTabNavigation.tsx @@ -1,4 +1,4 @@ -import { FC, ReactNode } from "react"; +import { ReactNode } from "react"; import WithTooltip from "../tooltip/WithTooltip"; @@ -10,20 +10,20 @@ interface TabOption { tooltip?: string; } -interface PropsT { - className?: string; - size?: "M" | "L"; - options: TabOption[]; - selectedTab: string; - onSelectTab: (tab: string) => void; -} - -const SmallTabNavigation: FC = ({ +const SmallTabNavigation = ({ className, size = "M", + variant = "secondary", options, selectedTab, onSelectTab, +}: { + className?: string; + size?: "M" | "L"; + variant?: "primary" | "secondary"; + options: TabOption[]; + selectedTab: string; + onSelectTab: (tab: string) => void; }) => { return (
@@ -33,6 +33,7 @@ const SmallTabNavigation: FC = ({ return ( ` position: relative; - border: 0; - background-color: transparent; + + border-top-left-radius: ${({ theme }) => theme.borderRadius}; + border-top-right-radius: ${({ theme }) => theme.borderRadius}; + + border: ${({ primary, theme, selected }) => + primary && selected + ? `1px solid ${theme.col.gray}` + : "1px solid transparent"}; + border-bottom: none; + + background-color: ${({ primary, theme, selected }) => + selected && primary ? theme.col.bg : "transparent"}; margin: 0 2px; height: ${({ size }) => (size === "L" ? "30px" : "26px")}; - padding: ${({ size }) => (size === "L" ? "0px 6px" : "0px 3px")}; + padding: ${({ size }) => (size === "L" ? "0px 10px" : "0px 3px")}; font-size: ${({ theme, size }) => - size === "L" ? theme.font.md : theme.font.xs}; + size === "L" ? theme.font.sm : theme.font.xs}; + + transform: ${({ primary }) => (primary ? "translateY(1px)" : "none")}; ${({ size }) => size === "M" && @@ -32,16 +46,18 @@ const Button = styled("button")<{ text-transform: uppercase; `}; - ${({ selected, highlightColor }) => + ${({ selected, primary, highlightColor }) => selected && + !primary && css` &::after { ${bottomBorderBase}; background-color: ${highlightColor}; } `} - ${({ theme, selected }) => + ${({ theme, selected, primary }) => !selected && + !primary && css` color: ${theme.col.gray}; &:hover { @@ -53,14 +69,6 @@ const Button = styled("button")<{ `}; `; -interface PropsT { - value: string; - size: "M" | "L"; - isSelected?: boolean; - onClick: () => void; - children?: React.ReactNode; -} - const valueToColor = (theme: Theme, value: string) => { switch (value) { case "own": @@ -72,26 +80,35 @@ const valueToColor = (theme: Theme, value: string) => { } }; -const SmallTabNavigationButton = forwardRef( - ({ value, children, size, isSelected, onClick }, ref) => { - const theme = useTheme(); - const highlightColor = valueToColor(theme, value); +const SmallTabNavigationButton = forwardRef< + HTMLButtonElement, + { + value: string; + size: "M" | "L"; + isSelected?: boolean; + onClick: () => void; + children?: React.ReactNode; + variant: "primary" | "secondary"; + } +>(({ value, children, size, isSelected, onClick, variant }, ref) => { + const theme = useTheme(); + const highlightColor = valueToColor(theme, value); - return ( - - - - ); - }, -); + return ( + + + + ); +}); export default SmallTabNavigationButton; From d50fa070b67ee5e896ca8a22e5f894299af13b47 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 11 Apr 2023 15:15:44 +0200 Subject: [PATCH 210/679] Perform fieldname transformation earlier - right after loading forms - right after loading a saved form config - directly before submitting a form to the API --- .../js/external-forms/FormConfigLoader.tsx | 6 +- frontend/src/js/external-forms/FormsTab.tsx | 8 +-- .../FormConceptCopyModal.tsx | 24 +++----- .../form-concept-group/FormConceptGroup.tsx | 1 - frontend/src/js/external-forms/form/Field.tsx | 54 ++++-------------- frontend/src/js/external-forms/helper.ts | 4 +- frontend/src/js/external-forms/reducer.ts | 42 +++++++++++++- .../src/js/external-forms/stateSelectors.ts | 25 +++------ .../js/external-forms/transformQueryToApi.ts | 56 +++++++++---------- 9 files changed, 101 insertions(+), 119 deletions(-) diff --git a/frontend/src/js/external-forms/FormConfigLoader.tsx b/frontend/src/js/external-forms/FormConfigLoader.tsx index 195366591a..3d75b8b77f 100644 --- a/frontend/src/js/external-forms/FormConfigLoader.tsx +++ b/frontend/src/js/external-forms/FormConfigLoader.tsx @@ -124,7 +124,9 @@ const FormConfigLoader: FC = ({ // from string, e.g. 'next' // to SelectValueT, e.g. { value: 'next', label: 'Next' } const field = collectAllFormFields(formConfig.fields).find( - (f): f is Field | Tabs => f.type !== "GROUP" && f.name === fieldname, + (f): f is Field | Tabs => + f.type !== "GROUP" && + f.name === getUniqueFieldname(formConfig.type, fieldname), ); if (!field) continue; @@ -135,7 +137,7 @@ const FormConfigLoader: FC = ({ }); // -------------------------- - setValue(getUniqueFieldname(formConfig.type, field), fieldValue, { + setValue(field.name, fieldValue, { shouldValidate: true, shouldDirty: true, shouldTouch: true, diff --git a/frontend/src/js/external-forms/FormsTab.tsx b/frontend/src/js/external-forms/FormsTab.tsx index fd54111a03..3070c5db7b 100644 --- a/frontend/src/js/external-forms/FormsTab.tsx +++ b/frontend/src/js/external-forms/FormsTab.tsx @@ -14,11 +14,7 @@ import FormsQueryRunner from "./FormsQueryRunner"; import { loadFormsSuccess, setExternalForm } from "./actions"; import type { Field, Form, Tabs } from "./config-types"; import type { DynamicFormValues } from "./form/Form"; -import { - collectAllFormFields, - getInitialValue, - getUniqueFieldname, -} from "./helper"; +import { collectAllFormFields, getInitialValue } from "./helper"; import { selectFormConfig } from "./stateSelectors"; const useLoadForms = ({ datasetId }: { datasetId: DatasetT["id"] | null }) => { @@ -83,7 +79,7 @@ const useInitializeForm = () => { activeLang, }); - return [getUniqueFieldname(config.type, field), initialValue]; + return [field.name, initialValue]; }), ); }, [allFields, datasetOptions, activeLang, config]); diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptCopyModal.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptCopyModal.tsx index 60c7b27895..ee8fb0c438 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptCopyModal.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptCopyModal.tsx @@ -11,7 +11,6 @@ import { useActiveLang } from "../../localization/useActiveLang"; import Modal from "../../modal/Modal"; import InputCheckbox from "../../ui-components/InputCheckbox"; import InputSelect from "../../ui-components/InputSelect/InputSelect"; -import { getUniqueFieldname } from "../helper"; import { useVisibleConceptListFields } from "../stateSelectors"; import type { FormConceptGroupT } from "./formConceptGroupState"; @@ -39,19 +38,15 @@ const SxInputCheckbox = styled(InputCheckbox)` margin: 5px 0; `; -interface PropsT { - formType: string; - targetFieldname: string; - onAccept: (selectedNodes: FormConceptGroupT[]) => void; - onClose: () => void; -} - const FormConceptCopyModal = ({ - formType, targetFieldname, onAccept, onClose, -}: PropsT) => { +}: { + targetFieldname: string; + onAccept: (selectedNodes: FormConceptGroupT[]) => void; + onClose: () => void; +}) => { const { t } = useTranslation(); const activeLang = useActiveLang(); const { getValues } = useFormContext(); @@ -60,11 +55,10 @@ const FormConceptCopyModal = ({ const conceptListFieldOptions = visibleConceptListFields .filter((field) => { - const uniqueFieldname = getUniqueFieldname(formType, field); - const isAnotherField = uniqueFieldname !== targetFieldname; + const isAnotherField = field.name !== targetFieldname; const hasValues = - formValues[uniqueFieldname] && - formValues[uniqueFieldname].some((value: FormConceptGroupT) => + formValues[field.name] && + formValues[field.name].some((value: FormConceptGroupT) => value.concepts.some(exists), ); @@ -72,7 +66,7 @@ const FormConceptCopyModal = ({ }) .map((field) => ({ label: field.label[activeLang] || "-", - value: getUniqueFieldname(formType, field), + value: field.name, })); // Since the modal is only rendered when there exists more than one concept list field diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index a072c9527e..72a44648e3 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -364,7 +364,6 @@ const FormConceptGroup = (props: Props) => { /> {isCopyModalOpen && ( setIsCopyModalOpen(false)} diff --git a/frontend/src/js/external-forms/form/Field.tsx b/frontend/src/js/external-forms/form/Field.tsx index 322d023aa4..680f347e36 100644 --- a/frontend/src/js/external-forms/form/Field.tsx +++ b/frontend/src/js/external-forms/form/Field.tsx @@ -32,7 +32,6 @@ import FormTabNavigation from "../form-tab-navigation/FormTabNavigation"; import { getFieldKey, getInitialValue, - getUniqueFieldname, isFormField, isOptionalField, } from "../helper"; @@ -67,7 +66,6 @@ const BOTTOM_MARGIN = 7; type Props = T & { children: (props: ControllerRenderProps) => ReactNode; control: Control; - formType: string; formField: FieldT | Tabs; defaultValue?: any; noContainer?: boolean; @@ -83,7 +81,6 @@ const FieldContainer = styled("div")<{ noLabel?: boolean }>` const ConnectedField = ({ children, control, - formType, formField, defaultValue, noContainer, @@ -92,7 +89,7 @@ const ConnectedField = ({ }: Props) => { const { t } = useTranslation(); const { field } = useController({ - name: getUniqueFieldname(formType, formField), + name: formField.name, defaultValue, control, rules: { @@ -163,11 +160,6 @@ const Field = ({ field, ...commonProps }: PropsT) => { ? getInitialValue(field, { availableDatasets, activeLang: locale }) : null; - const uniqueFieldname = - isFormField(field) && field.type !== "GROUP" - ? getUniqueFieldname(formType, field) - : ""; // To avoid null checks. We won't use this value for non-form-fields and GROUP fields. - switch (field.type) { case "HEADLINE": return ( @@ -184,7 +176,6 @@ const Field = ({ field, ...commonProps }: PropsT) => { case "STRING": return ( { } fullWidth={field.style ? field.style.fullWidth : false} value={fieldProps.value as string} - onChange={(value) => - setValue(uniqueFieldname, value, setValueConfig) - } + onChange={(value) => setValue(field.name, value, setValueConfig)} tooltip={field.tooltip ? field.tooltip[locale] : undefined} optional={optional} /> @@ -211,7 +200,6 @@ const Field = ({ field, ...commonProps }: PropsT) => { case "TEXTAREA": return ( { rows={field.style?.rows ?? 4} value={fieldProps.value as string} onChange={(value) => { - setValue(uniqueFieldname, value, setValueConfig); + setValue(field.name, value, setValueConfig); }} tooltip={field.tooltip ? field.tooltip[locale] : undefined} optional={optional} @@ -237,7 +225,6 @@ const Field = ({ field, ...commonProps }: PropsT) => { case "NUMBER": return ( { (field.placeholder && field.placeholder[locale]) || "" } value={fieldProps.value as number | null} - onChange={(value) => - setValue(uniqueFieldname, value, setValueConfig) - } + onChange={(value) => setValue(field.name, value, setValueConfig)} inputProps={{ step: field.step || "1", pattern: field.pattern, @@ -269,7 +254,6 @@ const Field = ({ field, ...commonProps }: PropsT) => { case "DATE_RANGE": return ( { optional={optional} value={fieldProps.value as DateStringMinMax} onChange={(value) => - setValue(uniqueFieldname, value, setValueConfig) + setValue(field.name, value, setValueConfig) } /> ); @@ -293,7 +277,6 @@ const Field = ({ field, ...commonProps }: PropsT) => { case "RESULT_GROUP": return ( { tooltip={field.tooltip ? field.tooltip[locale] : undefined} optional={optional} value={fieldProps.value as DragItemQuery} - onChange={(value) => - setValue(uniqueFieldname, value, setValueConfig) - } + onChange={(value) => setValue(field.name, value, setValueConfig)} /> )} @@ -315,7 +296,6 @@ const Field = ({ field, ...commonProps }: PropsT) => { case "CHECKBOX": return ( { {({ ref, ...fieldProps }) => ( - setValue(uniqueFieldname, value, setValueConfig) - } + onChange={(value) => setValue(field.name, value, setValueConfig)} label={field.label[locale] || ""} infoTooltip={field.tooltip ? field.tooltip[locale] : undefined} /> @@ -341,7 +319,6 @@ const Field = ({ field, ...commonProps }: PropsT) => { return ( { tooltip={field.tooltip ? field.tooltip[locale] : undefined} optional={optional} value={fieldProps.value as SelectOptionT | null} - onChange={(value) => - setValue(uniqueFieldname, value, setValueConfig) - } + onChange={(value) => setValue(field.name, value, setValueConfig)} /> )} @@ -369,7 +344,6 @@ const Field = ({ field, ...commonProps }: PropsT) => { return ( { optional={optional} value={fieldProps.value as SelectOptionT | null} onChange={(value) => - setValue(uniqueFieldname, value, setValueConfig) + setValue(field.name, value, setValueConfig) } /> ); @@ -425,7 +399,6 @@ const Field = ({ field, ...commonProps }: PropsT) => { case "TABS": return ( { - setValue(uniqueFieldname, tab, setValueConfig) + setValue(field.name, tab, setValueConfig) } options={field.tabs.map((tab) => ({ label: () => tab.title[locale] || "", @@ -476,18 +449,15 @@ const Field = ({ field, ...commonProps }: PropsT) => { case "CONCEPT_LIST": return ( {({ ref, ...fieldProps }) => ( - setValue(uniqueFieldname, value, setValueConfig) - } + onChange={(value) => setValue(field.name, value, setValueConfig)} label={field.label[locale] || ""} tooltip={field.tooltip ? field.tooltip[locale] : undefined} conceptDropzoneText={ diff --git a/frontend/src/js/external-forms/helper.ts b/frontend/src/js/external-forms/helper.ts index 25d32eb0e1..eac325912e 100644 --- a/frontend/src/js/external-forms/helper.ts +++ b/frontend/src/js/external-forms/helper.ts @@ -15,8 +15,8 @@ const nonFormFieldTypes = new Set(["HEADLINE", "DESCRIPTION"]); // We want to remember values of fields of form A when switching to form B, // so users may come back to form A and see their previous values. // So in order to avoid field name clashes, we need unique field names -export const getUniqueFieldname = (formType: string, field: Field | Tabs) => { - return `${formType}--${field.name}`; +export const getUniqueFieldname = (formType: string, rawFieldName: string) => { + return `${formType}--${rawFieldName}`; }; export const getRawFieldname = (uniqueFieldname: string) => { return uniqueFieldname.split("--").at(-1); diff --git a/frontend/src/js/external-forms/reducer.ts b/frontend/src/js/external-forms/reducer.ts index 0db1606039..e407beb3a2 100644 --- a/frontend/src/js/external-forms/reducer.ts +++ b/frontend/src/js/external-forms/reducer.ts @@ -3,7 +3,39 @@ import { getType } from "typesafe-actions"; import type { Action } from "../app/actions"; import { loadFormsSuccess, setExternalForm } from "./actions"; -import type { Form } from "./config-types"; +import type { Form, GeneralField } from "./config-types"; +import { getUniqueFieldname } from "./helper"; + +const transformToUniqueFieldnames = ( + formType: string, + fields: GeneralField[], +): GeneralField[] => { + return fields.map((field) => { + switch (field.type) { + case "HEADLINE": + case "DESCRIPTION": + return field; + case "GROUP": + return { + ...field, + fields: transformToUniqueFieldnames(formType, field.fields), + }; + case "TABS": + return { + ...field, + tabs: field.tabs.map((tab) => ({ + ...tab, + fields: transformToUniqueFieldnames(formType, tab.fields), + })), + }; + default: + return { + ...field, + name: getUniqueFieldname(formType, field.name), + }; + } + }); +}; export const availableFormsReducer = ( state: { @@ -14,7 +46,13 @@ export const availableFormsReducer = ( switch (action.type) { case getType(loadFormsSuccess): return Object.fromEntries( - action.payload.forms.map((form) => [form.type, form]), + action.payload.forms.map((form) => [ + form.type, + { + ...form, + fields: transformToUniqueFieldnames(form.type, form.fields), + }, + ]), ); default: return state; diff --git a/frontend/src/js/external-forms/stateSelectors.ts b/frontend/src/js/external-forms/stateSelectors.ts index 78a9eb5d56..95bb645cb2 100644 --- a/frontend/src/js/external-forms/stateSelectors.ts +++ b/frontend/src/js/external-forms/stateSelectors.ts @@ -7,7 +7,6 @@ import { useActiveLang } from "../localization/useActiveLang"; import { ConceptListField, Form, GeneralField } from "./config-types"; import type { FormConceptGroupT } from "./form-concept-group/formConceptGroupState"; -import { getUniqueFieldname } from "./helper"; export const selectAvailableForms = (state: StateT) => state.externalForms ? state.externalForms.availableForms : {}; @@ -32,7 +31,6 @@ export const selectRunningQuery = (state: StateT) => { }; function getVisibleConceptListFields( - config: Form, fields: GeneralField[], values: Record, ): ConceptListField[] { @@ -42,13 +40,13 @@ function getVisibleConceptListFields( case "GROUP": return field.fields; case "TABS": - const activeTabName = values[getUniqueFieldname(config.type, field)]; + const activeTabName = values[field.name]; const activeTab = field.tabs.find( (tab) => tab.name === activeTabName, ); return activeTab - ? getVisibleConceptListFields(config, activeTab.fields, values) + ? getVisibleConceptListFields(activeTab.fields, values) : []; default: return [field]; @@ -67,34 +65,25 @@ export const useVisibleConceptListFields = () => { if (!config) return []; - return getVisibleConceptListFields(config, config.fields, values); + return getVisibleConceptListFields(config.fields, values); }; export const useAllowExtendedCopying = ( targetFieldname: string, visibleConceptListFields: ConceptListField[], ) => { - const config = useSelector((state) => - selectFormConfig(state), - ); const values = useWatch({}); - const otherConceptListFields = visibleConceptListFields.filter((field) => - !config - ? false - : getUniqueFieldname(config.type, field) !== targetFieldname, + const otherConceptListFields = visibleConceptListFields.filter( + (field) => field.name !== targetFieldname, ); // Need to have min 2 fields to copy from one to another if (otherConceptListFields.length < 1) return false; const fieldHasFilledConcept = (field: ConceptListField) => { - if (!config) return false; - - const uniqueFieldname = getUniqueFieldname(config.type, field); - return ( - !!values[uniqueFieldname] && - values[uniqueFieldname].some((value: FormConceptGroupT) => + !!values[field.name] && + values[field.name].some((value: FormConceptGroupT) => value.concepts.some(exists), ) ); diff --git a/frontend/src/js/external-forms/transformQueryToApi.ts b/frontend/src/js/external-forms/transformQueryToApi.ts index 1cef12d5f1..91522e5768 100644 --- a/frontend/src/js/external-forms/transformQueryToApi.ts +++ b/frontend/src/js/external-forms/transformQueryToApi.ts @@ -7,12 +7,7 @@ import type { DragItemQuery } from "../standard-query-editor/types"; import type { Form, GeneralField } from "./config-types"; import type { FormConceptGroupT } from "./form-concept-group/formConceptGroupState"; import type { DynamicFormValues } from "./form/Form"; -import { - collectAllFormFields, - getRawFieldname, - getUniqueFieldname, - isFormField, -} from "./helper"; +import { collectAllFormFields, getRawFieldname, isFormField } from "./helper"; function transformElementGroupsToApi(elementGroups: FormConceptGroupT[]) { const elementGroupsWithAtLeastOneElement = elementGroups @@ -35,7 +30,6 @@ function transformElementGroupsToApi(elementGroups: FormConceptGroupT[]) { } function transformFieldToApiEntries( - configType: string, fieldConfig: GeneralField, formValues: DynamicFormValues, ): [string, any][] { @@ -43,34 +37,40 @@ function transformFieldToApiEntries( return []; } const formValue = + fieldConfig.type === "GROUP" ? null : formValues[fieldConfig.name]; + + const rawFieldname = fieldConfig.type === "GROUP" - ? null - : formValues[getUniqueFieldname(configType, fieldConfig)]; + ? "" // Group fields don't have a raw fieldname of their own + : getRawFieldname(fieldConfig.name); + + if (!exists(rawFieldname)) { + throw new Error( + `No raw fieldname found for ${fieldConfig.type}, this shouldn't happen`, + ); + } switch (fieldConfig.type) { case "CHECKBOX": - return [[fieldConfig.name, formValue || false]]; + return [[rawFieldname, formValue || false]]; case "TEXTAREA": case "STRING": case "NUMBER": - return [[fieldConfig.name, formValue ?? null]]; + return [[rawFieldname, formValue ?? null]]; case "DATASET_SELECT": case "SELECT": return [ - [ - fieldConfig.name, - formValue ? (formValue as SelectOptionT).value : null, - ], + [rawFieldname, formValue ? (formValue as SelectOptionT).value : null], ]; case "RESULT_GROUP": // A RESULT_GROUP field may allow null / be optional return [ - [fieldConfig.name, formValue ? (formValue as DragItemQuery).id : null], + [rawFieldname, formValue ? (formValue as DragItemQuery).id : null], ]; case "DATE_RANGE": return [ [ - fieldConfig.name, + rawFieldname, { min: (formValue as DateStringMinMax).min, max: (formValue as DateStringMinMax).max, @@ -80,13 +80,13 @@ function transformFieldToApiEntries( case "CONCEPT_LIST": return [ [ - fieldConfig.name, + rawFieldname, transformElementGroupsToApi(formValue as FormConceptGroupT[]), ], ]; case "GROUP": return fieldConfig.fields.flatMap((f) => - transformFieldToApiEntries(configType, f, formValues), + transformFieldToApiEntries(f, formValues), ); case "TABS": const selectedTab = fieldConfig.tabs.find( @@ -95,17 +95,17 @@ function transformFieldToApiEntries( if (!selectedTab) { throw new Error( - `No tab selected for ${fieldConfig.name}, this shouldn't happen`, + `No tab selected for ${rawFieldname}, this shouldn't happen`, ); } return [ [ - fieldConfig.name, + rawFieldname, { value: formValue, // Only include field values from the selected tab - ...transformFieldsToApi(configType, selectedTab.fields, formValues), + ...transformFieldsToApi(selectedTab.fields, formValues), }, ], ]; @@ -113,14 +113,11 @@ function transformFieldToApiEntries( } function transformFieldsToApi( - configType: string, fields: GeneralField[], formValues: DynamicFormValues, ): DynamicFormValues { return Object.fromEntries( - fields.flatMap((field) => - transformFieldToApiEntries(configType, field, formValues), - ), + fields.flatMap((field) => transformFieldToApiEntries(field, formValues)), ); } @@ -132,10 +129,7 @@ const transformQueryToApi = ( const formSpecificValuesToSave = Object.fromEntries( Object.entries(formValues) .filter(([k]) => - formFields.some( - (f) => - f.type !== "GROUP" && getUniqueFieldname(formConfig.type, f) === k, - ), + formFields.some((f) => f.type !== "GROUP" && f.name === k), ) .map(([k, v]) => [getRawFieldname(k), v]), ); @@ -143,7 +137,7 @@ const transformQueryToApi = ( return { type: formConfig.type, values: formSpecificValuesToSave, - ...transformFieldsToApi(formConfig.type, formConfig.fields, formValues), + ...transformFieldsToApi(formConfig.fields, formValues), }; }; From 447f96ef9363ce6a0e96ba90ff684c25a4c92bda Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 11 Apr 2023 15:39:08 +0200 Subject: [PATCH 211/679] Fix imports --- frontend/src/js/external-forms/helper.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/frontend/src/js/external-forms/helper.ts b/frontend/src/js/external-forms/helper.ts index eac325912e..f39980f1c4 100644 --- a/frontend/src/js/external-forms/helper.ts +++ b/frontend/src/js/external-forms/helper.ts @@ -1,13 +1,7 @@ import type { SelectOptionT } from "../api/types"; import type { Language } from "../localization/useActiveLang"; -import type { - Field, - FormField, - GeneralField, - Group, - Tabs, -} from "./config-types"; +import type { FormField, GeneralField, Group } from "./config-types"; const nonFormFieldTypes = new Set(["HEADLINE", "DESCRIPTION"]); From d5438aa8c57e42dc0e951472de8e9e1f06b6cd1b Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 11 Apr 2023 15:46:36 +0200 Subject: [PATCH 212/679] Clear up hasValues condition --- .../form-concept-group/FormConceptCopyModal.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptCopyModal.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptCopyModal.tsx index ee8fb0c438..bc219477b1 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptCopyModal.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptCopyModal.tsx @@ -58,9 +58,9 @@ const FormConceptCopyModal = ({ const isAnotherField = field.name !== targetFieldname; const hasValues = formValues[field.name] && - formValues[field.name].some((value: FormConceptGroupT) => - value.concepts.some(exists), - ); + formValues[field.name] + .flatMap((v: FormConceptGroupT) => v.concepts) + .some(exists); return isAnotherField && hasValues; }) From cff5606b186c259469117e971a30e75dc25033b0 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 11 Apr 2023 15:57:36 +0200 Subject: [PATCH 213/679] Re add unselected form tab hover effect --- .../SmallTabNavigationButton.tsx | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/frontend/src/js/small-tab-navigation/SmallTabNavigationButton.tsx b/frontend/src/js/small-tab-navigation/SmallTabNavigationButton.tsx index 764663aeab..44d2aabec9 100644 --- a/frontend/src/js/small-tab-navigation/SmallTabNavigationButton.tsx +++ b/frontend/src/js/small-tab-navigation/SmallTabNavigationButton.tsx @@ -24,10 +24,23 @@ const Button = styled("button")<{ border-top-left-radius: ${({ theme }) => theme.borderRadius}; border-top-right-radius: ${({ theme }) => theme.borderRadius}; + transition: border 0.1s ease-in-out; + border: ${({ primary, theme, selected }) => primary && selected ? `1px solid ${theme.col.gray}` - : "1px solid transparent"}; + : primary + ? "1px solid transparent" + : "none"}; + &:hover { + border: ${({ primary, theme, selected }) => + primary && selected + ? `1px solid ${theme.col.gray}` + : primary + ? `1px solid ${theme.col.grayMediumLight}` + : "none"}; + border-bottom: none; + } border-bottom: none; background-color: ${({ primary, theme, selected }) => From 48bbaa9b13cd1eed45d76f7a33e2b441dd148ea9 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 11 Apr 2023 16:04:43 +0200 Subject: [PATCH 214/679] add emptyLabel as first entry in list endpoint. Also add emptyLabel es entry for searching --- .../conquery/apiv1/FilterTemplate.java | 2 + .../conquery/models/datasets/Column.java | 6 +++ .../models/datasets/concepts/Searchable.java | 2 + .../filters/specific/SelectFilter.java | 47 ++++++++++--------- .../models/index/FrontendValueIndex.java | 2 - .../resources/api/ConceptsProcessor.java | 13 +++-- 6 files changed, 44 insertions(+), 28 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java index a24ea2d384..436c98b81d 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java @@ -108,6 +108,8 @@ public List> getSearches(IndexConfig config, Namespace emptyLabel )); + search.addItem(new FrontendValue("", getEmptyLabel()), List.of(getEmptyLabel())); + return List.of(search); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java index 1819a3f59c..215645bd68 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java @@ -54,6 +54,7 @@ public class Column extends Labeled implements NamespacedIdentifiable< private int minSuffixLength = 3; private boolean generateSuffixes; private boolean searchDisabled = false; + private String emptyLabel; @JsonIgnore @Getter(lazy = true) @@ -157,6 +158,8 @@ public List> getSearches(IndexConfig config, Namespace final TrieSearch search = new TrieSearch<>(suffixLength, config.getSearchSplitChars()); + search.addItem(new FrontendValue("", getEmptyLabel()), List.of(getEmptyLabel())); + storage.getAllImports().stream() .filter(imp -> imp.getTable().equals(getTable())) .flatMap(imp -> { @@ -168,6 +171,9 @@ public List> getSearches(IndexConfig config, Namespace .onClose(() -> log.debug("DONE processing values for {}", getId())) .forEach(feValue -> search.addItem(feValue, FilterSearch.extractKeywords(feValue))); + + search.shrinkToFit(); + return List.of(search); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java index 1047c74a4c..84db1fcee7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java @@ -39,6 +39,8 @@ default List> getSearchReferences() { return List.of(this); } + String getEmptyLabel(); + /** * Parameter used in the construction of {@link com.bakdata.conquery.util.search.TrieSearch}, defining the shortest suffix to create. * Ignored if isGenerateSuffixes is true. diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java index 79f22f7657..9c15afd489 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java @@ -47,9 +47,10 @@ public abstract class SelectFilter extends SingleColumnFilter @NsIdRef @View.ApiManagerPersistence private FilterTemplate template; + private int searchMinSuffixLength = 3; + private boolean generateSearchSuffixes = true; - @JsonIgnore - public abstract String getFilterType(); + private String emptyLabel; @Override public EnumSet getAcceptedColumnTypes() { @@ -68,26 +69,8 @@ public void configureFrontend(FrontendFilterConfiguration.Top f) throws ConceptC //TODO FK add empty label } - @NotNull - protected List collectLabels() { - return labels.entrySet().stream() - .map(entry -> new FrontendValue(entry.getKey(), entry.getValue())) - .collect(Collectors.toList()); - } - @JsonIgnore - @ValidationMethod(message = "Cannot use both labels and template.") - public boolean isNotUsingTemplateAndLabels() { - // Technically it's possible it just doesn't make much sense and would lead to Single-Point-of-Truth confusion. - if (getTemplate() == null && labels.isEmpty()) { - return true; - } - - return (getTemplate() == null) != labels.isEmpty(); - } - - private int searchMinSuffixLength = 3; - private boolean generateSearchSuffixes = true; + public abstract String getFilterType(); @Override public List> getSearchReferences() { @@ -106,6 +89,22 @@ public List> getSearchReferences() { return out; } + @NotNull + protected List collectLabels() { + return labels.entrySet().stream().map(entry -> new FrontendValue(entry.getKey(), entry.getValue())).collect(Collectors.toList()); + } + + @JsonIgnore + @ValidationMethod(message = "Cannot use both labels and template.") + public boolean isNotUsingTemplateAndLabels() { + // Technically it's possible it just doesn't make much sense and would lead to Single-Point-of-Truth confusion. + if (getTemplate() == null && labels.isEmpty()) { + return true; + } + + return (getTemplate() == null) != labels.isEmpty(); + } + @Override @JsonIgnore public boolean isGenerateSuffixes() { @@ -130,11 +129,15 @@ public boolean isSearchDisabled() { @Override public List> getSearches(IndexConfig config, NamespaceStorage storage) { - TrieSearch search = new TrieSearch<>(config.getSearchSuffixLength(), config.getSearchSplitChars()); + final TrieSearch search = new TrieSearch<>(config.getSearchSuffixLength(), config.getSearchSplitChars()); + + search.addItem(new FrontendValue("", getEmptyLabel()), List.of(getEmptyLabel())); + labels.entrySet() .stream() .map(entry -> new FrontendValue(entry.getKey(), entry.getValue())) .forEach(feValue -> search.addItem(feValue, FilterSearch.extractKeywords(feValue))); + search.shrinkToFit(); return List.of(search); diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java index 69f272c206..8411436581 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.models.index; -import java.util.List; import java.util.Map; import com.bakdata.conquery.apiv1.FilterTemplate; @@ -30,7 +29,6 @@ public FrontendValueIndex(int suffixCutoff, String split, String valueTemplate, super(suffixCutoff, split); this.valueTemplate = valueTemplate; this.optionValueTemplate = optionValueTemplate; - addItem(new FrontendValue("", emptyLabel), List.of(emptyLabel, "")); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java index ea69f3d37e..6921468d3d 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java @@ -233,9 +233,15 @@ private Cursor listAllValues(Searchable searchable) { See: https://stackoverflow.com/questions/61114380/java-streams-buffering-huge-streams */ - final Iterator - iterators = - Iterators.concat(Iterators.transform(namespace.getFilterSearch().getSearchesFor(searchable).iterator(), TrieSearch::iterator)); + + final Iterator iterators = + Iterators.concat( + // We are always leading with the empty value. + Iterators.singletonIterator(new FrontendValue("", searchable.getEmptyLabel())), + Iterators.concat(Iterators.transform(namespace.getFilterSearch() + .getSearchesFor(searchable) + .iterator(), TrieSearch::iterator)) + ); // Use Set to accomplish distinct values final Set seen = new HashSet<>(); @@ -246,7 +252,6 @@ private Cursor listAllValues(Searchable searchable) { private long countAllValues(Searchable searchable) { final Namespace namespace = namespaces.get(searchable.getDataset().getId()); - return namespace.getFilterSearch().getTotal(searchable); } From f67d9f8d48c388646e30baacf9162e83df412abc Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 11 Apr 2023 17:06:50 +0200 Subject: [PATCH 215/679] Start displaying time stratified infos --- frontend/src/js/api/types.ts | 3 + frontend/src/js/entity-history/EntityCard.tsx | 61 +++++++++++++++++++ .../src/js/entity-history/EntityHeader.tsx | 6 +- .../src/js/entity-history/EntityInfos.tsx | 8 +-- frontend/src/js/entity-history/History.tsx | 10 ++- frontend/src/js/entity-history/Timeline.tsx | 15 +++++ frontend/src/js/entity-history/actions.ts | 5 +- frontend/src/js/entity-history/reducer.ts | 5 ++ 8 files changed, 102 insertions(+), 11 deletions(-) create mode 100644 frontend/src/js/entity-history/EntityCard.tsx diff --git a/frontend/src/js/api/types.ts b/frontend/src/js/api/types.ts index 7fd1c376b3..ab45ee7f29 100644 --- a/frontend/src/js/api/types.ts +++ b/frontend/src/js/api/types.ts @@ -561,6 +561,9 @@ export interface TimeStratifiedInfoYear { export interface TimeStratifiedInfo { label: string; description: string | null; + totals: { + [label: string]: string; + }; columns: { label: string; // Matches `label` with `year.values` and `year.quarters[].values` defaultLabel: string; // Probably not used by us diff --git a/frontend/src/js/entity-history/EntityCard.tsx b/frontend/src/js/entity-history/EntityCard.tsx new file mode 100644 index 0000000000..9589a9a854 --- /dev/null +++ b/frontend/src/js/entity-history/EntityCard.tsx @@ -0,0 +1,61 @@ +import styled from "@emotion/styled"; + +import { EntityInfo, TimeStratifiedInfo } from "../api/types"; + +import EntityInfos from "./EntityInfos"; + +const Container = styled("div")` + display: grid; + grid-template-columns: 2fr 1fr; + gap: 30px; + padding: 20px; + background-color: ${({ theme }) => theme.col.bg}; + border-radius: ${({ theme }) => theme.borderRadius}; + border: 1px solid ${({ theme }) => theme.col.grayLight}; +`; + +const Centered = styled("div")` + display: flex; + align-items: center; +`; + +const Grid = styled("div")` + display: flex; + gap: 10px; + flex-direction: column; +`; +const Label = styled("div")` + font-size: ${({ theme }) => theme.font.xs}; +`; +const Value = styled("div")` + font-size: ${({ theme }) => theme.font.sm}; + font-weight: 400; +`; + +export const EntityCard = ({ + className, + infos, + timeStratifiedInfos, +}: { + className?: string; + infos: EntityInfo[]; + timeStratifiedInfos: TimeStratifiedInfo[]; +}) => { + return ( + + + + + {timeStratifiedInfos.map((timeStratifiedInfo) => ( + + {Object.entries(timeStratifiedInfo.totals).map(([k, v]) => ( +
+ {v} + +
+ ))} + + ))} + + ); +}; diff --git a/frontend/src/js/entity-history/EntityHeader.tsx b/frontend/src/js/entity-history/EntityHeader.tsx index 10ff327e6a..d3caa665cf 100644 --- a/frontend/src/js/entity-history/EntityHeader.tsx +++ b/frontend/src/js/entity-history/EntityHeader.tsx @@ -2,12 +2,11 @@ import styled from "@emotion/styled"; import { useTranslation } from "react-i18next"; import { useSelector } from "react-redux"; -import type { EntityInfo, SelectOptionT } from "../api/types"; +import type { SelectOptionT } from "../api/types"; import type { StateT } from "../app/reducers"; import { BadgeToggleButton } from "../button/BadgeToggleButton"; import { Heading3 } from "../headings/Headings"; -import EntityInfos from "./EntityInfos"; import { EntityId } from "./reducer"; const Root = styled("div")` @@ -52,7 +51,6 @@ interface Props { className?: string; currentEntityIndex: number; currentEntityId: EntityId; - currentEntityInfos: EntityInfo[]; status: SelectOptionT[]; setStatus: (value: SelectOptionT[]) => void; entityStatusOptions: SelectOptionT[]; @@ -62,7 +60,6 @@ export const EntityHeader = ({ className, currentEntityIndex, currentEntityId, - currentEntityInfos, status, setStatus, entityStatusOptions, @@ -96,7 +93,6 @@ export const EntityHeader = ({ {totalEvents} {t("history.events", { count: totalEvents })}
- {entityStatusOptions.map((option, i) => ( diff --git a/frontend/src/js/entity-history/EntityInfos.tsx b/frontend/src/js/entity-history/EntityInfos.tsx index 316a054259..345193b01f 100644 --- a/frontend/src/js/entity-history/EntityInfos.tsx +++ b/frontend/src/js/entity-history/EntityInfos.tsx @@ -4,16 +4,16 @@ import { Fragment, memo } from "react"; import { EntityInfo } from "../api/types"; const Grid = styled("div")` - display: grid; + display: inline-grid; grid-template-columns: 1fr auto; - gap: 0 10px; + gap: 0 16px; `; const Label = styled("div")` font-size: ${({ theme }) => theme.font.xs}; - font-weight: 400; `; const Value = styled("div")` - font-size: ${({ theme }) => theme.font.xs}; + font-size: ${({ theme }) => theme.font.sm}; + font-weight: 400; `; const EntityInfos = ({ infos }: { infos: EntityInfo[] }) => { diff --git a/frontend/src/js/entity-history/History.tsx b/frontend/src/js/entity-history/History.tsx index 48272b2c6d..983b753deb 100644 --- a/frontend/src/js/entity-history/History.tsx +++ b/frontend/src/js/entity-history/History.tsx @@ -11,6 +11,7 @@ import type { HistorySources, ResultUrlWithLabel, SelectOptionT, + TimeStratifiedInfo, } from "../api/types"; import type { StateT } from "../app/reducers"; import ErrorFallback from "../error-fallback/ErrorFallback"; @@ -111,6 +112,10 @@ export const History = () => { const currentEntityInfos = useSelector( (state) => state.entityHistory.currentEntityInfos, ); + const currentEntityTimeStratifiedInfos = useSelector< + StateT, + TimeStratifiedInfo[] + >((state) => state.entityHistory.currentEntityTimeStratifiedInfos); const resultUrls = useSelector( (state) => state.entityHistory.resultUrls, ); @@ -203,7 +208,6 @@ export const History = () => { { detailLevel={detailLevel} sources={sourcesSet} contentFilter={contentFilter} + currentEntityInfos={currentEntityInfos} + currentEntityTimeStratifiedInfos={ + currentEntityTimeStratifiedInfos + } getIsOpen={getIsOpen} toggleOpenYear={toggleOpenYear} toggleOpenQuarter={toggleOpenQuarter} diff --git a/frontend/src/js/entity-history/Timeline.tsx b/frontend/src/js/entity-history/Timeline.tsx index cb1a904d14..9d45dccee9 100644 --- a/frontend/src/js/entity-history/Timeline.tsx +++ b/frontend/src/js/entity-history/Timeline.tsx @@ -7,12 +7,15 @@ import { ColumnDescriptionSemanticConceptColumn, ConceptIdT, CurrencyConfigT, + EntityInfo, + TimeStratifiedInfo, } from "../api/types"; import type { StateT } from "../app/reducers"; import { useDatasetId } from "../dataset/selectors"; import { ContentFilterValue } from "./ContentControl"; import type { DetailLevel } from "./DetailControl"; +import { EntityCard } from "./EntityCard"; import type { EntityHistoryStateT, EntityEvent } from "./reducer"; import { TimelineEmptyPlaceholder } from "./timeline/TimelineEmptyPlaceholder"; import Year from "./timeline/Year"; @@ -36,8 +39,14 @@ const Root = styled("div")` width: 100%; `; +const SxEntityCard = styled(EntityCard)` + grid-column: span 2; +`; + interface Props { className?: string; + currentEntityInfos: EntityInfo[]; + currentEntityTimeStratifiedInfos: TimeStratifiedInfo[]; detailLevel: DetailLevel; sources: Set; contentFilter: ContentFilterValue; @@ -48,6 +57,8 @@ interface Props { const Timeline = ({ className, + currentEntityInfos, + currentEntityTimeStratifiedInfos, detailLevel, sources, contentFilter, @@ -79,6 +90,10 @@ const Timeline = ({ return ( + {eventsByQuarterWithGroups.map(({ year, quarterwiseData }) => ( Date: Wed, 12 Apr 2023 09:34:31 +0200 Subject: [PATCH 216/679] Implementation of emptyLabel as part Searchable. --- .../conquery/apiv1/FilterTemplate.java | 14 +- .../conquery/models/config/IndexConfig.java | 4 + .../conquery/models/datasets/Column.java | 8 +- .../models/datasets/concepts/Searchable.java | 4 +- .../filters/specific/SelectFilter.java | 18 +-- .../models/index/AbstractIndexKey.java | 1 - .../models/index/FrontendValueIndex.java | 3 +- .../models/index/FrontendValueIndexKey.java | 6 +- .../conquery/models/index/IndexKey.java | 2 +- .../conquery/models/index/IndexService.java | 26 ++-- .../conquery/models/index/MapIndexKey.java | 2 +- .../models/index/MapInternToExternMapper.java | 2 + .../models/jobs/UpdateFilterSearchJob.java | 9 +- .../resources/api/ConceptsProcessor.java | 8 +- .../conquery/util/search/TrieSearch.java | 131 +++++++++--------- .../tests/FilterAutocompleteTest.java | 3 +- .../tests/FilterResolutionTest.java | 2 +- .../models/index/IndexServiceTest.java | 43 +++--- 18 files changed, 134 insertions(+), 152 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java index 436c98b81d..03cda498ba 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/FilterTemplate.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.apiv1; import java.net.URI; -import java.util.List; import javax.validation.constraints.NotEmpty; import javax.validation.constraints.NotNull; @@ -74,8 +73,6 @@ public class FilterTemplate extends IdentifiableImpl implements S @NotEmpty private final String optionValue; - private final String emptyLabel; - private int minSuffixLength = 3; private boolean generateSuffixes = true; @@ -93,24 +90,21 @@ public boolean isSearchDisabled() { return false; } - public List> getSearches(IndexConfig config, NamespaceStorage storage) { + public TrieSearch createTrieSearch(IndexConfig config, NamespaceStorage storage) { final URI resolvedURI = FileUtil.getResolvedUri(config.getBaseUrl(), getFilePath()); log.trace("Resolved filter template reference url for search '{}': {}", this.getId(), resolvedURI); - FrontendValueIndex search = indexService.getIndex(new FrontendValueIndexKey( + final FrontendValueIndex search = indexService.getIndex(new FrontendValueIndexKey( resolvedURI, columnValue, value, optionValue, isGenerateSuffixes() ? getMinSuffixLength() : Integer.MAX_VALUE, - config.getSearchSplitChars(), - emptyLabel + config.getSearchSplitChars() )); - search.addItem(new FrontendValue("", getEmptyLabel()), List.of(getEmptyLabel())); - - return List.of(search); + return search; } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/IndexConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/IndexConfig.java index 9fc7a17102..c027587ff4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/IndexConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/IndexConfig.java @@ -5,6 +5,7 @@ import javax.annotation.Nullable; import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; import com.bakdata.conquery.models.index.IndexKey; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -30,6 +31,9 @@ public class IndexConfig { @Nullable private String searchSplitChars = "(),;.:\"'/"; + @NotNull + private String emptyLabel = "No Value"; + @JsonIgnore @ValidationMethod(message = "Specified baseUrl is not valid") public boolean isValidUrl() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java index 215645bd68..bea6fd2583 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java @@ -54,7 +54,6 @@ public class Column extends Labeled implements NamespacedIdentifiable< private int minSuffixLength = 3; private boolean generateSuffixes; private boolean searchDisabled = false; - private String emptyLabel; @JsonIgnore @Getter(lazy = true) @@ -152,14 +151,12 @@ private String computeDefaultDictionaryName(String importName) { @Override - public List> getSearches(IndexConfig config, NamespaceStorage storage) { + public TrieSearch createTrieSearch(IndexConfig config, NamespaceStorage storage) { final int suffixLength = isGenerateSuffixes() ? config.getSearchSuffixLength() : Integer.MAX_VALUE; final TrieSearch search = new TrieSearch<>(suffixLength, config.getSearchSplitChars()); - search.addItem(new FrontendValue("", getEmptyLabel()), List.of(getEmptyLabel())); - storage.getAllImports().stream() .filter(imp -> imp.getTable().equals(getTable())) .flatMap(imp -> { @@ -172,9 +169,8 @@ public List> getSearches(IndexConfig config, Namespace .forEach(feValue -> search.addItem(feValue, FilterSearch.extractKeywords(feValue))); - search.shrinkToFit(); - return List.of(search); + return search; } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java index 84db1fcee7..ee8c13d5ab 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Searchable.java @@ -26,7 +26,7 @@ public interface Searchable>> /** * All available {@link FrontendValue}s for searching in a {@link TrieSearch}. */ - List> getSearches(IndexConfig config, NamespaceStorage storage); + TrieSearch createTrieSearch(IndexConfig config, NamespaceStorage storage); /** * The actual Searchables to use, if there is potential for deduplication/pooling. @@ -39,8 +39,6 @@ default List> getSearchReferences() { return List.of(this); } - String getEmptyLabel(); - /** * Parameter used in the construction of {@link com.bakdata.conquery.util.search.TrieSearch}, defining the shortest suffix to create. * Ignored if isGenerateSuffixes is true. diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java index 9c15afd489..5a92cad3d6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java @@ -50,8 +50,6 @@ public abstract class SelectFilter extends SingleColumnFilter private int searchMinSuffixLength = 3; private boolean generateSearchSuffixes = true; - private String emptyLabel; - @Override public EnumSet getAcceptedColumnTypes() { return EnumSet.of(MajorTypeId.STRING); @@ -66,7 +64,6 @@ public void configureFrontend(FrontendFilterConfiguration.Top f) throws ConceptC f.setCreatable(getSearchReferences().stream().noneMatch(Predicate.not(Searchable::isSearchDisabled))); f.setOptions(collectLabels()); - //TODO FK add empty label } @JsonIgnore @@ -91,7 +88,8 @@ public List> getSearchReferences() { @NotNull protected List collectLabels() { - return labels.entrySet().stream().map(entry -> new FrontendValue(entry.getKey(), entry.getValue())).collect(Collectors.toList()); + return labels.entrySet().stream() + .map(entry -> new FrontendValue(entry.getKey(), entry.getValue())).collect(Collectors.toList()); } @JsonIgnore @@ -127,19 +125,13 @@ public boolean isSearchDisabled() { } @Override - public List> getSearches(IndexConfig config, NamespaceStorage storage) { + public TrieSearch createTrieSearch(IndexConfig config, NamespaceStorage storage) { final TrieSearch search = new TrieSearch<>(config.getSearchSuffixLength(), config.getSearchSplitChars()); - search.addItem(new FrontendValue("", getEmptyLabel()), List.of(getEmptyLabel())); - - labels.entrySet() - .stream() - .map(entry -> new FrontendValue(entry.getKey(), entry.getValue())) - .forEach(feValue -> search.addItem(feValue, FilterSearch.extractKeywords(feValue))); + collectLabels().forEach(feValue -> search.addItem(feValue, FilterSearch.extractKeywords(feValue))); - search.shrinkToFit(); - return List.of(search); + return search; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/AbstractIndexKey.java b/backend/src/main/java/com/bakdata/conquery/models/index/AbstractIndexKey.java index dff8a9996f..df6b9cee45 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/AbstractIndexKey.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/AbstractIndexKey.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.models.index; import java.net.URI; -import java.net.URL; import lombok.Data; diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java index 8411436581..2951859937 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndex.java @@ -25,7 +25,7 @@ public class FrontendValueIndex extends TrieSearch implements Ind */ private final String optionValueTemplate; - public FrontendValueIndex(int suffixCutoff, String split, String valueTemplate, String optionValueTemplate, String emptyLabel) { + public FrontendValueIndex(int suffixCutoff, String split, String valueTemplate, String optionValueTemplate) { super(suffixCutoff, split); this.valueTemplate = valueTemplate; this.optionValueTemplate = optionValueTemplate; @@ -54,6 +54,5 @@ public int size() { @Override public void finalizer() { - shrinkToFit(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java index b4d91a293f..6cc020c4d9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/FrontendValueIndexKey.java @@ -28,16 +28,14 @@ public class FrontendValueIndexKey extends AbstractIndexKey */ private final String optionValueTemplate; - private final String emptyLabel; - public FrontendValueIndexKey(URI csv, String internalColumn, String valueTemplate, String optionValueTemplate, int suffixCutoff, String splitPattern, String emptyLabel) { + public FrontendValueIndexKey(URI csv, String internalColumn, String valueTemplate, String optionValueTemplate, int suffixCutoff, String splitPattern) { super(csv, internalColumn); this.suffixCutoff = suffixCutoff; this.splitPattern = splitPattern; this.valueTemplate = valueTemplate; this.optionValueTemplate = optionValueTemplate; - this.emptyLabel = emptyLabel; } @Override @@ -47,6 +45,6 @@ public List getExternalTemplates() { @Override public FrontendValueIndex createIndex() { - return new FrontendValueIndex(suffixCutoff, splitPattern, valueTemplate, optionValueTemplate, emptyLabel); + return new FrontendValueIndex(suffixCutoff, splitPattern, valueTemplate, optionValueTemplate); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/IndexKey.java b/backend/src/main/java/com/bakdata/conquery/models/index/IndexKey.java index fda0bedaf1..9c4b8bffd7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/IndexKey.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/IndexKey.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.models.index; import java.net.URI; -import java.net.URL; import java.util.List; /** @@ -26,4 +25,5 @@ public interface IndexKey>> { List getExternalTemplates(); I createIndex(); + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java b/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java index a8c3e9bd54..bff5f1498a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/IndexService.java @@ -34,9 +34,10 @@ public class IndexService implements Injectable { private final CsvParserSettings csvParserSettings; + private final LoadingCache, Index> mappings = CacheBuilder.newBuilder().build(new CacheLoader<>() { @Override - public Index load(@NotNull IndexKey key) throws Exception { + public Index load(@NotNull IndexKey key) throws Exception { log.info("Started to parse mapping {}", key); final Map emptyDefaults = computeEmptyDefaults(key); @@ -70,12 +71,9 @@ public Index load(@NotNull IndexKey key) throws Exception { int2ext.put(internalValue, externalValue); } catch (IllegalArgumentException e) { - log.warn( - "Skipping mapping '{}'->'{}' in row {}, because there was already a mapping", - internalValue, - externalValue, - csvParser.getContext().currentLine(), - (Exception) (log.isTraceEnabled() ? e : null) + log.warn("Skipping mapping '{}'->'{}' in row {}, because there was already a mapping", + internalValue, externalValue, csvParser.getContext().currentLine(), + (Exception) (log.isTraceEnabled() ? e : null) ); } } @@ -106,10 +104,10 @@ private Pair> computeInternalExternal(@NotNull Index final String internalValue = row.getString(key.getInternalColumn()); if (internalValue == null) { - log.trace( - "Could not create a mapping for row {} because the cell for the internal value was empty. Row: {}", - csvParser.getContext().currentLine(), - log.isTraceEnabled() ? StringUtils.join(row.toFieldMap()) : null + log.trace("Could not create a mapping for row {} because the cell for the internal value was empty. Row: {}", csvParser.getContext().currentLine(), + log.isTraceEnabled() + ? StringUtils.join(row.toFieldMap()) + : null ); return null; } @@ -127,11 +125,7 @@ private Map computeTemplates(StringSubstitutor substitutor, List return externalTemplates.stream() .distinct() - .collect(Collectors.toMap( - Functions.identity(), - value -> whitespaceMatcher.trimAndCollapseFrom(substitutor.replace(value), ' ') - ) - ); + .collect(Collectors.toMap(Functions.identity(), value -> whitespaceMatcher.trimAndCollapseFrom(substitutor.replace(value), ' '))); } private Map computeEmptyDefaults(IndexKey key) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/MapIndexKey.java b/backend/src/main/java/com/bakdata/conquery/models/index/MapIndexKey.java index cdaaad9619..2fb207bec6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/MapIndexKey.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/MapIndexKey.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.models.index; import java.net.URI; -import java.net.URL; import java.util.List; import lombok.EqualsAndHashCode; @@ -13,6 +12,7 @@ public class MapIndexKey extends AbstractIndexKey { private final String externalTemplate; + public MapIndexKey(URI csv, String internalColumn, String externalTemplate) { super(csv, internalColumn); this.externalTemplate = externalTemplate; diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java b/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java index 0ee8181697..74087d0f3e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java @@ -61,6 +61,8 @@ public class MapInternToExternMapper extends NamedImpl i @NotEmpty private final String externalTemplate; + private final String emptyLabel; + //Manager only @JsonIgnore diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java index 2b315e92bc..438667822d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java @@ -85,12 +85,15 @@ public void execute() throws Exception { log.info("BEGIN collecting entries for `{}`", searchable); try { - final List> values = searchable.getSearches(indexConfig, storage); + final TrieSearch search = searchable.createTrieSearch(indexConfig, storage); - for (TrieSearch search : values) { - synchronizedResult.put(searchable, search); + if(search.findExact(List.of(""), 1).isEmpty()){ + search.addItem(new FrontendValue("", indexConfig.getEmptyLabel()), List.of(indexConfig.getEmptyLabel())); } + search.shrinkToFit(); + synchronizedResult.put(searchable, search); + log.debug( "DONE collecting entries for `{}`, within {}", searchable, diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java index 6921468d3d..d316295a46 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java @@ -27,6 +27,7 @@ import com.bakdata.conquery.models.auth.entities.Subject; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.PreviewConfig; import com.bakdata.conquery.models.datasets.concepts.Concept; @@ -64,6 +65,8 @@ public class ConceptsProcessor { private final DatasetRegistry namespaces; private final Validator validator; + private final ConqueryConfig config; + private final LoadingCache, FrontendList> nodeCache = CacheBuilder.newBuilder().softValues().expireAfterWrite(10, TimeUnit.MINUTES).build(new CacheLoader<>() { @Override @@ -218,7 +221,7 @@ public AutoCompleteResult autocompleteTextFilter(Searchable searchable, Optio return new AutoCompleteResult(fullResult.subList(startIncl, Math.min(fullResult.size(), endExcl)), fullResult.size()); } catch (ExecutionException e) { - log.warn("Failed to search for \"{}\".", maybeText, log.isTraceEnabled() ? e : null); + log.warn("Failed to search for \"{}\".", maybeText, (Exception) (log.isTraceEnabled() ? e : null)); return new AutoCompleteResult(Collections.emptyList(), 0); } } @@ -233,11 +236,10 @@ private Cursor listAllValues(Searchable searchable) { See: https://stackoverflow.com/questions/61114380/java-streams-buffering-huge-streams */ - final Iterator iterators = Iterators.concat( // We are always leading with the empty value. - Iterators.singletonIterator(new FrontendValue("", searchable.getEmptyLabel())), + Iterators.singletonIterator(new FrontendValue("", config.getIndex().getEmptyLabel())), Iterators.concat(Iterators.transform(namespace.getFilterSearch() .getSearchesFor(searchable) .iterator(), TrieSearch::iterator)) diff --git a/backend/src/main/java/com/bakdata/conquery/util/search/TrieSearch.java b/backend/src/main/java/com/bakdata/conquery/util/search/TrieSearch.java index 2877975714..d5c8622836 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/search/TrieSearch.java +++ b/backend/src/main/java/com/bakdata/conquery/util/search/TrieSearch.java @@ -54,7 +54,10 @@ public class TrieSearch> { private final int suffixCutoff; private final Pattern splitPattern; - + /** + * Maps from keywords to associated items. + */ + private final PatriciaTrie> trie = new PatriciaTrie<>(); private boolean shrunk = false; private long size = -1; @@ -67,18 +70,41 @@ public TrieSearch(int suffixCutoff, String split) { splitPattern = Pattern.compile(String.format("[\\s%s]+", Pattern.quote(Objects.requireNonNullElse(split, "") + WHOLE_WORD_MARKER))); } - /** - * Maps from keywords to associated items. - */ - private final PatriciaTrie> trie = new PatriciaTrie<>(); + public List findItems(Collection keywords, int limit) { + final Object2DoubleMap itemWeights = new Object2DoubleAVLTreeMap<>(); - Stream suffixes(String word) { - return Stream.concat( - // We append a special character here marking original words as we want to favor them in weighing. - Stream.of(word + WHOLE_WORD_MARKER), - IntStream.range(1, Math.max(1, word.length() - suffixCutoff)) - .mapToObj(word::substring) - ); + // We are not guaranteed to have split keywords incoming, so we normalize them for searching + keywords = keywords.stream().flatMap(this::split).collect(Collectors.toSet()); + + for (String keyword : keywords) { + // Query trie for all items associated with extensions of keywords + final SortedMap> hits = trie.prefixMap(keyword); + + for (Map.Entry> entry : hits.entrySet()) { + + // calculate and update weights for all queried items + final String itemWord = entry.getKey(); + + final double weight = weightWord(keyword, itemWord); + + entry.getValue() + .forEach(item -> + { + // We combine hits multiplicative to favor items with multiple hits + final double currentWeight = itemWeights.getOrDefault(item, 1); + itemWeights.put(item, currentWeight * weight); + }); + } + } + + // Sort items according to their weight, then limit. + // Note that sorting is in ascending order, meaning lower-scores are better. + return itemWeights.object2DoubleEntrySet() + .stream() + .sorted(Comparator.comparingDouble(Object2DoubleMap.Entry::getDoubleValue)) + .limit(limit) + .map(Map.Entry::getKey) + .collect(Collectors.toList()); } private Stream split(String keyword) { @@ -129,43 +155,6 @@ private boolean isOriginal(String itemWord) { return itemWord.endsWith(WHOLE_WORD_MARKER); } - public List findItems(Collection keywords, int limit) { - final Object2DoubleMap itemWeights = new Object2DoubleAVLTreeMap<>(); - - // We are not guaranteed to have split keywords incoming, so we normalize them for searching - keywords = keywords.stream().flatMap(this::split).collect(Collectors.toSet()); - - for (String keyword : keywords) { - // Query trie for all items associated with extensions of keywords - final SortedMap> hits = trie.prefixMap(keyword); - - for (Map.Entry> entry : hits.entrySet()) { - - // calculate and update weights for all queried items - final String itemWord = entry.getKey(); - - final double weight = weightWord(keyword, itemWord); - - entry.getValue() - .forEach(item -> - { - // We combine hits multiplicative to favor items with multiple hits - final double currentWeight = itemWeights.getOrDefault(item, 1); - itemWeights.put(item, currentWeight * weight); - }); - } - } - - // Sort items according to their weight, then limit. - // Note that sorting is in ascending order, meaning lower-scores are better. - return itemWeights.object2DoubleEntrySet() - .stream() - .sorted(Comparator.comparingDouble(Object2DoubleMap.Entry::getDoubleValue)) - .limit(limit) - .map(Map.Entry::getKey) - .collect(Collectors.toList()); - } - public List findExact(Collection keywords, int limit) { return keywords.stream() .flatMap(this::split) @@ -180,6 +169,24 @@ private Stream doGet(String kw) { return trie.getOrDefault(kw, Collections.emptyList()).stream(); } + public void addItem(T item, List keywords) { + // Associate item with all extracted keywords + keywords.stream() + .filter(Predicate.not(Strings::isNullOrEmpty)) + .flatMap(this::split) + .flatMap(this::suffixes) + .distinct() + .forEach(kw -> doPut(kw, item)); + } + + Stream suffixes(String word) { + return Stream.concat( + // We append a special character here marking original words as we want to favor them in weighing. + Stream.of(word + WHOLE_WORD_MARKER), + IntStream.range(1, Math.max(1, word.length() - suffixCutoff)) + .mapToObj(word::substring) + ); + } private void doPut(String kw, T item) { ensureWriteable(); @@ -195,16 +202,6 @@ private void ensureWriteable() { throw new IllegalStateException("Cannot alter a shrunk search."); } - public void addItem(T item, List keywords) { - // Associate item with all extracted keywords - keywords.stream() - .filter(Predicate.not(Strings::isNullOrEmpty)) - .flatMap(this::split) - .flatMap(this::suffixes) - .distinct() - .forEach(kw -> doPut(kw, item)); - } - public Collection listItems() { //TODO this a pretty dangerous operation, I'd rather see a session based iterator instead return trie.values().stream() @@ -214,14 +211,6 @@ public Collection listItems() { .collect(Collectors.toList()); } - public long calculateSize() { - if (size != -1) { - return size; - } - - return trie.values().stream().distinct().count(); - } - /** * Since growth of ArrayList might be excessive, we can shrink the internal lists to only required size instead. * @@ -238,6 +227,14 @@ public void shrinkToFit() { shrunk = true; } + public long calculateSize() { + if (size != -1) { + return size; + } + + return trie.values().stream().distinct().count(); + } + public void logStats() { final IntSummaryStatistics statistics = trie.values() diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java index a379e4b375..efd5af334d 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java @@ -120,7 +120,8 @@ public void execute(StandaloneSupport conquery) throws Exception { ), MediaType.APPLICATION_JSON_TYPE)); final ConceptsProcessor.AutoCompleteResult resolvedFromCsv = fromCsvResponse.readEntity(ConceptsProcessor.AutoCompleteResult.class); - assertThat(resolvedFromCsv.values().stream().map(FrontendValue::getValue)).containsExactly("a", "aaa", "aab", "baaa"); + assertThat(resolvedFromCsv.values().stream().map(FrontendValue::getValue)) + .containsExactly("a", "aaa", "aab", "baaa", "" /* `No V*a*lue` :^) */); } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterResolutionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterResolutionTest.java index a5602a7cc0..a2e9d46975 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterResolutionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterResolutionTest.java @@ -79,7 +79,7 @@ public void execute(StandaloneSupport conquery) throws Exception { final IndexService indexService = new IndexService(conquery.getConfig().getCsv().createCsvParserSettings()); - filter.setTemplate(new FilterTemplate(conquery.getDataset(), "test", tmpCSv.toUri(), "HEADER", "", "", 2, true, indexService)); + filter.setTemplate(new FilterTemplate(conquery.getDataset(), "test", tmpCSv.toUri(), "HEADER", "", "", 2, true, indexService)); final URI matchingStatsUri = HierarchyHelper.hierarchicalPath(conquery.defaultAdminURIBuilder() , AdminDatasetResource.class, "updateMatchingStats") diff --git a/backend/src/test/java/com/bakdata/conquery/models/index/IndexServiceTest.java b/backend/src/test/java/com/bakdata/conquery/models/index/IndexServiceTest.java index da2ac24c67..a1309e05e9 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/index/IndexServiceTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/index/IndexServiceTest.java @@ -29,11 +29,10 @@ @Slf4j public class IndexServiceTest { - private final IndexService indexService = new IndexService(new CsvParserSettings()); private final static Dataset DATASET = new Dataset("dataset"); private final static ConqueryConfig CONFIG = new ConqueryConfig(); - private final static ClientAndServer REF_SERVER = ClientAndServer.startClientAndServer(); + private final IndexService indexService = new IndexService(new CsvParserSettings()); @BeforeAll @SneakyThrows @@ -63,21 +62,24 @@ void testLoading() throws NoSuchFieldException, IllegalAccessException, URISynta "test1", new URI("classpath:/tests/aggregator/FIRST_MAPPED_AGGREGATOR/mapping.csv"), "internal", - "{{external}}" + "{{external}}", + "no value" ); final MapInternToExternMapper mapperUrlAbsolute = new MapInternToExternMapper( "testUrlAbsolute", new URI(String.format("http://localhost:%d/mapping.csv", REF_SERVER.getPort())), "internal", - "{{external}}" + "{{external}}", + "no value" ); final MapInternToExternMapper mapperUrlRelative = new MapInternToExternMapper( "testUrlRelative", new URI("./mapping.csv"), "internal", - "{{external}}" + "{{external}}", + "no value" ); @@ -100,6 +102,20 @@ void testLoading() throws NoSuchFieldException, IllegalAccessException, URISynta } + private static void injectComponents(MapInternToExternMapper mapInternToExternMapper, IndexService indexService, ConqueryConfig config) + throws NoSuchFieldException, IllegalAccessException { + + final Field indexServiceField = MapInternToExternMapper.class.getDeclaredField(MapInternToExternMapper.Fields.mapIndex); + indexServiceField.setAccessible(true); + indexServiceField.set(mapInternToExternMapper, indexService); + + final Field configField = MapInternToExternMapper.class.getDeclaredField(MapInternToExternMapper.Fields.config); + configField.setAccessible(true); + configField.set(mapInternToExternMapper, config); + + mapInternToExternMapper.setDataset(DATASET); + } + @Test @Order(2) void testEvictOnMapper() @@ -109,7 +125,8 @@ void testEvictOnMapper() "test1", new URI("classpath:/tests/aggregator/FIRST_MAPPED_AGGREGATOR/mapping.csv"), "internal", - "{{external}}" + "{{external}}", + "no value" ); injectComponents(mapInternToExternMapper, indexService, CONFIG); @@ -135,18 +152,4 @@ void testEvictOnMapper() .isNotSameAs(mappingAfterEvict); } - private static void injectComponents(MapInternToExternMapper mapInternToExternMapper, IndexService indexService, ConqueryConfig config) - throws NoSuchFieldException, IllegalAccessException { - - final Field indexServiceField = MapInternToExternMapper.class.getDeclaredField(MapInternToExternMapper.Fields.mapIndex); - indexServiceField.setAccessible(true); - indexServiceField.set(mapInternToExternMapper, indexService); - - final Field configField = MapInternToExternMapper.class.getDeclaredField(MapInternToExternMapper.Fields.config); - configField.setAccessible(true); - configField.set(mapInternToExternMapper, config); - - mapInternToExternMapper.setDataset(DATASET); - } - } From 19b823ec6c12208dbc1e3e37d02969345a8d66d3 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 12 Apr 2023 09:43:16 +0200 Subject: [PATCH 217/679] adds test for listing all values --- .../tests/FilterAutocompleteTest.java | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java index efd5af334d..2882416181 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/FilterAutocompleteTest.java @@ -50,26 +50,25 @@ public class FilterAutocompleteTest extends IntegrationTest.Simple implements Pr @Override public void execute(StandaloneSupport conquery) throws Exception { //read test specification - String - testJson = + final String testJson = In.resource("/tests/query/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY/MULTI_SELECT_DATE_RESTRICTION_OR_CONCEPT_QUERY.test.json") .withUTF8() .readAll(); - DatasetId dataset = conquery.getDataset().getId(); + final DatasetId dataset = conquery.getDataset().getId(); - ConqueryTestSpec test = JsonIntegrationTest.readJson(dataset, testJson); + final ConqueryTestSpec test = JsonIntegrationTest.readJson(dataset, testJson); ValidatorHelper.failOnError(log, conquery.getValidator().validate(test)); test.importRequiredData(conquery); - CSVConfig csvConf = conquery.getConfig().getCsv(); + final CSVConfig csvConf = conquery.getConfig().getCsv(); conquery.waitUntilWorkDone(); - Concept concept = conquery.getNamespace().getStorage().getAllConcepts().iterator().next(); - Connector connector = concept.getConnectors().iterator().next(); - SelectFilter filter = (SelectFilter) connector.getFilters().iterator().next(); + final Concept concept = conquery.getNamespace().getStorage().getAllConcepts().iterator().next(); + final Connector connector = concept.getConnectors().iterator().next(); + final SelectFilter filter = (SelectFilter) connector.getFilters().iterator().next(); // Copy search csv from resources to tmp folder. final Path tmpCSv = Files.createTempFile("conquery_search", "csv"); @@ -141,5 +140,21 @@ public void execute(StandaloneSupport conquery) throws Exception { assertThat(resolvedFromValues.values().stream().map(FrontendValue::getValue)) .containsExactly("f", "fm"); } + + + // Data starting with a is in reference csv + { + final Response fromCsvResponse = conquery.getClient().target(autocompleteUri) + .request(MediaType.APPLICATION_JSON_TYPE) + .post(Entity.entity(new FilterResource.AutocompleteRequest( + Optional.of(""), + OptionalInt.empty(), + OptionalInt.empty() + ), MediaType.APPLICATION_JSON_TYPE)); + + final ConceptsProcessor.AutoCompleteResult resolvedFromCsv = fromCsvResponse.readEntity(ConceptsProcessor.AutoCompleteResult.class); + assertThat(resolvedFromCsv.values().stream().map(FrontendValue::getValue)) + .containsExactly("", "aaa", "a", "baaa", "aab", "b", "f", "fm", "m", "mf"); + } } } \ No newline at end of file From 89e1be64e29f3632af2f44fd8f8315336baae7f4 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 12 Apr 2023 10:02:21 +0200 Subject: [PATCH 218/679] use isEmpty not isBlank for empty test --- .../models/query/filter/event/MultiSelectFilterNode.java | 2 +- .../conquery/models/query/filter/event/SelectFilterNode.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/MultiSelectFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/MultiSelectFilterNode.java index 445e528a1e..42e2601419 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/MultiSelectFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/MultiSelectFilterNode.java @@ -42,7 +42,7 @@ public MultiSelectFilterNode(Column column, String[] filterValue) { super(filterValue); this.column = column; selectedValuesCache = new ConcurrentHashMap<>(); - empty = Arrays.stream(filterValue).anyMatch(Strings::isBlank); + empty = Arrays.stream(filterValue).anyMatch(Strings::isEmpty); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/SelectFilterNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/SelectFilterNode.java index 17ea23a913..2af64b1942 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/SelectFilterNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/filter/event/SelectFilterNode.java @@ -32,7 +32,7 @@ public SelectFilterNode(Column column, String filterValue) { super(filterValue); this.column = column; - empty = Strings.isBlank(filterValue); + empty = Strings.isEmpty(filterValue); } @Override From 4566cd583c857cf93e28fe472b879f2ac9bebf49 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 12 Apr 2023 16:14:40 +0200 Subject: [PATCH 219/679] remove unused emptyLabel field --- .../bakdata/conquery/models/index/MapInternToExternMapper.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java b/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java index 74087d0f3e..0ee8181697 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java +++ b/backend/src/main/java/com/bakdata/conquery/models/index/MapInternToExternMapper.java @@ -61,8 +61,6 @@ public class MapInternToExternMapper extends NamedImpl i @NotEmpty private final String externalTemplate; - private final String emptyLabel; - //Manager only @JsonIgnore From f6c1484b0c47af032c5e63226cd01fea26fd7594 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 12 Apr 2023 16:17:25 +0200 Subject: [PATCH 220/679] remove unused parameter --- .../models/index/IndexServiceTest.java | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/backend/src/test/java/com/bakdata/conquery/models/index/IndexServiceTest.java b/backend/src/test/java/com/bakdata/conquery/models/index/IndexServiceTest.java index a1309e05e9..de70f28240 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/index/IndexServiceTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/index/IndexServiceTest.java @@ -25,13 +25,13 @@ import org.mockserver.model.HttpResponse; import org.mockserver.model.MediaType; -@TestMethodOrder(value = MethodOrderer.OrderAnnotation.class) +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) @Slf4j public class IndexServiceTest { - private final static Dataset DATASET = new Dataset("dataset"); - private final static ConqueryConfig CONFIG = new ConqueryConfig(); - private final static ClientAndServer REF_SERVER = ClientAndServer.startClientAndServer(); + private static final Dataset DATASET = new Dataset("dataset"); + private static final ConqueryConfig CONFIG = new ConqueryConfig(); + private static final ClientAndServer REF_SERVER = ClientAndServer.startClientAndServer(); private final IndexService indexService = new IndexService(new CsvParserSettings()); @BeforeAll @@ -62,24 +62,21 @@ void testLoading() throws NoSuchFieldException, IllegalAccessException, URISynta "test1", new URI("classpath:/tests/aggregator/FIRST_MAPPED_AGGREGATOR/mapping.csv"), "internal", - "{{external}}", - "no value" + "{{external}}" ); final MapInternToExternMapper mapperUrlAbsolute = new MapInternToExternMapper( "testUrlAbsolute", new URI(String.format("http://localhost:%d/mapping.csv", REF_SERVER.getPort())), "internal", - "{{external}}", - "no value" + "{{external}}" ); final MapInternToExternMapper mapperUrlRelative = new MapInternToExternMapper( "testUrlRelative", new URI("./mapping.csv"), "internal", - "{{external}}", - "no value" + "{{external}}" ); @@ -125,8 +122,7 @@ void testEvictOnMapper() "test1", new URI("classpath:/tests/aggregator/FIRST_MAPPED_AGGREGATOR/mapping.csv"), "internal", - "{{external}}", - "no value" + "{{external}}" ); injectComponents(mapInternToExternMapper, indexService, CONFIG); @@ -136,7 +132,7 @@ void testEvictOnMapper() assertThat(mapInternToExternMapper.external("int1")).as("Internal Value").isEqualTo("hello"); - MapIndex mappingBeforeEvict = mapInternToExternMapper.getInt2ext(); + final MapIndex mappingBeforeEvict = mapInternToExternMapper.getInt2ext(); indexService.evictCache(); @@ -145,7 +141,7 @@ void testEvictOnMapper() mapInternToExternMapper.init(); - MapIndex mappingAfterEvict = mapInternToExternMapper.getInt2ext(); + final MapIndex mappingAfterEvict = mapInternToExternMapper.getInt2ext(); // Check that the mapping reinitialized assertThat(mappingBeforeEvict).as("Mapping before and after eviction") From 7a0817dceb94c6934da23d00c9292cded7d40e72 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 12 Apr 2023 17:46:51 +0200 Subject: [PATCH 221/679] fixes writing to closed search when it is deduplicated --- .../bakdata/conquery/models/jobs/UpdateFilterSearchJob.java | 4 ++-- .../java/com/bakdata/conquery/util/search/TrieSearch.java | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java index 438667822d..2ec42deefd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java @@ -87,11 +87,11 @@ public void execute() throws Exception { try { final TrieSearch search = searchable.createTrieSearch(indexConfig, storage); - if(search.findExact(List.of(""), 1).isEmpty()){ + if(search.isWriteable() && search.findExact(List.of(""), 1).isEmpty()){ search.addItem(new FrontendValue("", indexConfig.getEmptyLabel()), List.of(indexConfig.getEmptyLabel())); + search.shrinkToFit(); } - search.shrinkToFit(); synchronizedResult.put(searchable, search); log.debug( diff --git a/backend/src/main/java/com/bakdata/conquery/util/search/TrieSearch.java b/backend/src/main/java/com/bakdata/conquery/util/search/TrieSearch.java index d5c8622836..a10f3453d6 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/search/TrieSearch.java +++ b/backend/src/main/java/com/bakdata/conquery/util/search/TrieSearch.java @@ -196,7 +196,7 @@ private void doPut(String kw, T item) { } private void ensureWriteable() { - if (!shrunk) { + if (isWriteable()) { return; } throw new IllegalStateException("Cannot alter a shrunk search."); @@ -266,4 +266,8 @@ public Iterator iterator() { seen::add ); } + + public boolean isWriteable() { + return !shrunk; + } } From 13223d6b49d4b57531a5914f8ee1bf180a9d77ce Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 17 Apr 2023 10:41:18 +0200 Subject: [PATCH 222/679] removed the possibility to place the same concept twice --- frontend/src/js/query-node-editor/ConceptDropzone.tsx | 4 ++-- frontend/src/js/standard-query-editor/QueryNode.tsx | 5 ++++- frontend/src/js/standard-query-editor/queryReducer.ts | 8 +------- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/frontend/src/js/query-node-editor/ConceptDropzone.tsx b/frontend/src/js/query-node-editor/ConceptDropzone.tsx index d05e5ece5c..6669ca0585 100644 --- a/frontend/src/js/query-node-editor/ConceptDropzone.tsx +++ b/frontend/src/js/query-node-editor/ConceptDropzone.tsx @@ -30,8 +30,8 @@ const ConceptDropzone: FC = ({ node, onDropConcept }) => { const conceptId = (item as DragItemConceptTreeNode).ids[0]; return ( - (item as DragItemConceptTreeNode).tree === node.tree - // !node.ids.some((id) => id === conceptId) + (item as DragItemConceptTreeNode).tree === node.tree && + !node.ids.some((id) => id === conceptId) ); }} > diff --git a/frontend/src/js/standard-query-editor/QueryNode.tsx b/frontend/src/js/standard-query-editor/QueryNode.tsx index 86c01b1fb4..7147b0f810 100644 --- a/frontend/src/js/standard-query-editor/QueryNode.tsx +++ b/frontend/src/js/standard-query-editor/QueryNode.tsx @@ -204,7 +204,10 @@ const QueryNode = ({ ) { return false; } - return item.tree === node.tree; + const conceptId = item.ids[0]; + const itemAlreadyInNode = node.ids.includes(conceptId); + const itemHasConceptRoot = item.tree === node.tree; + return itemHasConceptRoot && !itemAlreadyInNode; }} highlightDroppable={true} > diff --git a/frontend/src/js/standard-query-editor/queryReducer.ts b/frontend/src/js/standard-query-editor/queryReducer.ts index 0adbcfa6be..45f5bf4fd2 100644 --- a/frontend/src/js/standard-query-editor/queryReducer.ts +++ b/frontend/src/js/standard-query-editor/queryReducer.ts @@ -688,14 +688,8 @@ const onAddConceptToNode = ( if (!nodeIsConceptQueryNode(node)) return state; - let ids = concept.ids; - - node.ids.forEach(nodeId => { - if(!ids.includes(nodeId)) ids.push(nodeId); - }); - return setElementProperties(state, andIdx, orIdx, { - ids: ids, + ids: [...concept.ids, ...node.ids], }); }; From ec0ce9519d18af9cee5605dbe842467ff5c7926e Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 17 Apr 2023 10:41:39 +0200 Subject: [PATCH 223/679] formatting --- .../src/js/small-tab-navigation/HoverNavigatable.tsx | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx b/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx index c6454025d5..c3d7222ed7 100644 --- a/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx +++ b/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx @@ -22,7 +22,11 @@ const Root = styled("div")<{ highlightDroppable?: boolean; }>` background-color: ${({ theme, isDroppable, highlightDroppable, isOver }) => - isOver&& isDroppable ? `${theme.col.grayLight}` : highlightDroppable && isDroppable ? `${theme.col.grayVeryLight}` : "inherit"}; + isOver && isDroppable + ? `${theme.col.grayLight}` + : highlightDroppable && isDroppable + ? `${theme.col.grayVeryLight}` + : "inherit"}; position: relative; border-radius: ${({ theme }) => theme.borderRadius}; display: inline-flex; @@ -36,7 +40,7 @@ export const HoverNavigatable = ({ children, className, canDrop, - highlightDroppable + highlightDroppable, }: PropsT) => { const [timeoutVar, setTimeoutVar] = useState(null); @@ -55,7 +59,7 @@ export const HoverNavigatable = ({ setTimeout(() => { setTimeoutVar(null); if (monitor.isOver()) { - triggerNavigate(); + triggerNavigate(); } }, TIME_UNTIL_NAVIGATE), ); From 8ca6a64d9b66df7315f1497b329aa94c3e649b6c Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 17 Apr 2023 11:32:40 +0200 Subject: [PATCH 224/679] use lambda expression instead of two ternary expressions --- .../src/js/small-tab-navigation/HoverNavigatable.tsx | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx b/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx index c3d7222ed7..7064930f41 100644 --- a/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx +++ b/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx @@ -21,12 +21,11 @@ const Root = styled("div")<{ isDroppable?: boolean; highlightDroppable?: boolean; }>` - background-color: ${({ theme, isDroppable, highlightDroppable, isOver }) => - isOver && isDroppable - ? `${theme.col.grayLight}` - : highlightDroppable && isDroppable - ? `${theme.col.grayVeryLight}` - : "inherit"}; + background-color: ${({ theme, isDroppable, highlightDroppable, isOver }) => { + if (isOver && isDroppable) return `${theme.col.grayLight}`; + if (highlightDroppable && isDroppable) return `${theme.col.grayVeryLight}`; + return "inherit"; + }}; position: relative; border-radius: ${({ theme }) => theme.borderRadius}; display: inline-flex; From ae174a3ff6887abea85eb2266159381c7f17e49c Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 17 Apr 2023 12:00:23 +0200 Subject: [PATCH 225/679] rename suffix to unit, add space in FE --- .../com/bakdata/conquery/models/config/FrontendConfig.java | 2 +- frontend/mock-api/config.json | 2 +- frontend/src/js/api/types.ts | 2 +- frontend/src/js/entity-history/timeline/EventCard.tsx | 5 ++++- frontend/src/js/startup/reducer.ts | 2 +- frontend/src/js/ui-components/CurrencyInput.tsx | 5 ++++- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index 1d11526c02..7487780d0f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -60,7 +60,7 @@ public class FrontendConfig { @Data public static class CurrencyConfig { - private String suffix = " €"; + private String unit = "€"; private String thousandSeparator = "."; private String decimalSeparator = ","; private int decimalScale = 2; diff --git a/frontend/mock-api/config.json b/frontend/mock-api/config.json index 04acc163ce..13753d49f7 100644 --- a/frontend/mock-api/config.json +++ b/frontend/mock-api/config.json @@ -3,7 +3,7 @@ "production": true, "currency": { "factor": 1, - "suffix": " €", + "unit": "€", "thousandSeparator": ".", "decimalSeparator": ",", "decimalScale": 2 diff --git a/frontend/src/js/api/types.ts b/frontend/src/js/api/types.ts index dfcd1287a7..91b0e5f237 100644 --- a/frontend/src/js/api/types.ts +++ b/frontend/src/js/api/types.ts @@ -28,7 +28,7 @@ export interface DateRangeT { } export interface CurrencyConfigT { - suffix: string; + unit: string; thousandSeparator: string; decimalSeparator: string; decimalScale: number; diff --git a/frontend/src/js/entity-history/timeline/EventCard.tsx b/frontend/src/js/entity-history/timeline/EventCard.tsx index 42bd19b759..b16cd5d1e2 100644 --- a/frontend/src/js/entity-history/timeline/EventCard.tsx +++ b/frontend/src/js/entity-history/timeline/EventCard.tsx @@ -155,7 +155,10 @@ const EventCard = ({ {column.defaultLabel} diff --git a/frontend/src/js/startup/reducer.ts b/frontend/src/js/startup/reducer.ts index 812d664438..e8cfb15a29 100644 --- a/frontend/src/js/startup/reducer.ts +++ b/frontend/src/js/startup/reducer.ts @@ -20,7 +20,7 @@ const initialState: StartupStateT = { ids: [], }, currency: { - suffix: " €", + unit: "€", thousandSeparator: ".", decimalSeparator: ",", decimalScale: 2, diff --git a/frontend/src/js/ui-components/CurrencyInput.tsx b/frontend/src/js/ui-components/CurrencyInput.tsx index 3fa51f7556..657411ef6e 100644 --- a/frontend/src/js/ui-components/CurrencyInput.tsx +++ b/frontend/src/js/ui-components/CurrencyInput.tsx @@ -50,7 +50,10 @@ const CurrencyInput: FC = ({ }, [value]); return ( Date: Mon, 17 Apr 2023 12:13:11 +0200 Subject: [PATCH 226/679] Add yearly time stratified info --- frontend/src/js/entity-history/EntityCard.tsx | 38 +++++++--- .../src/js/entity-history/EntityInfos.tsx | 1 + frontend/src/js/entity-history/Timeline.tsx | 7 +- .../src/js/entity-history/timeline/Year.tsx | 4 + .../js/entity-history/timeline/YearHead.tsx | 74 ++++++++++++++++++- .../src/js/entity-history/timeline/util.ts | 9 ++- 6 files changed, 116 insertions(+), 17 deletions(-) diff --git a/frontend/src/js/entity-history/EntityCard.tsx b/frontend/src/js/entity-history/EntityCard.tsx index 9589a9a854..824624f8cf 100644 --- a/frontend/src/js/entity-history/EntityCard.tsx +++ b/frontend/src/js/entity-history/EntityCard.tsx @@ -1,8 +1,11 @@ import styled from "@emotion/styled"; +import { useSelector } from "react-redux"; import { EntityInfo, TimeStratifiedInfo } from "../api/types"; +import { StateT } from "../app/reducers"; import EntityInfos from "./EntityInfos"; +import { getColumnType } from "./timeline/util"; const Container = styled("div")` display: grid; @@ -16,7 +19,7 @@ const Container = styled("div")` const Centered = styled("div")` display: flex; - align-items: center; + align-items: flex-start; `; const Grid = styled("div")` @@ -41,21 +44,34 @@ export const EntityCard = ({ infos: EntityInfo[]; timeStratifiedInfos: TimeStratifiedInfo[]; }) => { + const currencyPrefix = useSelector( + (state) => state.startup.config.currency.prefix, + ); + return ( - {timeStratifiedInfos.map((timeStratifiedInfo) => ( - - {Object.entries(timeStratifiedInfo.totals).map(([k, v]) => ( -
- {v} - -
- ))} - - ))} + {timeStratifiedInfos.map((timeStratifiedInfo) => { + return ( + + {Object.entries(timeStratifiedInfo.totals).map(([label, value]) => { + const columnType = getColumnType(timeStratifiedInfo, label); + + return ( +
+ + {value} + {columnType === "MONEY" ? " " + currencyPrefix : ""} + + +
+ ); + })} + + ); + })} ); }; diff --git a/frontend/src/js/entity-history/EntityInfos.tsx b/frontend/src/js/entity-history/EntityInfos.tsx index 345193b01f..28a3781850 100644 --- a/frontend/src/js/entity-history/EntityInfos.tsx +++ b/frontend/src/js/entity-history/EntityInfos.tsx @@ -7,6 +7,7 @@ const Grid = styled("div")` display: inline-grid; grid-template-columns: 1fr auto; gap: 0 16px; + place-items: center start; `; const Label = styled("div")` font-size: ${({ theme }) => theme.font.xs}; diff --git a/frontend/src/js/entity-history/Timeline.tsx b/frontend/src/js/entity-history/Timeline.tsx index 9d45dccee9..c8e9b67501 100644 --- a/frontend/src/js/entity-history/Timeline.tsx +++ b/frontend/src/js/entity-history/Timeline.tsx @@ -33,9 +33,9 @@ const Root = styled("div")` -webkit-overflow-scrolling: touch; padding: 0 20px 0 10px; display: inline-grid; - grid-template-columns: 125px auto; + grid-template-columns: 200px auto; grid-auto-rows: minmax(min-content, max-content); - gap: 20px 4px; + gap: 12px 4px; width: 100%; `; @@ -100,6 +100,7 @@ const Timeline = ({ year={year} datasetId={datasetId} quarterwiseData={quarterwiseData} + timeStratifiedInfos={currentEntityTimeStratifiedInfos} getIsOpen={getIsOpen} toggleOpenYear={toggleOpenYear} toggleOpenQuarter={toggleOpenQuarter} @@ -293,7 +294,7 @@ const useTimeBucketedSortedData = ( while (sortedEvents[0].year < currentYear) { sortedEvents.unshift({ year: sortedEvents[0].year + 1, - quarterwiseData: [1, 2, 3, 4].map((q) => ({ + quarterwiseData: [4, 3, 2, 1].map((q) => ({ quarter: q, events: [], })), diff --git a/frontend/src/js/entity-history/timeline/Year.tsx b/frontend/src/js/entity-history/timeline/Year.tsx index e37350aefa..ccc43d5250 100644 --- a/frontend/src/js/entity-history/timeline/Year.tsx +++ b/frontend/src/js/entity-history/timeline/Year.tsx @@ -6,6 +6,7 @@ import { ConceptIdT, CurrencyConfigT, DatasetT, + TimeStratifiedInfo, } from "../../api/types"; import { ContentFilterValue } from "../ContentControl"; import { DetailLevel } from "../DetailControl"; @@ -33,6 +34,7 @@ const Year = ({ columnBuckets, currencyConfig, rootConceptIdsByColumn, + timeStratifiedInfos, }: { datasetId: DatasetT["id"]; year: number; @@ -46,6 +48,7 @@ const Year = ({ currencyConfig: CurrencyConfigT; columnBuckets: ColumnBuckets; columns: Record; + timeStratifiedInfos: TimeStratifiedInfo[]; }) => { const isYearOpen = getIsOpen(year); const totalEvents = quarterwiseData.reduce( @@ -61,6 +64,7 @@ const Year = ({ year={year} totalEvents={totalEvents} onClick={() => toggleOpenYear(year)} + timeStratifiedInfos={timeStratifiedInfos} /> {quarterwiseData.map(({ quarter, groupedEvents, differences }) => { diff --git a/frontend/src/js/entity-history/timeline/YearHead.tsx b/frontend/src/js/entity-history/timeline/YearHead.tsx index 4247d5faf3..856a6c9c90 100644 --- a/frontend/src/js/entity-history/timeline/YearHead.tsx +++ b/frontend/src/js/entity-history/timeline/YearHead.tsx @@ -2,10 +2,15 @@ import styled from "@emotion/styled"; import { faCaretDown, faCaretRight } from "@fortawesome/free-solid-svg-icons"; import { memo } from "react"; import { useTranslation } from "react-i18next"; +import { useSelector } from "react-redux"; +import { TimeStratifiedInfo } from "../../api/types"; +import { StateT } from "../../app/reducers"; +import { exists } from "../../common/helpers/exists"; import FaIcon from "../../icon/FaIcon"; import { SmallHeading } from "./SmallHeading"; +import { getColumnType } from "./util"; const Root = styled("div")` font-size: ${({ theme }) => theme.font.xs}; @@ -16,10 +21,11 @@ const StickyWrap = styled("div")` position: sticky; top: 0; left: 0; - padding: 5px 0; + padding: 5px; cursor: pointer; display: grid; - grid-template-columns: 20px 1fr; + grid-template-columns: 16px 1fr; + gap: 8px 0; border-radius: ${({ theme }) => theme.borderRadius}; border: 1px solid transparent; &:hover { @@ -27,16 +33,76 @@ const StickyWrap = styled("div")` } `; +const Grid = styled("div")` + display: grid; + grid-template-columns: 1fr auto; + gap: 2px 10px; +`; + +const Value = styled("div")` + font-size: ${({ theme }) => theme.font.tiny}; + font-weight: 400; + white-space: nowrap; +`; + +const Label = styled("div")` + font-size: ${({ theme }) => theme.font.tiny}; + max-width: 100%; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +`; + +const TimeStratifiedInfos = ({ + year, + timeStratifiedInfos, +}: { + year: number; + timeStratifiedInfos: TimeStratifiedInfo[]; +}) => { + const currencyUnit = useSelector( + (state) => state.startup.config.currency.prefix, + ); + const yearInfos = timeStratifiedInfos + .map((i) => i.years.find((info) => info.year === year)) + .filter(exists); + + return ( + <> + {" "} + {yearInfos.map((info) => ( + + {Object.entries(info.values).map(([label, value]) => { + const columnType = getColumnType(timeStratifiedInfos[0], label); + + return ( + <> + + {value} + {columnType === "MONEY" ? " " + currencyUnit : ""} + + + + ); + })} + + ))} + + ); +}; + const YearHead = ({ year, totalEvents, onClick, isOpen, + timeStratifiedInfos, }: { isOpen: boolean; year: number; totalEvents: number; onClick: () => void; + timeStratifiedInfos: TimeStratifiedInfo[]; }) => { const { t } = useTranslation(); @@ -50,6 +116,10 @@ const YearHead = ({ {totalEvents} {t("history.events", { count: totalEvents })} + ); diff --git a/frontend/src/js/entity-history/timeline/util.ts b/frontend/src/js/entity-history/timeline/util.ts index 9d9bd143d3..eef88d1c25 100644 --- a/frontend/src/js/entity-history/timeline/util.ts +++ b/frontend/src/js/entity-history/timeline/util.ts @@ -1,4 +1,4 @@ -import { ColumnDescription } from "../../api/types"; +import { ColumnDescription, TimeStratifiedInfo } from "../../api/types"; export const isIdColumn = (columnDescription: ColumnDescription) => columnDescription.semantics.some((s) => s.type === "ID"); @@ -18,3 +18,10 @@ export const isMoneyColumn = (columnDescription: ColumnDescription) => export const isSecondaryIdColumn = (columnDescription: ColumnDescription) => columnDescription.semantics.some((s) => s.type === "SECONDARY_ID"); + +export const getColumnType = ( + timeStratifiedInfo: TimeStratifiedInfo, + label: string, +) => { + return timeStratifiedInfo.columns.find((c) => c.label === label)?.type; +}; From 849d59b3844662159b86210f4037e2d1f34e7365 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 17 Apr 2023 12:21:19 +0200 Subject: [PATCH 227/679] Fix dataset form fields not correctly initializing after dataset change --- frontend/src/js/external-forms/FormsTab.tsx | 13 ++++++++++--- frontend/src/js/external-forms/helper.ts | 14 +++++++++++--- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/frontend/src/js/external-forms/FormsTab.tsx b/frontend/src/js/external-forms/FormsTab.tsx index 3070c5db7b..794586107b 100644 --- a/frontend/src/js/external-forms/FormsTab.tsx +++ b/frontend/src/js/external-forms/FormsTab.tsx @@ -56,7 +56,11 @@ export const useDatasetOptions = () => { ); }; -const useInitializeForm = () => { +const useInitializeForm = ({ + datasetId, +}: { + datasetId: DatasetT["id"] | null; +}) => { const activeLang = useActiveLang(); const config = useSelector(selectFormConfig); const allFields: (Field | Tabs)[] = useMemo(() => { @@ -77,12 +81,13 @@ const useInitializeForm = () => { const initialValue = getInitialValue(field, { availableDatasets: datasetOptions, activeLang, + datasetId, }); return [field.name, initialValue]; }), ); - }, [allFields, datasetOptions, activeLang, config]); + }, [allFields, datasetOptions, activeLang, config, datasetId]); const methods = useForm({ defaultValues, @@ -104,7 +109,9 @@ const FormsTab = () => { useLoadForms({ datasetId }); - const { methods, config, datasetOptions, onReset } = useInitializeForm(); + const { methods, config, datasetOptions, onReset } = useInitializeForm({ + datasetId, + }); useEffect( function resetOnDatasetChange() { diff --git a/frontend/src/js/external-forms/helper.ts b/frontend/src/js/external-forms/helper.ts index 11d867553e..0efdfc901a 100644 --- a/frontend/src/js/external-forms/helper.ts +++ b/frontend/src/js/external-forms/helper.ts @@ -1,4 +1,4 @@ -import type { SelectOptionT } from "../api/types"; +import type { DatasetT, SelectOptionT } from "../api/types"; import type { Language } from "../localization/useActiveLang"; import type { FormField, GeneralField, Group } from "./config-types"; @@ -72,7 +72,11 @@ export function collectAllFormFields(fields: GeneralField[]): FormField[] { export function getInitialValue( field: Exclude, - context: { availableDatasets: SelectOptionT[]; activeLang: Language }, + context: { + availableDatasets: SelectOptionT[]; + activeLang: Language; + datasetId: DatasetT["id"] | null; + }, ): | string | number @@ -84,7 +88,11 @@ export function getInitialValue( switch (field.type) { case "DATASET_SELECT": if (context.availableDatasets.length > 0) { - return context.availableDatasets[0]; + return ( + context.availableDatasets.find( + (opt) => opt.value === context.datasetId, + ) || context.availableDatasets[0] + ); } else { return undefined; } From 32e7d064a1bb0f63e4ce117daa0a1dad6abf1790 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 17 Apr 2023 12:31:01 +0200 Subject: [PATCH 228/679] Fix lint --- frontend/src/js/external-forms/form/Field.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/js/external-forms/form/Field.tsx b/frontend/src/js/external-forms/form/Field.tsx index 88cbc3962c..c39a051775 100644 --- a/frontend/src/js/external-forms/form/Field.tsx +++ b/frontend/src/js/external-forms/form/Field.tsx @@ -168,7 +168,11 @@ const Field = ({ field, ...commonProps }: PropsT) => { const defaultValue = isFormField(field) && field.type !== "GROUP" - ? getInitialValue(field, { availableDatasets, activeLang: locale }) + ? getInitialValue(field, { + availableDatasets, + activeLang: locale, + datasetId, + }) : null; switch (field.type) { From abbec044a39cd43982bc252f7c4922cd3ec39cd2 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 17 Apr 2023 15:04:18 +0200 Subject: [PATCH 229/679] Fix expandable condition --- .../js/external-forms/form-concept-group/FormConceptGroup.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 72a44648e3..9e343b181c 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -311,7 +311,8 @@ const FormConceptGroup = (props: Props) => { ), ), expandable: - !props.isSingle && hasConceptChildren(concept), + !props.disallowMultipleColumns && + hasConceptChildren(concept), active: !!concept.includeSubnodes, }} /> From db69e4ffe6607aeaf35a44bf17b78d4450a83e55 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Tue, 18 Apr 2023 10:04:55 +0200 Subject: [PATCH 230/679] use spread for currency config and then override --- frontend/src/js/entity-history/timeline/EventCard.tsx | 4 +--- frontend/src/js/ui-components/CurrencyInput.tsx | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/frontend/src/js/entity-history/timeline/EventCard.tsx b/frontend/src/js/entity-history/timeline/EventCard.tsx index b16cd5d1e2..60e3b85868 100644 --- a/frontend/src/js/entity-history/timeline/EventCard.tsx +++ b/frontend/src/js/entity-history/timeline/EventCard.tsx @@ -155,9 +155,7 @@ const EventCard = ({ {column.defaultLabel} = ({ }, [value]); return ( Date: Tue, 18 Apr 2023 10:31:19 +0200 Subject: [PATCH 231/679] cleanup, update tab hovering color to lighter grey --- frontend/src/js/small-tab-navigation/HoverNavigatable.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx b/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx index 7064930f41..20aa1946e3 100644 --- a/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx +++ b/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx @@ -22,8 +22,9 @@ const Root = styled("div")<{ highlightDroppable?: boolean; }>` background-color: ${({ theme, isDroppable, highlightDroppable, isOver }) => { - if (isOver && isDroppable) return `${theme.col.grayLight}`; - if (highlightDroppable && isDroppable) return `${theme.col.grayVeryLight}`; + if (isOver && isDroppable) + return highlightDroppable ? theme.col.grayLight : theme.col.grayVeryLight; + if (highlightDroppable && isDroppable) return theme.col.grayVeryLight; return "inherit"; }}; position: relative; From bdb7f2f1ff41985ae8efd94063b44eb71221fa89 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 17 Apr 2023 16:29:13 +0200 Subject: [PATCH 232/679] Iterate yearly stratified infos --- frontend/src/js/entity-history/EntityCard.tsx | 56 ++++++++++----- .../js/entity-history/timeline/YearHead.tsx | 68 ++++++++++++------- 2 files changed, 83 insertions(+), 41 deletions(-) diff --git a/frontend/src/js/entity-history/EntityCard.tsx b/frontend/src/js/entity-history/EntityCard.tsx index 824624f8cf..b5eb54e8d0 100644 --- a/frontend/src/js/entity-history/EntityCard.tsx +++ b/frontend/src/js/entity-history/EntityCard.tsx @@ -1,15 +1,17 @@ import styled from "@emotion/styled"; +import { Fragment } from "react"; import { useSelector } from "react-redux"; import { EntityInfo, TimeStratifiedInfo } from "../api/types"; import { StateT } from "../app/reducers"; +import { exists } from "../common/helpers/exists"; import EntityInfos from "./EntityInfos"; import { getColumnType } from "./timeline/util"; const Container = styled("div")` display: grid; - grid-template-columns: 2fr 1fr; + grid-template-columns: 1.618fr 1fr; gap: 30px; padding: 20px; background-color: ${({ theme }) => theme.col.bg}; @@ -23,9 +25,9 @@ const Centered = styled("div")` `; const Grid = styled("div")` - display: flex; - gap: 10px; - flex-direction: column; + display: inline-grid; + gap: 4px 10px; + grid-template-columns: auto auto; `; const Label = styled("div")` font-size: ${({ theme }) => theme.font.xs}; @@ -35,13 +37,9 @@ const Value = styled("div")` font-weight: 400; `; -export const EntityCard = ({ - className, - infos, +const TimeStratifiedInfos = ({ timeStratifiedInfos, }: { - className?: string; - infos: EntityInfo[]; timeStratifiedInfos: TimeStratifiedInfo[]; }) => { const currencyPrefix = useSelector( @@ -49,29 +47,53 @@ export const EntityCard = ({ ); return ( - - - - +
{timeStratifiedInfos.map((timeStratifiedInfo) => { return ( - {Object.entries(timeStratifiedInfo.totals).map(([label, value]) => { - const columnType = getColumnType(timeStratifiedInfo, label); + {timeStratifiedInfo.columns.map((column) => { + const columnType = getColumnType( + timeStratifiedInfo, + column.label, + ); + + const label = column.label; + const value = timeStratifiedInfo.totals[column.label]; + + if (!exists(value)) return <>; return ( -
+ {value} {columnType === "MONEY" ? " " + currencyPrefix : ""} -
+ ); })} ); })} +
+ ); +}; + +export const EntityCard = ({ + className, + infos, + timeStratifiedInfos, +}: { + className?: string; + infos: EntityInfo[]; + timeStratifiedInfos: TimeStratifiedInfo[]; +}) => { + return ( + + + + + ); }; diff --git a/frontend/src/js/entity-history/timeline/YearHead.tsx b/frontend/src/js/entity-history/timeline/YearHead.tsx index 856a6c9c90..c5c515c0f5 100644 --- a/frontend/src/js/entity-history/timeline/YearHead.tsx +++ b/frontend/src/js/entity-history/timeline/YearHead.tsx @@ -1,12 +1,11 @@ import styled from "@emotion/styled"; import { faCaretDown, faCaretRight } from "@fortawesome/free-solid-svg-icons"; -import { memo } from "react"; +import { Fragment, memo } from "react"; import { useTranslation } from "react-i18next"; import { useSelector } from "react-redux"; import { TimeStratifiedInfo } from "../../api/types"; import { StateT } from "../../app/reducers"; -import { exists } from "../../common/helpers/exists"; import FaIcon from "../../icon/FaIcon"; import { SmallHeading } from "./SmallHeading"; @@ -35,8 +34,8 @@ const StickyWrap = styled("div")` const Grid = styled("div")` display: grid; - grid-template-columns: 1fr auto; - gap: 2px 10px; + grid-template-columns: 60px auto; + gap: 0px 10px; `; const Value = styled("div")` @@ -63,30 +62,50 @@ const TimeStratifiedInfos = ({ const currencyUnit = useSelector( (state) => state.startup.config.currency.prefix, ); - const yearInfos = timeStratifiedInfos - .map((i) => i.years.find((info) => info.year === year)) - .filter(exists); + + const infos = timeStratifiedInfos + .map((info) => { + return { + info, + yearInfo: info.years.find((i) => i.year === year), + }; + }) + .filter( + ( + i, + ): i is { + info: TimeStratifiedInfo; + yearInfo: TimeStratifiedInfo["years"][number]; + } => !!i.yearInfo?.values && Object.entries(i.yearInfo.values).length > 0, + ); return ( <> - {" "} - {yearInfos.map((info) => ( - - {Object.entries(info.values).map(([label, value]) => { - const columnType = getColumnType(timeStratifiedInfos[0], label); + {infos.map(({ info, yearInfo }) => { + return ( + + {Object.entries(yearInfo.values) + .sort( + ([l1], [l2]) => + info.columns.findIndex((c) => c.label === l1) - + info.columns.findIndex((c) => c.label === l2), + ) + .map(([label, value]) => { + const columnType = getColumnType(info, label); - return ( - <> - - {value} - {columnType === "MONEY" ? " " + currencyUnit : ""} - - - - ); - })} - - ))} + return ( + + + {value} + {columnType === "MONEY" ? " " + currencyUnit : ""} + + + + ); + })} + + ); + })} ); }; @@ -116,6 +135,7 @@ const YearHead = ({ {totalEvents} {t("history.events", { count: totalEvents })} + Date: Tue, 18 Apr 2023 11:51:00 +0200 Subject: [PATCH 233/679] unify can drop + Form editor prevent opening, when cannot drop --- .../form-concept-group/FormConceptNode.tsx | 7 ++++- .../js/query-node-editor/ConceptDropzone.tsx | 29 +++++++++++++++++-- .../small-tab-navigation/HoverNavigatable.tsx | 1 - .../js/standard-query-editor/QueryNode.tsx | 27 ++--------------- 4 files changed, 36 insertions(+), 28 deletions(-) diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx index c16363c7f7..eb74a00aad 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx @@ -9,6 +9,7 @@ import { useTranslation } from "react-i18next"; import { getWidthAndHeight } from "../../app/DndProvider"; import IconButton from "../../button/IconButton"; +import { canDropConceptTreeNodeBeDropped } from "../../query-node-editor/ConceptDropzone"; import { HoverNavigatable } from "../../small-tab-navigation/HoverNavigatable"; import { getRootNodeLabel } from "../../standard-query-editor/helper"; import type { DragItemConceptTreeNode } from "../../standard-query-editor/types"; @@ -126,7 +127,11 @@ const FormConceptNode: FC = ({ : undefined; return ( - + { ref.current = instance; diff --git a/frontend/src/js/query-node-editor/ConceptDropzone.tsx b/frontend/src/js/query-node-editor/ConceptDropzone.tsx index 6669ca0585..2eaf19db92 100644 --- a/frontend/src/js/query-node-editor/ConceptDropzone.tsx +++ b/frontend/src/js/query-node-editor/ConceptDropzone.tsx @@ -3,8 +3,12 @@ import { FC } from "react"; import { useTranslation } from "react-i18next"; import { DNDType } from "../common/constants/dndTypes"; -import type { DragItemConceptTreeNode } from "../standard-query-editor/types"; -import Dropzone from "../ui-components/Dropzone"; +import { nodeIsConceptQueryNode } from "../model/node"; +import type { + DragItemConceptTreeNode, + StandardQueryNodeT, +} from "../standard-query-editor/types"; +import Dropzone, { PossibleDroppableObject } from "../ui-components/Dropzone"; const SxDropzone = styled(Dropzone)` width: 100%; @@ -17,6 +21,27 @@ interface PropsT { onDropConcept: (concept: DragItemConceptTreeNode) => void; } +export const droppableObjectIsConceptTreeNode = ( + node: PossibleDroppableObject, +): node is DragItemConceptTreeNode => { + return node.type === DNDType.CONCEPT_TREE_NODE; +}; + +export const canDropConceptTreeNodeBeDropped = (node: StandardQueryNodeT) => { + return (item: PossibleDroppableObject) => { + if ( + !droppableObjectIsConceptTreeNode(item) || + !nodeIsConceptQueryNode(node) + ) { + return false; + } + const conceptId = item.ids[0]; + const itemAlreadyInNode = node.ids.includes(conceptId); + const itemHasConceptRoot = item.tree === node.tree; + return itemHasConceptRoot && !itemAlreadyInNode; + }; +}; + const ConceptDropzone: FC = ({ node, onDropConcept }) => { const { t } = useTranslation(); diff --git a/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx b/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx index 20aa1946e3..d4680b148d 100644 --- a/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx +++ b/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx @@ -28,7 +28,6 @@ const Root = styled("div")<{ return "inherit"; }}; position: relative; - border-radius: ${({ theme }) => theme.borderRadius}; display: inline-flex; `; diff --git a/frontend/src/js/standard-query-editor/QueryNode.tsx b/frontend/src/js/standard-query-editor/QueryNode.tsx index 7147b0f810..94542c21cb 100644 --- a/frontend/src/js/standard-query-editor/QueryNode.tsx +++ b/frontend/src/js/standard-query-editor/QueryNode.tsx @@ -7,7 +7,6 @@ import { useSelector } from "react-redux"; import type { QueryT } from "../api/types"; import { getWidthAndHeight } from "../app/DndProvider"; import type { StateT } from "../app/reducers"; -import { DNDType } from "../common/constants/dndTypes"; import { getConceptById } from "../concept-trees/globalTreeStoreHelper"; import { nodeHasNonDefaultSettings, @@ -15,14 +14,14 @@ import { nodeIsConceptQueryNode, } from "../model/node"; import { isQueryExpandable } from "../model/query"; +import { canDropConceptTreeNodeBeDropped } from "../query-node-editor/ConceptDropzone"; import { HoverNavigatable } from "../small-tab-navigation/HoverNavigatable"; import AdditionalInfoHoverable from "../tooltip/AdditionalInfoHoverable"; -import { PossibleDroppableObject } from "../ui-components/Dropzone"; import QueryNodeActions from "./QueryNodeActions"; import QueryNodeContent from "./QueryNodeContent"; import { getRootNodeLabel } from "./helper"; -import { DragItemConceptTreeNode, StandardQueryNodeT } from "./types"; +import { StandardQueryNodeT } from "./types"; const FlexHoverNavigatable = styled(HoverNavigatable)` display: flex; @@ -48,9 +47,6 @@ const Root = styled("div")<{ active ? `2px solid ${theme.col.blueGrayDark}` : `1px solid ${theme.col.grayMediumLight}`}; - &:hover { - background-color: ${({ theme }) => theme.col.bg}; - } `; interface PropsT { @@ -87,12 +83,6 @@ const nodeHasActiveSecondaryId = ( } }; -export const droppableObjectIsConceptTreeNode = ( - node: PossibleDroppableObject, -): node is DragItemConceptTreeNode => { - return node.type === DNDType.CONCEPT_TREE_NODE; -}; - const QueryNode = ({ node, andIdx, @@ -197,18 +187,7 @@ const QueryNode = ({ const QueryNodeRoot = ( { - if ( - !droppableObjectIsConceptTreeNode(item) || - !nodeIsConceptQueryNode(node) - ) { - return false; - } - const conceptId = item.ids[0]; - const itemAlreadyInNode = node.ids.includes(conceptId); - const itemHasConceptRoot = item.tree === node.tree; - return itemHasConceptRoot && !itemAlreadyInNode; - }} + canDrop={canDropConceptTreeNodeBeDropped(node)} highlightDroppable={true} > Date: Tue, 18 Apr 2023 12:06:00 +0200 Subject: [PATCH 234/679] Update to new currency unit api --- frontend/src/js/entity-history/EntityCard.tsx | 6 +++--- frontend/src/js/entity-history/timeline/YearHead.tsx | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/js/entity-history/EntityCard.tsx b/frontend/src/js/entity-history/EntityCard.tsx index b5eb54e8d0..4998cb1567 100644 --- a/frontend/src/js/entity-history/EntityCard.tsx +++ b/frontend/src/js/entity-history/EntityCard.tsx @@ -42,8 +42,8 @@ const TimeStratifiedInfos = ({ }: { timeStratifiedInfos: TimeStratifiedInfo[]; }) => { - const currencyPrefix = useSelector( - (state) => state.startup.config.currency.prefix, + const currencyUnit = useSelector( + (state) => state.startup.config.currency.unit, ); return ( @@ -66,7 +66,7 @@ const TimeStratifiedInfos = ({ {value} - {columnType === "MONEY" ? " " + currencyPrefix : ""} + {columnType === "MONEY" ? " " + currencyUnit : ""} diff --git a/frontend/src/js/entity-history/timeline/YearHead.tsx b/frontend/src/js/entity-history/timeline/YearHead.tsx index c5c515c0f5..d3dcc971b9 100644 --- a/frontend/src/js/entity-history/timeline/YearHead.tsx +++ b/frontend/src/js/entity-history/timeline/YearHead.tsx @@ -60,7 +60,7 @@ const TimeStratifiedInfos = ({ timeStratifiedInfos: TimeStratifiedInfo[]; }) => { const currencyUnit = useSelector( - (state) => state.startup.config.currency.prefix, + (state) => state.startup.config.currency.unit, ); const infos = timeStratifiedInfos From 8a3cd6366bf58669f2b8ebe49a14e6c45388ffd3 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 18 Apr 2023 12:13:47 +0200 Subject: [PATCH 235/679] Remove raw data badge tooltip --- .../entity-history/timeline/RawDataBadge.tsx | 37 +++++-------------- 1 file changed, 9 insertions(+), 28 deletions(-) diff --git a/frontend/src/js/entity-history/timeline/RawDataBadge.tsx b/frontend/src/js/entity-history/timeline/RawDataBadge.tsx index 570039e423..9b4997d02f 100644 --- a/frontend/src/js/entity-history/timeline/RawDataBadge.tsx +++ b/frontend/src/js/entity-history/timeline/RawDataBadge.tsx @@ -1,6 +1,5 @@ import styled from "@emotion/styled"; -import WithTooltip from "../../tooltip/WithTooltip"; import { EntityEvent } from "../reducer"; const Badge = styled("div")` @@ -19,33 +18,15 @@ interface Props { export const RawDataBadge = ({ className, event }: Props) => { return ( - - {JSON.stringify(event, null, 2)} - - } + { + if (navigator.clipboard) { + navigator.clipboard.writeText(JSON.stringify(event, null, 2)); + } + }} > - { - if (navigator.clipboard) { - navigator.clipboard.writeText(JSON.stringify(event, null, 2)); - } - }} - > - {event.source} - - + {event.source} + ); }; From 0a4621289597582c376d3b68975a2c7c62cf908a Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Tue, 18 Apr 2023 13:33:20 +0200 Subject: [PATCH 236/679] cleanup, use function --- .../form-concept-group/FormConceptNode.tsx | 2 +- frontend/src/js/query-node-editor/ConceptDropzone.tsx | 11 +---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx index eb74a00aad..212068da77 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx @@ -130,7 +130,7 @@ const FormConceptNode: FC = ({ { diff --git a/frontend/src/js/query-node-editor/ConceptDropzone.tsx b/frontend/src/js/query-node-editor/ConceptDropzone.tsx index 2eaf19db92..19d10163aa 100644 --- a/frontend/src/js/query-node-editor/ConceptDropzone.tsx +++ b/frontend/src/js/query-node-editor/ConceptDropzone.tsx @@ -49,16 +49,7 @@ const ConceptDropzone: FC = ({ node, onDropConcept }) => { >> */ acceptedDropTypes={DROP_TYPES} onDrop={(item) => onDropConcept(item as DragItemConceptTreeNode)} - canDrop={(item) => { - // The dragged item should contain exactly one id - // since it was dragged from the tree - const conceptId = (item as DragItemConceptTreeNode).ids[0]; - - return ( - (item as DragItemConceptTreeNode).tree === node.tree && - !node.ids.some((id) => id === conceptId) - ); - }} + canDrop={canDropConceptTreeNodeBeDropped(node)} > {() => t("queryNodeEditor.dropConcept")} From 36b6a20e5781d0b9937f85d16fc9fd47b88a1cd8 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Tue, 18 Apr 2023 13:55:36 +0200 Subject: [PATCH 237/679] add tooltip to analysis layer --- .../js/standard-query-editor/SecondaryIdSelector.tsx | 12 ++++++++---- frontend/src/localization/de.json | 1 + frontend/src/localization/en.json | 1 + 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/frontend/src/js/standard-query-editor/SecondaryIdSelector.tsx b/frontend/src/js/standard-query-editor/SecondaryIdSelector.tsx index eefe507b00..64441064cd 100644 --- a/frontend/src/js/standard-query-editor/SecondaryIdSelector.tsx +++ b/frontend/src/js/standard-query-editor/SecondaryIdSelector.tsx @@ -9,6 +9,7 @@ import type { StateT } from "../app/reducers"; import { exists } from "../common/helpers/exists"; import FaIcon from "../icon/FaIcon"; import { nodeIsConceptQueryNode } from "../model/node"; +import WithTooltip from "../tooltip/WithTooltip"; import ToggleButton from "../ui-components/ToggleButton"; import { setSelectedSecondaryId } from "./actions"; @@ -22,6 +23,7 @@ const Headline = styled.h3<{ active?: boolean }>` transition: color ${({ theme }) => theme.transitionTime}; color: ${({ theme, active }) => active ? theme.col.blueGrayDark : theme.col.gray}; + width: fit-content; `; const SxFaIcon = styled(FaIcon)<{ active?: boolean }>` @@ -152,10 +154,12 @@ const SecondaryIdSelectorUI = memo( return (
- - - {t("queryEditor.secondaryId")} - + + + + {t("queryEditor.secondaryId")} + + Date: Tue, 18 Apr 2023 13:57:31 +0200 Subject: [PATCH 238/679] make de.json text consistent with other texts --- frontend/src/localization/de.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/localization/de.json b/frontend/src/localization/de.json index 263a4aedf0..bc8c000c42 100644 --- a/frontend/src/localization/de.json +++ b/frontend/src/localization/de.json @@ -58,7 +58,7 @@ "previousQuery": "Anfrage", "loadingPreviousQuery": "Lade Anfragedetails", "secondaryId": "Analyse-Ebene", - "secondaryIdTootlip": "Hier können zusätzliche Verknüpfungen zwischen den Konzepten bestimmt werden. Bei Standard erfolgt die Verknüpfung auf Versicherten-Ebene, bei den weiteren Schaltflächen auf Ebene des Versicherten + die zusätzlich gewählte Verknüpfung", + "secondaryIdTootlip": "Hier können zusätzliche Verknüpfungen zwischen den Konzepten bestimmt werden. Bei Standard erfolgt die Verknüpfung auf Versicherten-Ebene, bei den weiteren Schaltflächen auf Ebene des Versicherten + die zusätzlich gewählte Verknüpfung.", "secondaryIdStandard": "Standard", "hasSecondaryId": "Analyse-Ebene aktiv", "removeNode": "Knoten entfernen", From 66ac1fb31f1a269701ba98c59e85f6d032b76860 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Tue, 18 Apr 2023 14:56:57 +0200 Subject: [PATCH 239/679] hide copy button attribute --- frontend/src/js/external-forms/config-types.ts | 1 + frontend/src/js/external-forms/stateSelectors.ts | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/frontend/src/js/external-forms/config-types.ts b/frontend/src/js/external-forms/config-types.ts index df13f49d91..31a2627e27 100644 --- a/frontend/src/js/external-forms/config-types.ts +++ b/frontend/src/js/external-forms/config-types.ts @@ -209,6 +209,7 @@ export type ConceptListField = CommonField & { conceptColumnDropzoneLabel?: TranslatableString; rowPrefixField?: SelectField; isTwoDimensional?: boolean; // Default: False + showCopyButton?: boolean; isSingle?: boolean; // Default: False defaults?: ConceptListDefaults; validations?: ConceptListFieldValidation[]; diff --git a/frontend/src/js/external-forms/stateSelectors.ts b/frontend/src/js/external-forms/stateSelectors.ts index 95bb645cb2..d195acbb4b 100644 --- a/frontend/src/js/external-forms/stateSelectors.ts +++ b/frontend/src/js/external-forms/stateSelectors.ts @@ -80,6 +80,12 @@ export const useAllowExtendedCopying = ( // Need to have min 2 fields to copy from one to another if (otherConceptListFields.length < 1) return false; + const targetField = visibleConceptListFields.find( + (field) => field.name === targetFieldname, + ); + if (targetField?.showCopyButton !== undefined && !targetField.showCopyButton) + return false; + const fieldHasFilledConcept = (field: ConceptListField) => { return ( !!values[field.name] && From 64c7c5eb8041c6667816df6940f9d456dfcc80b7 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 18 Apr 2023 17:06:41 +0200 Subject: [PATCH 240/679] log some headers to find out why they are weird --- .../bakdata/conquery/apiv1/RequestHelper.java | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/RequestHelper.java b/backend/src/main/java/com/bakdata/conquery/apiv1/RequestHelper.java index 8e5aba7332..3a031120cd 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/RequestHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/RequestHelper.java @@ -6,23 +6,27 @@ import javax.servlet.http.HttpServletRequest; import javax.ws.rs.container.ContainerRequestContext; import javax.ws.rs.core.MultivaluedMap; -import javax.ws.rs.core.UriInfo; +import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.eclipse.jetty.http.HttpHeader; @Slf4j +@UtilityClass public class RequestHelper { public static String getRequestURL(HttpServletRequest req) { - if (req.getHeader(AdditionalHeaders.HTTP_HEADER_REAL_HOST) != null) { + if (req.getHeader(HttpHeader.X_FORWARDED_HOST.asString()) != null) { try { - return new URL( - req.getHeader(AdditionalHeaders.HTTP_HEADER_REAL_PROTO), - req.getHeader(AdditionalHeaders.HTTP_HEADER_REAL_HOST), - "" - ).toString(); - } catch (Exception e) { + final String host = req.getHeader(HttpHeader.X_FORWARDED_HOST.asString()); + final String protocol = req.getHeader(HttpHeader.X_FORWARDED_PROTO.asString()); + + log.debug("Proto=`{}` Fwd-Host=`{}`", protocol, host); + + return new URL(protocol, host, "").toString(); + } + catch (Exception e) { log.warn("Failed to build response URL from X-Forward headers", e); } } @@ -44,7 +48,8 @@ public static URI getRequestURL(ContainerRequestContext req) { headers.getFirst(AdditionalHeaders.HTTP_HEADER_REAL_HOST), "" ).toURI(); - } catch (Exception e) { + } + catch (Exception e) { log.warn("Failed to build response URL from X-Forward headers", e); } } From 13cd7ec9be54c012abf6761e43b38718772548f8 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 18 Apr 2023 17:42:23 +0200 Subject: [PATCH 241/679] fixes building URI, otherwise we'd have weird IPv6 adresses --- .../bakdata/conquery/apiv1/RequestHelper.java | 26 ++++++++++++------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/RequestHelper.java b/backend/src/main/java/com/bakdata/conquery/apiv1/RequestHelper.java index 3a031120cd..da93e28bd6 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/RequestHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/RequestHelper.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.apiv1; import java.net.URI; -import java.net.URL; import javax.servlet.http.HttpServletRequest; import javax.ws.rs.container.ContainerRequestContext; @@ -10,6 +9,7 @@ import lombok.experimental.UtilityClass; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.apache.http.client.utils.URIBuilder; import org.eclipse.jetty.http.HttpHeader; @Slf4j @@ -22,9 +22,12 @@ public static String getRequestURL(HttpServletRequest req) { final String host = req.getHeader(HttpHeader.X_FORWARDED_HOST.asString()); final String protocol = req.getHeader(HttpHeader.X_FORWARDED_PROTO.asString()); - log.debug("Proto=`{}` Fwd-Host=`{}`", protocol, host); + log.trace("Proto=`{}` Fwd-Host=`{}`", protocol, host); - return new URL(protocol, host, "").toString(); + return new URIBuilder() + .setHost(host) + .setScheme(protocol) + .toString(); } catch (Exception e) { log.warn("Failed to build response URL from X-Forward headers", e); @@ -41,13 +44,18 @@ public static String getRequestURL(HttpServletRequest req) { */ public static URI getRequestURL(ContainerRequestContext req) { final MultivaluedMap headers = req.getHeaders(); - if (headers.getFirst(AdditionalHeaders.HTTP_HEADER_REAL_HOST) != null) { + if (headers.getFirst(HttpHeader.X_FORWARDED_HOST.asString()) != null) { try { - return new URL( - headers.getFirst(AdditionalHeaders.HTTP_HEADER_REAL_PROTO), - headers.getFirst(AdditionalHeaders.HTTP_HEADER_REAL_HOST), - "" - ).toURI(); + + final String host = headers.getFirst(HttpHeader.X_FORWARDED_HOST.asString()); + final String protocol = headers.getFirst(HttpHeader.X_FORWARDED_PROTO.asString()); + + log.trace("Proto=`{}` Fwd-Host=`{}`", protocol, host); + + return new URIBuilder() + .setHost(host) + .setScheme(protocol) + .build(); } catch (Exception e) { log.warn("Failed to build response URL from X-Forward headers", e); From 412d8c9e31bcc7928290d6d4c6afc1f241485366 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Tue, 18 Apr 2023 17:43:46 +0200 Subject: [PATCH 242/679] move canNodeBeDropped, renamed it from canDropConceptTreeNodeBeDropped, changed to returning lambda to a normal function, removed change to remove border-radius --- .../form-concept-group/FormConceptNode.tsx | 4 +-- frontend/src/js/model/node.ts | 23 +++++++++++++ .../js/query-node-editor/ConceptDropzone.tsx | 32 +++---------------- .../small-tab-navigation/HoverNavigatable.tsx | 1 + .../js/standard-query-editor/QueryNode.tsx | 4 +-- 5 files changed, 32 insertions(+), 32 deletions(-) diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx index 212068da77..a56cdcaafb 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx @@ -9,7 +9,7 @@ import { useTranslation } from "react-i18next"; import { getWidthAndHeight } from "../../app/DndProvider"; import IconButton from "../../button/IconButton"; -import { canDropConceptTreeNodeBeDropped } from "../../query-node-editor/ConceptDropzone"; +import { canNodeBeDropped } from "../../model/node"; import { HoverNavigatable } from "../../small-tab-navigation/HoverNavigatable"; import { getRootNodeLabel } from "../../standard-query-editor/helper"; import type { DragItemConceptTreeNode } from "../../standard-query-editor/types"; @@ -129,7 +129,7 @@ const FormConceptNode: FC = ({ return ( canNodeBeDropped(conceptNode, item)} highlightDroppable > { + return node.type === DNDType.CONCEPT_TREE_NODE; +}; + +export const canNodeBeDropped = ( + node: StandardQueryNodeT, + item: PossibleDroppableObject, +) => { + if ( + !droppableObjectIsConceptTreeNode(item) || + !nodeIsConceptQueryNode(node) + ) { + return false; + } + const conceptId = item.ids[0]; + const itemAlreadyInNode = node.ids.includes(conceptId); + const itemHasConceptRoot = item.tree === node.tree; + return itemHasConceptRoot && !itemAlreadyInNode; +}; diff --git a/frontend/src/js/query-node-editor/ConceptDropzone.tsx b/frontend/src/js/query-node-editor/ConceptDropzone.tsx index 19d10163aa..247226bbfa 100644 --- a/frontend/src/js/query-node-editor/ConceptDropzone.tsx +++ b/frontend/src/js/query-node-editor/ConceptDropzone.tsx @@ -3,12 +3,9 @@ import { FC } from "react"; import { useTranslation } from "react-i18next"; import { DNDType } from "../common/constants/dndTypes"; -import { nodeIsConceptQueryNode } from "../model/node"; -import type { - DragItemConceptTreeNode, - StandardQueryNodeT, -} from "../standard-query-editor/types"; -import Dropzone, { PossibleDroppableObject } from "../ui-components/Dropzone"; +import { canNodeBeDropped } from "../model/node"; +import type { DragItemConceptTreeNode } from "../standard-query-editor/types"; +import Dropzone from "../ui-components/Dropzone"; const SxDropzone = styled(Dropzone)` width: 100%; @@ -21,27 +18,6 @@ interface PropsT { onDropConcept: (concept: DragItemConceptTreeNode) => void; } -export const droppableObjectIsConceptTreeNode = ( - node: PossibleDroppableObject, -): node is DragItemConceptTreeNode => { - return node.type === DNDType.CONCEPT_TREE_NODE; -}; - -export const canDropConceptTreeNodeBeDropped = (node: StandardQueryNodeT) => { - return (item: PossibleDroppableObject) => { - if ( - !droppableObjectIsConceptTreeNode(item) || - !nodeIsConceptQueryNode(node) - ) { - return false; - } - const conceptId = item.ids[0]; - const itemAlreadyInNode = node.ids.includes(conceptId); - const itemHasConceptRoot = item.tree === node.tree; - return itemHasConceptRoot && !itemAlreadyInNode; - }; -}; - const ConceptDropzone: FC = ({ node, onDropConcept }) => { const { t } = useTranslation(); @@ -49,7 +25,7 @@ const ConceptDropzone: FC = ({ node, onDropConcept }) => { >> */ acceptedDropTypes={DROP_TYPES} onDrop={(item) => onDropConcept(item as DragItemConceptTreeNode)} - canDrop={canDropConceptTreeNodeBeDropped(node)} + canDrop={(item) => canNodeBeDropped(node, item)} > {() => t("queryNodeEditor.dropConcept")} diff --git a/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx b/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx index d4680b148d..20aa1946e3 100644 --- a/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx +++ b/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx @@ -28,6 +28,7 @@ const Root = styled("div")<{ return "inherit"; }}; position: relative; + border-radius: ${({ theme }) => theme.borderRadius}; display: inline-flex; `; diff --git a/frontend/src/js/standard-query-editor/QueryNode.tsx b/frontend/src/js/standard-query-editor/QueryNode.tsx index 94542c21cb..2349e81ad8 100644 --- a/frontend/src/js/standard-query-editor/QueryNode.tsx +++ b/frontend/src/js/standard-query-editor/QueryNode.tsx @@ -12,9 +12,9 @@ import { nodeHasNonDefaultSettings, nodeHasFilterValues, nodeIsConceptQueryNode, + canNodeBeDropped, } from "../model/node"; import { isQueryExpandable } from "../model/query"; -import { canDropConceptTreeNodeBeDropped } from "../query-node-editor/ConceptDropzone"; import { HoverNavigatable } from "../small-tab-navigation/HoverNavigatable"; import AdditionalInfoHoverable from "../tooltip/AdditionalInfoHoverable"; @@ -187,7 +187,7 @@ const QueryNode = ({ const QueryNodeRoot = ( canNodeBeDropped(node, item)} highlightDroppable={true} > Date: Tue, 18 Apr 2023 18:00:20 +0200 Subject: [PATCH 243/679] change to entity use InfoTooltip --- .../src/js/standard-query-editor/SecondaryIdSelector.tsx | 6 ++---- frontend/src/localization/de.json | 2 +- frontend/src/localization/en.json | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/frontend/src/js/standard-query-editor/SecondaryIdSelector.tsx b/frontend/src/js/standard-query-editor/SecondaryIdSelector.tsx index 64441064cd..7cf95f783f 100644 --- a/frontend/src/js/standard-query-editor/SecondaryIdSelector.tsx +++ b/frontend/src/js/standard-query-editor/SecondaryIdSelector.tsx @@ -9,12 +9,12 @@ import type { StateT } from "../app/reducers"; import { exists } from "../common/helpers/exists"; import FaIcon from "../icon/FaIcon"; import { nodeIsConceptQueryNode } from "../model/node"; -import WithTooltip from "../tooltip/WithTooltip"; import ToggleButton from "../ui-components/ToggleButton"; import { setSelectedSecondaryId } from "./actions"; import type { StandardQueryStateT } from "./queryReducer"; import type { SelectedSecondaryIdStateT } from "./selectedSecondaryIdReducer"; +import InfoTooltip from "../tooltip/InfoTooltip"; const Headline = styled.h3<{ active?: boolean }>` font-size: ${({ theme }) => theme.font.sm}; @@ -23,7 +23,6 @@ const Headline = styled.h3<{ active?: boolean }>` transition: color ${({ theme }) => theme.transitionTime}; color: ${({ theme, active }) => active ? theme.col.blueGrayDark : theme.col.gray}; - width: fit-content; `; const SxFaIcon = styled(FaIcon)<{ active?: boolean }>` @@ -154,12 +153,11 @@ const SecondaryIdSelectorUI = memo( return (
- {t("queryEditor.secondaryId")} + - Date: Tue, 18 Apr 2023 18:03:15 +0200 Subject: [PATCH 244/679] format --- .../js/standard-query-editor/SecondaryIdSelector.tsx | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/frontend/src/js/standard-query-editor/SecondaryIdSelector.tsx b/frontend/src/js/standard-query-editor/SecondaryIdSelector.tsx index 7cf95f783f..fdcd2fde10 100644 --- a/frontend/src/js/standard-query-editor/SecondaryIdSelector.tsx +++ b/frontend/src/js/standard-query-editor/SecondaryIdSelector.tsx @@ -9,12 +9,12 @@ import type { StateT } from "../app/reducers"; import { exists } from "../common/helpers/exists"; import FaIcon from "../icon/FaIcon"; import { nodeIsConceptQueryNode } from "../model/node"; +import InfoTooltip from "../tooltip/InfoTooltip"; import ToggleButton from "../ui-components/ToggleButton"; import { setSelectedSecondaryId } from "./actions"; import type { StandardQueryStateT } from "./queryReducer"; import type { SelectedSecondaryIdStateT } from "./selectedSecondaryIdReducer"; -import InfoTooltip from "../tooltip/InfoTooltip"; const Headline = styled.h3<{ active?: boolean }>` font-size: ${({ theme }) => theme.font.sm}; @@ -153,11 +153,11 @@ const SecondaryIdSelectorUI = memo( return (
- - - {t("queryEditor.secondaryId")} - - + + + {t("queryEditor.secondaryId")} + + Date: Wed, 19 Apr 2023 10:56:26 +0200 Subject: [PATCH 245/679] creates logs directory and defines it as volume (new log-lib is not able to create it manually) --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index 34775b5f14..c1d50f95aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,6 +46,9 @@ ENV CLUSTER_PORT=${CLUSTER_PORT:-8082} ENV ADMIN_PORT=${ADMIN_PORT:-8081} ENV API_PORT=${API_PORT:-8080} +RUN mkdir /app/logs +VOLUME /app/logs + ENTRYPOINT [ "java", "-jar", "conquery.jar" ] CMD [ "standalone" ] From d7a54595251263ec98caff9415019b4a5f72a635 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 19 Apr 2023 11:23:37 +0200 Subject: [PATCH 246/679] removes PREFIX_LIST as it is a dupe and deprecated --- tutorial/mimic_iii_demo/icd9.ipynb | 1 - 1 file changed, 1 deletion(-) diff --git a/tutorial/mimic_iii_demo/icd9.ipynb b/tutorial/mimic_iii_demo/icd9.ipynb index e10bc82dba..547875081c 100644 --- a/tutorial/mimic_iii_demo/icd9.ipynb +++ b/tutorial/mimic_iii_demo/icd9.ipynb @@ -287,7 +287,6 @@ " \"label\": f\"{match.group('start')}-{match.group('end')}\",\n", " \"description\": match.group(\"name\").title(),\n", " \"condition\": {\n", - " \"type\": \"PREFIX_RANGE\",\n", " \"type\": \"EQUAL\",\n", " \"values\": []\n", " },\n", From d7e444afb526d5d9eedffe463f4f459aea81db9f Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 19 Apr 2023 12:57:42 +0200 Subject: [PATCH 247/679] adds logging around FilterSearch --- .../java/com/bakdata/conquery/models/datasets/Column.java | 7 ------- .../conquery/models/jobs/UpdateFilterSearchJob.java | 7 ++++--- .../com/bakdata/conquery/models/query/FilterSearch.java | 8 ++++++-- .../bakdata/conquery/resources/api/ConceptsProcessor.java | 8 +++++--- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java index bea6fd2583..f7303d091b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/Column.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.models.datasets; -import java.util.List; import java.util.Map; import javax.annotation.Nullable; @@ -172,10 +171,4 @@ public TrieSearch createTrieSearch(IndexConfig config, NamespaceS return search; } - - @Override - public List> getSearchReferences() { - return List.of(this); - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java index 2ec42deefd..2821fe0577 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/UpdateFilterSearchJob.java @@ -82,7 +82,7 @@ public void execute() throws Exception { final StopWatch watch = StopWatch.createStarted(); - log.info("BEGIN collecting entries for `{}`", searchable); + log.info("BEGIN collecting entries for `{}`", searchable.getId()); try { final TrieSearch search = searchable.createTrieSearch(indexConfig, storage); @@ -95,8 +95,9 @@ public void execute() throws Exception { synchronizedResult.put(searchable, search); log.debug( - "DONE collecting entries for `{}`, within {}", - searchable, + "DONE collecting {} entries for `{}`, within {}", + search.calculateSize(), + searchable.getId(), Duration.ofMillis(watch.getTime()) ); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java b/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java index 63850ff7e2..b67a908a2c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java @@ -62,8 +62,12 @@ public static List extractKeywords(FrontendValue value) { /** * For a {@link SelectFilter} collect all relevant {@link TrieSearch}. */ - public List> getSearchesFor(Searchable searchable) { - return searchable.getSearchReferences().stream() + public final List> getSearchesFor(Searchable searchable) { + final List> references = searchable.getSearchReferences(); + + log.debug("Got {} as searchables for {}", references.stream().map(Searchable::getId).collect(Collectors.toList()), searchable.getId()); + + return references.stream() .map(searchCache::get) .filter(Objects::nonNull) .collect(Collectors.toList()); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java index a049a16437..7f15a02e81 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java @@ -25,7 +25,6 @@ import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.models.auth.entities.Subject; import com.bakdata.conquery.models.auth.permissions.Ability; -import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.PreviewConfig; @@ -235,12 +234,15 @@ private Cursor listAllValues(Searchable searchable) { See: https://stackoverflow.com/questions/61114380/java-streams-buffering-huge-streams */ + List> searches = namespace.getFilterSearch().getSearchesFor(searchable); + + log.debug(""); + final Iterator iterators = Iterators.concat( // We are always leading with the empty value. Iterators.singletonIterator(new FrontendValue("", config.getIndex().getEmptyLabel())), - Iterators.concat(Iterators.transform(namespace.getFilterSearch() - .getSearchesFor(searchable) + Iterators.concat(Iterators.transform(searches .iterator(), TrieSearch::iterator)) ); From a905ff153a82bb93794c4af27f28b4b9cd1d52bc Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 19 Apr 2023 14:54:15 +0200 Subject: [PATCH 248/679] log collectLabels content --- .../datasets/concepts/filters/specific/SelectFilter.java | 5 ++++- .../bakdata/conquery/resources/api/ConceptsProcessor.java | 7 ++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java index 5a92cad3d6..9b4eadb002 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java @@ -89,7 +89,8 @@ public List> getSearchReferences() { @NotNull protected List collectLabels() { return labels.entrySet().stream() - .map(entry -> new FrontendValue(entry.getKey(), entry.getValue())).collect(Collectors.toList()); + .map(entry -> new FrontendValue(entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); } @JsonIgnore @@ -129,6 +130,8 @@ public TrieSearch createTrieSearch(IndexConfig config, NamespaceS final TrieSearch search = new TrieSearch<>(config.getSearchSuffixLength(), config.getSearchSplitChars()); + log.debug("Labels for {}: `{}`", getId(), collectLabels().stream().map(FrontendValue::toString).collect(Collectors.toList())); + collectLabels().forEach(feValue -> search.addItem(feValue, FilterSearch.extractKeywords(feValue))); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java index 7f15a02e81..b2e13edbfd 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java @@ -234,16 +234,13 @@ private Cursor listAllValues(Searchable searchable) { See: https://stackoverflow.com/questions/61114380/java-streams-buffering-huge-streams */ - List> searches = namespace.getFilterSearch().getSearchesFor(searchable); - - log.debug(""); + final List> searches = namespace.getFilterSearch().getSearchesFor(searchable); final Iterator iterators = Iterators.concat( // We are always leading with the empty value. Iterators.singletonIterator(new FrontendValue("", config.getIndex().getEmptyLabel())), - Iterators.concat(Iterators.transform(searches - .iterator(), TrieSearch::iterator)) + Iterators.concat(Iterators.transform(searches.iterator(), TrieSearch::iterator)) ); // Use Set to accomplish distinct values From 9b476fe6b74a0c0e85d6158a34b6eec3d4b866b5 Mon Sep 17 00:00:00 2001 From: Marco Korinth Date: Wed, 19 Apr 2023 17:35:27 +0200 Subject: [PATCH 249/679] add datepicker for date inputs --- frontend/package.json | 2 + frontend/src/js/ui-components/BaseInput.tsx | 5 +- frontend/src/js/ui-components/InputDate.tsx | 112 ++++++++++++++++++ .../src/js/ui-components/InputDateRange.tsx | 17 ++- frontend/yarn.lock | 61 +++++++++- 5 files changed, 188 insertions(+), 9 deletions(-) create mode 100644 frontend/src/js/ui-components/InputDate.tsx diff --git a/frontend/package.json b/frontend/package.json index b6bea94ee9..6442e257c9 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -53,6 +53,7 @@ "mustache": "^4.2.0", "nodemon": "^2.0.21", "react": "^18.1.0", + "react-datepicker": "^4.11.0", "react-dnd": "^16.0.1", "react-dnd-html5-backend": "^16.0.1", "react-dnd-multi-backend": "^8.0.0", @@ -100,6 +101,7 @@ "@types/mustache": "^4.1.2", "@types/node": "^18.15.3", "@types/papaparse": "^5.3.2", + "@types/react-datepicker": "^4.10.0", "@types/react-dom": "^18.0.3", "@types/react-list": "^0.8.7", "@types/react-router-dom": "^5.3.3", diff --git a/frontend/src/js/ui-components/BaseInput.tsx b/frontend/src/js/ui-components/BaseInput.tsx index 45bf40a438..31c726eb7c 100644 --- a/frontend/src/js/ui-components/BaseInput.tsx +++ b/frontend/src/js/ui-components/BaseInput.tsx @@ -74,7 +74,7 @@ interface InputProps { onKeyPress?: (e: KeyboardEvent) => void; } -interface Props { +export interface Props { className?: string; inputType: string; money?: boolean; @@ -86,6 +86,7 @@ interface Props { large?: boolean; inputProps?: InputProps; currencyConfig?: CurrencyConfigT; + onFocus?: (e: FocusEvent) => void; onBlur?: (e: FocusEvent) => void; onChange: (val: string | number | null) => void; } @@ -120,6 +121,7 @@ const BaseInput = forwardRef( money, value, onChange, + onFocus, onBlur, placeholder, large, @@ -175,6 +177,7 @@ const BaseInput = forwardRef( }} value={exists(value) ? value : ""} large={large} + onFocus={onFocus} onBlur={onBlur} onWheel={ (e) => diff --git a/frontend/src/js/ui-components/InputDate.tsx b/frontend/src/js/ui-components/InputDate.tsx new file mode 100644 index 0000000000..71abff0f98 --- /dev/null +++ b/frontend/src/js/ui-components/InputDate.tsx @@ -0,0 +1,112 @@ +import styled from "@emotion/styled"; +import { createElement, createRef, forwardRef, useState } from "react"; +import ReactDatePicker from "react-datepicker"; + +import { formatDate, parseDate } from "../common/helpers/dateHelper"; + +import BaseInput, { Props as BaseInputProps } from "./BaseInput"; + +const Root = styled("div")` + position: relative; + + .react-datepicker-wrapper { + position: absolute; + top: 0; + bottom: 0; + z-index: -1; + } + .react-datepicker-popper[data-placement^="bottom"] { + padding-top: 0.25rem; + transform: translate3d(0, 2rem, 0) !important; + } + .react-datepicker-popper[data-placement^="top"] { + padding-bottom: 0; + transform: translate3d(0, -2rem, 0) !important; + } +`; + +const HiddenInput = styled("input")` + display: none; +`; + +const StyledCalendar = styled("div")` + .react-datepicker__day--selected { + background: ${({ theme }) => theme.col.green}; + } +`; + +type Props = Omit & { + value: string | null; + dateFormat: string; + className?: string; + onChange: (val: string | null) => void; + onCalendarSelect?: () => void; +}; + +const InputDate = forwardRef( + ( + { + className, + value, + dateFormat, + onChange, + onCalendarSelect, + onFocus, + ...props + }, + ref, + ) => { + const datePickerRef = createRef(); + const [focusBlocked, setFocusBlocked] = useState(false); + + return ( + { + if (e.key === "Escape") datePickerRef.current?.setOpen(false); + }} + > + { + onChange(val as string); + }} + onFocus={(e) => { + if (focusBlocked) { + e.currentTarget.blur(); + setFocusBlocked(false); + } else { + onFocus?.(e); + datePickerRef.current?.setOpen(true); + } + }} + inputProps={{ + ...props?.inputProps, + onKeyPress: (e) => { + datePickerRef.current?.setOpen(false); + props.inputProps?.onKeyPress?.(e); + }, + }} + /> + { + val && onChange(formatDate(val as Date, dateFormat)); + onCalendarSelect?.(); + setFocusBlocked(true); + datePickerRef.current?.setOpen(false); + }} + onClickOutside={() => datePickerRef.current?.setOpen(false)} + customInput={createElement(HiddenInput)} + calendarContainer={StyledCalendar} + /> + + ); + }, +); + +export default InputDate; diff --git a/frontend/src/js/ui-components/InputDateRange.tsx b/frontend/src/js/ui-components/InputDateRange.tsx index 1946f3d9d6..cddb404e7d 100644 --- a/frontend/src/js/ui-components/InputDateRange.tsx +++ b/frontend/src/js/ui-components/InputDateRange.tsx @@ -1,6 +1,7 @@ import { css } from "@emotion/react"; import styled from "@emotion/styled"; -import { FC, ReactNode, useMemo } from "react"; +import { FC, ReactNode, createRef, useMemo } from "react"; +import "react-datepicker/dist/react-datepicker.css"; import { useTranslation } from "react-i18next"; import { IndexPrefix } from "../common/components/IndexPrefix"; @@ -14,7 +15,7 @@ import { import { exists } from "../common/helpers/exists"; import InfoTooltip from "../tooltip/InfoTooltip"; -import BaseInput from "./BaseInput"; +import InputDate from "./InputDate"; import Label from "./Label"; import Labeled from "./Labeled"; import Optional from "./Optional"; @@ -165,6 +166,8 @@ const InputDateRange: FC = ({ const min = getDisplayDate("min", value, displayDateFormat); const max = getDisplayDate("max", value, displayDateFormat); + const maxRef = createRef(); + const isMinValid = exists(value.min && parseDate(min, displayDateFormat)); const isMaxValid = exists(value.max && parseDate(max, displayDateFormat)); @@ -199,9 +202,9 @@ const InputDateRange: FC = ({ {labelWithSuffix} - = ({ onChange={(val) => onChangeRaw("min", val as string, displayDateFormat) } + onCalendarSelect={() => maxRef.current?.focus()} onBlur={(e) => applyDate("min", e.target.value, displayDateFormat)} inputProps={{ autoFocus, @@ -216,9 +220,10 @@ const InputDateRange: FC = ({ /> - Date: Thu, 20 Apr 2023 09:51:25 +0200 Subject: [PATCH 250/679] fixes a bug where labels were empty --- .../filters/specific/BigMultiSelectFilter.java | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java index 7fa0bdb7bb..f17e80851a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/BigMultiSelectFilter.java @@ -1,10 +1,6 @@ package com.bakdata.conquery.models.datasets.concepts.filters.specific; -import java.util.Collections; -import java.util.List; - import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; -import com.bakdata.conquery.apiv1.frontend.FrontendValue; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.query.filter.event.MultiSelectFilterNode; @@ -12,7 +8,6 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.Setter; -import org.jetbrains.annotations.NotNull; /** * This filter represents a select in the front end. This means that the user can select one or more values from a list of values. @@ -35,10 +30,4 @@ public String getFilterType() { public FilterNode createFilterNode(String[] value) { return new MultiSelectFilterNode(getColumn(), value); } - - @NotNull - protected List collectLabels() { - // Frontend expects no Labels when encountering BIG_MULTI_SELECT - return Collections.emptyList(); - } } From 3e431ca3d3a4eeca8050a408c816c38e02c708ab Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 20 Apr 2023 09:54:03 +0200 Subject: [PATCH 251/679] reduces verbose logging to trace --- .../datasets/concepts/filters/specific/SelectFilter.java | 4 +++- .../java/com/bakdata/conquery/models/query/FilterSearch.java | 4 +++- .../com/bakdata/conquery/resources/api/ConceptsProcessor.java | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java index 9b4eadb002..8a547e5308 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java @@ -130,7 +130,9 @@ public TrieSearch createTrieSearch(IndexConfig config, NamespaceS final TrieSearch search = new TrieSearch<>(config.getSearchSuffixLength(), config.getSearchSplitChars()); - log.debug("Labels for {}: `{}`", getId(), collectLabels().stream().map(FrontendValue::toString).collect(Collectors.toList())); + if(log.isTraceEnabled()) { + log.trace("Labels for {}: `{}`", getId(), collectLabels().stream().map(FrontendValue::toString).collect(Collectors.toList())); + } collectLabels().forEach(feValue -> search.addItem(feValue, FilterSearch.extractKeywords(feValue))); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java b/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java index b67a908a2c..fabf329c07 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/FilterSearch.java @@ -65,7 +65,9 @@ public static List extractKeywords(FrontendValue value) { public final List> getSearchesFor(Searchable searchable) { final List> references = searchable.getSearchReferences(); - log.debug("Got {} as searchables for {}", references.stream().map(Searchable::getId).collect(Collectors.toList()), searchable.getId()); + if(log.isTraceEnabled()) { + log.trace("Got {} as searchables for {}", references.stream().map(Searchable::getId).collect(Collectors.toList()), searchable.getId()); + } return references.stream() .map(searchCache::get) diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java index b2e13edbfd..c59be20ec6 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java @@ -98,7 +98,7 @@ public List load(Pair, String> filterAndSearch) { private final LoadingCache, CursorAndLength> listResults = CacheBuilder.newBuilder().softValues().build(new CacheLoader<>() { @Override public CursorAndLength load(Searchable searchable) { - log.debug("Creating cursor for `{}`", searchable.getId()); + log.trace("Creating cursor for `{}`", searchable.getId()); return new CursorAndLength(listAllValues(searchable), countAllValues(searchable)); } From eb47b434b9a0c38fc514370b2f617c5e61719048 Mon Sep 17 00:00:00 2001 From: Marco Korinth Date: Thu, 20 Apr 2023 10:35:58 +0200 Subject: [PATCH 252/679] moved datepicker styling and changed onCalendarSelect callback --- frontend/src/js/ui-components/InputDate.tsx | 8 +++++--- frontend/src/js/ui-components/InputDateRange.tsx | 1 - 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/frontend/src/js/ui-components/InputDate.tsx b/frontend/src/js/ui-components/InputDate.tsx index 71abff0f98..2e33932372 100644 --- a/frontend/src/js/ui-components/InputDate.tsx +++ b/frontend/src/js/ui-components/InputDate.tsx @@ -1,6 +1,7 @@ import styled from "@emotion/styled"; import { createElement, createRef, forwardRef, useState } from "react"; import ReactDatePicker from "react-datepicker"; +import "react-datepicker/dist/react-datepicker.css"; import { formatDate, parseDate } from "../common/helpers/dateHelper"; @@ -40,7 +41,7 @@ type Props = Omit & { dateFormat: string; className?: string; onChange: (val: string | null) => void; - onCalendarSelect?: () => void; + onCalendarSelect?: (val: string | null) => void; }; const InputDate = forwardRef( @@ -95,8 +96,9 @@ const InputDate = forwardRef( ref={datePickerRef} selected={value ? parseDate(value, dateFormat) : new Date()} onChange={(val) => { - val && onChange(formatDate(val as Date, dateFormat)); - onCalendarSelect?.(); + const selectedDate = formatDate(val as Date, dateFormat); + val && onChange(selectedDate); + onCalendarSelect?.(selectedDate); setFocusBlocked(true); datePickerRef.current?.setOpen(false); }} diff --git a/frontend/src/js/ui-components/InputDateRange.tsx b/frontend/src/js/ui-components/InputDateRange.tsx index cddb404e7d..e14e0458b8 100644 --- a/frontend/src/js/ui-components/InputDateRange.tsx +++ b/frontend/src/js/ui-components/InputDateRange.tsx @@ -1,7 +1,6 @@ import { css } from "@emotion/react"; import styled from "@emotion/styled"; import { FC, ReactNode, createRef, useMemo } from "react"; -import "react-datepicker/dist/react-datepicker.css"; import { useTranslation } from "react-i18next"; import { IndexPrefix } from "../common/components/IndexPrefix"; From 5ccd28252189fdd1dde1fa666271a36cbfcc4a10 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 20 Apr 2023 12:15:30 +0200 Subject: [PATCH 253/679] Implement rendering to json object in EntityPreviewExecution --- .../query/preview/EntityPreviewExecution.java | 314 +++++++++++------- .../integration/tests/EntityExportTest.java | 10 +- 2 files changed, 193 insertions(+), 131 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java index 7bb69b2470..8b99872422 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java @@ -19,6 +19,7 @@ import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.common.CDate; import com.bakdata.conquery.models.common.QuarterUtils; +import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.PreviewConfig; import com.bakdata.conquery.models.execution.ManagedExecution; @@ -37,13 +38,17 @@ import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.MultilineEntityResult; +import com.bakdata.conquery.models.types.ResultType; import com.bakdata.conquery.models.types.SemanticType; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.OptBoolean; +import com.fasterxml.jackson.databind.node.BooleanNode; +import com.fasterxml.jackson.databind.node.DoubleNode; +import com.fasterxml.jackson.databind.node.IntNode; +import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.collect.MoreCollectors; import lombok.ToString; -import org.apache.logging.log4j.util.Strings; import org.jetbrains.annotations.NotNull; /** @@ -65,6 +70,178 @@ public EntityPreviewExecution(EntityPreviewForm entityPreviewQuery, User user, D super(entityPreviewQuery, user, submittedDataset, storage); } + /** + * Query contains both YEARS and QUARTERS lines: Group them. + * + * @return + */ + private static Map> getQuarterLines(EntityResult entityResult) { + final Map> quarterLines = new HashMap<>(); + + for (Object[] line : entityResult.listResultLines()) { + if (Resolution.valueOf((String) line[AbsoluteFormQuery.RESOLUTION_INDEX]) != Resolution.QUARTERS) { + continue; + } + + // Since we know the dates are always aligned we need to only respect their starts. + final LocalDate date = CDate.toLocalDate(((List) line[AbsoluteFormQuery.TIME_INDEX]).get(0)); + + final int year = date.getYear(); + final int quarter = QuarterUtils.getQuarter(date); + + quarterLines.computeIfAbsent(year, (ignored) -> new HashMap<>(4)).put(quarter, line); + } + + return quarterLines; + } + + /** + * Query contains both YEARS and QUARTERS lines: Group them. + * + * @return + */ + private static Map getYearLines(EntityResult entityResult) { + + final Map yearLines = new HashMap<>(); + + for (Object[] line : entityResult.listResultLines()) { + + if (Resolution.valueOf((String) line[AbsoluteFormQuery.RESOLUTION_INDEX]) != Resolution.YEARS) { + continue; + } + + // Since we know the dates are always aligned we need to only respect their starts. + final LocalDate date = CDate.toLocalDate(((List) line[AbsoluteFormQuery.TIME_INDEX]).get(0)); + + final int year = date.getYear(); + + yearLines.put(year, line); + } + + return yearLines; + } + + /** + * Creates a transformer printing lines, transformed into a Map of label->value. + * Null values are omitted. + */ + private static Function> createLineToMapTransformer(List resultInfos, Map select2desc, PrintSettings printSettings) { + + + final int size = resultInfos.size(); + final String[] columnNames = new String[size]; + + for (int index = 0; index < size; index++) { + final ResultInfo resultInfo = resultInfos.get(index); + + if (resultInfo instanceof SelectResultInfo selectResultInfo) { + columnNames[index] = select2desc.get(selectResultInfo.getSelect().getId()).label(); + } + } + + return line -> { + final Map out = new HashMap<>(size); + + for (int column = 0; column < size; column++) { + final String columnName = columnNames[column]; + + if (columnName == null) { + continue; + } + + + final Object value = renderValue(line[column], resultInfos.get(column).getType(), printSettings); + + if (value == null) { + continue; + } + + out.put(columnName, value); + } + + return out; + }; + } + + /** + * Instead of outputting only String values, render to Json equivalents + */ + private static Object renderValue(Object value, ResultType type, PrintSettings printSettings) { + if (value == null) { + return null; + } + + if (type instanceof ResultType.StringT stringT) { + + // StringT may have a mapping that translates values + final String string = stringT.printNullable(printSettings, value); + + if (string.isBlank()) { + return null; + } + + return new TextNode(string); + } + + if (type instanceof ResultType.DateT) { + return new TextNode(CDate.toLocalDate((Integer) value).toString()); + } + + if (type instanceof ResultType.IntegerT) { + return new IntNode((Integer) value); + } + + if (type instanceof ResultType.BooleanT) { + return BooleanNode.valueOf((Boolean) value); + } + + if (type instanceof ResultType.MoneyT) { + return DoubleNode.valueOf((Double) value); + } + + if (type instanceof ResultType.NumericT) { + return DoubleNode.valueOf((Double) value); + } + + if (type instanceof ResultType.DateRangeT) { + final List values = (List) value; + return CDateRange.of(values.get(0), values.get(1)).toSimpleRange(); + } + + + if (type instanceof ResultType.ListT listT) { + return ((List) value).stream().map(entry -> renderValue(entry, listT.getElementType(), printSettings)).collect(Collectors.toList()); + } + + throw new IllegalArgumentException(String.format("Don't know how to handle %s", type)); + } + + /** + * For the selects in result infos, build ColumnDescriptors using definitions (label and description) from PreviewConfig. + */ + private static List createChronoColumnDescriptors(SingleTableResult query, Map select2desc) { + + final List columnDescriptions = new ArrayList<>(); + + for (ResultInfo info : query.getResultInfos()) { + if (info instanceof SelectResultInfo selectResultInfo) { + final PreviewConfig.InfoCardSelect desc = select2desc.get(selectResultInfo.getSelect().getId()); + + // We build these by hand because they are labeled and described by config. + columnDescriptions.add(ColumnDescriptor.builder() + .label(desc.label()) + .defaultLabel(desc.label()) + .type(info.getType().typeInfo()) + .semantics(info.getSemantics()) + .description(Objects.requireNonNullElse(desc.description(), selectResultInfo.getDescription())) + .build()); + } + } + + + return columnDescriptions; + } + @Override public boolean isSystem() { // This Form should NEVER be started manually. Nor persisted @@ -100,11 +277,6 @@ public FullExecutionStatus buildStatusFull(Subject subject) { return status; } - @JsonIgnore - private ManagedQuery getValuesQuery() { - return getSubQueries().get(EntityPreviewForm.VALUES_QUERY_NAME); - } - /** * Takes a ManagedQuery, and transforms its result into a List of {@link EntityPreviewStatus.Info}. * The format of the query is an {@link AbsoluteFormQuery} containing a single line for one person. This should correspond to {@link EntityPreviewForm#VALUES_QUERY_NAME}. @@ -122,7 +294,8 @@ private List transformQueryResultToInfos(ManagedQuery for (int index = AbsoluteFormQuery.FEATURES_OFFSET; index < infoCardExecution.getResultInfos().size(); index++) { final ResultInfo resultInfo = infoCardExecution.getResultInfos().get(index); - final String printed = resultInfo.getType().printNullable(printSettings, values[index]); + + final Object printed = renderValue(values[index], resultInfo.getType(), printSettings); extraInfos.add(new EntityPreviewStatus.Info( resultInfo.userColumnName(printSettings), @@ -164,7 +337,9 @@ private List toChronoInfos(PreviewConfi final List columnDescriptors = createChronoColumnDescriptors(query, select2desc); - final EntityPreviewStatus.TimeStratifiedInfos infos = new EntityPreviewStatus.TimeStratifiedInfos(description.label(), description.description(), columnDescriptors, lineTransformer.apply(completeResult), yearEntries); + final EntityPreviewStatus.TimeStratifiedInfos + infos = + new EntityPreviewStatus.TimeStratifiedInfos(description.label(), description.description(), columnDescriptors, lineTransformer.apply(completeResult), yearEntries); timeStratifiedInfos.add(infos); } @@ -212,124 +387,6 @@ private Object[] getCompleteLine(EntityResult entityResult) { throw new IllegalStateException("Result has no row for COMPLETE"); } - /** - * Query contains both YEARS and QUARTERS lines: Group them. - * - * @return - */ - private static Map> getQuarterLines(EntityResult entityResult) { - final Map> quarterLines = new HashMap<>(); - - for (Object[] line : entityResult.listResultLines()) { - if (Resolution.valueOf((String) line[AbsoluteFormQuery.RESOLUTION_INDEX]) != Resolution.QUARTERS) { - continue; - } - - // Since we know the dates are always aligned we need to only respect their starts. - final LocalDate date = CDate.toLocalDate(((List) line[AbsoluteFormQuery.TIME_INDEX]).get(0)); - - final int year = date.getYear(); - final int quarter = QuarterUtils.getQuarter(date); - - quarterLines.computeIfAbsent(year, (ignored) -> new HashMap<>(4)).put(quarter, line); - } - - return quarterLines; - } - - /** - * Query contains both YEARS and QUARTERS lines: Group them. - * - * @return - */ - private static Map getYearLines(EntityResult entityResult) { - - final Map yearLines = new HashMap<>(); - - for (Object[] line : entityResult.listResultLines()) { - - if (Resolution.valueOf((String) line[AbsoluteFormQuery.RESOLUTION_INDEX]) != Resolution.YEARS) { - continue; - } - - // Since we know the dates are always aligned we need to only respect their starts. - final LocalDate date = CDate.toLocalDate(((List) line[AbsoluteFormQuery.TIME_INDEX]).get(0)); - - final int year = date.getYear(); - - yearLines.put(year, line); - } - - return yearLines; - } - - /** - * Creates a transformer printing lines, transformed into a Map of label->value. - * Null values are omitted. - */ - private static Function> createLineToMapTransformer(List resultInfos, Map select2desc, PrintSettings printSettings) { - - - final int size = resultInfos.size(); - final String[] columnNames = new String[size]; - - for (int index = 0; index < size; index++) { - final ResultInfo resultInfo = resultInfos.get(index); - - if (resultInfo instanceof SelectResultInfo selectResultInfo) { - columnNames[index] = select2desc.get(selectResultInfo.getSelect().getId()).label(); - } - } - - return line -> { - final Map out = new HashMap<>(size); - - for (int column = 0; column < size; column++) { - final String columnName = columnNames[column]; - - if (columnName == null) { - continue; - } - - final String value = resultInfos.get(column).getType().printNullable(printSettings, line[column]); - - if (Strings.isBlank(value)) { - continue; - } - - out.put(columnName, value); - } - - return out; - }; - } - - /** - * For the selects in result infos, build ColumnDescriptors using definitions (label and description) from PreviewConfig. - */ - private static List createChronoColumnDescriptors(SingleTableResult query, Map select2desc) { - - final List columnDescriptions = new ArrayList<>(); - - for (ResultInfo info : query.getResultInfos()) { - if (info instanceof SelectResultInfo selectResultInfo) { - final PreviewConfig.InfoCardSelect desc = select2desc.get(selectResultInfo.getSelect().getId()); - - // We build these by hand because they are labeled and described by config. - columnDescriptions.add(ColumnDescriptor.builder() - .label(desc.label()) - .defaultLabel(desc.label()) - .type(info.getType().typeInfo()) - .semantics(info.getSemantics()) - .description(Objects.requireNonNullElse(desc.description(), selectResultInfo.getDescription())) - .build()); - } - } - - - return columnDescriptions; - } - protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject, FullExecutionStatus status) { status.setColumnDescriptions(generateColumnDescriptions()); } @@ -359,6 +416,11 @@ public List generateColumnDescriptions() { return descriptors; } + @JsonIgnore + private ManagedQuery getValuesQuery() { + return getSubQueries().get(EntityPreviewForm.VALUES_QUERY_NAME); + } + @Override protected void setAdditionalFieldsForStatusWithSource(Subject subject, FullExecutionStatus status) { status.setColumnDescriptions(generateColumnDescriptions()); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/EntityExportTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/EntityExportTest.java index f730e98406..ab862e3de4 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/EntityExportTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/EntityExportTest.java @@ -171,31 +171,31 @@ public void execute(String name, TestConquery testConquery) throws Exception { // assert only 2010 as the other years are empty assertThat(infos.years().get(0)) .isEqualTo(new EntityPreviewStatus.YearEntry( - 2010, Map.of("Values", "B2"), + 2010, Map.of("Values", List.of("B2")), List.of( new EntityPreviewStatus.QuarterEntry(1, Collections.emptyMap()), new EntityPreviewStatus.QuarterEntry(2, Collections.emptyMap()), - new EntityPreviewStatus.QuarterEntry(3, Map.of("Values", "B2")), + new EntityPreviewStatus.QuarterEntry(3, Map.of("Values", List.of("B2"))), new EntityPreviewStatus.QuarterEntry(4, Collections.emptyMap()) ) )); assertThat(result.getTimeStratifiedInfos().get(0).totals()).isEqualTo( - Map.of("Values", "A1 ; B2") + Map.of("Values", List.of("A1", "B2")) ); assertThat(result.getInfos()).containsExactly( new EntityPreviewStatus.Info( "Age", - "9", + 9, ResultType.IntegerT.INSTANCE.typeInfo(), null, Set.of(new SemanticType.SelectResultT(conquery.getDatasetRegistry().resolve(SelectId.Parser.INSTANCE.parsePrefixed(dataset.getName(), "tree1.connector.age")))) ), new EntityPreviewStatus.Info( "Values", - "A1 ; B2", + List.of("A1", "B2"), new ResultType.ListT(ResultType.StringT.INSTANCE).typeInfo(), null, Set.of( From bc83ff4c149a9eb95cf1917dfbbf7745e04fc198 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 20 Apr 2023 15:36:51 +0200 Subject: [PATCH 254/679] fix forSelect not respecting UniversalSelects on concepts --- .../query/concept/specific/CQConcept.java | 93 +++++++++++-------- openapi.yaml | 33 +++++++ 2 files changed, 85 insertions(+), 41 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java index 4c0a2004fd..f9ea072431 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java @@ -81,16 +81,52 @@ public class CQConcept extends CQElement implements NamespacedIdentifiableHoldin @NsIdRefCollection private List cSelects = new ArrayList<>(getSelects()); + final List conSelects = new ArrayList<>(t.getSelects()); + final List selects = new ArrayList<>(); - private boolean excludeFromTimeAggregation = false; + private boolean excludeFromTimeAggregation; //TODO FK 2.12.2021: remove this after successful recode. @JsonAlias("excludeFromSecondaryIdQuery") - private boolean excludeFromSecondaryId = false; + private boolean excludeFromSecondaryId; @JsonView(View.InternalCommunication.class) private boolean aggregateEventDates; + public static CQConcept forSelect(Select select) { + final CQConcept cqConcept = new CQConcept(); + cqConcept.setElements(List.of(select.getHolder().findConcept())); + + if (select.getHolder() instanceof Connector) { + final CQTable table = new CQTable(); + cqConcept.setTables(List.of(table)); + + table.setConnector(((Connector) select.getHolder())); + + table.setSelects(List.of(select)); + } + else { + cqConcept.setTables(((Concept) select.getHolder()) + .getConnectors().stream() + .map(conn -> { + final CQTable table = new CQTable(); + table.setConnector(conn); + return table; + }).toList()); + + cqConcept.setSelects(List.of(select)); + } + + return cqConcept; + } + public static CQConcept forConnector(Connector source) { + final CQConcept cqConcept = new CQConcept(); + cqConcept.setElements(List.of(source.getConcept())); + final CQTable cqTable = new CQTable(); + cqTable.setConcept(cqConcept); + cqTable.setConnector(source); + cqConcept.setTables(List.of(cqTable)); + + return cqConcept; + } @Override public String defaultLabel(Locale locale) { @@ -155,15 +191,15 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { final List> conceptAggregators = createAggregators(plan, selects); - List tableNodes = new ArrayList<>(); + final List tableNodes = new ArrayList<>(); for (CQTable table : tables) { - List> filters = table.getFilters().stream() - .map(FilterValue::createNode) - .collect(Collectors.toList()); + final List> filters = table.getFilters().stream() + .map(FilterValue::createNode) + .collect(Collectors.toList()); //add filter to children - List> aggregators = new ArrayList<>(); + final List> aggregators = new ArrayList<>(); aggregators.addAll(conceptAggregators); @@ -171,7 +207,7 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { // Exists aggregators hold a reference to their parent FiltersNode so they need to be treated separately. // They also don't need aggregation as they simply imitate their reference. - List existsAggregators = + final List existsAggregators = connectorAggregators.stream() .filter(ExistsAggregator.class::isInstance) .map(ExistsAggregator.class::cast) @@ -182,7 +218,7 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { aggregators.removeIf(ExistsAggregator.class::isInstance); - List> eventDateUnionAggregators = + final List> eventDateUnionAggregators = aggregateEventDates ? List.of(new EventDateUnionAggregator(Set.of(table.getConnector().getTable()))) : Collections.emptyList(); @@ -203,7 +239,6 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { .anyMatch(o -> Objects.equals(context.getSelectedSecondaryId(), o)); - final ConceptNode node = new ConceptNode( conceptSpecificNode, elements, @@ -252,7 +287,7 @@ private Column selectValidityDateColumn(CQTable table) { @Override public List getResultInfos() { - List resultInfos = new ArrayList<>(); + final List resultInfos = new ArrayList<>(); for (Select select : selects) { resultInfos.add(select.getResultInfo(this)); @@ -276,57 +311,33 @@ public void collectNamespacedObjects(Set> identifiable @Override public void resolve(QueryResolveContext context) { - this.aggregateEventDates = !(excludeFromTimeAggregation || DateAggregationMode.NONE.equals(context.getDateAggregationMode())); + aggregateEventDates = !(excludeFromTimeAggregation || DateAggregationMode.NONE.equals(context.getDateAggregationMode())); tables.forEach(t -> t.resolve(context)); } @Override public void setDefaultExists() { - boolean allTablesEmpty = getTables().stream() - .map(CQTable::getSelects) - .allMatch(List::isEmpty); + final boolean allTablesEmpty = getTables().stream() + .map(CQTable::getSelects) + .allMatch(List::isEmpty); if (!(getSelects().isEmpty() && (tables.isEmpty() || allTablesEmpty))) { // Don't fill if there are any selects on concept level or on any table level return; } - List cSelects = new ArrayList<>(getSelects()); cSelects.addAll(getConcept().getDefaultSelects()); setSelects(cSelects); for (CQTable t : getTables()) { - List conSelects = new ArrayList<>(t.getSelects()); conSelects.addAll(t.getConnector().getDefaultSelects()); t.setSelects(conSelects); } } - public static CQConcept forSelect(Select select) { - CQConcept cqConcept = new CQConcept(); - cqConcept.setElements(List.of(select.getHolder().findConcept())); - CQTable table = new CQTable(); - cqConcept.setTables(List.of(table)); - - table.setConnector(((Connector) select.getHolder())); - - table.setSelects(List.of(select)); - - return cqConcept; - } - - public static CQConcept forConnector(Connector source) { - final CQConcept cqConcept = new CQConcept(); - cqConcept.setElements(List.of(source.getConcept())); - final CQTable cqTable = new CQTable(); - cqTable.setConcept(cqConcept); - cqTable.setConnector(source); - cqConcept.setTables(List.of(cqTable)); - - return cqConcept; - } - @Override public RequiredEntities collectRequiredEntities(QueryExecutionContext context) { final Set connectors = getTables().stream().map(CQTable::getConnector).collect(Collectors.toSet()); diff --git a/openapi.yaml b/openapi.yaml index 1675cdefeb..0c87090b27 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -98,6 +98,38 @@ components: - RUNNING - FAILED - DONE + Column: + type: object + properties: + name: + type: string + label: + type: string + type: + $ref: "#/components/schemas/ColumnType" + description: + type: string + minSuffixLength: + type: integer + generateSuffixes: + type: boolean + searchDisabled: + type: boolean + sharedDictionary: + type: string + secondaryId: + $ref: "#/components/schemas/SecondaryIdId" + Table: + type: object + properties: + name: + type: string + label: + type: string + columns: + type: array + items: + $ref: "#/components/schemas/Column" FrontendDatasetId: type: object properties: @@ -2043,6 +2075,7 @@ components: type: array items: $ref: "#/components/schemas/SecondaryIdId" + paths: /frontend/config: get: From f89f6ee1bf75b5c2db07ef2a812da6c85518d6a0 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 24 Apr 2023 12:16:45 +0200 Subject: [PATCH 260/679] remove background color so form editor highlighting works again --- .../js/external-forms/form-concept-group/FormConceptNode.tsx | 3 --- 1 file changed, 3 deletions(-) diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx index a56cdcaafb..b4e118c9a2 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx @@ -20,7 +20,6 @@ const Root = styled("div")<{ }>` padding: 5px 10px; cursor: pointer; - background-color: white; max-width: 200px; border-radius: ${({ theme }) => theme.borderRadius}; transition: background-color ${({ theme }) => theme.transitionTime}; @@ -31,10 +30,8 @@ const Root = styled("div")<{ &:hover { background-color: ${({ theme }) => theme.col.bgAlt}; } - display: grid; grid-template-columns: 1fr auto; - font-size: ${({ theme }) => theme.font.sm}; `; From 2eec67fa4c82a7b1969ea0f817df41b7808eee44 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 24 Apr 2023 12:42:18 +0200 Subject: [PATCH 261/679] On reset click, do a partial reset only --- frontend/src/js/external-forms/FormsTab.tsx | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/frontend/src/js/external-forms/FormsTab.tsx b/frontend/src/js/external-forms/FormsTab.tsx index 3070c5db7b..d5301a0e37 100644 --- a/frontend/src/js/external-forms/FormsTab.tsx +++ b/frontend/src/js/external-forms/FormsTab.tsx @@ -93,7 +93,14 @@ const useInitializeForm = () => { methods.reset(defaultValues); }, [methods, defaultValues]); - return { methods, config, datasetOptions, onReset }; + const onResetActiveForm = useCallback(() => { + methods.reset({ + ...methods.getValues(), + ...defaultValues, + }); + }, [methods, defaultValues]); + + return { methods, config, datasetOptions, onReset, onResetActiveForm }; }; const FormsTab = () => { @@ -104,7 +111,8 @@ const FormsTab = () => { useLoadForms({ datasetId }); - const { methods, config, datasetOptions, onReset } = useInitializeForm(); + const { methods, config, datasetOptions, onReset, onResetActiveForm } = + useInitializeForm(); useEffect( function resetOnDatasetChange() { @@ -117,7 +125,7 @@ const FormsTab = () => { return ( - + Date: Mon, 24 Apr 2023 13:25:58 +0200 Subject: [PATCH 262/679] Also fix hide on click --- .../src/js/external-forms/FormsNavigation.tsx | 16 ++-------------- frontend/src/js/external-forms/FormsTab.tsx | 2 +- frontend/src/js/tooltip/ConfirmableTooltip.tsx | 16 ++++++++++++++-- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/frontend/src/js/external-forms/FormsNavigation.tsx b/frontend/src/js/external-forms/FormsNavigation.tsx index 92fc676ece..0e9832423b 100644 --- a/frontend/src/js/external-forms/FormsNavigation.tsx +++ b/frontend/src/js/external-forms/FormsNavigation.tsx @@ -39,10 +39,7 @@ const SxIconButton = styled(IconButton)` padding: 7px 10px; `; -interface Props { - reset: () => void; -} -const FormsNavigation = ({ reset }: Props) => { +const FormsNavigation = ({ onReset }: { onReset: () => void }) => { const language = useActiveLang(); const { t } = useTranslation(); @@ -70,15 +67,6 @@ const FormsNavigation = ({ reset }: Props) => { })) .sort((a, b) => (a.label < b.label ? -1 : 1)); - const activeFormType = useSelector((state) => - selectActiveFormType(state), - ); - const onClear = () => { - if (activeFormType) { - reset(); - } - }; - return ( @@ -97,7 +85,7 @@ const FormsNavigation = ({ reset }: Props) => { }} /> diff --git a/frontend/src/js/external-forms/FormsTab.tsx b/frontend/src/js/external-forms/FormsTab.tsx index d5301a0e37..78a38f9a41 100644 --- a/frontend/src/js/external-forms/FormsTab.tsx +++ b/frontend/src/js/external-forms/FormsTab.tsx @@ -125,7 +125,7 @@ const FormsTab = () => { return ( - + void; red?: boolean; }) => { + const tippyRef = useRef(null); const { t } = useTranslation(); const dropdown = useMemo(() => { return ( { + onConfirm(); + + // TODO: Find a better way to get the tippy instance / to hide it + // https://github.com/atomiks/tippyjs-react/issues/324 + const tippyInstance = (tippyRef.current as any)?._tippy as Instance; + if (tippyInstance) { + tippyInstance.hide(); + } + }} small bgHover red={red} @@ -60,6 +71,7 @@ export const ConfirmableTooltip = ({ trigger="click" offset={offset} hideOnClick + ref={tippyRef} > {children} From 1ebbbb21352c8484d028f55822b291686bb06560 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 24 Apr 2023 14:32:09 +0200 Subject: [PATCH 263/679] Fix Typo --- frontend/src/js/standard-query-editor/SecondaryIdSelector.tsx | 2 +- frontend/src/localization/de.json | 2 +- frontend/src/localization/en.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/js/standard-query-editor/SecondaryIdSelector.tsx b/frontend/src/js/standard-query-editor/SecondaryIdSelector.tsx index fdcd2fde10..a8006d8820 100644 --- a/frontend/src/js/standard-query-editor/SecondaryIdSelector.tsx +++ b/frontend/src/js/standard-query-editor/SecondaryIdSelector.tsx @@ -156,7 +156,7 @@ const SecondaryIdSelectorUI = memo( {t("queryEditor.secondaryId")} - + Date: Mon, 24 Apr 2023 14:43:58 +0200 Subject: [PATCH 264/679] simplify if statement --- frontend/src/js/external-forms/stateSelectors.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/js/external-forms/stateSelectors.ts b/frontend/src/js/external-forms/stateSelectors.ts index d195acbb4b..b8b6511c66 100644 --- a/frontend/src/js/external-forms/stateSelectors.ts +++ b/frontend/src/js/external-forms/stateSelectors.ts @@ -83,7 +83,7 @@ export const useAllowExtendedCopying = ( const targetField = visibleConceptListFields.find( (field) => field.name === targetFieldname, ); - if (targetField?.showCopyButton !== undefined && !targetField.showCopyButton) + if (targetField?.showCopyButton === false) return false; const fieldHasFilledConcept = (field: ConceptListField) => { From 161a9c3acf6f09fdd03de12d8f12fd13abe7b7f5 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 24 Apr 2023 14:56:25 +0200 Subject: [PATCH 265/679] fix formatting --- frontend/src/js/external-forms/stateSelectors.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/frontend/src/js/external-forms/stateSelectors.ts b/frontend/src/js/external-forms/stateSelectors.ts index b8b6511c66..301f7b4592 100644 --- a/frontend/src/js/external-forms/stateSelectors.ts +++ b/frontend/src/js/external-forms/stateSelectors.ts @@ -83,8 +83,7 @@ export const useAllowExtendedCopying = ( const targetField = visibleConceptListFields.find( (field) => field.name === targetFieldname, ); - if (targetField?.showCopyButton === false) - return false; + if (targetField?.showCopyButton === false) return false; const fieldHasFilledConcept = (field: ConceptListField) => { return ( From 6262594fdb4ce5e59c1981e24454fa19e6db413d Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Mon, 24 Apr 2023 18:27:16 +0200 Subject: [PATCH 266/679] migrate to NotNull as NotEmpty is not allowed on URL --- .../com/bakdata/conquery/models/config/FormBackendConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java index 1d0889b094..da0e4f9ae9 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java @@ -67,7 +67,7 @@ public class FormBackendConfig implements PluginConfig, MultiInstancePlugin { @NotEmpty private String healthCheckPath = "health"; - @NotEmpty + @NotNull private URL conqueryApiUrl; @Valid From 56f66a4d267ca7e1db7b1ba38039a7d10ac703a3 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 25 Apr 2023 09:53:03 +0200 Subject: [PATCH 267/679] adds alias for unit: `prefix` to avoid annoying version mismatches --- .../java/com/bakdata/conquery/models/config/FrontendConfig.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index 7487780d0f..e50fb7008f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -9,6 +9,7 @@ import javax.validation.constraints.NotNull; import com.bakdata.conquery.models.forms.frontendconfiguration.FormScanner; +import com.fasterxml.jackson.annotation.JsonAlias; import groovy.transform.ToString; import lombok.AllArgsConstructor; import lombok.Data; @@ -60,6 +61,7 @@ public class FrontendConfig { @Data public static class CurrencyConfig { + @JsonAlias("prefix") private String unit = "€"; private String thousandSeparator = "."; private String decimalSeparator = ","; From be592534cfc196eee5e880df12797f6c79cda0af Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 20 Apr 2023 15:36:51 +0200 Subject: [PATCH 268/679] fix forSelect not respecting UniversalSelects on concepts --- .../query/concept/specific/CQConcept.java | 93 +++++++++++-------- openapi.yaml | 33 +++++++ 2 files changed, 85 insertions(+), 41 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java index 4c0a2004fd..f9ea072431 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java @@ -81,16 +81,52 @@ public class CQConcept extends CQElement implements NamespacedIdentifiableHoldin @NsIdRefCollection private List cSelects = new ArrayList<>(getSelects()); + final List conSelects = new ArrayList<>(t.getSelects()); + final List getDefaultSelects() { - return getSelects() - .stream() - .filter(Select::isDefault) - .collect(Collectors.toList()); + return getSelects().stream().filter(Select::isDefault).collect(Collectors.toList()); } public abstract List getSelects(); @@ -91,7 +90,7 @@ public int countElements() { * Allows concepts to create their own altered FiltersNode if necessary. */ public QPNode createConceptQuery(QueryPlanContext context, List> filters, List> aggregators, List> eventDateAggregators, Column validityDateColumn) { - QPNode child = filters.isEmpty() && aggregators.isEmpty() ? new Leaf() : FiltersNode.create(filters, aggregators, eventDateAggregators); + final QPNode child = filters.isEmpty() && aggregators.isEmpty() ? new Leaf() : FiltersNode.create(filters, aggregators, eventDateAggregators); // Only if a validityDateColumn exists, capsule children in ValidityDateNode diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java index d453f7e981..f064c864bc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java @@ -2,6 +2,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.Objects; @@ -62,7 +63,7 @@ public static FrontendRoot createRoot(NamespaceStorage storage, Subject subject) for (int i = 0; i < allConcepts.size(); i++) { if (isPermitted[i]) { - roots.put(allConcepts.get(i).getId(), createCTRoot(allConcepts.get(i), storage.getStructure())); + roots.put(allConcepts.get(i).getId(), createConceptRoot(allConcepts.get(i), storage.getStructure())); } } if (roots.isEmpty()) { @@ -83,61 +84,44 @@ public static FrontendRoot createRoot(NamespaceStorage storage, Subject subject) } //add all secondary IDs root.getSecondaryIds() - .addAll( - storage.getSecondaryIds().stream() + .addAll(storage.getSecondaryIds() + .stream() .filter(sid -> !sid.isHidden()) .map(sid -> new FrontendSecondaryId(sid.getId().toString(), sid.getLabel(), sid.getDescription())) - .collect(Collectors.toSet()) - ); + .collect(Collectors.toSet())); return root; } - private static FrontendNode createCTRoot(Concept c, StructureNode[] structureNodes) { - - final MatchingStats matchingStats = c.getMatchingStats(); - - final StructureNodeId structureParent = Arrays - .stream(structureNodes) - .filter(sn -> sn.getContainedRoots().contains(c.getId())) - .findAny() - .map(StructureNode::getId) - .orElse(null); - - final FrontendNode n = FrontendNode.builder() - .active(true) - .description(c.getDescription()) - .label(c.getLabel()) - .additionalInfos(c.getAdditionalInfos()) - .matchingEntries(matchingStats != null ? matchingStats.countEvents() : 0) - .matchingEntities(matchingStats != null ? matchingStats.countEntities() : 0) - .dateRange(matchingStats != null && matchingStats.spanEvents() != null ? matchingStats.spanEvents().toSimpleRange() : null) - .detailsAvailable(Boolean.TRUE) - .codeListResolvable(c.countElements() > 1) - .parent(structureParent) - .selects(c - .getSelects() - .stream() - .map(FrontEndConceptBuilder::createSelect) - .collect(Collectors.toList()) - ) - .tables(c - .getConnectors() - .stream() - .map(FrontEndConceptBuilder::createTable) - .collect(Collectors.toList()) - ) - .build(); - - if (c instanceof ConceptTreeNode tree && tree.getChildren() != null) { - n.setChildren( - tree.getChildren() - .stream() - .map(ConceptTreeChild::getId) - .toArray(ConceptTreeChildId[]::new) - ); + private static FrontendNode createConceptRoot(Concept concept, StructureNode[] structureNodes) { + + final MatchingStats matchingStats = concept.getMatchingStats(); + + final StructureNodeId + structureParent = + Arrays.stream(structureNodes).filter(sn -> sn.getContainedRoots().contains(concept.getId())).findAny().map(StructureNode::getId).orElse(null); + + final FrontendNode node = + FrontendNode.builder() + .active(true) + .description(concept.getDescription()) + .label(concept.getLabel()) + .additionalInfos(concept.getAdditionalInfos()) + .matchingEntries(matchingStats != null ? matchingStats.countEvents() : 0) + .matchingEntities(matchingStats != null ? matchingStats.countEntities() : 0) + .dateRange(matchingStats != null && matchingStats.spanEvents() != null ? matchingStats.spanEvents().toSimpleRange() : null) + .detailsAvailable(Boolean.TRUE) + .codeListResolvable(concept.countElements() > 1) + .parent(structureParent) + .excludeFromTimeAggregation(concept.isDefaultExcludeFromTimeAggregation() || concept.getConnectors().stream().map(Connector::getValidityDates).flatMap(Collection::stream).count() == 0) + .selects(concept.getSelects().stream().map(FrontEndConceptBuilder::createSelect).collect(Collectors.toList())) + .tables(concept.getConnectors().stream().map(FrontEndConceptBuilder::createTable).collect(Collectors.toList())) + .build(); + + if (concept instanceof ConceptTreeNode tree && tree.getChildren() != null) { + node.setChildren(tree.getChildren().stream().map(ConceptTreeChild::getId).toArray(ConceptTreeChildId[]::new)); } - return n; + return node; } @Nullable @@ -163,66 +147,43 @@ private static FrontendNode createStructureNode(StructureNode cn, Map, Fro .codeListResolvable(false) .additionalInfos(cn.getAdditionalInfos()) .parent(cn.getParent() == null ? null : cn.getParent().getId()) - .children( - Stream.concat( - cn.getChildren().stream().map(IdentifiableImpl::getId), - unstructured.stream() - ).toArray(Id[]::new) - ) + .children(Stream.concat(cn.getChildren().stream().map(IdentifiableImpl::getId), unstructured.stream()).toArray(Id[]::new)) .build(); } public static FrontendSelect createSelect(Select select) { - return FrontendSelect - .builder() - .id(select.getId()) - .label(select.getLabel()) - .description(select.getDescription()) - .resultType(select.getResultType()) - .isDefault(select.isDefault()) - .build(); + return FrontendSelect.builder() + .id(select.getId()) + .label(select.getLabel()) + .description(select.getDescription()) + .resultType(select.getResultType()) + .isDefault(select.isDefault()) + .build(); } public static FrontendTable createTable(Connector con) { - final FrontendTable result = + final FrontendTable + result = FrontendTable.builder() .id(con.getTable().getId()) .connectorId(con.getId()) .label(con.getLabel()) .isDefault(con.isDefault()) - .filters( - con.collectAllFilters() - .stream() - .map(FrontEndConceptBuilder::createFilter) - .collect(Collectors.toList()) - ) - .selects( - con.getSelects() - .stream() - .map(FrontEndConceptBuilder::createSelect) - .collect(Collectors.toList()) - ) - .supportedSecondaryIds( - Arrays.stream(con.getTable().getColumns()) - .map(Column::getSecondaryId) - .filter(Objects::nonNull) - .map(Identifiable::getId) - .collect(Collectors.toSet()) - ) + .filters(con.collectAllFilters().stream().map(FrontEndConceptBuilder::createFilter).collect(Collectors.toList())) + .selects(con.getSelects().stream().map(FrontEndConceptBuilder::createSelect).collect(Collectors.toList())) + .supportedSecondaryIds(Arrays.stream(con.getTable().getColumns()) + .map(Column::getSecondaryId) + .filter(Objects::nonNull) + .map(Identifiable::getId) + .collect(Collectors.toSet())) .build(); if (con.getValidityDates().size() > 1) { - result.setDateColumn( - new FrontendValidityDate( - con.getValidityDatesDescription(), - null, - con - .getValidityDates() - .stream() - .map(vd -> new FrontendValue(vd.getId().toString(), vd.getLabel())) - .collect(Collectors.toList()) - ) - ); + result.setDateColumn(new FrontendValidityDate(con.getValidityDatesDescription(), null, con.getValidityDates() + .stream() + .map(vd -> new FrontendValue(vd.getId() + .toString(), vd.getLabel())) + .collect(Collectors.toList()))); if (!result.getDateColumn().getOptions().isEmpty()) { result.getDateColumn().setDefaultValue(result.getDateColumn().getOptions().get(0).getValue()); @@ -243,15 +204,17 @@ public static FrontendFilterConfiguration.Top createFilter(Filter filter) { private static FrontendNode createCTNode(ConceptElement ce) { final MatchingStats matchingStats = ce.getMatchingStats(); - final FrontendNode n = FrontendNode.builder() - .active(null) - .description(ce.getDescription()) - .label(ce.getLabel()) - .additionalInfos(ce.getAdditionalInfos()) - .matchingEntries(matchingStats != null ? matchingStats.countEvents() : 0) - .matchingEntities(matchingStats != null ? matchingStats.countEntities() : 0) - .dateRange(matchingStats != null && matchingStats.spanEvents() != null ? matchingStats.spanEvents().toSimpleRange() : null) - .build(); + final FrontendNode + n = + FrontendNode.builder() + .active(null) + .description(ce.getDescription()) + .label(ce.getLabel()) + .additionalInfos(ce.getAdditionalInfos()) + .matchingEntries(matchingStats != null ? matchingStats.countEvents() : 0) + .matchingEntities(matchingStats != null ? matchingStats.countEntities() : 0) + .dateRange(matchingStats != null && matchingStats.spanEvents() != null ? matchingStats.spanEvents().toSimpleRange() : null) + .build(); if (ce instanceof ConceptTreeNode tree) { if (tree.getChildren() != null) { From 6ef5da8c59ec090e394c214c0135461d8958205a Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 9 May 2023 18:20:39 +0200 Subject: [PATCH 309/679] map progress as double, and fix sorting of progress --- .../conquery/models/jobs/JobManager.java | 18 ++++++++++-------- .../conquery/models/jobs/JobStatus.java | 5 ++--- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManager.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManager.java index a42ee21647..1783db1ae2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManager.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManager.java @@ -8,12 +8,14 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public class JobManager implements Closeable{ +public class JobManager implements Closeable { private final JobExecutor slowExecutor; private final JobExecutor fastExecutor; - private final Thread.UncaughtExceptionHandler notifyExecutorDied = (thread, ex) -> { System.exit(1);}; + private final Thread.UncaughtExceptionHandler notifyExecutorDied = (thread, ex) -> { + System.exit(1); + }; public JobManager(String name, boolean failOnError) { @@ -31,25 +33,25 @@ public void addSlowJob(Job job) { log.trace("Added job {}", job.getLabel()); slowExecutor.add(job); } - + public void addFastJob(Job job) { fastExecutor.add(job); } - - public List getSlowJobs() { - return slowExecutor.getJobs(); - } public JobManagerStatus reportStatus() { return new JobManagerStatus( getSlowJobs() .stream() - .map(job -> new JobStatus(job.getJobId(), job.getProgressReporter(), job.getLabel(), job.isCancelled())) + .map(job -> new JobStatus(job.getJobId(), job.getProgressReporter().getProgress(), job.getLabel(), job.isCancelled())) .collect(Collectors.toList()) ); } + public List getSlowJobs() { + return slowExecutor.getJobs(); + } + public boolean isSlowWorkerBusy() { return slowExecutor.isBusy(); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobStatus.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobStatus.java index 73916d2345..1184a4298d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobStatus.java @@ -3,7 +3,6 @@ import java.util.Comparator; import java.util.UUID; -import com.bakdata.conquery.util.progressreporter.ProgressReporter; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -14,13 +13,13 @@ @AllArgsConstructor public class JobStatus implements Comparable { - public static final Comparator BY_PROGRESS = (left, right) -> Double.compare(right.progressReporter.getProgress(), left.progressReporter.getProgress()); + public static final Comparator BY_PROGRESS = Comparator.comparing(JobStatus::getProgress).reversed(); public static final Comparator BY_LABEL = Comparator.comparing(JobStatus::getLabel); public static final Comparator BY_UUID = Comparator.comparing(JobStatus::getJobId); private UUID jobId; - private ProgressReporter progressReporter; + private double progress; private String label; private boolean cancelled; From 417b18465e99e205deccd1967f2901817620e4e5 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 10 May 2023 15:05:40 +0200 Subject: [PATCH 310/679] fix not setting excludeFromTimeAggregation for bare nodes --- .../concepts/FrontEndConceptBuilder.java | 64 +++++++++++++------ openapi.yaml | 3 + 2 files changed, 46 insertions(+), 21 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java index f064c864bc..c88586659b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java @@ -62,9 +62,11 @@ public static FrontendRoot createRoot(NamespaceStorage storage, Subject subject) final boolean[] isPermitted = subject.isPermitted(allConcepts, Ability.READ); for (int i = 0; i < allConcepts.size(); i++) { - if (isPermitted[i]) { - roots.put(allConcepts.get(i).getId(), createConceptRoot(allConcepts.get(i), storage.getStructure())); + if (!isPermitted[i]) { + continue; } + + roots.put(allConcepts.get(i).getId(), createConceptRoot(allConcepts.get(i), storage.getStructure())); } if (roots.isEmpty()) { log.warn("No concepts could be collected for {} on dataset {}. The subject is possibly lacking the permission to use them.", subject.getId(), storage.getDataset() @@ -113,7 +115,12 @@ private static FrontendNode createConceptRoot(Concept concept, StructureNode[ .detailsAvailable(Boolean.TRUE) .codeListResolvable(concept.countElements() > 1) .parent(structureParent) - .excludeFromTimeAggregation(concept.isDefaultExcludeFromTimeAggregation() || concept.getConnectors().stream().map(Connector::getValidityDates).flatMap(Collection::stream).count() == 0) + .excludeFromTimeAggregation(concept.isDefaultExcludeFromTimeAggregation() || concept.getConnectors() + .stream() + .map(Connector::getValidityDates) + .flatMap(Collection::stream) + .findAny() + .isEmpty()) .selects(concept.getSelects().stream().map(FrontEndConceptBuilder::createSelect).collect(Collectors.toList())) .tables(concept.getConnectors().stream().map(FrontEndConceptBuilder::createTable).collect(Collectors.toList())) .build(); @@ -125,9 +132,9 @@ private static FrontendNode createConceptRoot(Concept concept, StructureNode[ } @Nullable - private static FrontendNode createStructureNode(StructureNode cn, Map, FrontendNode> roots) { + private static FrontendNode createStructureNode(StructureNode structureNode, Map, FrontendNode> roots) { final List unstructured = new ArrayList<>(); - for (ConceptId id : cn.getContainedRoots()) { + for (ConceptId id : structureNode.getContainedRoots()) { if (!roots.containsKey(id)) { log.trace("Concept from structure node can not be found: {}", id); continue; @@ -141,13 +148,13 @@ private static FrontendNode createStructureNode(StructureNode cn, Map, Fro return FrontendNode.builder() .active(false) - .description(cn.getDescription()) - .label(cn.getLabel()) + .description(structureNode.getDescription()) + .label(structureNode.getLabel()) .detailsAvailable(Boolean.FALSE) .codeListResolvable(false) - .additionalInfos(cn.getAdditionalInfos()) - .parent(cn.getParent() == null ? null : cn.getParent().getId()) - .children(Stream.concat(cn.getChildren().stream().map(IdentifiableImpl::getId), unstructured.stream()).toArray(Id[]::new)) + .additionalInfos(structureNode.getAdditionalInfos()) + .parent(structureNode.getParent() == null ? null : structureNode.getParent().getId()) + .children(Stream.concat(structureNode.getChildren().stream().map(IdentifiableImpl::getId), unstructured.stream()).toArray(Id[]::new)) .build(); } @@ -204,17 +211,32 @@ public static FrontendFilterConfiguration.Top createFilter(Filter filter) { private static FrontendNode createCTNode(ConceptElement ce) { final MatchingStats matchingStats = ce.getMatchingStats(); - final FrontendNode - n = - FrontendNode.builder() - .active(null) - .description(ce.getDescription()) - .label(ce.getLabel()) - .additionalInfos(ce.getAdditionalInfos()) - .matchingEntries(matchingStats != null ? matchingStats.countEvents() : 0) - .matchingEntities(matchingStats != null ? matchingStats.countEntities() : 0) - .dateRange(matchingStats != null && matchingStats.spanEvents() != null ? matchingStats.spanEvents().toSimpleRange() : null) - .build(); + FrontendNode.FrontendNodeBuilder nodeBuilder = FrontendNode.builder() + .active(null) + .description(ce.getDescription()) + .label(ce.getLabel()) + .additionalInfos(ce.getAdditionalInfos()) + .matchingEntries(matchingStats != null ? matchingStats.countEvents() : 0) + .matchingEntities(matchingStats != null ? matchingStats.countEntities() : 0) + .dateRange(matchingStats != null && matchingStats.spanEvents() != null + ? matchingStats.spanEvents().toSimpleRange() + : null); + + + if (ce instanceof Concept concept) { + final boolean anyValidityDates = concept.getConnectors() + .stream() + .map(Connector::getValidityDates) + .flatMap(Collection::stream) + .findAny() + .isEmpty(); + + nodeBuilder = nodeBuilder.excludeFromTimeAggregation(concept.isDefaultExcludeFromTimeAggregation() || anyValidityDates); + } + + + final FrontendNode n = nodeBuilder.build(); + if (ce instanceof ConceptTreeNode tree) { if (tree.getChildren() != null) { diff --git a/openapi.yaml b/openapi.yaml index 0c87090b27..7f4d1e577b 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -1008,6 +1008,9 @@ components: type: array items: $ref: "#/components/schemas/UniversalSelect" + defaultExcludeFromTimeAggregation: + type: boolean + description: "If true, excludeFromTimeAggregation will be set to true by default for the users." hidden: description: "If true, the concept is not visible in the API or the frontend." type: boolean From e5b5084fd2115c7ed123a5a23434b4af3d0929ad Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 10 May 2023 17:34:13 +0200 Subject: [PATCH 311/679] fix wrong usage of progress --- .../com/bakdata/conquery/resources/admin/ui/jobs.html.ftl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl index 55a0604f8f..26e0847442 100644 --- a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl +++ b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl @@ -21,8 +21,7 @@
-
- 
${job.progressReporter.estimate}
 +
From c6b76b48ef63f4acdbe7eae7cfceecd3e021b24a Mon Sep 17 00:00:00 2001 From: Marco Korinth Date: Thu, 11 May 2023 10:08:55 +0200 Subject: [PATCH 312/679] added year and month selection to Datepicker --- .../ui-components/InputDate/CustomHeader.tsx | 150 ++++++++++++++++++ .../InputDate/CustomHeaderComponents.tsx | 41 +++++ .../{ => InputDate}/InputDate.tsx | 7 +- .../src/js/ui-components/InputDateRange.tsx | 2 +- 4 files changed, 197 insertions(+), 3 deletions(-) create mode 100644 frontend/src/js/ui-components/InputDate/CustomHeader.tsx create mode 100644 frontend/src/js/ui-components/InputDate/CustomHeaderComponents.tsx rename frontend/src/js/ui-components/{ => InputDate}/InputDate.tsx (92%) diff --git a/frontend/src/js/ui-components/InputDate/CustomHeader.tsx b/frontend/src/js/ui-components/InputDate/CustomHeader.tsx new file mode 100644 index 0000000000..5f5c1a89a2 --- /dev/null +++ b/frontend/src/js/ui-components/InputDate/CustomHeader.tsx @@ -0,0 +1,150 @@ +import { + faChevronLeft, + faChevronRight, +} from "@fortawesome/free-solid-svg-icons"; +import { useState } from "react"; +import { ReactDatePickerCustomHeaderProps } from "react-datepicker"; + +import { SelectOptionT } from "../../api/types"; +import IconButton from "../../button/IconButton"; +import { Menu } from "../InputSelect/InputSelectComponents"; + +import { + MonthYearLabel, + OptionButton, + OptionList, + Root, + SelectMenuContainer, +} from "./CustomHeaderComponents"; + +const yearOptions: SelectOptionT[] = [...Array(10).keys()] + .map((n) => new Date().getFullYear() - n) + .map((year) => ({ + label: String(year), + value: year, + })) + .reverse(); + +const months = [ + "Januar", + "Februar", + "März", + "April", + "Mai", + "Juni", + "Juli", + "August", + "September", + "Oktober", + "November", + "Dezember", +]; +const monthOptions: SelectOptionT[] = months.map((month, i) => ({ + label: month, + value: i, +})); + +const SelectMenu = ({ + date, + options, + onSelect, +}: Pick & { + options: SelectOptionT[]; + onSelect: (n: number) => void; +}) => ( + + + + {options.map((option) => ( + onSelect(option.value as number)} + > + {option.label} + + ))} + + + +); + +const YearMonthSelect = ({ + date, + changeMonth, + changeYear, +}: Pick< + ReactDatePickerCustomHeaderProps, + "date" | "changeYear" | "changeMonth" +>) => { + const [yearSelectOpen, setYearSelectOpen] = useState(false); + const [monthSelectOpen, setMonthSelectOpen] = useState(false); + const handleClick = () => { + if (yearSelectOpen || monthSelectOpen) { + setYearSelectOpen(false); + setMonthSelectOpen(false); + } else { + setYearSelectOpen(true); + } + }; + + return ( + <> + + {months[date.getMonth()]} {date.getFullYear()} + + {yearSelectOpen && ( + { + changeYear(year); + setYearSelectOpen(false); + setMonthSelectOpen(true); + }} + /> + )} + {monthSelectOpen && ( + { + changeMonth(month); + setMonthSelectOpen(false); + }} + /> + )} + + ); +}; + +export const CustomHeader = ({ + date, + changeYear, + changeMonth, + decreaseMonth, + increaseMonth, + prevMonthButtonDisabled, + nextMonthButtonDisabled, +}: ReactDatePickerCustomHeaderProps) => { + return ( + + + + + + ); +}; diff --git a/frontend/src/js/ui-components/InputDate/CustomHeaderComponents.tsx b/frontend/src/js/ui-components/InputDate/CustomHeaderComponents.tsx new file mode 100644 index 0000000000..f669e40aae --- /dev/null +++ b/frontend/src/js/ui-components/InputDate/CustomHeaderComponents.tsx @@ -0,0 +1,41 @@ +import styled from "@emotion/styled"; + +import BasicButton from "../../button/BasicButton"; +import { List } from "../InputSelect/InputSelectComponents"; + +export const Root = styled("div")` + display: flex; + justify-content: space-between; + align-items: center; +`; + +export const SelectMenuContainer = styled("div")` + position: absolute; + top: 40px; + left: 0; + right: 0; +`; + +export const OptionList = styled(List)` + display: grid; + grid-template-columns: auto auto; + gap: 5px; +`; + +export const OptionButton = styled(BasicButton)` + font-size: 14px; + background: ${({ theme, active }) => + active ? theme.col.blueGrayDark : "inherit"}; + color: ${({ active }) => (active ? "white" : "inherit")}; + border-radius: ${({ theme }) => theme.borderRadius}; + border: ${({ theme }) => "1px solid " + theme.col.gray}; + &:hover { + background: ${({ theme }) => theme.col.blueGrayDark}; + color: white; + } +`; + +export const MonthYearLabel = styled("div")` + font-weight: bold; + cursor: pointer; +`; diff --git a/frontend/src/js/ui-components/InputDate.tsx b/frontend/src/js/ui-components/InputDate/InputDate.tsx similarity index 92% rename from frontend/src/js/ui-components/InputDate.tsx rename to frontend/src/js/ui-components/InputDate/InputDate.tsx index 3bd58ce481..99cc141a3d 100644 --- a/frontend/src/js/ui-components/InputDate.tsx +++ b/frontend/src/js/ui-components/InputDate/InputDate.tsx @@ -3,9 +3,10 @@ import { createElement, forwardRef, useRef, useState } from "react"; import ReactDatePicker from "react-datepicker"; import "react-datepicker/dist/react-datepicker.css"; -import { formatDate, parseDate } from "../common/helpers/dateHelper"; +import { formatDate, parseDate } from "../../common/helpers/dateHelper"; +import BaseInput, { Props as BaseInputProps } from "../BaseInput"; -import BaseInput, { Props as BaseInputProps } from "./BaseInput"; +import { CustomHeader } from "./CustomHeader"; const Root = styled("div")` position: relative; @@ -121,8 +122,10 @@ const InputDate = forwardRef( datePickerRef.current?.setOpen(false); }} onClickOutside={() => datePickerRef.current?.setOpen(false)} + renderCustomHeader={CustomHeader} customInput={createElement(HiddenInput)} calendarContainer={StyledCalendar} + showFullMonthYearPicker={true} /> ); diff --git a/frontend/src/js/ui-components/InputDateRange.tsx b/frontend/src/js/ui-components/InputDateRange.tsx index e14e0458b8..f8e0e46662 100644 --- a/frontend/src/js/ui-components/InputDateRange.tsx +++ b/frontend/src/js/ui-components/InputDateRange.tsx @@ -14,7 +14,7 @@ import { import { exists } from "../common/helpers/exists"; import InfoTooltip from "../tooltip/InfoTooltip"; -import InputDate from "./InputDate"; +import InputDate from "./InputDate/InputDate"; import Label from "./Label"; import Labeled from "./Labeled"; import Optional from "./Optional"; From 063cb45ed70ca1215cfa59dd098de6fc3993ae3e Mon Sep 17 00:00:00 2001 From: Marco Korinth Date: Thu, 11 May 2023 10:24:42 +0200 Subject: [PATCH 313/679] removed unneccessary line --- frontend/src/js/ui-components/InputDate/InputDate.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/src/js/ui-components/InputDate/InputDate.tsx b/frontend/src/js/ui-components/InputDate/InputDate.tsx index 99cc141a3d..a3cf0aeb52 100644 --- a/frontend/src/js/ui-components/InputDate/InputDate.tsx +++ b/frontend/src/js/ui-components/InputDate/InputDate.tsx @@ -125,7 +125,6 @@ const InputDate = forwardRef( renderCustomHeader={CustomHeader} customInput={createElement(HiddenInput)} calendarContainer={StyledCalendar} - showFullMonthYearPicker={true} /> ); From 632f3e2f5a1676a30a6017b211596d557455fcb7 Mon Sep 17 00:00:00 2001 From: Marco Korinth Date: Thu, 11 May 2023 17:01:18 +0200 Subject: [PATCH 314/679] changed absolute positioning to anchor-dimensions pattern --- .../src/js/ui-components/InputDate/CustomHeaderComponents.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/js/ui-components/InputDate/CustomHeaderComponents.tsx b/frontend/src/js/ui-components/InputDate/CustomHeaderComponents.tsx index f669e40aae..72d159246a 100644 --- a/frontend/src/js/ui-components/InputDate/CustomHeaderComponents.tsx +++ b/frontend/src/js/ui-components/InputDate/CustomHeaderComponents.tsx @@ -13,7 +13,7 @@ export const SelectMenuContainer = styled("div")` position: absolute; top: 40px; left: 0; - right: 0; + width: 100%; `; export const OptionList = styled(List)` From 9434a2596e544d70a356452692e0da29003839e9 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Mon, 15 May 2023 10:35:29 +0200 Subject: [PATCH 315/679] Documentation for .import.json (#3062) Documentation for .import.json --- openapi.yaml | 176 ++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 166 insertions(+), 10 deletions(-) diff --git a/openapi.yaml b/openapi.yaml index 7f4d1e577b..d1c019e358 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -130,6 +130,160 @@ components: type: array items: $ref: "#/components/schemas/Column" + + OutputDescriptionBase: + type: object + properties: + name: + description: "Name of the `table.column` to be imported into." + type: string + required: + type: boolean + LineOutput: + description: "Emits the line number. The line-number is not guaranteed to be in file order, just an incrementing unique number: If multiple files are preprocessed, the line-number will not reset to zero." + allOf: + - $ref: "#/components/schemas/OutputDescriptionBase" + - properties: + operation: + type: string + enum: [ LINE ] + NullOutput: + description: "Emits a `null`/NA value. Can be used to unify among multiple input sources with differing file schemas." + allOf: + - $ref: "#/components/schemas/OutputDescriptionBase" + - properties: + operation: + type: string + enum: [ NULL ] + + CopyOutput: + description: "Parses and emits the values in inputColumn as the provided type." + allOf: + - $ref: "#/components/schemas/OutputDescriptionBase" + - properties: + inputColumn: + type: string + description: "Name of the csv column to process." + inputType: + $ref: "#/components/schemas/MajorType" + operation: + type: string + enum: [ COPY ] + CompoundDateRangeOutput: + description: "Creates a virtual daterange column as a composite of its neighboring DATE-columns (not csv columns). + This helps avoid copying multiple values of the same column, when they can be reused." + allOf: + - $ref: "#/components/schemas/OutputDescriptionBase" + - properties: + startColumn: + type: string + endColumn: + type: string + allowOpen: + type: boolean + operation: + type: string + enum: [ COMPOUND_DATE_RANGE ] + + DateRangeOutput: + description: "Parses `startColumn` and `endColumn` as dates, then emits them as a daterange." + allOf: + - $ref: "#/components/schemas/OutputDescriptionBase" + - properties: + startColumn: + type: string + endColumn: + type: string + allowOpen: + type: boolean + operation: + type: string + enum: [ DATE_RANGE ] + + EpochDateRangeOutput: + deprecated: true + description: "Parses `startColumn` and `endColumn` as integers, interpreting them as dates in the Unix-Epoch (i.e. days since 01-01-1970), then emits them as a daterange." + allOf: + - $ref: "#/components/schemas/OutputDescriptionBase" + - properties: + startColumn: + type: string + endColumn: + type: string + allowOpen: + type: boolean + operation: + type: string + enum: [ EPOCH_DATE_RANGE ] + + EpochOutput: + deprecated: true + description: "Parses `startColumn` integer, emitting it as dates in the Unix-Epoch (i.e. days since 01-01-1970)" + allOf: + - $ref: "#/components/schemas/OutputDescriptionBase" + - properties: + inputColumn: + type: string + operation: + type: string + enum: [ EPOCH ] + + OutputDescription: + description: "Operations to transform, then import, values into tables." + discriminator: + propertyName: operation + mapping: { + # FK: I'm using JSON notation here, because the key `NULL` breaks some tools. + "COMPOUND_DATE_RANGE": "#/components/schemas/CompoundDateRangeOutput", + "COPY": "#/components/schemas/CopyOutput", + "DATE_RANGE": "#/components/schemas/DateRangeOutput", + "EPOCH": "#/components/schemas/EpochOutput", + "EPOCH_DATE_RANGE": "#/components/schemas/EpochDateRangeOutput", + "LINE": "#/components/schemas/LineOutput", + "NULL": "#/components/schemas/NullOutput", + } + oneOf: + - $ref: "#/components/schemas/CompoundDateRangeOutput" + - $ref: "#/components/schemas/CopyOutput" + - $ref: "#/components/schemas/DateRangeOutput" + - $ref: "#/components/schemas/EpochDateRangeOutput" + - $ref: "#/components/schemas/EpochOutput" + - $ref: "#/components/schemas/LineOutput" + - $ref: "#/components/schemas/NullOutput" + + Import: + description: "Describes how one or multiple CSV files are to be transformed into a .cqpp file compatible with the designated table." + type: object + properties: + table: + $ref: "#/components/schemas/TableId" + name: + description: "Name of the import. When preprocessed with tags, will be overwritten to the respective tag." + type: string + label: + deprecated: true + type: string + inputs: + description: "One or more descriptions to preprocess a `.csv` or `.csv.gz` into a single cqpp." + type: array + items: + type: object + properties: + sourceFile: + description: "Filename of the to preprocess file, relative to the path supplied in `preprocess`." + type: string + filter: + description: "Groovy script which can be used to filter the csv rows. + The row is passed as `row` array, the fields can be accessed as integer variables to the index in the file. + For example accessing `value` in `id,value` can be done with `row[value]` (no string up-ticks), or `row[1]`." + type: string + primary: + $ref: "#/components/schemas/OutputDescription" + output: + type: array + items: + $ref: "#/components/schemas/OutputDescription" + FrontendDatasetId: type: object properties: @@ -1026,20 +1180,22 @@ components: type: string value: type: string - + MajorType: + type: string + enum: + - STRING + - INTEGER + - BOOLEAN + - NUMERIC + - MONEY + - DATE + - DATE_RANGE ColumnType: description: Available types for input and output of conquery. Additionally list, which is a nested type of ColumnType. oneOf: + - $ref: "#/components/schemas/MajorType" - type: string - enum: - - STRING - - INTEGER - - BOOLEAN - - NUMERIC - - MONEY - - DATE - - DATE_RANGE - - type: string + description: "Arbitrary nesting of ColumnTypes including List." pattern: "LIST\\[.*\\]" SemanticsEventDate: description: Values contain the primary date of a row. From e4b24f73dfc6f2882eca87428513b2ed5afeb804 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 15 May 2023 11:22:39 +0200 Subject: [PATCH 316/679] initial Version of dropping concepts between each other --- .../DropzoneBetweenElements.tsx | 91 ++++++ .../form-concept-group/FormConceptGroup.tsx | 265 ++++++++++-------- .../formConceptGroupState.ts | 6 + 3 files changed, 247 insertions(+), 115 deletions(-) create mode 100644 frontend/src/js/external-forms/form-concept-group/DropzoneBetweenElements.tsx diff --git a/frontend/src/js/external-forms/form-concept-group/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-concept-group/DropzoneBetweenElements.tsx new file mode 100644 index 0000000000..181abedf24 --- /dev/null +++ b/frontend/src/js/external-forms/form-concept-group/DropzoneBetweenElements.tsx @@ -0,0 +1,91 @@ +import { DropTargetMonitor, useDrop } from "react-dnd"; +import Dropzone, { ChildArgs, PossibleDroppableObject } from "../../ui-components/Dropzone"; +import styled from "@emotion/styled"; +import { DNDType } from "../../common/constants/dndTypes"; +import { ReactNode } from "react"; + +interface PropsT { + onDrop: ( + item: DroppableObject, + monitor: DropTargetMonitor, + ) => void; + acceptedDropTypes: string[]; + children?: (args: ChildArgs) => ReactNode; + } + +const DropzoneBetweenElements = ({onDrop, children, acceptedDropTypes}: PropsT) => { + const Root = styled("div")<{ + isHovered: boolean; + }>` + width: 100%; + left: 0; + top: -17px; + right: 0; + position: absolute; + bottom: 90%; + border-radius: ${({ theme }) => theme.borderRadius}; + `; + + + const DropzoneRoot = styled("div")` + width: 100%; + left: 0; + top: -17px; + right: 0; + position: absolute; + bottom: 90%; + z-index: 2; + background-color: ${({ theme }) => theme.col.bg}; + `; + + const [{ isOver, isDroppable }, drop] = useDrop({ + accept: [ + DNDType.FORM_CONFIG, + DNDType.CONCEPT_TREE_NODE, + DNDType.PREVIOUS_QUERY, + DNDType.PREVIOUS_SECONDARY_ID_QUERY, + ], + hover: (_, __) => { + if (!isDroppable) return; + + }, + collect: (monitor) => ({ + isOver: monitor.isOver(), + isDroppable: monitor.canDrop(), + }), + }); + + + const [{ isOver: isOver2, isDroppable : isDroppable2 }, drop2] = useDrop({ + accept: [ + DNDType.FORM_CONFIG, + DNDType.CONCEPT_TREE_NODE, + DNDType.PREVIOUS_QUERY, + DNDType.PREVIOUS_SECONDARY_ID_QUERY, + ], + hover: (_, __) => { + if (!isDroppable2) return; + }, + collect: (monitor) => ({ + isOver: monitor.isOver(), + isDroppable: monitor.canDrop(), + }), + }); + + return( + <> + {!isOver && !isOver2 && } + {/* Show when hovered with text and dropzone */} + {(isOver || isOver2) && + ( + + {children} + + ) + + } + + ) +} + +export default DropzoneBetweenElements; \ No newline at end of file diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 9e343b181c..027b4d75cc 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -32,6 +32,7 @@ import { useVisibleConceptListFields, } from "../stateSelectors"; +import DropzoneBetweenElements from "./DropzoneBetweenElements"; import FormConceptCopyModal from "./FormConceptCopyModal"; import FormConceptNode from "./FormConceptNode"; import { @@ -40,6 +41,7 @@ import { copyConcept, FormConceptGroupT, initializeConcept, + insertValue, onToggleIncludeSubnodes, removeConcept, removeValue, @@ -237,130 +239,163 @@ const FormConceptGroup = (props: Props) => { ); }} items={props.value.map((row, i) => ( - - {props.renderRowPrefix - ? props.renderRowPrefix({ - value: props.value, - onChange: props.onChange, - row, - i, - }) - : null} - {row.concepts.length > 1 && ( - - - {t("externalForms.common.connectedWith")}: - - { - props.onChange( - setValueProperties(props.value, i, { - connector: val, - }), - ); - }} - options={[ - { value: "OR", label: t("common.or") }, - { value: "AND", label: t("common.and") }, - ]} - /> - - )} - - props.onChange(addConcept(props.value, i, null)) - } - onRemoveClick={(j) => - props.onChange( - props.value && props.value[i].concepts.length === 1 - ? removeValue(props.value, i) - : removeConcept(props.value, i, j), - ) - } - items={row.concepts.map((concept, j) => - concept ? ( - - setEditedFormQueryNodePosition({ - valueIdx: i, - conceptIdx: j, - }) - } - expand={{ - onClick: () => - props.onChange( - onToggleIncludeSubnodes( - props.value, - i, - j, - !concept.includeSubnodes, - newValue, - ), - ), - expandable: - !props.disallowMultipleColumns && - hasConceptChildren(concept), - active: !!concept.includeSubnodes, + <> + { + console.log("onDrop", item); + console.log(row); + if (isMovedObject(item)) { + return props.onChange( + addConcept( + insertValue(props.value, i, newValue), + i, + copyConcept(item), + ), + ); + } + + if (props.isValidConcept && !props.isValidConcept(item)) + return null; + + return props.onChange( + addConcept( + insertValue(props.value, i, newValue), + i, + initializeConcept(item, defaults, tableConfig), + ), + ); + }} + > + {() => props.conceptDropzoneText} + + + {props.renderRowPrefix + ? props.renderRowPrefix({ + value: props.value, + onChange: props.onChange, + row, + i, + }) + : null} + {row.concepts.length > 1 && ( + + + {t("externalForms.common.connectedWith")}: + + { + props.onChange( + setValueProperties(props.value, i, { + connector: val, + }), + ); }} + options={[ + { value: "OR", label: t("common.or") }, + { value: "AND", label: t("common.and") }, + ]} /> - ) : ( - */ - acceptedDropTypes={DROP_TYPES} - onImportLines={(lines) => - onImportLines(lines, { valueIdx: i, conceptIdx: j }) - } - onDrop={(item: DragItemConceptTreeNode | DragItemFile) => { - if (item.type === "__NATIVE_FILE__") { - onDropFile(item.files[0], { + + )} + + props.onChange(addConcept(props.value, i, null)) + } + onRemoveClick={(j) => + props.onChange( + props.value && props.value[i].concepts.length === 1 + ? removeValue(props.value, i) + : removeConcept(props.value, i, j), + ) + } + items={row.concepts.map((concept, j) => + concept ? ( + + setEditedFormQueryNodePosition({ valueIdx: i, conceptIdx: j, - }); - - return; + }) + } + expand={{ + onClick: () => + props.onChange( + onToggleIncludeSubnodes( + props.value, + i, + j, + !concept.includeSubnodes, + newValue, + ), + ), + expandable: + !props.disallowMultipleColumns && + hasConceptChildren(concept), + active: !!concept.includeSubnodes, + }} + /> + ) : ( + */ + acceptedDropTypes={DROP_TYPES} + onImportLines={(lines) => + onImportLines(lines, { valueIdx: i, conceptIdx: j }) } + onDrop={( + item: DragItemConceptTreeNode | DragItemFile, + ) => { + if (item.type === "__NATIVE_FILE__") { + onDropFile(item.files[0], { + valueIdx: i, + conceptIdx: j, + }); + + return; + } + + if (isMovedObject(item)) { + return props.onChange( + setConcept(props.value, i, j, copyConcept(item)), + ); + } + + if (props.isValidConcept && !props.isValidConcept(item)) + return null; - if (isMovedObject(item)) { return props.onChange( - setConcept(props.value, i, j, copyConcept(item)), + setConcept( + props.value, + i, + j, + initializeConcept(item, defaults, tableConfig), + ), ); + }} + > + {({ isOver, item }) => + isOver && isMovedObject(item) + ? t("externalForms.common.concept.copying") + : props.conceptDropzoneText } - - if (props.isValidConcept && !props.isValidConcept(item)) - return null; - - return props.onChange( - setConcept( - props.value, - i, - j, - initializeConcept(item, defaults, tableConfig), - ), - ); - }} - > - {({ isOver, item }) => - isOver && isMovedObject(item) - ? t("externalForms.common.concept.copying") - : props.conceptDropzoneText - } - - ), - )} - /> - + + ), + )} + /> + + ))} /> {isCopyModalOpen && ( diff --git a/frontend/src/js/external-forms/form-concept-group/formConceptGroupState.ts b/frontend/src/js/external-forms/form-concept-group/formConceptGroupState.ts index da2cbc3706..d24fe3c441 100644 --- a/frontend/src/js/external-forms/form-concept-group/formConceptGroupState.ts +++ b/frontend/src/js/external-forms/form-concept-group/formConceptGroupState.ts @@ -47,6 +47,12 @@ export const addValue = ( newValue: FormConceptGroupT, ) => [...value, newValue]; +export const insertValue = ( + value: FormConceptGroupT[], + valueIdx: number, + newValue: FormConceptGroupT, +) => [...value.slice(0, valueIdx), newValue, ...value.slice(valueIdx)]; + export const removeValue = (value: FormConceptGroupT[], valueIdx: number) => { return [...value.slice(0, valueIdx), ...value.slice(valueIdx + 1)]; }; From 4fa714da4f130d96f3581706305abf87a35c34a9 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 15 May 2023 11:37:13 +0200 Subject: [PATCH 317/679] cleanup --- .../DropzoneBetweenElements.tsx | 165 +++++++++--------- .../form-concept-group/FormConceptGroup.tsx | 2 - 2 files changed, 84 insertions(+), 83 deletions(-) diff --git a/frontend/src/js/external-forms/form-concept-group/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-concept-group/DropzoneBetweenElements.tsx index 181abedf24..0613fd1d35 100644 --- a/frontend/src/js/external-forms/form-concept-group/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-concept-group/DropzoneBetweenElements.tsx @@ -1,91 +1,94 @@ -import { DropTargetMonitor, useDrop } from "react-dnd"; -import Dropzone, { ChildArgs, PossibleDroppableObject } from "../../ui-components/Dropzone"; import styled from "@emotion/styled"; -import { DNDType } from "../../common/constants/dndTypes"; import { ReactNode } from "react"; +import { DropTargetMonitor, useDrop } from "react-dnd"; -interface PropsT { - onDrop: ( - item: DroppableObject, - monitor: DropTargetMonitor, - ) => void; - acceptedDropTypes: string[]; - children?: (args: ChildArgs) => ReactNode; - } - -const DropzoneBetweenElements = ({onDrop, children, acceptedDropTypes}: PropsT) => { - const Root = styled("div")<{ - isHovered: boolean; - }>` - width: 100%; - left: 0; - top: -17px; - right: 0; - position: absolute; - bottom: 90%; - border-radius: ${({ theme }) => theme.borderRadius}; - `; +import { DNDType } from "../../common/constants/dndTypes"; +import Dropzone, { + ChildArgs, + PossibleDroppableObject, +} from "../../ui-components/Dropzone"; +interface PropsT { + onDrop: (item: DroppableObject, monitor: DropTargetMonitor) => void; + acceptedDropTypes: string[]; + children?: (args: ChildArgs) => ReactNode; +} - const DropzoneRoot = styled("div")` - width: 100%; - left: 0; - top: -17px; - right: 0; - position: absolute; - bottom: 90%; - z-index: 2; - background-color: ${({ theme }) => theme.col.bg}; - `; +const DropzoneBetweenElements = < + DroppableObject extends PossibleDroppableObject, +>({ + onDrop, + children, + acceptedDropTypes, +}: PropsT) => { + const Root = styled("div")<{ + isHovered: boolean; + }>` + width: 100%; + left: 0; + top: -17px; + right: 0; + position: absolute; + bottom: 90%; + border-radius: ${({ theme }) => theme.borderRadius}; + `; - const [{ isOver, isDroppable }, drop] = useDrop({ - accept: [ - DNDType.FORM_CONFIG, - DNDType.CONCEPT_TREE_NODE, - DNDType.PREVIOUS_QUERY, - DNDType.PREVIOUS_SECONDARY_ID_QUERY, - ], - hover: (_, __) => { - if (!isDroppable) return; - - }, - collect: (monitor) => ({ - isOver: monitor.isOver(), - isDroppable: monitor.canDrop(), - }), - }); + const DropzoneRoot = styled("div")` + width: 100%; + left: 0; + top: -17px; + right: 0; + position: absolute; + bottom: 90%; + z-index: 2; + background-color: ${({ theme }) => theme.col.bg}; + `; + const [{ isOver, isDroppable }, drop] = useDrop({ + accept: [ + DNDType.FORM_CONFIG, + DNDType.CONCEPT_TREE_NODE, + DNDType.PREVIOUS_QUERY, + DNDType.PREVIOUS_SECONDARY_ID_QUERY, + ], + hover: (_, __) => { + if (!isDroppable) return; + }, + collect: (monitor) => ({ + isOver: monitor.isOver(), + isDroppable: monitor.canDrop(), + }), + }); - const [{ isOver: isOver2, isDroppable : isDroppable2 }, drop2] = useDrop({ - accept: [ - DNDType.FORM_CONFIG, - DNDType.CONCEPT_TREE_NODE, - DNDType.PREVIOUS_QUERY, - DNDType.PREVIOUS_SECONDARY_ID_QUERY, - ], - hover: (_, __) => { - if (!isDroppable2) return; - }, - collect: (monitor) => ({ - isOver: monitor.isOver(), - isDroppable: monitor.canDrop(), - }), - }); + const [{ isOver: isOver2, isDroppable: isDroppable2 }, drop2] = useDrop({ + accept: [ + DNDType.FORM_CONFIG, + DNDType.CONCEPT_TREE_NODE, + DNDType.PREVIOUS_QUERY, + DNDType.PREVIOUS_SECONDARY_ID_QUERY, + ], + hover: (_, __) => { + if (!isDroppable2) return; + }, + collect: (monitor) => ({ + isOver: monitor.isOver(), + isDroppable: monitor.canDrop(), + }), + }); - return( - <> - {!isOver && !isOver2 && } - {/* Show when hovered with text and dropzone */} - {(isOver || isOver2) && - ( - - {children} - - ) - - } - - ) -} + return ( + <> + {!isOver && !isOver2 && } + {/* Show when hovered with text and dropzone */} + {(isOver || isOver2) && ( + + + {children} + + + )} + + ); +}; -export default DropzoneBetweenElements; \ No newline at end of file +export default DropzoneBetweenElements; diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 027b4d75cc..5b8e9788b5 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -243,8 +243,6 @@ const FormConceptGroup = (props: Props) => { { - console.log("onDrop", item); - console.log(row); if (isMovedObject(item)) { return props.onChange( addConcept( From 2ba5888205de22ac80c153ae4c134ae73495b79c Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 15 May 2023 12:13:24 +0200 Subject: [PATCH 318/679] format --- .../form-concept-group/FormConceptGroup.tsx | 44 ++++++++++--------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 5b8e9788b5..6d55b3f636 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -240,33 +240,35 @@ const FormConceptGroup = (props: Props) => { }} items={props.value.map((row, i) => ( <> - { - if (isMovedObject(item)) { + {!props.disallowMultipleColumns && ( + { + if (isMovedObject(item)) { + return props.onChange( + addConcept( + insertValue(props.value, i, newValue), + i, + copyConcept(item), + ), + ); + } + + if (props.isValidConcept && !props.isValidConcept(item)) + return null; + return props.onChange( addConcept( insertValue(props.value, i, newValue), i, - copyConcept(item), + initializeConcept(item, defaults, tableConfig), ), ); - } - - if (props.isValidConcept && !props.isValidConcept(item)) - return null; - - return props.onChange( - addConcept( - insertValue(props.value, i, newValue), - i, - initializeConcept(item, defaults, tableConfig), - ), - ); - }} - > - {() => props.conceptDropzoneText} - + }} + > + {() => props.conceptDropzoneText} + + )} {props.renderRowPrefix ? props.renderRowPrefix({ From fd38fa801a26253dc5a138f0d1e158dd177aca7d Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 15 May 2023 13:36:41 +0200 Subject: [PATCH 319/679] Add query runner and start trying to submit query --- frontend/src/js/api/api.ts | 6 +- frontend/src/js/api/apiHelper.ts | 36 ++- frontend/src/js/app/reducers.ts | 2 + frontend/src/js/editor-v2/EditorV2.tsx | 251 +++++++++--------- .../src/js/editor-v2/EditorV2QueryRunner.tsx | 43 +++ frontend/src/js/editor-v2/types.ts | 4 + .../js/external-forms/FormsQueryRunner.tsx | 5 +- frontend/src/js/query-runner/QueryRunner.tsx | 10 +- frontend/src/js/query-runner/actions.ts | 13 +- .../StandardQueryRunner.tsx | 6 +- .../TimebasedQueryRunner.tsx | 2 +- 11 files changed, 241 insertions(+), 137 deletions(-) create mode 100644 frontend/src/js/editor-v2/EditorV2QueryRunner.tsx diff --git a/frontend/src/js/api/api.ts b/frontend/src/js/api/api.ts index b651aedcac..5b0a7f5836 100644 --- a/frontend/src/js/api/api.ts +++ b/frontend/src/js/api/api.ts @@ -1,5 +1,6 @@ import { useCallback } from "react"; +import { EditorV2Query } from "../editor-v2/types"; import { EntityId } from "../entity-history/reducer"; import { apiUrl } from "../environment"; import type { FormConfigT } from "../previous-queries/list/reducer"; @@ -95,7 +96,10 @@ export const usePostQueries = () => { return useCallback( ( datasetId: DatasetT["id"], - query: StandardQueryStateT | ValidatedTimebasedQueryStateT, + query: + | StandardQueryStateT + | ValidatedTimebasedQueryStateT + | EditorV2Query, options: { queryType: string; selectedSecondaryId?: string | null }, ) => api({ diff --git a/frontend/src/js/api/apiHelper.ts b/frontend/src/js/api/apiHelper.ts index 852e4185b3..0b18c8a12e 100644 --- a/frontend/src/js/api/apiHelper.ts +++ b/frontend/src/js/api/apiHelper.ts @@ -6,6 +6,7 @@ // Some keys are added (e.g. the query type attribute) import { isEmpty } from "../common/helpers/commonHelper"; import { exists } from "../common/helpers/exists"; +import { EditorV2Query, Tree } from "../editor-v2/types"; import { nodeIsConceptQueryNode } from "../model/node"; import { isLabelPristine } from "../standard-query-editor/helper"; import type { StandardQueryStateT } from "../standard-query-editor/queryReducer"; @@ -204,11 +205,42 @@ const transformTimebasedQueryToApi = (query: ValidatedTimebasedQueryStateT) => ), ); +const transformTreeToApi = (tree: Tree) => { + let dateRestriction; + if (tree.dates?.restriction) { + dateRestriction = createDateRestriction(tree.dates.restriction, tree); + } + + let negation; + if (tree.negation) { + negation = createNegation(tree); + } + + let combined; + if (dateRestriction && negation) { + combined = dateRestriction; + combined.child = negation; + } else if (dateRestriction) { + combined = dateRestriction; + } else if (negation) { + combined = negation; + } else { + combined = tree; + + tree.dates; +}; + +const transformEditorV2QueryToApi = (query: EditorV2Query) => { + if (!query.tree) return null; + + return transformTreeToApi(query.tree); +}; + // The query state already contains the query. // But small additions are made (properties allowlisted), empty things filtered out // to make it compatible with the backend API export const transformQueryToApi = ( - query: StandardQueryStateT | ValidatedTimebasedQueryStateT, + query: StandardQueryStateT | ValidatedTimebasedQueryStateT | EditorV2Query, options: { queryType: string; selectedSecondaryId?: string | null }, ) => { switch (options.queryType) { @@ -221,6 +253,8 @@ export const transformQueryToApi = ( query as StandardQueryStateT, options.selectedSecondaryId, ); + case "editorV2": + return transformEditorV2QueryToApi(query as EditorV2Query); default: return null; } diff --git a/frontend/src/js/app/reducers.ts b/frontend/src/js/app/reducers.ts index daf3f1c56e..af19ae2b33 100644 --- a/frontend/src/js/app/reducers.ts +++ b/frontend/src/js/app/reducers.ts @@ -64,6 +64,7 @@ export type StateT = { previousQueriesFolderFilter: PreviousQueriesFolderFilterStateT; preview: PreviewStateT; snackMessage: SnackMessageStateT; + editorV2QueryRunner: QueryRunnerStateT; queryEditor: { query: StandardQueryStateT; selectedSecondaryId: SelectedSecondaryIdStateT; @@ -101,6 +102,7 @@ const buildAppReducer = () => { preview, user, entityHistory, + editorV2QueryRunner: createQueryRunnerReducer("editorV2"), queryEditor: combineReducers({ query: queryReducer, selectedSecondaryId: selectedSecondaryIdsReducer, diff --git a/frontend/src/js/editor-v2/EditorV2.tsx b/frontend/src/js/editor-v2/EditorV2.tsx index 3f93c20d80..ea4c3fa014 100644 --- a/frontend/src/js/editor-v2/EditorV2.tsx +++ b/frontend/src/js/editor-v2/EditorV2.tsx @@ -20,6 +20,7 @@ import { import Dropzone from "../ui-components/Dropzone"; import { Connector, Grid } from "./EditorLayout"; +import { EditorV2QueryRunner } from "./EditorV2QueryRunner"; import { TreeNode } from "./TreeNode"; import { EDITOR_DROP_TYPES } from "./config"; import { useConnectorEditing } from "./connector-update/useConnectorRotation"; @@ -31,6 +32,13 @@ import { Tree } from "./types"; import { findNodeById, useTranslatedConnection } from "./util"; const Root = styled("div")` + flex-grow: 1; + height: 100%; + display: flex; + flex-direction: column; +`; + +const Main = styled("div")` flex-grow: 1; height: 100%; padding: 8px 10px 10px 10px; @@ -200,129 +208,132 @@ export function EditorV2({ setSelectedNodeId(undefined); }} > - {showModal && selectedNode && ( - { - updateTreeNode(selectedNode.id, (node) => { - if (!node.dates) node.dates = {}; - node.dates.excluded = excluded; - }); - }} - onResetDates={() => - updateTreeNode(selectedNode.id, (node) => { - if (!node.dates) return; - node.dates.restriction = undefined; - }) - } - setDateRange={(dateRange) => { - updateTreeNode(selectedNode.id, (node) => { - if (!node.dates) node.dates = {}; - node.dates.restriction = dateRange; - }); - }} - /> - )} - - - {featureDates && selectedNode && ( - { - e.stopPropagation(); - onOpen(); - }} - > - {t("editorV2.dates")} - - )} - {featureNegate && selectedNode && ( - { - e.stopPropagation(); - onNegateClick(); - }} - > - {t("editorV2.negate")} - - )} - {selectedNode?.children && ( - { - e.stopPropagation(); - onFlip(); - }} - > - {t("editorV2.flip")} - - )} - {featureConnectorRotate && selectedNode?.children && ( - { - e.stopPropagation(); - onRotateConnector(); - }} - > - {t("editorV2.connector")} - {connection} - - )} - {canExpand && ( - { - e.stopPropagation(); - onExpand(); - }} - > - {t("editorV2.expand")} - - )} - {selectedNode && ( - { - e.stopPropagation(); - onDelete(); - }} - > - {t("editorV2.delete")} - - )} - - - {t("editorV2.clear")} - - - - {tree ? ( - - ) : ( - { - setTree({ - id: createId(), - data: item as DragItemConceptTreeNode | DragItemQuery, +
+ {showModal && selectedNode && ( + { + updateTreeNode(selectedNode.id, (node) => { + if (!node.dates) node.dates = {}; + node.dates.excluded = excluded; + }); + }} + onResetDates={() => + updateTreeNode(selectedNode.id, (node) => { + if (!node.dates) return; + node.dates.restriction = undefined; + }) + } + setDateRange={(dateRange) => { + updateTreeNode(selectedNode.id, (node) => { + if (!node.dates) node.dates = {}; + node.dates.restriction = dateRange; }); }} - acceptedDropTypes={EDITOR_DROP_TYPES} - > - {() =>
{t("editorV2.initialDropText")}
} - + /> )} - + + + {featureDates && selectedNode && ( + { + e.stopPropagation(); + onOpen(); + }} + > + {t("editorV2.dates")} + + )} + {featureNegate && selectedNode && ( + { + e.stopPropagation(); + onNegateClick(); + }} + > + {t("editorV2.negate")} + + )} + {selectedNode?.children && ( + { + e.stopPropagation(); + onFlip(); + }} + > + {t("editorV2.flip")} + + )} + {featureConnectorRotate && selectedNode?.children && ( + { + e.stopPropagation(); + onRotateConnector(); + }} + > + {t("editorV2.connector")} + {connection} + + )} + {canExpand && ( + { + e.stopPropagation(); + onExpand(); + }} + > + {t("editorV2.expand")} + + )} + {selectedNode && ( + { + e.stopPropagation(); + onDelete(); + }} + > + {t("editorV2.delete")} + + )} + + + {t("editorV2.clear")} + + + + {tree ? ( + + ) : ( + { + setTree({ + id: createId(), + data: item as DragItemConceptTreeNode | DragItemQuery, + }); + }} + acceptedDropTypes={EDITOR_DROP_TYPES} + > + {() =>
{t("editorV2.initialDropText")}
} + + )} + +
+ ); } diff --git a/frontend/src/js/editor-v2/EditorV2QueryRunner.tsx b/frontend/src/js/editor-v2/EditorV2QueryRunner.tsx new file mode 100644 index 0000000000..432e007845 --- /dev/null +++ b/frontend/src/js/editor-v2/EditorV2QueryRunner.tsx @@ -0,0 +1,43 @@ +import { useSelector } from "react-redux"; + +import { StateT } from "../app/reducers"; +import { useDatasetId } from "../dataset/selectors"; +import QueryRunner from "../query-runner/QueryRunner"; +import { useStartQuery, useStopQuery } from "../query-runner/actions"; +import { QueryRunnerStateT } from "../query-runner/reducer"; + +import { EditorV2Query } from "./types"; + +export const EditorV2QueryRunner = ({ query }: { query: EditorV2Query }) => { + const datasetId = useDatasetId(); + const queryRunner = useSelector( + (state) => state.editorV2QueryRunner, + ); + const startStandardQuery = useStartQuery("editorV2"); + const stopStandardQuery = useStopQuery("editorV2"); + + const startQuery = () => { + if (datasetId) { + startStandardQuery(datasetId, query); + } + }; + + const queryId = queryRunner.runningQuery; + const stopQuery = () => { + if (queryId) { + stopStandardQuery(queryId); + } + }; + + const disabled = !query.tree; + + return ( + + ); +}; diff --git a/frontend/src/js/editor-v2/types.ts b/frontend/src/js/editor-v2/types.ts index 8237edc155..d4f7a82a11 100644 --- a/frontend/src/js/editor-v2/types.ts +++ b/frontend/src/js/editor-v2/types.ts @@ -22,3 +22,7 @@ export interface Tree { items: Tree[]; }; } + +export interface EditorV2Query { + tree?: Tree; +} diff --git a/frontend/src/js/external-forms/FormsQueryRunner.tsx b/frontend/src/js/external-forms/FormsQueryRunner.tsx index b79d022a09..0bbcff5a14 100644 --- a/frontend/src/js/external-forms/FormsQueryRunner.tsx +++ b/frontend/src/js/external-forms/FormsQueryRunner.tsx @@ -1,4 +1,3 @@ -import { FC } from "react"; import { useFormContext, useFormState } from "react-hook-form"; import { useSelector } from "react-redux"; @@ -37,7 +36,7 @@ const isButtonEnabled = ({ ); }; -const FormQueryRunner: FC = () => { +const FormQueryRunner = () => { const datasetId = useDatasetId(); const queryRunner = useSelector( selectQueryRunner, @@ -83,7 +82,7 @@ const FormQueryRunner: FC = () => { return ( void; stopQuery: () => void; @@ -52,7 +52,7 @@ const QueryRunner: FC = ({ stopQuery, buttonTooltip, isQueryRunning, - isButtonEnabled, + disabled, }) => { const btnAction = isQueryRunning ? stopQuery : startQuery; const isStartStopLoading = @@ -64,9 +64,9 @@ const QueryRunner: FC = ({ useHotkeys( "shift+enter", () => { - if (isButtonEnabled) btnAction(); + if (!disabled) btnAction(); }, - [isButtonEnabled, btnAction], + [disabled, btnAction], ); return ( @@ -77,7 +77,7 @@ const QueryRunner: FC = ({ onClick={btnAction} isStartStopLoading={isStartStopLoading} isQueryRunning={isQueryRunning} - disabled={!isButtonEnabled} + disabled={disabled} /> diff --git a/frontend/src/js/query-runner/actions.ts b/frontend/src/js/query-runner/actions.ts index ac51774bb9..1c713af6a1 100644 --- a/frontend/src/js/query-runner/actions.ts +++ b/frontend/src/js/query-runner/actions.ts @@ -23,6 +23,7 @@ import { errorPayload, successPayload, } from "../common/actions/genericActions"; +import { EditorV2Query } from "../editor-v2/types"; import { getExternalSupportedErrorMessage } from "../environment"; import { useLoadFormConfigs, @@ -58,7 +59,11 @@ export type QueryRunnerActions = ActionType< by sending a DELETE request for that query ID */ -export type QueryTypeT = "standard" | "timebased" | "externalForms"; +export type QueryTypeT = + | "standard" + | "editorV2" + | "timebased" + | "externalForms"; export const startQuery = createAsyncAction( "query-runners/START_QUERY_START", @@ -80,6 +85,7 @@ export const useStartQuery = (queryType: QueryTypeT) => { datasetId: DatasetT["id"], query: | StandardQueryStateT + | EditorV2Query | ValidatedTimebasedQueryStateT | FormQueryPostPayload, { @@ -96,7 +102,10 @@ export const useStartQuery = (queryType: QueryTypeT) => { : () => postQueries( datasetId, - query as StandardQueryStateT | ValidatedTimebasedQueryStateT, + query as + | StandardQueryStateT + | EditorV2Query + | ValidatedTimebasedQueryStateT, { queryType, selectedSecondaryId, diff --git a/frontend/src/js/standard-query-editor/StandardQueryRunner.tsx b/frontend/src/js/standard-query-editor/StandardQueryRunner.tsx index 56b94ce5a1..8ebc3a42bb 100644 --- a/frontend/src/js/standard-query-editor/StandardQueryRunner.tsx +++ b/frontend/src/js/standard-query-editor/StandardQueryRunner.tsx @@ -49,7 +49,7 @@ const StandardQueryRunner = () => { const isDatasetValid = validateDataset(datasetId); const hasQueryValidDates = validateQueryDates(query); const isQueryValid = validateQueryLength(query) && hasQueryValidDates; - const isQueryNotStartedOrStopped = validateQueryStartStop(queryRunner); + const queryStartStopReady = validateQueryStartStop(queryRunner); const buttonTooltip = useButtonTooltip(hasQueryValidDates); @@ -73,9 +73,7 @@ const StandardQueryRunner = () => { { return ( Date: Tue, 16 May 2023 15:42:00 +0200 Subject: [PATCH 320/679] add function to retrieve month names, move styled components inside CustomHeader.tsx, add constant for year selection span and reuse TransparentButton component --- frontend/src/js/common/helpers/dateHelper.ts | 14 ++++ .../ui-components/InputDate/CustomHeader.tsx | 65 +++++++++++-------- .../InputDate/CustomHeaderComponents.tsx | 41 ------------ 3 files changed, 51 insertions(+), 69 deletions(-) delete mode 100644 frontend/src/js/ui-components/InputDate/CustomHeaderComponents.tsx diff --git a/frontend/src/js/common/helpers/dateHelper.ts b/frontend/src/js/common/helpers/dateHelper.ts index 789f669afc..b522810f02 100644 --- a/frontend/src/js/common/helpers/dateHelper.ts +++ b/frontend/src/js/common/helpers/dateHelper.ts @@ -10,6 +10,7 @@ import { formatDistance, } from "date-fns"; import { de, enGB } from "date-fns/locale"; +import i18next from "i18next"; import { useTranslation } from "react-i18next"; // To save the date in this format in the state @@ -213,3 +214,16 @@ export function getFirstAndLastDateOfRange(dateRangeStr: string): { return { first, last }; } + +export function getMonthName(date: Date): string { + const locale = i18next.language === "de" ? de : enGB; + return format(date, "LLLL", { locale }); +} + +export function getMonthNames(): string[] { + return [...Array(12).keys()].map((month) => { + const date = new Date(); + date.setMonth(month); + return getMonthName(date); + }); +} diff --git a/frontend/src/js/ui-components/InputDate/CustomHeader.tsx b/frontend/src/js/ui-components/InputDate/CustomHeader.tsx index 5f5c1a89a2..71071a3732 100644 --- a/frontend/src/js/ui-components/InputDate/CustomHeader.tsx +++ b/frontend/src/js/ui-components/InputDate/CustomHeader.tsx @@ -1,3 +1,4 @@ +import styled from "@emotion/styled"; import { faChevronLeft, faChevronRight, @@ -7,17 +8,36 @@ import { ReactDatePickerCustomHeaderProps } from "react-datepicker"; import { SelectOptionT } from "../../api/types"; import IconButton from "../../button/IconButton"; -import { Menu } from "../InputSelect/InputSelectComponents"; +import { TransparentButton } from "../../button/TransparentButton"; +import { getMonthName, getMonthNames } from "../../common/helpers/dateHelper"; +import { List, Menu } from "../InputSelect/InputSelectComponents"; -import { - MonthYearLabel, - OptionButton, - OptionList, - Root, - SelectMenuContainer, -} from "./CustomHeaderComponents"; +export const Root = styled("div")` + display: flex; + justify-content: space-between; + align-items: center; +`; + +export const SelectMenuContainer = styled("div")` + position: absolute; + top: 40px; + left: 0; + width: 100%; +`; + +export const OptionList = styled(List)` + display: grid; + grid-template-columns: auto auto; + gap: 5px; +`; + +export const MonthYearLabel = styled("div")` + font-weight: bold; + cursor: pointer; +`; -const yearOptions: SelectOptionT[] = [...Array(10).keys()] +const yearSelectionSpan = 10; +const yearOptions: SelectOptionT[] = [...Array(yearSelectionSpan).keys()] .map((n) => new Date().getFullYear() - n) .map((year) => ({ label: String(year), @@ -25,21 +45,7 @@ const yearOptions: SelectOptionT[] = [...Array(10).keys()] })) .reverse(); -const months = [ - "Januar", - "Februar", - "März", - "April", - "Mai", - "Juni", - "Juli", - "August", - "September", - "Oktober", - "November", - "Dezember", -]; -const monthOptions: SelectOptionT[] = months.map((month, i) => ({ +const monthOptions: SelectOptionT[] = getMonthNames().map((month, i) => ({ label: month, value: i, })); @@ -56,14 +62,17 @@ const SelectMenu = ({ {options.map((option) => ( - onSelect(option.value as number)} > {option.label} - + ))} @@ -92,7 +101,7 @@ const YearMonthSelect = ({ return ( <> - {months[date.getMonth()]} {date.getFullYear()} + {getMonthName(date)} {date.getFullYear()} {yearSelectOpen && ( - active ? theme.col.blueGrayDark : "inherit"}; - color: ${({ active }) => (active ? "white" : "inherit")}; - border-radius: ${({ theme }) => theme.borderRadius}; - border: ${({ theme }) => "1px solid " + theme.col.gray}; - &:hover { - background: ${({ theme }) => theme.col.blueGrayDark}; - color: white; - } -`; - -export const MonthYearLabel = styled("div")` - font-weight: bold; - cursor: pointer; -`; From 94f74609f71654c33ecfaebb2fc86df1d8d7160a Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 16 May 2023 17:55:52 +0200 Subject: [PATCH 321/679] Add keyboard shortcut tooltips, make send query possible --- frontend/src/js/api/apiHelper.ts | 72 +++++++-- frontend/src/js/button/IconButton.tsx | 6 +- .../src/js/common/components/KeyboardKey.tsx | 16 ++ frontend/src/js/editor-v2/EditorV2.tsx | 153 ++++++++++-------- .../js/editor-v2/KeyboardShortcutTooltip.tsx | 50 ++++++ frontend/src/js/editor-v2/TreeNode.tsx | 7 +- frontend/src/js/editor-v2/config.ts | 10 ++ frontend/src/localization/de.json | 3 +- frontend/src/localization/en.json | 3 +- 9 files changed, 236 insertions(+), 84 deletions(-) create mode 100644 frontend/src/js/common/components/KeyboardKey.tsx create mode 100644 frontend/src/js/editor-v2/KeyboardShortcutTooltip.tsx diff --git a/frontend/src/js/api/apiHelper.ts b/frontend/src/js/api/apiHelper.ts index 0b18c8a12e..7db40ec3ab 100644 --- a/frontend/src/js/api/apiHelper.ts +++ b/frontend/src/js/api/apiHelper.ts @@ -115,6 +115,11 @@ const createConceptQuery = (root: T) => ({ root, }); +const createOr = (children: T) => ({ + type: "OR" as const, + children, +}); + const createAnd = (children: T) => ({ type: "AND" as const, children, @@ -205,35 +210,76 @@ const transformTimebasedQueryToApi = (query: ValidatedTimebasedQueryStateT) => ), ); -const transformTreeToApi = (tree: Tree) => { +const transformTreeToApi = (tree: Tree): unknown => { let dateRestriction; if (tree.dates?.restriction) { - dateRestriction = createDateRestriction(tree.dates.restriction, tree); + dateRestriction = createDateRestriction(tree.dates.restriction, null); } let negation; if (tree.negation) { - negation = createNegation(tree); + negation = createNegation(null); + } + + let node; + if (!tree.children) { + if (!tree.data) { + throw new Error( + "Tree has no children and no data, this shouldn't happen.", + ); + } + + node = nodeIsConceptQueryNode(tree.data) + ? createQueryConcept(tree.data) + : createSavedQuery(tree.data.id); + } else { + switch (tree.children.connection) { + case "and": + node = createAnd(tree.children.items.map(transformTreeToApi)); + break; + case "or": + node = createOr(tree.children.items.map(transformTreeToApi)); + break; + case "before": + node = { + type: "BEFORE", + // TODO: + // ...days, + preceding: { + sampler: "EARLIEST", + child: transformTreeToApi(tree.children.items[0]), + }, + index: { + sampler: "EARLIEST", + child: transformTreeToApi(tree.children.items[1]), + }, + }; + break; + } } - let combined; if (dateRestriction && negation) { - combined = dateRestriction; - combined.child = negation; + return { + ...dateRestriction, + child: { + ...negation, + child: node, + }, + }; } else if (dateRestriction) { - combined = dateRestriction; - } else if (negation) { - combined = negation; + return { + ...dateRestriction, + child: node, + }; } else { - combined = tree; - - tree.dates; + return node; + } }; const transformEditorV2QueryToApi = (query: EditorV2Query) => { if (!query.tree) return null; - return transformTreeToApi(query.tree); + return createConceptQuery(transformTreeToApi(query.tree)); }; // The query state already contains the query. diff --git a/frontend/src/js/button/IconButton.tsx b/frontend/src/js/button/IconButton.tsx index 7e333f8f37..2de24b10fb 100644 --- a/frontend/src/js/button/IconButton.tsx +++ b/frontend/src/js/button/IconButton.tsx @@ -45,12 +45,12 @@ const SxBasicButton = styled(BasicButton)<{ }>` background-color: transparent; color: ${({ theme, active, secondary, red }) => - active + red + ? theme.col.red + : active ? theme.col.blueGrayDark : secondary ? theme.col.orange - : red - ? theme.col.red : theme.col.black}; opacity: ${({ frame }) => (frame ? 1 : 0.75)}; transition: opacity ${({ theme }) => theme.transitionTime}, diff --git a/frontend/src/js/common/components/KeyboardKey.tsx b/frontend/src/js/common/components/KeyboardKey.tsx new file mode 100644 index 0000000000..ba19a4788a --- /dev/null +++ b/frontend/src/js/common/components/KeyboardKey.tsx @@ -0,0 +1,16 @@ +import styled from "@emotion/styled"; +import { ReactNode } from "react"; + +const KeyShape = styled("kbd")` + padding: 2px 4px; + border: 1px solid ${({ theme }) => theme.col.grayLight}; + box-shadow: 0 0 3px 0 ${({ theme }) => theme.col.grayLight}; + font-size: ${({ theme }) => theme.font.xs}; + line-height: 1; + border-radius: ${({ theme }) => theme.borderRadius}; + text-transform: uppercase; +`; + +export const KeyboardKey = ({ children }: { children: ReactNode }) => ( + {children} +); diff --git a/frontend/src/js/editor-v2/EditorV2.tsx b/frontend/src/js/editor-v2/EditorV2.tsx index ea4c3fa014..bcfcbeee24 100644 --- a/frontend/src/js/editor-v2/EditorV2.tsx +++ b/frontend/src/js/editor-v2/EditorV2.tsx @@ -21,8 +21,9 @@ import Dropzone from "../ui-components/Dropzone"; import { Connector, Grid } from "./EditorLayout"; import { EditorV2QueryRunner } from "./EditorV2QueryRunner"; +import { KeyboardShortcutTooltip } from "./KeyboardShortcutTooltip"; import { TreeNode } from "./TreeNode"; -import { EDITOR_DROP_TYPES } from "./config"; +import { EDITOR_DROP_TYPES, HOTKEYS } from "./config"; import { useConnectorEditing } from "./connector-update/useConnectorRotation"; import { DateModal } from "./date-restriction/DateModal"; import { useDateEditing } from "./date-restriction/useDateEditing"; @@ -81,9 +82,9 @@ const useEditorState = () => { return findNodeById(tree, selectedNodeId); }, [tree, selectedNodeId]); - const onReset = () => { + const onReset = useCallback(() => { setTree(undefined); - }; + }, []); const updateTreeNode = useCallback( (id: string, update: (node: Tree) => void) => { @@ -164,9 +165,10 @@ export function EditorV2({ } }, [selectedNode, setTree, updateTreeNode]); - useHotkeys("del", onDelete, [onDelete]); - useHotkeys("backspace", onDelete, [onDelete]); - useHotkeys("f", onFlip, [onFlip]); + useHotkeys(HOTKEYS.delete[0].keyname, onDelete, [onDelete]); + useHotkeys(HOTKEYS.delete[1].keyname, onDelete, [onDelete]); + useHotkeys(HOTKEYS.flip.keyname, onFlip, [onFlip]); + useHotkeys(HOTKEYS.reset.keyname, onReset, [onReset]); const { canExpand, onExpand } = useExpandQuery({ enabled: featureExpand, @@ -238,76 +240,101 @@ export function EditorV2({ {featureDates && selectedNode && ( - { - e.stopPropagation(); - onOpen(); - }} - > - {t("editorV2.dates")} - + + { + e.stopPropagation(); + onOpen(); + }} + > + {t("editorV2.dates")} + + )} {featureNegate && selectedNode && ( - { - e.stopPropagation(); - onNegateClick(); - }} - > - {t("editorV2.negate")} - + + { + e.stopPropagation(); + onNegateClick(); + }} + > + {t("editorV2.negate")} + + )} {selectedNode?.children && ( - { - e.stopPropagation(); - onFlip(); - }} - > - {t("editorV2.flip")} - + + { + e.stopPropagation(); + onFlip(); + }} + > + {t("editorV2.flip")} + + )} {featureConnectorRotate && selectedNode?.children && ( - { - e.stopPropagation(); - onRotateConnector(); - }} + - {t("editorV2.connector")} - {connection} - + { + e.stopPropagation(); + onRotateConnector(); + }} + > + {t("editorV2.connector")} + {connection} + + )} {canExpand && ( - { - e.stopPropagation(); - onExpand(); - }} - > - {t("editorV2.expand")} - + + { + e.stopPropagation(); + onExpand(); + }} + > + {t("editorV2.expand")} + + )} {selectedNode && ( - { - e.stopPropagation(); - onDelete(); - }} - > - {t("editorV2.delete")} - + + { + e.stopPropagation(); + onDelete(); + }} + > + {t("editorV2.delete")} + + )} - - {t("editorV2.clear")} - + + + {t("editorV2.clear")} + + {tree ? ( diff --git a/frontend/src/js/editor-v2/KeyboardShortcutTooltip.tsx b/frontend/src/js/editor-v2/KeyboardShortcutTooltip.tsx new file mode 100644 index 0000000000..8d7124de45 --- /dev/null +++ b/frontend/src/js/editor-v2/KeyboardShortcutTooltip.tsx @@ -0,0 +1,50 @@ +import styled from "@emotion/styled"; +import { ReactElement } from "react"; +import { useTranslation } from "react-i18next"; + +import { KeyboardKey } from "../common/components/KeyboardKey"; +import WithTooltip from "../tooltip/WithTooltip"; + +const KeyTooltip = styled("div")` + padding: 8px 15px; + display: flex; + align-items: center; + gap: 5px; +`; + +const Keys = styled("div")` + display: flex; + align-items: center; + gap: 2px; +`; + +export const KeyboardShortcutTooltip = ({ + keyname, + children, +}: { + keyname: string; + children: ReactElement; +}) => { + const { t } = useTranslation(); + const keynames = keyname.split("+"); + + return ( + + {t("common.shortcut")}:{" "} + + {keynames.map((keyPart, i) => ( + <> + {keyPart} + {i < keynames.length - 1 && "+"} + + ))} + + + } + > + {children} + + ); +}; diff --git a/frontend/src/js/editor-v2/TreeNode.tsx b/frontend/src/js/editor-v2/TreeNode.tsx index 6838f1dc7a..dda372fc5b 100644 --- a/frontend/src/js/editor-v2/TreeNode.tsx +++ b/frontend/src/js/editor-v2/TreeNode.tsx @@ -1,6 +1,6 @@ import styled from "@emotion/styled"; import { createId } from "@paralleldrive/cuid2"; -import { memo, useMemo } from "react"; +import { memo } from "react"; import { useTranslation } from "react-i18next"; import { DNDType } from "../common/constants/dndTypes"; @@ -24,7 +24,7 @@ const Node = styled("div")<{ negated?: boolean; leaf?: boolean; }>` - padding: ${({ leaf }) => (leaf ? "8px 10px" : "12px")}; + padding: ${({ leaf }) => (leaf ? "8px 10px" : "20px")}; border: 1px solid ${({ negated, theme, selected }) => negated ? "red" : selected ? theme.col.gray : theme.col.grayMediumLight}; @@ -32,7 +32,7 @@ const Node = styled("div")<{ selected ? `inset 0px 0px 0px 1px ${theme.col.gray}` : "none"}; border-radius: ${({ theme }) => theme.borderRadius}; - width: ${({ leaf }) => (leaf ? "150px" : "inherit")}; + width: ${({ leaf }) => (leaf ? "180px" : "inherit")}; background-color: ${({ leaf, theme }) => (leaf ? "white" : theme.col.bg)}; cursor: pointer; display: flex; @@ -166,6 +166,7 @@ export function TreeNode({ node.id = newParentId; node.data = undefined; node.dates = undefined; + node.negation = false; const connection = treeParent?.children?.connection || tree.children?.connection; diff --git a/frontend/src/js/editor-v2/config.ts b/frontend/src/js/editor-v2/config.ts index 5dd3382b72..7fd22a71a3 100644 --- a/frontend/src/js/editor-v2/config.ts +++ b/frontend/src/js/editor-v2/config.ts @@ -5,3 +5,13 @@ export const EDITOR_DROP_TYPES = [ DNDType.PREVIOUS_QUERY, DNDType.PREVIOUS_SECONDARY_ID_QUERY, ]; + +export const HOTKEYS = { + expand: { keyname: "x" }, + negate: { keyname: "n" }, + editDates: { keyname: "d" }, + delete: [{ keyname: "backspace" }, { keyname: "del" }], + flip: { keyname: "f" }, + rotateConnector: { keyname: "c" }, + reset: { keyname: "shift+backspace" }, +}; diff --git a/frontend/src/localization/de.json b/frontend/src/localization/de.json index 83d55cd51c..669d114f28 100644 --- a/frontend/src/localization/de.json +++ b/frontend/src/localization/de.json @@ -170,7 +170,8 @@ "dateInvalid": "Ungültiges Datum", "missingLabel": "Unbenannt", "import": "Importieren", - "openFileDialog": "Datei auswählen" + "openFileDialog": "Datei auswählen", + "shortcut": "Kürzel" }, "tooltip": { "headline": "Info", diff --git a/frontend/src/localization/en.json b/frontend/src/localization/en.json index fa8c4deac6..4c79979470 100644 --- a/frontend/src/localization/en.json +++ b/frontend/src/localization/en.json @@ -170,7 +170,8 @@ "dateInvalid": "Invalid date", "missingLabel": "Unknown", "import": "Import", - "openFileDialog": "Select file" + "openFileDialog": "Select file", + "shortcut": "Key" }, "tooltip": { "headline": "Info", From 79015c2e3ebdb62ac09e9247d749a98890e753c7 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 17 May 2023 11:51:45 +0200 Subject: [PATCH 322/679] set prettyPrint=false for EntityPreviewExecution output (#3072) --- .../conquery/models/query/preview/EntityPreviewExecution.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java index 5963941685..6010af221a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewExecution.java @@ -268,7 +268,7 @@ public FullExecutionStatus buildStatusFull(Subject subject) { setStatusFull(status, subject); status.setQuery(getValuesQuery().getQuery()); - final PrintSettings printSettings = new PrintSettings(true, I18n.LOCALE.get(), getNamespace(), getConfig(), null, previewConfig::resolveSelectLabel); + final PrintSettings printSettings = new PrintSettings(false, I18n.LOCALE.get(), getNamespace(), getConfig(), null, previewConfig::resolveSelectLabel); status.setInfos(transformQueryResultToInfos(getInfoCardExecution(), printSettings)); From 7b23878a3e55b5ddf931add4a0103f86410f9ca2 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 17 May 2023 13:32:28 +0200 Subject: [PATCH 323/679] Test build images in ci (#3073) * Cleanup of unsued files * adds tasks testing building of frontend and backend container --- .../workflows/test_build_backend_image.yml | 45 +++ .../workflows/test_build_frontend_image.yml | 40 +++ .restyled.yaml | 19 -- checkstyle.xml | 163 --------- heroku.yml | 4 - utilities/eclipse.cleanup.xml | 61 ---- utilities/eclipse.formatter.xml | 315 ------------------ utilities/intellij.formatter.xml | 58 ---- 8 files changed, 85 insertions(+), 620 deletions(-) create mode 100644 .github/workflows/test_build_backend_image.yml create mode 100644 .github/workflows/test_build_frontend_image.yml delete mode 100644 .restyled.yaml delete mode 100644 checkstyle.xml delete mode 100644 heroku.yml delete mode 100644 utilities/eclipse.cleanup.xml delete mode 100644 utilities/eclipse.formatter.xml delete mode 100644 utilities/intellij.formatter.xml diff --git a/.github/workflows/test_build_backend_image.yml b/.github/workflows/test_build_backend_image.yml new file mode 100644 index 0000000000..d7991ae1e5 --- /dev/null +++ b/.github/workflows/test_build_backend_image.yml @@ -0,0 +1,45 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Test building backend image + +on: + pull_request: + types: [opened, reopened, synchronize] + branches: + - develop + - master + paths: + - 'backend/**' + - 'executable/**' + - 'Dockerfile' + - 'pom.xml' + - 'lombok.config' + - 'scripts/**' + - '.github/workflows/test_build_backend_image.yml' + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}-backend + +jobs: + test-build-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + # Fetch all tags + fetch-depth: 0 + + - name: Build docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + push: false diff --git a/.github/workflows/test_build_frontend_image.yml b/.github/workflows/test_build_frontend_image.yml new file mode 100644 index 0000000000..53f901f2f5 --- /dev/null +++ b/.github/workflows/test_build_frontend_image.yml @@ -0,0 +1,40 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Test building frontend image + +on: + pull_request: + types: [opened, reopened, synchronize] + branches: + - develop + - master + paths: + - 'frontend/*' + - 'scripts/*' + - '.github/workflows/test_build_frontend_image.yml' +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}-frontend + +jobs: + test-build-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + # Fetch all tags + fetch-depth: 0 + + - name: Build docker image + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: ./frontend + push: false diff --git a/.restyled.yaml b/.restyled.yaml deleted file mode 100644 index 8c27fc9961..0000000000 --- a/.restyled.yaml +++ /dev/null @@ -1,19 +0,0 @@ -enabled: false -#auto: false -#pull_requests: true -#comments: true -request_review: author -labels: ["code-style"] - -restylers: - # - astyle: - # include: - # - "**/*.java" - # arguments: - # - --style=java - # - --indent=force-tab=4 - # - --lineend=linux - # - --indent-after-parens - # - --break-closing-braces - # - --break-one-line-headers - # - --add-braces \ No newline at end of file diff --git a/checkstyle.xml b/checkstyle.xml deleted file mode 100644 index 843c435536..0000000000 --- a/checkstyle.xml +++ /dev/null @@ -1,163 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/heroku.yml b/heroku.yml deleted file mode 100644 index f38927c6d9..0000000000 --- a/heroku.yml +++ /dev/null @@ -1,4 +0,0 @@ -#deploys develop pushes to https://conquery-dev.herokuapp.com/app/static and master to https://conquery.herokuapp.com/app/static -build: - docker: - web: frontend/Dockerfile diff --git a/utilities/eclipse.cleanup.xml b/utilities/eclipse.cleanup.xml deleted file mode 100644 index ab7122abc5..0000000000 --- a/utilities/eclipse.cleanup.xml +++ /dev/null @@ -1,61 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/utilities/eclipse.formatter.xml b/utilities/eclipse.formatter.xml deleted file mode 100644 index 280d2afdef..0000000000 --- a/utilities/eclipse.formatter.xml +++ /dev/null @@ -1,315 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/utilities/intellij.formatter.xml b/utilities/intellij.formatter.xml deleted file mode 100644 index a106c86d86..0000000000 --- a/utilities/intellij.formatter.xml +++ /dev/null @@ -1,58 +0,0 @@ - - \ No newline at end of file From b6bb51ecb6e1ce193123932ce5e538cb1b3bf430 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 17 May 2023 13:39:15 +0200 Subject: [PATCH 324/679] adds `ConceptColumnT` to ConceptColumnSelect if asIds=true --- .../select/concept/ConceptColumnSelect.java | 16 ++++++++++++++++ .../value/ConceptElementsAggregator.java | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java index e20a546551..9cdcccbd74 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java @@ -1,5 +1,8 @@ package com.bakdata.conquery.models.datasets.concepts.select.concept; +import java.util.Set; + +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.models.datasets.concepts.select.Select; @@ -7,6 +10,8 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.ConceptElementsAggregator; import com.bakdata.conquery.models.query.queryplan.aggregators.specific.value.ConceptValuesAggregator; +import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; +import com.bakdata.conquery.models.types.SemanticType; import com.fasterxml.jackson.annotation.JsonIgnore; import io.dropwizard.validation.ValidationMethod; import lombok.Data; @@ -32,6 +37,17 @@ public Aggregator createAggregator() { return new ConceptValuesAggregator(((TreeConcept) getHolder().findConcept())); } + @Override + public SelectResultInfo getResultInfo(CQConcept cqConcept) { + Set additionalSemantics = null; + + if (isAsIds()) { + additionalSemantics = Set.of(new SemanticType.ConceptColumnT(cqConcept.getConcept())); + } + + return new SelectResultInfo(this, cqConcept, additionalSemantics); + } + @JsonIgnore @ValidationMethod(message = "Holder must be TreeConcept.") public boolean isHolderTreeConcept() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptElementsAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptElementsAggregator.java index e8118b950f..311e77c384 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptElementsAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/ConceptElementsAggregator.java @@ -49,7 +49,7 @@ public void collectRequiredTables(Set requiredTables) { @Override public void nextTable(QueryExecutionContext ctx, Table currentTable) { - Connector connector = tableConnectors.get(currentTable); + final Connector connector = tableConnectors.get(currentTable); if (connector == null) { column = null; From b1369a0f565554b264b9a422c88deb35678193ef Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 17 May 2023 15:28:19 +0200 Subject: [PATCH 325/679] split JobManagerStatus by datasets for better display --- .../bakdata/conquery/commands/ShardNode.java | 26 ++-- .../conquery/models/jobs/ImportJob.java | 2 +- .../conquery/models/jobs/JobManager.java | 14 +- .../models/jobs/JobManagerStatus.java | 32 +++-- .../specific/UpdateJobManagerStatus.java | 26 ++-- .../models/worker/ShardNodeInformation.java | 47 +++++-- .../resources/admin/rest/AdminProcessor.java | 128 ++++++++---------- .../resources/admin/rest/AdminResource.java | 4 +- .../conquery/resources/admin/ui/jobs.html.ftl | 61 +++++---- 9 files changed, 178 insertions(+), 162 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java index cde4efe2c8..a2bb9e62a2 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ShardNode.java @@ -314,23 +314,27 @@ private void reportJobManagerStatus() { // Collect the ShardNode and all its workers jobs into a single queue - final JobManagerStatus jobManagerStatus = jobManager.reportStatus(); for (Worker worker : workers.getWorkers().values()) { - jobManagerStatus.getJobs().addAll(worker.getJobManager().reportStatus().getJobs()); - } - + final JobManagerStatus jobManagerStatus = new JobManagerStatus( + null, worker.getInfo().getDataset(), + worker.getJobManager().getJobStatus() + ); - try { - context.trySend(new UpdateJobManagerStatus(jobManagerStatus)); - } - catch (Exception e) { - log.warn("Failed to report job manager status", e); + try { + context.trySend(new UpdateJobManagerStatus(jobManagerStatus)); + } + catch (Exception e) { + log.warn("Failed to report job manager status", e); - if (config.isFailOnError()) { - System.exit(1); + if (config.isFailOnError()) { + System.exit(1); + } } } + + + } public boolean isBusy() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/ImportJob.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/ImportJob.java index d02838e864..3e9aa63cbe 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/ImportJob.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/ImportJob.java @@ -350,7 +350,7 @@ private Map> sendBuckets(Map starts, M private void awaitFreeJobQueue(WorkerInformation responsibleWorker) { try { - responsibleWorker.getConnectedShardNode().waitForFreeJobqueue(); + responsibleWorker.getConnectedShardNode().waitForFreeJobQueue(); } catch (InterruptedException e) { log.error("Interrupted while waiting for worker[{}] to have free space in queue", responsibleWorker, e); diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManager.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManager.java index 1783db1ae2..b9be29b718 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManager.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManager.java @@ -38,14 +38,12 @@ public void addFastJob(Job job) { fastExecutor.add(job); } - public JobManagerStatus reportStatus() { - - return new JobManagerStatus( - getSlowJobs() - .stream() - .map(job -> new JobStatus(job.getJobId(), job.getProgressReporter().getProgress(), job.getLabel(), job.isCancelled())) - .collect(Collectors.toList()) - ); + public List getJobStatus() { + return getSlowJobs().stream() + .map(job -> new JobStatus(job.getJobId(), job.getProgressReporter().getProgress(), job.getLabel(), job.isCancelled())) + .sorted() + .collect(Collectors.toList()); + } public List getSlowJobs() { diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManagerStatus.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManagerStatus.java index b2524f9b01..4c6329b4e4 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManagerStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManagerStatus.java @@ -3,29 +3,36 @@ import java.time.Duration; import java.time.LocalDateTime; import java.util.Collection; -import java.util.SortedSet; -import java.util.TreeSet; +import javax.annotation.Nullable; import javax.validation.constraints.NotNull; +import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Data; -import lombok.NonNull; +import lombok.EqualsAndHashCode; import lombok.RequiredArgsConstructor; +import lombok.With; +import org.apache.commons.lang3.time.DurationFormatUtils; @Data @RequiredArgsConstructor(onConstructor_ = @JsonCreator) public class JobManagerStatus { - @NonNull + @With + @Nullable + private final String origin; + @Nullable + private final DatasetId dataset; @NotNull - private final LocalDateTime timestamp = LocalDateTime.now(); + @EqualsAndHashCode.Exclude + private final LocalDateTime timestamp; @NotNull - private final SortedSet jobs = new TreeSet<>(); - - public JobManagerStatus(Collection jobs) { - this.jobs.addAll(jobs); + @EqualsAndHashCode.Exclude + private final Collection jobs; + public JobManagerStatus(String origin, DatasetId dataset, Collection statuses) { + this(origin, dataset, LocalDateTime.now(), statuses); } public int size() { @@ -35,11 +42,8 @@ public int size() { // Used in AdminUIResource/jobs @JsonIgnore public String getAgeString() { - Duration duration = Duration.between(timestamp, LocalDateTime.now()); + final Duration duration = Duration.between(timestamp, LocalDateTime.now()); - if (duration.toSeconds() > 0) { - return Long.toString(duration.toSeconds()) + " s"; - } - return Long.toString(duration.toMillis()) + " ms"; + return DurationFormatUtils.formatDurationWords(duration.toMillis(), true, true); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/UpdateJobManagerStatus.java b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/UpdateJobManagerStatus.java index 6668b6d446..86f50b302c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/UpdateJobManagerStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/UpdateJobManagerStatus.java @@ -8,31 +8,27 @@ import com.bakdata.conquery.models.messages.network.NetworkMessage; import com.bakdata.conquery.models.messages.network.NetworkMessageContext.ManagerNodeNetworkContext; import com.bakdata.conquery.models.worker.ShardNodeInformation; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.ToString; +import lombok.Data; import lombok.extern.slf4j.Slf4j; -@CPSType(id="UPDATE_JOB_MANAGER_STATUS", base=NetworkMessage.class) -@NoArgsConstructor @AllArgsConstructor @Getter @Setter @ToString(of = "status") +@CPSType(id = "UPDATE_JOB_MANAGER_STATUS", base = NetworkMessage.class) @Slf4j +@Data public class UpdateJobManagerStatus extends MessageToManagerNode { @NotNull - private JobManagerStatus status; + private final JobManagerStatus status; @Override public void react(ManagerNodeNetworkContext context) throws Exception { - ShardNodeInformation node = context.getNamespaces() - .getShardNodes() - .get(context.getRemoteAddress()); + final ShardNodeInformation node = context.getNamespaces() + .getShardNodes() + .get(context.getRemoteAddress()); if (node == null) { - log.error("Could not find ShardNode {}, I only know of {}", context.getRemoteAddress(), context.getNamespaces().getShardNodes().keySet()); - } - else { - node.setJobManagerStatus(status); + log.error("Could not find ShardNode `{}`, I only know of {}", context.getRemoteAddress(), context.getNamespaces().getShardNodes().keySet()); + return; } + // The shards don't know their own name so we attach it here + node.addJobManagerStatus(status.withOrigin(context.getRemoteAddress().toString())); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java b/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java index ff1b27d619..e369c9819b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java @@ -2,6 +2,8 @@ import java.time.LocalDateTime; import java.time.temporal.ChronoUnit; +import java.util.HashSet; +import java.util.Set; import com.bakdata.conquery.io.mina.MessageSender; import com.bakdata.conquery.io.mina.NetworkSession; @@ -20,7 +22,11 @@ public class ShardNodeInformation extends MessageSender.Simple jobManagerStatus = new HashSet<>(); - /** - * Used to await/notify for full job-queues. - */ - @JsonIgnore - private final transient Object jobManagerSync = new Object(); + private LocalDateTime lastStatusTime = LocalDateTime.now(); public ShardNodeInformation(NetworkSession session, int backpressure) { super(session); @@ -55,7 +57,11 @@ private String getLatenessMetricName() { * Calculate the time in Milliseconds since we last received a {@link JobManagerStatus} from the corresponding shard. */ private long getMillisSinceLastStatus() { - return getJobManagerStatus().getTimestamp().until(LocalDateTime.now(), ChronoUnit.MILLIS); + if(getJobManagerStatus().isEmpty()){ + return -1; + } + + return lastStatusTime.until(LocalDateTime.now(), ChronoUnit.MILLIS); } @Override @@ -65,17 +71,32 @@ public void awaitClose() { SharedMetricRegistries.getDefault().remove(getLatenessMetricName()); } - public void setJobManagerStatus(JobManagerStatus status) { - jobManagerStatus = status; - if (status.size() < backpressure) { + public long calculatePressure() { + return jobManagerStatus.stream().mapToLong(status -> status.getJobs().size()).sum(); + } + + public void addJobManagerStatus(JobManagerStatus incoming) { + lastStatusTime = LocalDateTime.now(); + + synchronized (jobManagerStatus) { + // replace with new status + jobManagerStatus.remove(incoming); + jobManagerStatus.add(incoming); + } + + if (calculatePressure() < backpressure) { synchronized (jobManagerSync) { jobManagerSync.notifyAll(); } } } - public void waitForFreeJobqueue() throws InterruptedException { - if (jobManagerStatus.size() >= backpressure) { + public void waitForFreeJobQueue() throws InterruptedException { + if (jobManagerStatus.isEmpty()) { + return; + } + + if (calculatePressure() >= backpressure) { log.trace("Have to wait for free JobQueue (size = {})", jobManagerStatus.size()); synchronized (jobManagerSync) { jobManagerSync.wait(); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminProcessor.java index 2aa0277602..76261723dd 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminProcessor.java @@ -4,7 +4,6 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.Objects; import java.util.SortedSet; import java.util.TreeSet; import java.util.concurrent.ScheduledExecutorService; @@ -28,10 +27,10 @@ import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.models.jobs.JobManagerStatus; import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.models.worker.ShardNodeInformation; import com.bakdata.conquery.util.ConqueryEscape; import com.fasterxml.jackson.databind.ObjectWriter; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.Multimap; import com.univocity.parsers.csv.CsvWriter; import groovy.lang.GroovyShell; @@ -58,14 +57,6 @@ public class AdminProcessor { private final Validator validator; private final ObjectWriter jsonWriter = Jackson.MAPPER.writer(); - - - public synchronized void addRole(Role role) throws JSONException { - ValidatorHelper.failOnError(log, validator.validate(role)); - log.trace("New role:\tLabel: {}\tName: {}\tId: {} ", role.getLabel(), role.getName(), role.getId()); - storage.addRole(role); - } - public void addRoles(List roles) { for (Role role : roles) { @@ -78,6 +69,12 @@ public void addRoles(List roles) { } } + public synchronized void addRole(Role role) throws JSONException { + ValidatorHelper.failOnError(log, validator.validate(role)); + log.trace("New role:\tLabel: {}\tName: {}\tId: {} ", role.getLabel(), role.getName(), role.getId()); + storage.addRole(role); + } + /** * Deletes the mandator, that is identified by the id. Its references are * removed from the users, the groups, and from the storage. @@ -106,7 +103,7 @@ public SortedSet getAllRoles() { /** * Handles creation of permissions. * - * @param owner to which the permission is assigned + * @param owner to which the permission is assigned * @param permission The permission to create. * @throws JSONException is thrown upon processing JSONs. */ @@ -117,8 +114,7 @@ public void createPermission(PermissionOwner owner, ConqueryPermission permis /** * Handles deletion of permissions. * - * - * @param owner the owner of the permission + * @param owner the owner of the permission * @param permission The permission to delete. */ public void deletePermission(PermissionOwner owner, ConqueryPermission permission) { @@ -138,11 +134,6 @@ public synchronized void deleteUser(User user) { log.trace("Removed user {} from the storage.", user.getId()); } - public void addUser(User user) { - storage.addUser(user); - log.trace("New user:\tLabel: {}\tName: {}\tId: {} ", user.getLabel(), user.getName(), user.getId()); - } - public void addUsers(List users) { for (User user : users) { @@ -155,15 +146,13 @@ public void addUsers(List users) { } } - public TreeSet getAllGroups() { - return new TreeSet<>(storage.getAllGroups()); + public void addUser(User user) { + storage.addUser(user); + log.trace("New user:\tLabel: {}\tName: {}\tId: {} ", user.getLabel(), user.getName(), user.getId()); } - public synchronized void addGroup(Group group) throws JSONException { - ValidatorHelper.failOnError(log, validator.validate(group)); - storage.addGroup(group); - log.trace("New group:\tLabel: {}\tName: {}\tId: {} ", group.getLabel(), group.getName(), group.getId()); - + public TreeSet getAllGroups() { + return new TreeSet<>(storage.getAllGroups()); } public void addGroups(List groups) { @@ -178,6 +167,13 @@ public void addGroups(List groups) { } } + public synchronized void addGroup(Group group) throws JSONException { + ValidatorHelper.failOnError(log, validator.validate(group)); + storage.addGroup(group); + log.trace("New group:\tLabel: {}\tName: {}\tId: {} ", group.getLabel(), group.getName(), group.getId()); + + } + public void addUserToGroup(Group group, User user) { group.addMember(user); log.trace("Added user {} to group {}", user, group); @@ -193,12 +189,12 @@ public void deleteGroup(Group group) { log.trace("Removed group {}", group); } - public void deleteRoleFrom(RoleOwner owner, Role role) { + public void deleteRoleFrom(RoleOwner owner, Role role) { owner.removeRole(role); log.trace("Removed role {} from {}", role, owner); } - public void addRoleTo(RoleOwner owner, Role role) { + public void addRoleTo(RoleOwner owner, Role role) { owner.addRole(role); log.trace("Added role {} to {}", role, owner); } @@ -210,23 +206,15 @@ public String getPermissionOverviewAsCSV() { return getPermissionOverviewAsCSV(storage.getAllUsers()); } - - /** - * Renders the permission overview for all users in a certain {@link Group} in form of a CSV. - */ - public String getPermissionOverviewAsCSV(Group group) { - return getPermissionOverviewAsCSV(group.getMembers().stream().map(storage::getUser).collect(Collectors.toList())); - } - /** * Renders the permission overview for certain {@link User} in form of a CSV. */ public String getPermissionOverviewAsCSV(Collection users) { - StringWriter sWriter = new StringWriter(); - CsvWriter writer = config.getCsv().createWriter(sWriter); - List scope = config - .getAuthorizationRealms() - .getOverviewScope(); + final StringWriter sWriter = new StringWriter(); + final CsvWriter writer = config.getCsv().createWriter(sWriter); + final List scope = config + .getAuthorizationRealms() + .getOverviewScope(); // Header writeAuthOverviewHeader(writer, scope); // Body @@ -240,7 +228,7 @@ public String getPermissionOverviewAsCSV(Collection users) { * Writes the header of the CSV auth overview to the specified writer. */ private static void writeAuthOverviewHeader(CsvWriter writer, List scope) { - List headers = new ArrayList<>(); + final List headers = new ArrayList<>(); headers.add("User"); headers.addAll(scope); writer.writeHeaders(headers); @@ -254,7 +242,7 @@ private static void writeAuthOverviewUser(CsvWriter writer, List scope, writer.addValue(String.format("%s %s", user.getLabel(), ConqueryEscape.unescape(user.getName()))); // Print the permission per domain in the remaining columns - Multimap permissions = AuthorizationHelper.getEffectiveUserPermissions(user, scope, storage); + final Multimap permissions = AuthorizationHelper.getEffectiveUserPermissions(user, scope, storage); for (String domain : scope) { writer.addValue(permissions.get(domain).stream() .map(Object::toString) @@ -262,41 +250,45 @@ private static void writeAuthOverviewUser(CsvWriter writer, List scope, } writer.writeValuesToRow(); } - public ImmutableMap getJobs() { - return ImmutableMap.builder() - .put("ManagerNode", getJobManager().reportStatus()) - // Namespace JobManagers on ManagerNode - .putAll( - getDatasetRegistry().getDatasets().stream() - .collect(Collectors.toMap( - ns -> String.format("ManagerNode::%s", ns.getDataset().getId()), - ns -> ns.getJobManager().reportStatus() - ))) - // Remote Worker JobManagers - .putAll( - getDatasetRegistry() - .getShardNodes() - .values() - .stream() - .collect(Collectors.toMap( - si -> Objects.toString(si.getRemoteAddress()), - ShardNodeInformation::getJobManagerStatus - )) - ) - .build(); + + /** + * Renders the permission overview for all users in a certain {@link Group} in form of a CSV. + */ + public String getPermissionOverviewAsCSV(Group group) { + return getPermissionOverviewAsCSV(group.getMembers().stream().map(storage::getUser).collect(Collectors.toList())); } public boolean isBusy() { //Note that this does not and cannot check for fast jobs! - return getJobs().values().stream() + return getJobs().stream() .map(JobManagerStatus::getJobs) .anyMatch(Predicate.not(Collection::isEmpty)); } + public Collection getJobs() { + final List out = new ArrayList<>(); + + out.add(new JobManagerStatus("Manager", null, getJobManager().getJobStatus())); + + for (Namespace namespace : getDatasetRegistry().getDatasets()) { + out.add(new JobManagerStatus( + "Manager", namespace.getDataset().getId(), + namespace.getJobManager().getJobStatus() + )); + } + + for (ShardNodeInformation si : getDatasetRegistry().getShardNodes().values()) { + out.addAll(si.getJobManagerStatus()); + } + + return out; + } public Object executeScript(String script) { - CompilerConfiguration config = new CompilerConfiguration(); - GroovyShell groovy = new GroovyShell(config); + + final CompilerConfiguration config = new CompilerConfiguration(); + final GroovyShell groovy = new GroovyShell(config); + groovy.setProperty("datasetRegistry", getDatasetRegistry()); groovy.setProperty("jobManager", getJobManager()); groovy.setProperty("config", getConfig()); @@ -305,7 +297,7 @@ public Object executeScript(String script) { try { return groovy.evaluate(script); } - catch(Exception e) { + catch (Exception e) { return ExceptionUtils.getStackTrace(e); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java index dec6699bb0..479e0e8751 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java @@ -3,6 +3,7 @@ import static com.bakdata.conquery.resources.ResourceConstants.JOB_ID; import java.time.LocalDate; +import java.util.Collection; import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; @@ -35,7 +36,6 @@ import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.ShardNodeInformation; import com.bakdata.conquery.resources.admin.ui.AdminUIResource; -import com.google.common.collect.ImmutableMap; import io.dropwizard.auth.Auth; import lombok.RequiredArgsConstructor; @@ -89,7 +89,7 @@ public Response cancelJob(@PathParam(JOB_ID) UUID jobId) { @GET @Path("/jobs/") - public ImmutableMap getJobs() { + public Collection getJobs() { return processor.getJobs(); } diff --git a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl index 26e0847442..b6992afffd 100644 --- a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl +++ b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl @@ -1,12 +1,41 @@ <#import "templates/template.html.ftl" as layout> <@layout.layout> - <#list c as node, status> +
+
+ +
+ +
+
+ + + <#list c as status>
- ${node} + ${status.origin} ${status.dataset?} updated ${status.ageString} ago ${status.jobs?size} @@ -40,32 +69,4 @@
- -
-
- -
- -
-
\ No newline at end of file From 8b879912335b0c61ba7cd6aa3df8c0ddf6484eed Mon Sep 17 00:00:00 2001 From: Marco Korinth Date: Wed, 17 May 2023 15:34:40 +0200 Subject: [PATCH 326/679] turn date helper functions into hooks --- frontend/src/js/common/helpers/dateHelper.ts | 10 +++--- .../ui-components/InputDate/CustomHeader.tsx | 33 ++++++++++--------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/frontend/src/js/common/helpers/dateHelper.ts b/frontend/src/js/common/helpers/dateHelper.ts index b522810f02..c5fc13fc8c 100644 --- a/frontend/src/js/common/helpers/dateHelper.ts +++ b/frontend/src/js/common/helpers/dateHelper.ts @@ -10,7 +10,6 @@ import { formatDistance, } from "date-fns"; import { de, enGB } from "date-fns/locale"; -import i18next from "i18next"; import { useTranslation } from "react-i18next"; // To save the date in this format in the state @@ -215,15 +214,16 @@ export function getFirstAndLastDateOfRange(dateRangeStr: string): { return { first, last }; } -export function getMonthName(date: Date): string { - const locale = i18next.language === "de" ? de : enGB; +export function useMonthName(date: Date): string { + const locale = useDateLocale(); return format(date, "LLLL", { locale }); } -export function getMonthNames(): string[] { +export function useMonthNames(): string[] { + const locale = useDateLocale(); return [...Array(12).keys()].map((month) => { const date = new Date(); date.setMonth(month); - return getMonthName(date); + return format(date, "LLLL", { locale }); }); } diff --git a/frontend/src/js/ui-components/InputDate/CustomHeader.tsx b/frontend/src/js/ui-components/InputDate/CustomHeader.tsx index 71071a3732..1ca9009a4b 100644 --- a/frontend/src/js/ui-components/InputDate/CustomHeader.tsx +++ b/frontend/src/js/ui-components/InputDate/CustomHeader.tsx @@ -9,7 +9,7 @@ import { ReactDatePickerCustomHeaderProps } from "react-datepicker"; import { SelectOptionT } from "../../api/types"; import IconButton from "../../button/IconButton"; import { TransparentButton } from "../../button/TransparentButton"; -import { getMonthName, getMonthNames } from "../../common/helpers/dateHelper"; +import { useMonthName, useMonthNames } from "../../common/helpers/dateHelper"; import { List, Menu } from "../InputSelect/InputSelectComponents"; export const Root = styled("div")` @@ -36,20 +36,6 @@ export const MonthYearLabel = styled("div")` cursor: pointer; `; -const yearSelectionSpan = 10; -const yearOptions: SelectOptionT[] = [...Array(yearSelectionSpan).keys()] - .map((n) => new Date().getFullYear() - n) - .map((year) => ({ - label: String(year), - value: year, - })) - .reverse(); - -const monthOptions: SelectOptionT[] = getMonthNames().map((month, i) => ({ - label: month, - value: i, -})); - const SelectMenu = ({ date, options, @@ -87,6 +73,21 @@ const YearMonthSelect = ({ ReactDatePickerCustomHeaderProps, "date" | "changeYear" | "changeMonth" >) => { + const yearSelectionSpan = 10; + const yearOptions: SelectOptionT[] = [...Array(yearSelectionSpan).keys()] + .map((n) => new Date().getFullYear() - n) + .map((year) => ({ + label: String(year), + value: year, + })) + .reverse(); + + const monthNames = useMonthNames(); + const monthOptions: SelectOptionT[] = monthNames.map((month, i) => ({ + label: month, + value: i, + })); + const [yearSelectOpen, setYearSelectOpen] = useState(false); const [monthSelectOpen, setMonthSelectOpen] = useState(false); const handleClick = () => { @@ -101,7 +102,7 @@ const YearMonthSelect = ({ return ( <> - {getMonthName(date)} {date.getFullYear()} + {useMonthName(date)} {date.getFullYear()} {yearSelectOpen && ( Date: Wed, 17 May 2023 16:38:51 +0200 Subject: [PATCH 327/679] adds missing JsonCreator to UpdateJobManagerStatus --- .../conquery/models/jobs/JobManagerStatus.java | 11 ++++++----- .../network/specific/UpdateJobManagerStatus.java | 7 ++++--- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManagerStatus.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManagerStatus.java index 4c6329b4e4..ec2397604c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManagerStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/JobManagerStatus.java @@ -2,7 +2,7 @@ import java.time.Duration; import java.time.LocalDateTime; -import java.util.Collection; +import java.util.List; import javax.annotation.Nullable; import javax.validation.constraints.NotNull; @@ -10,15 +10,15 @@ import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; import lombok.Data; import lombok.EqualsAndHashCode; -import lombok.RequiredArgsConstructor; import lombok.With; import org.apache.commons.lang3.time.DurationFormatUtils; @Data -@RequiredArgsConstructor(onConstructor_ = @JsonCreator) +@AllArgsConstructor(onConstructor_ = @JsonCreator) public class JobManagerStatus { @With @Nullable @@ -30,8 +30,9 @@ public class JobManagerStatus { private final LocalDateTime timestamp; @NotNull @EqualsAndHashCode.Exclude - private final Collection jobs; - public JobManagerStatus(String origin, DatasetId dataset, Collection statuses) { + private final List jobs; + + public JobManagerStatus(String origin, DatasetId dataset, List statuses) { this(origin, dataset, LocalDateTime.now(), statuses); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/UpdateJobManagerStatus.java b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/UpdateJobManagerStatus.java index 86f50b302c..3f2c339ecf 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/UpdateJobManagerStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/UpdateJobManagerStatus.java @@ -8,21 +8,22 @@ import com.bakdata.conquery.models.messages.network.NetworkMessage; import com.bakdata.conquery.models.messages.network.NetworkMessageContext.ManagerNodeNetworkContext; import com.bakdata.conquery.models.worker.ShardNodeInformation; +import com.fasterxml.jackson.annotation.JsonCreator; import lombok.Data; +import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @CPSType(id = "UPDATE_JOB_MANAGER_STATUS", base = NetworkMessage.class) @Slf4j @Data +@RequiredArgsConstructor(onConstructor_ = {@JsonCreator}) public class UpdateJobManagerStatus extends MessageToManagerNode { @NotNull private final JobManagerStatus status; @Override public void react(ManagerNodeNetworkContext context) throws Exception { - final ShardNodeInformation node = context.getNamespaces() - .getShardNodes() - .get(context.getRemoteAddress()); + final ShardNodeInformation node = context.getNamespaces().getShardNodes().get(context.getRemoteAddress()); if (node == null) { log.error("Could not find ShardNode `{}`, I only know of {}", context.getRemoteAddress(), context.getNamespaces().getShardNodes().keySet()); From 490697b9cbb9923ae4cdecddee5c4bed66915e1a Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 17 May 2023 16:50:20 +0200 Subject: [PATCH 328/679] removes faulty `?` in ftl --- .../com/bakdata/conquery/resources/admin/ui/jobs.html.ftl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl index b6992afffd..cc00ba8a93 100644 --- a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl +++ b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl @@ -35,7 +35,7 @@
- ${status.origin} ${status.dataset?} + ${status.origin} ${status.dataset} updated ${status.ageString} ago ${status.jobs?size} From 89ebd1fe25ae5efe1ce6a6ecb280d813389aa42a Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 17 May 2023 17:02:58 +0200 Subject: [PATCH 329/679] fixes null handling of dataset --- .../com/bakdata/conquery/resources/admin/ui/jobs.html.ftl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl index cc00ba8a93..a08129260e 100644 --- a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl +++ b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl @@ -35,7 +35,7 @@
- ${status.origin} ${status.dataset} + ${status.origin} ${(status.dataset)!} updated ${status.ageString} ago ${status.jobs?size} From 615536e8de72c5fe0ca7151fa513caee2ef2eb14 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 17 May 2023 17:19:20 +0200 Subject: [PATCH 330/679] fix mapping of progress --- .../com/bakdata/conquery/resources/admin/ui/jobs.html.ftl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl index a08129260e..a29f0563d0 100644 --- a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl +++ b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl @@ -50,7 +50,7 @@
+
{headline}
- +
+ + + {t("queryNodeEditor.excludeTimestamps")} + + +
); diff --git a/frontend/src/js/editor-v2/date-restriction/DateRange.tsx b/frontend/src/js/editor-v2/date-restriction/DateRange.tsx index 2371c75f49..f420539f06 100644 --- a/frontend/src/js/editor-v2/date-restriction/DateRange.tsx +++ b/frontend/src/js/editor-v2/date-restriction/DateRange.tsx @@ -1,6 +1,8 @@ import styled from "@emotion/styled"; +import { useTranslation } from "react-i18next"; import { DateRangeT } from "../../api/types"; +import { formatDate } from "../../common/helpers/dateHelper"; const Root = styled("div")` font-size: ${({ theme }) => theme.font.xs}; @@ -17,19 +19,35 @@ const Label = styled("div")` justify-self: flex-end; `; +const getFormattedDate = (date: string | undefined, dateFormat: string) => { + if (!date) return null; + + const d = new Date(date); + + if (isNaN(d.getTime())) return null; + + return formatDate(d, dateFormat); +}; + export const DateRange = ({ dateRange }: { dateRange: DateRangeT }) => { + const { t } = useTranslation(); + const dateFormat = t("inputDateRange.dateFormat"); + + const dateMin = getFormattedDate(dateRange.min, dateFormat); + const dateMax = getFormattedDate(dateRange.max, dateFormat); + return ( - {dateRange.min && ( + {dateMin && ( <> - - {dateRange.min} + + {dateMin} )} - {dateRange.max && dateRange.max !== dateRange.min && ( + {dateMax && dateMax !== dateMin && ( <> - - {dateRange.max} + + {dateMax} )} diff --git a/frontend/src/js/editor-v2/date-restriction/useDateEditing.ts b/frontend/src/js/editor-v2/date-restriction/useDateEditing.ts index ad205bef5f..5ce236992e 100644 --- a/frontend/src/js/editor-v2/date-restriction/useDateEditing.ts +++ b/frontend/src/js/editor-v2/date-restriction/useDateEditing.ts @@ -31,7 +31,7 @@ export const useDateEditing = ({ return ( selectedNode.data?.label || - (selectedNode.children?.items || []).map((c) => c.data?.label).join(" ") + (selectedNode.children?.items || []).map((c) => c.data?.label).join(" - ") ); }, [selectedNode]); diff --git a/frontend/src/js/icon/FaIcon.tsx b/frontend/src/js/icon/FaIcon.tsx index d970f310d7..36497900a4 100644 --- a/frontend/src/js/icon/FaIcon.tsx +++ b/frontend/src/js/icon/FaIcon.tsx @@ -10,6 +10,7 @@ export interface IconStyleProps { center?: boolean; right?: boolean; white?: boolean; + red?: boolean; light?: boolean; gray?: boolean; main?: boolean; @@ -48,9 +49,11 @@ export const Icon = styled(FontAwesomeIcon, { text-align: ${({ center }) => (center ? "center" : "left")}; font-size: ${({ theme, large, tiny }) => large ? theme.font.md : tiny ? theme.font.tiny : theme.font.sm}; - color: ${({ theme, white, gray, light, main, active, disabled }) => + color: ${({ theme, white, gray, red, light, main, active, disabled }) => disabled ? theme.col.grayMediumLight + : red + ? theme.col.red : gray ? theme.col.gray : active diff --git a/frontend/src/js/ui-components/InputDateRange.tsx b/frontend/src/js/ui-components/InputDateRange.tsx index e14e0458b8..82cc216f7b 100644 --- a/frontend/src/js/ui-components/InputDateRange.tsx +++ b/frontend/src/js/ui-components/InputDateRange.tsx @@ -1,5 +1,6 @@ import { css } from "@emotion/react"; import styled from "@emotion/styled"; +import { faCalendar } from "@fortawesome/free-regular-svg-icons"; import { FC, ReactNode, createRef, useMemo } from "react"; import { useTranslation } from "react-i18next"; @@ -12,6 +13,7 @@ import { DateStringMinMax, } from "../common/helpers/dateHelper"; import { exists } from "../common/helpers/exists"; +import { Icon } from "../icon/FaIcon"; import InfoTooltip from "../tooltip/InfoTooltip"; import InputDate from "./InputDate"; @@ -33,7 +35,6 @@ const StyledLabel = styled(Label)<{ large?: boolean }>` large && css` font-size: ${theme.font.md}; - margin: 20px 0 10px; `} `; @@ -175,6 +176,7 @@ const InputDateRange: FC = ({ return ( + {exists(indexPrefix) && # {indexPrefix}} {optional && } {label} diff --git a/frontend/src/localization/de.json b/frontend/src/localization/de.json index 4bf397fb7e..71dcdad502 100644 --- a/frontend/src/localization/de.json +++ b/frontend/src/localization/de.json @@ -528,6 +528,7 @@ "delete": "Löschen", "expand": "Expandieren", "edit": "Details", - "initialDropText": "Ziehe ein Konzept oder eine Anfrage hier hinein." + "initialDropText": "Ziehe ein Konzept oder eine Anfrage hier hinein.", + "datesExcluded": "Keine Datumswerte" } } diff --git a/frontend/src/localization/en.json b/frontend/src/localization/en.json index f4eb721b5b..99ce82dd82 100644 --- a/frontend/src/localization/en.json +++ b/frontend/src/localization/en.json @@ -527,6 +527,7 @@ "delete": "Delete", "expand": "Expand", "edit": "Details", - "initialDropText": "Drop a concept or query here." + "initialDropText": "Drop a concept or query here.", + "datesExcluded": "No dates" } } From dd60564b8fc2609e355f4efcb05fb05bfa80d2d8 Mon Sep 17 00:00:00 2001 From: Marco Korinth Date: Wed, 24 May 2023 12:57:42 +0200 Subject: [PATCH 336/679] set first day of the week to monday in datepicker and added hover effect for yearmonth selection --- frontend/src/js/ui-components/InputDate/CustomHeader.tsx | 6 ++++++ frontend/src/js/ui-components/InputDate/InputDate.tsx | 1 + 2 files changed, 7 insertions(+) diff --git a/frontend/src/js/ui-components/InputDate/CustomHeader.tsx b/frontend/src/js/ui-components/InputDate/CustomHeader.tsx index 1ca9009a4b..ca1269e39f 100644 --- a/frontend/src/js/ui-components/InputDate/CustomHeader.tsx +++ b/frontend/src/js/ui-components/InputDate/CustomHeader.tsx @@ -34,6 +34,12 @@ export const OptionList = styled(List)` export const MonthYearLabel = styled("div")` font-weight: bold; cursor: pointer; + transition: opacity ${({ theme }) => theme.transitionTime}; + opacity: 0.75; + + &:hover { + opacity: 1; + } `; const SelectMenu = ({ diff --git a/frontend/src/js/ui-components/InputDate/InputDate.tsx b/frontend/src/js/ui-components/InputDate/InputDate.tsx index a3cf0aeb52..d99ec77f55 100644 --- a/frontend/src/js/ui-components/InputDate/InputDate.tsx +++ b/frontend/src/js/ui-components/InputDate/InputDate.tsx @@ -125,6 +125,7 @@ const InputDate = forwardRef( renderCustomHeader={CustomHeader} customInput={createElement(HiddenInput)} calendarContainer={StyledCalendar} + calendarStartDay={1} /> ); From 263c0db09710b186821efe6b8c2e1ed3b18bcea1 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Fri, 26 May 2023 11:20:13 +0200 Subject: [PATCH 337/679] add X to close button --- .../DropzoneBetweenElements.tsx} | 47 ++++++++++++++----- .../form-components/DropzoneList.tsx | 19 ++++---- 2 files changed, 47 insertions(+), 19 deletions(-) rename frontend/src/js/external-forms/{form-concept-group/BetweenElement.tsx => form-components/DropzoneBetweenElements.tsx} (68%) diff --git a/frontend/src/js/external-forms/form-concept-group/BetweenElement.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx similarity index 68% rename from frontend/src/js/external-forms/form-concept-group/BetweenElement.tsx rename to frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index a7716cc89e..d6e5a9b8d2 100644 --- a/frontend/src/js/external-forms/form-concept-group/BetweenElement.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -1,8 +1,9 @@ import styled from "@emotion/styled"; -import { faPlus } from "@fortawesome/free-solid-svg-icons"; +import { faPlus, faTimes } from "@fortawesome/free-solid-svg-icons"; import { ReactNode, useState } from "react"; import { DropTargetMonitor, useDrop } from "react-dnd"; +import IconButton from "../../button/IconButton"; import FaIcon from "../../icon/FaIcon"; import Dropzone, { ChildArgs, @@ -13,16 +14,19 @@ interface Props { onDrop: (item: DroppableObject, monitor: DropTargetMonitor) => void; acceptedDropTypes: string[]; children?: (args: ChildArgs) => ReactNode; + isFirstElement: boolean; } const Root = styled("div")<{ isOver: boolean; isDroppable: boolean; + isFirstElement: boolean; }>` background-color: ${({ theme, isDroppable, isOver }) => { if (isOver && isDroppable) return theme.col.grayLight; return isDroppable ? theme.col.grayVeryLight : "inherit"; }}; + margin-top: ${({ isFirstElement }) => (isFirstElement ? "5px" : "0px")}; width: 100%; text-align: center; `; @@ -31,16 +35,31 @@ const PlusContainer = styled("div")` margin: auto; `; +const DropzoneContainer = styled("div")` + overflow: hidden; + height: 54px; +`; + const SxFaIcon = styled(FaIcon)` height: 15px; color: ${({ theme }) => theme.col.black}; opacity: 0.75; `; +const RemoveBtn = styled(IconButton)` + position: relative; + color: ${({ theme }) => theme.col.black}; + top: -64px; + left: 97%; + z-index: 2; + background-color: white; +`; + const BetweenElements = ({ acceptedDropTypes, children, onDrop, + isFirstElement, }: Props) => { const SxDropzone = styled(Dropzone)` margin: 5px 0 5px 0; @@ -65,11 +84,15 @@ const BetweenElements = ({ setShowDropzone(false); onDrop(item, monitor); }; - return ( <> {!(showDropzone || isOver || isOver2) && ( - + setShowDropzone(true)}> @@ -77,14 +100,16 @@ const BetweenElements = ({ )} {(showDropzone || isOver || isOver2) && ( - // TODO x - to close the dropzone - - {children} - + + + {children} + + + )} ); diff --git a/frontend/src/js/external-forms/form-components/DropzoneList.tsx b/frontend/src/js/external-forms/form-components/DropzoneList.tsx index 887e48d498..1e97d3c67f 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneList.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneList.tsx @@ -14,7 +14,8 @@ import DropzoneWithFileInput, { } from "../../ui-components/DropzoneWithFileInput"; import Label from "../../ui-components/Label"; import Optional from "../../ui-components/Optional"; -import BetweenElements from "../form-concept-group/BetweenElement"; + +import BetweenElements from "./DropzoneBetweenElements"; const ListItem = styled("div")` position: relative; @@ -22,7 +23,6 @@ const ListItem = styled("div")` box-shadow: 0 0 3px 0 rgba(0, 0, 0, 0.1); background-color: white; border-radius: ${({ theme }) => theme.borderRadius}; - margin-bottom: 5px; `; const StyledIconButton = styled(IconButton)` @@ -95,12 +95,15 @@ const DropzoneList = (
{items.map((item, i) => ( <> - - {() => conceptDropzoneText} - + {!disallowMultipleColumns && ( + + {() => conceptDropzoneText} + + )} onDelete(i)} /> {item} From 4be2eb43ddc85ec8296955d0c399cc71d20c38cd Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 29 May 2023 14:32:06 +0200 Subject: [PATCH 338/679] Fix multi-select race condition --- .../InputMultiSelect/InputMultiSelect.tsx | 35 ++++------ .../InputMultiSelect/SelectedItem.tsx | 13 ++-- .../useSyncWithValueFromAbove.ts | 65 ------------------- 3 files changed, 22 insertions(+), 91 deletions(-) delete mode 100644 frontend/src/js/ui-components/InputMultiSelect/useSyncWithValueFromAbove.ts diff --git a/frontend/src/js/ui-components/InputMultiSelect/InputMultiSelect.tsx b/frontend/src/js/ui-components/InputMultiSelect/InputMultiSelect.tsx index 5cfe3be56f..4fefe99913 100644 --- a/frontend/src/js/ui-components/InputMultiSelect/InputMultiSelect.tsx +++ b/frontend/src/js/ui-components/InputMultiSelect/InputMultiSelect.tsx @@ -38,7 +38,6 @@ import { useCloseOnClickOutside } from "./useCloseOnClickOutside"; import { useFilteredOptions } from "./useFilteredOptions"; import { useLoadMoreInitially } from "./useLoadMoreInitially"; import { useResolvableSelect } from "./useResolvableSelect"; -import { useSyncWithValueFromAbove } from "./useSyncWithValueFromAbove"; const MAX_SELECTED_ITEMS_LIMIT = 200; @@ -104,8 +103,6 @@ const InputMultiSelect = ({ const [inputValue, setInputValue] = useState(""); const { t } = useTranslation(); - const [syncingState, setSyncingState] = useState(false); - const { getSelectedItemProps, getDropdownProps, @@ -117,9 +114,9 @@ const InputMultiSelect = ({ activeIndex, } = useMultipleSelection({ initialSelectedItems: defaultValue || [], - onSelectedItemsChange: (changes) => { + selectedItems: value, + onStateChange: (changes) => { if (changes.selectedItems) { - setSyncingState(true); onChange(changes.selectedItems); } }, @@ -166,27 +163,31 @@ const InputMultiSelect = ({ case useCombobox.stateChangeTypes.InputKeyDownEnter: case useCombobox.stateChangeTypes.InputBlur: case useCombobox.stateChangeTypes.ItemClick: + // Support disabled items if (changes.selectedItem?.disabled) { return state; } + // Make sure we're staying around the index of the item that was just selected const stayAlmostAtTheSamePositionIndex = state.highlightedIndex === filteredOptions.length - 1 ? state.highlightedIndex - 1 : state.highlightedIndex; + // Determine the right item to be "chosen", supporting "creatable" items const hasChosenCreatableItem = creatable && state.highlightedIndex === 0 && inputValue.length > 0; - // The item that will be "chosen" const selectedItem = hasChosenCreatableItem ? { value: inputValue, label: inputValue } : changes.selectedItem; - if ( - selectedItem && - !selectedItems.find((item) => selectedItem.value === item.value) - ) { + const hasItemHighlighted = state.highlightedIndex > -1; + const isNotSelectedYet = + !!selectedItem && + !selectedItems.find((item) => selectedItem.value === item.value); + + if (isNotSelectedYet && hasItemHighlighted) { addSelectedItem(selectedItem); } @@ -246,14 +247,6 @@ const InputMultiSelect = ({ const clickOutsideRef = useCloseOnClickOutside({ isOpen, toggleMenu }); - useSyncWithValueFromAbove({ - value, - selectedItems, - setSelectedItems, - syncingState, - setSyncingState, - }); - const clearStaleSearch = () => { if (!isOpen) { setInputValue(""); @@ -288,12 +281,12 @@ const InputMultiSelect = ({ > - {selectedItems.map((option, index) => { + {selectedItems.map((item, index) => { return ( ( ( - { index, option, disabled, removeSelectedItem, getSelectedItemProps }, + { index, item, disabled, removeSelectedItem, getSelectedItemProps }, ref, ) => { - const label = option.selectedLabel || option.label || option.value; + const label = item.selectedLabel || item.label || item.value; const selectedItemProps = getSelectedItemProps({ - selectedItem: option, + selectedItem: item, index, }); @@ -57,7 +57,10 @@ const SelectedItem = forwardRef< removeSelectedItem(option)} + onClick={(e) => { + e.stopPropagation(); // otherwise the click handler on the Container overrides this + removeSelectedItem(item); + }} /> ); diff --git a/frontend/src/js/ui-components/InputMultiSelect/useSyncWithValueFromAbove.ts b/frontend/src/js/ui-components/InputMultiSelect/useSyncWithValueFromAbove.ts deleted file mode 100644 index 474a9a968b..0000000000 --- a/frontend/src/js/ui-components/InputMultiSelect/useSyncWithValueFromAbove.ts +++ /dev/null @@ -1,65 +0,0 @@ -import { useEffect } from "react"; - -import type { SelectOptionT } from "../../api/types"; -import { usePrevious } from "../../common/helpers/usePrevious"; - -/** - * The idea here is that we want to allow parent components to update the `value` state - * at any time – then the selected items should update accordingly. - * - * Unfortunately, it led to an infinite loop, when selecting items fast. - * 1) every click causes an internal change to `selectedItems` - * 2) this change is synced up using `onStateChange` from `useMultipleSelection` - * 3) which triggers a change to value and causes a rerender + this effect to re-run - * 4) but this effect will already have a new state for selectedItems - * 5) so we're calling setSelectedItems with an outdated value - * - * => To counter this, we introduced a "syncing" boolean, which we set true in 2) and double-check before 5) - * - * This works, but there is probably a better way to solve this, - * like trying to tie `value` to `selectedItems` more directly - * (= having more of a "controled component state"). - */ -export const useSyncWithValueFromAbove = ({ - value, - selectedItems, - setSelectedItems, - syncingState, - setSyncingState, -}: { - value: SelectOptionT[]; - setSelectedItems: (items: SelectOptionT[]) => void; - selectedItems: SelectOptionT[]; - syncingState: boolean; - setSyncingState: (syncing: boolean) => void; -}) => { - const prevValue = usePrevious(value); - - useEffect(() => { - const prevValueStr = JSON.stringify(prevValue); - const valueStr = JSON.stringify(value); - const selectedItemsStr = JSON.stringify(selectedItems); - - const valueChanged = prevValueStr !== valueStr; - const weDontHaveValueAlreadySelected = valueStr !== selectedItemsStr; - - const takeFromAbove = valueChanged && weDontHaveValueAlreadySelected; - - if (syncingState) { - // Helps prevent race conditions, when selecting options fast - setSyncingState(false); - return; - } - - if (takeFromAbove) { - setSelectedItems(value); - } - }, [ - selectedItems, - setSelectedItems, - prevValue, - value, - syncingState, - setSyncingState, - ]); -}; From c2be3f3dc8a68b2e3ebc0499d5c600bb067c546c Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 30 May 2023 10:58:29 +0200 Subject: [PATCH 339/679] Fix active state --- frontend/src/js/editor-v2/TreeNode.tsx | 225 +++++++++--------- frontend/src/js/model/node.ts | 26 ++ .../js/standard-query-editor/QueryNode.tsx | 15 +- 3 files changed, 149 insertions(+), 117 deletions(-) diff --git a/frontend/src/js/editor-v2/TreeNode.tsx b/frontend/src/js/editor-v2/TreeNode.tsx index 1c84aa310a..810eeb7aa4 100644 --- a/frontend/src/js/editor-v2/TreeNode.tsx +++ b/frontend/src/js/editor-v2/TreeNode.tsx @@ -6,8 +6,9 @@ import { useTranslation } from "react-i18next"; import { DNDType } from "../common/constants/dndTypes"; import { Icon } from "../icon/FaIcon"; -import { nodeIsConceptQueryNode } from "../model/node"; +import { nodeIsConceptQueryNode, useActiveState } from "../model/node"; import { getRootNodeLabel } from "../standard-query-editor/helper"; +import WithTooltip from "../tooltip/WithTooltip"; import Dropzone, { DropzoneProps } from "../ui-components/Dropzone"; import { Connector, Grid } from "./EditorLayout"; @@ -23,6 +24,7 @@ const NodeContainer = styled("div")` const Node = styled("div")<{ selected?: boolean; + active?: boolean; negated?: boolean; leaf?: boolean; isDragging?: boolean; @@ -30,10 +32,16 @@ const Node = styled("div")<{ padding: ${({ leaf, isDragging }) => leaf ? "8px 10px" : isDragging ? "5px" : "24px"}; border: 2px solid - ${({ negated, theme, selected }) => - negated ? "red" : selected ? theme.col.gray : theme.col.grayMediumLight}; + ${({ negated, theme, selected, active }) => + negated + ? theme.col.red + : active + ? theme.col.blueGrayDark + : selected + ? theme.col.gray + : theme.col.grayMediumLight}; box-shadow: ${({ selected, theme }) => - selected ? `inset 0px 0px 0px 2px ${theme.col.gray}` : "none"}; + selected ? `inset 0px 0px 0px 4px ${theme.col.blueGrayVeryLight}` : "none"}; border-radius: ${({ theme }) => theme.borderRadius}; width: ${({ leaf }) => (leaf ? "180px" : "inherit")}; @@ -148,6 +156,8 @@ export function TreeNode({ const rootNodeLabel = tree.data ? getRootNodeLabel(tree.data) : null; + const { active, tooltipText } = useActiveState(tree.data); + const onDropOutsideOfNode = ({ pos, direction, @@ -240,110 +250,113 @@ export function TreeNode({ onDrop={() => {}} > {({ canDrop }) => ( - { - e.stopPropagation(); - setSelectedNodeId(tree.id); - }} - > - {tree.dates?.restriction && ( - - - - )} - {tree.dates?.excluded && ( - - - {t("editorV2.datesExcluded")} - - )} - {(!tree.children || tree.data) && ( -
- {tree.data?.type !== DNDType.CONCEPT_TREE_NODE && ( - - {t("queryEditor.previousQuery")} - - )} - {rootNodeLabel && {rootNodeLabel}} - {tree.data?.label && {tree.data.label}} - {tree.data && nodeIsConceptQueryNode(tree.data) && ( - {tree.data?.description} - )} -
- )} - {tree.children && ( - - onDropAtChildrenIdx({ idx: 0, item })} - > - {() => ( - + + { + e.stopPropagation(); + setSelectedNodeId(tree.id); + }} + > + {tree.dates?.restriction && ( + + + + )} + {tree.dates?.excluded && ( + + + {t("editorV2.datesExcluded")} + + )} + {(!tree.children || tree.data) && ( +
+ {tree.data?.type !== DNDType.CONCEPT_TREE_NODE && ( + + {t("queryEditor.previousQuery")} + )} - - {tree.children.items.map((item, i, items) => ( - <> - - {i < items.length - 1 && ( - - onDropAtChildrenIdx({ idx: i + 1, item }) - } - > - {() => ( - - )} - - )} - - ))} - - onDropAtChildrenIdx({ - idx: tree.children!.items.length, - item, - }) - } - > - {() => ( - + {rootNodeLabel && {rootNodeLabel}} + {tree.data?.label && {tree.data.label}} + {tree.data && nodeIsConceptQueryNode(tree.data) && ( + {tree.data?.description} )} - - - )} - +
+ )} + {tree.children && ( + + onDropAtChildrenIdx({ idx: 0, item })} + > + {() => ( + + )} + + {tree.children.items.map((item, i, items) => ( + <> + + {i < items.length - 1 && ( + + onDropAtChildrenIdx({ idx: i + 1, item }) + } + > + {() => ( + + )} + + )} + + ))} + + onDropAtChildrenIdx({ + idx: tree.children!.items.length, + item, + }) + } + > + {() => ( + + )} + + + )} + + )} {droppable.h && ( diff --git a/frontend/src/js/model/node.ts b/frontend/src/js/model/node.ts index 92f2dc8012..4cf533d58e 100644 --- a/frontend/src/js/model/node.ts +++ b/frontend/src/js/model/node.ts @@ -7,6 +7,7 @@ import { faFolderOpen, faMinus, } from "@fortawesome/free-solid-svg-icons"; +import { useTranslation } from "react-i18next"; import { ConceptElementT, ConceptT } from "../api/types"; import { DNDType } from "../common/constants/dndTypes"; @@ -149,3 +150,28 @@ export const canNodeBeDropped = ( const itemHasConceptRoot = item.tree === node.tree; return itemHasConceptRoot && !itemAlreadyInNode; }; + +export const useActiveState = (node?: StandardQueryNodeT) => { + const { t } = useTranslation(); + + if (!node) { + return { + active: false, + tooltipText: undefined, + }; + } + + const hasNonDefaultSettings = !node.error && nodeHasNonDefaultSettings(node); + const hasFilterValues = nodeHasFilterValues(node); + + const tooltipText = hasNonDefaultSettings + ? t("queryEditor.hasNonDefaultSettings") + : hasFilterValues + ? t("queryEditor.hasDefaultSettings") + : undefined; + + return { + active: hasNonDefaultSettings || hasFilterValues, + tooltipText, + }; +}; diff --git a/frontend/src/js/standard-query-editor/QueryNode.tsx b/frontend/src/js/standard-query-editor/QueryNode.tsx index 73951a4ba5..353405fdd2 100644 --- a/frontend/src/js/standard-query-editor/QueryNode.tsx +++ b/frontend/src/js/standard-query-editor/QueryNode.tsx @@ -13,6 +13,7 @@ import { nodeHasFilterValues, nodeIsConceptQueryNode, canNodeBeDropped, + useActiveState, } from "../model/node"; import { isQueryExpandable } from "../model/query"; import { HoverNavigatable } from "../small-tab-navigation/HoverNavigatable"; @@ -93,21 +94,19 @@ const QueryNode = ({ onToggleTimestamps, onToggleSecondaryIdExclude, }: PropsT) => { - const { t } = useTranslation(); const rootNodeLabel = getRootNodeLabel(node); const ref = useRef(null); const activeSecondaryId = useSelector( (state) => state.queryEditor.selectedSecondaryId, ); - - const hasNonDefaultSettings = !node.error && nodeHasNonDefaultSettings(node); - const hasFilterValues = nodeHasFilterValues(node); const hasActiveSecondaryId = nodeHasActiveSecondaryId( node, activeSecondaryId, ); + const { active, tooltipText } = useActiveState(node); + const item: StandardQueryNodeT = { // Return the data describing the dragged item // NOT using `...node` since that would also spread `children` in. @@ -161,12 +160,6 @@ const QueryNode = ({ } as StandardQueryNodeT), }); - const tooltipText = hasNonDefaultSettings - ? t("queryEditor.hasNonDefaultSettings") - : hasFilterValues - ? t("queryEditor.hasDefaultSettings") - : undefined; - const expandClick = useCallback(() => { if (nodeIsConceptQueryNode(node) || !node.query) return; @@ -195,7 +188,7 @@ const QueryNode = ({ ref.current = instance; drag(instance); }} - active={hasNonDefaultSettings || hasFilterValues} + active={active} onClick={node.error ? undefined : () => onEditClick(andIdx, orIdx)} > Date: Tue, 30 May 2023 12:08:20 +0200 Subject: [PATCH 340/679] cleanup, improve text, fix small css issues --- .../DropzoneBetweenElements.tsx | 34 +++++++------------ .../form-components/DropzoneList.tsx | 16 ++++----- .../form-concept-group/FormConceptGroup.tsx | 1 - frontend/src/localization/de.json | 3 +- frontend/src/localization/en.json | 3 +- 5 files changed, 23 insertions(+), 34 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index d6e5a9b8d2..2c10931a94 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -1,19 +1,17 @@ import styled from "@emotion/styled"; -import { faPlus, faTimes } from "@fortawesome/free-solid-svg-icons"; -import { ReactNode, useState } from "react"; +import { faPlus } from "@fortawesome/free-solid-svg-icons"; +import { useState } from "react"; import { DropTargetMonitor, useDrop } from "react-dnd"; -import IconButton from "../../button/IconButton"; import FaIcon from "../../icon/FaIcon"; import Dropzone, { - ChildArgs, PossibleDroppableObject, } from "../../ui-components/Dropzone"; +import { useTranslation } from "react-i18next"; interface Props { onDrop: (item: DroppableObject, monitor: DropTargetMonitor) => void; acceptedDropTypes: string[]; - children?: (args: ChildArgs) => ReactNode; isFirstElement: boolean; } @@ -32,7 +30,11 @@ const Root = styled("div")<{ `; const PlusContainer = styled("div")` - margin: auto; + margin: 0; + display: flex; + justify-content: center; + align-items: center; + height: 21px; `; const DropzoneContainer = styled("div")` @@ -41,26 +43,18 @@ const DropzoneContainer = styled("div")` `; const SxFaIcon = styled(FaIcon)` - height: 15px; + height: 12px; color: ${({ theme }) => theme.col.black}; opacity: 0.75; `; -const RemoveBtn = styled(IconButton)` - position: relative; - color: ${({ theme }) => theme.col.black}; - top: -64px; - left: 97%; - z-index: 2; - background-color: white; -`; - const BetweenElements = ({ acceptedDropTypes, - children, onDrop, isFirstElement, }: Props) => { + const { t } = useTranslation(); + const SxDropzone = styled(Dropzone)` margin: 5px 0 5px 0; `; @@ -100,15 +94,13 @@ const BetweenElements = ({ )} {(showDropzone || isOver || isOver2) && ( - + setShowDropzone(false)}> - {children} + {() => t("externalForms.default.dropBetweenLabel")} - )} diff --git a/frontend/src/js/external-forms/form-components/DropzoneList.tsx b/frontend/src/js/external-forms/form-components/DropzoneList.tsx index 1e97d3c67f..4cf10621ac 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneList.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneList.tsx @@ -15,7 +15,7 @@ import DropzoneWithFileInput, { import Label from "../../ui-components/Label"; import Optional from "../../ui-components/Optional"; -import BetweenElements from "./DropzoneBetweenElements"; +import DropzoneBetweenElements from "./DropzoneBetweenElements"; const ListItem = styled("div")` position: relative; @@ -52,7 +52,6 @@ interface PropsT { ) => void; onDropFile: (file: File) => void; onImportLines: (lines: string[]) => void; - conceptDropzoneText: string; dropBetween: ( i: number, ) => (item: DroppableObject, monitor: DropTargetMonitor) => void; @@ -71,7 +70,6 @@ const DropzoneList = ( disallowMultipleColumns, onDrop, onImportLines, - conceptDropzoneText, dropBetween, }: PropsT, ref: Ref, @@ -94,21 +92,19 @@ const DropzoneList = ( {items && items.length > 0 && (
{items.map((item, i) => ( - <> +
{!disallowMultipleColumns && ( - - {() => conceptDropzoneText} - + /> )} - + onDelete(i)} /> {item} - +
))}
)} diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 94fcb4af78..4bf9a0e73a 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -261,7 +261,6 @@ const FormConceptGroup = (props: Props) => { ), ); }} - conceptDropzoneText={props.conceptDropzoneText} items={props.value.map((row, i) => ( <> diff --git a/frontend/src/localization/de.json b/frontend/src/localization/de.json index 1b5843d915..a4f1f256bb 100644 --- a/frontend/src/localization/de.json +++ b/frontend/src/localization/de.json @@ -273,7 +273,8 @@ }, "default": { "conceptDropzoneLabel": "Füge ein Konzept oder eine Konzeptliste hinzu", - "conceptColumnDropzoneLabel": "Füge ein Konzept oder eine Konzeptliste hinzu" + "conceptColumnDropzoneLabel": "Füge ein Konzept oder eine Konzeptliste hinzu", + "dropBetweenLabel": "Füge ein Konzept oder eine Konzeptliste hinzu. Clicke hier um abzubrechen" }, "copyModal": { "headline": "Kopieren von Konzepten aus anderem Feld", diff --git a/frontend/src/localization/en.json b/frontend/src/localization/en.json index 52d4c1f0ef..efa70d3c78 100644 --- a/frontend/src/localization/en.json +++ b/frontend/src/localization/en.json @@ -274,7 +274,8 @@ }, "default": { "conceptDropzoneLabel": "Add a concept or a concept list", - "conceptColumnDropzoneLabel": "Add a concept or a concept list" + "conceptColumnDropzoneLabel": "Add a concept or a concept list", + "dropBetweenLabel": "Add a concept or a concept list. Click here to cancel" }, "copyModal": { "headline": "Copy concepts form another field", From 70589c6b863d7e007108d51542a19c516e6a7895 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Tue, 30 May 2023 12:08:43 +0200 Subject: [PATCH 341/679] format --- .../form-components/DropzoneBetweenElements.tsx | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index 2c10931a94..f3ae29b124 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -2,12 +2,12 @@ import styled from "@emotion/styled"; import { faPlus } from "@fortawesome/free-solid-svg-icons"; import { useState } from "react"; import { DropTargetMonitor, useDrop } from "react-dnd"; +import { useTranslation } from "react-i18next"; import FaIcon from "../../icon/FaIcon"; import Dropzone, { PossibleDroppableObject, } from "../../ui-components/Dropzone"; -import { useTranslation } from "react-i18next"; interface Props { onDrop: (item: DroppableObject, monitor: DropTargetMonitor) => void; @@ -95,10 +95,7 @@ const BetweenElements = ({ {(showDropzone || isOver || isOver2) && ( setShowDropzone(false)}> - + {() => t("externalForms.default.dropBetweenLabel")} From 00cc78f2ebdea210d2bc6a911355cd6e1c082ff3 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Tue, 30 May 2023 12:26:01 +0200 Subject: [PATCH 342/679] add bottom Dropzone margin, renaming to make vars clearer --- .../form-components/DropzoneBetweenElements.tsx | 10 ++++++---- .../external-forms/form-components/DropzoneList.tsx | 13 ++++++++----- .../form-concept-group/FormConceptGroup.tsx | 2 +- 3 files changed, 15 insertions(+), 10 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index f3ae29b124..511857ed7a 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -50,7 +50,7 @@ const SxFaIcon = styled(FaIcon)` const BetweenElements = ({ acceptedDropTypes, - onDrop, + onDrop: onDropCallback, isFirstElement, }: Props) => { const { t } = useTranslation(); @@ -60,6 +60,7 @@ const BetweenElements = ({ `; const [showDropzone, setShowDropzone] = useState(false); + const [{ isOver, isDroppable }, drop] = useDrop({ accept: acceptedDropTypes, collect: (monitor) => ({ @@ -74,10 +75,11 @@ const BetweenElements = ({ }), }); - const onDropped = (item: DroppableObject, monitor: DropTargetMonitor) => { + const onDrop = (item: DroppableObject, monitor: DropTargetMonitor) => { setShowDropzone(false); - onDrop(item, monitor); + onDropCallback(item, monitor); }; + return ( <> {!(showDropzone || isOver || isOver2) && ( @@ -95,7 +97,7 @@ const BetweenElements = ({ {(showDropzone || isOver || isOver2) && ( setShowDropzone(false)}> - + {() => t("externalForms.default.dropBetweenLabel")} diff --git a/frontend/src/js/external-forms/form-components/DropzoneList.tsx b/frontend/src/js/external-forms/form-components/DropzoneList.tsx index 4cf10621ac..c9d556b3ad 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneList.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneList.tsx @@ -52,7 +52,7 @@ interface PropsT { ) => void; onDropFile: (file: File) => void; onImportLines: (lines: string[]) => void; - dropBetween: ( + dropInbetween: ( i: number, ) => (item: DroppableObject, monitor: DropTargetMonitor) => void; } @@ -70,10 +70,13 @@ const DropzoneList = ( disallowMultipleColumns, onDrop, onImportLines, - dropBetween, + dropInbetween, }: PropsT, ref: Ref, ) => { + const SxDropzoneWithFileInput = styled(DropzoneWithFileInput)` + margin-top: 5px; + `; // allow at least one column const showDropzone = (items && items.length === 0) || !disallowMultipleColumns; @@ -96,7 +99,7 @@ const DropzoneList = ( {!disallowMultipleColumns && ( )} @@ -110,13 +113,13 @@ const DropzoneList = ( )}
{showDropzone && onImportLines && ( - {dropzoneChildren} - + )}
diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 4bf9a0e73a..7304ef2d31 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -200,7 +200,7 @@ const FormConceptGroup = (props: Props) => { ? t("externalForms.common.concept.copying") : props.attributeDropzoneText } - dropBetween={(i: number) => { + dropInbetween={(i: number) => { return (item: DragItemConceptTreeNode) => { if (isMovedObject(item)) { return props.onChange( From 725ea78fa32444130bd64e3bc2b39bc3b5b575ed Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 30 May 2023 12:53:09 +0200 Subject: [PATCH 343/679] Add filter suggestions loading --- .../EditorV2QueryNodeEditor.tsx | 65 ++++++++++++++----- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/frontend/src/js/editor-v2/query-node-edit/EditorV2QueryNodeEditor.tsx b/frontend/src/js/editor-v2/query-node-edit/EditorV2QueryNodeEditor.tsx index 8793879912..7faecf403f 100644 --- a/frontend/src/js/editor-v2/query-node-edit/EditorV2QueryNodeEditor.tsx +++ b/frontend/src/js/editor-v2/query-node-edit/EditorV2QueryNodeEditor.tsx @@ -1,6 +1,12 @@ import { useCallback } from "react"; +import { + PostPrefixForSuggestionsParams, + usePostPrefixForSuggestions, +} from "../../api/api"; import { SelectOptionT } from "../../api/types"; +import { exists } from "../../common/helpers/exists"; +import { mergeFilterOptions } from "../../model/filter"; import { NodeResetConfig } from "../../model/node"; import { resetSelects } from "../../model/select"; import { @@ -9,6 +15,7 @@ import { tableWithDefaults, } from "../../model/table"; import QueryNodeEditor from "../../query-node-editor/QueryNodeEditor"; +import { filterSuggestionToSelectOption } from "../../query-node-editor/suggestionsHelper"; import { DragItemConceptTreeNode } from "../../standard-query-editor/types"; import { ModeT } from "../../ui-components/InputRange"; @@ -39,18 +46,6 @@ export const EditorV2QueryNodeEditor = ({ [node, onChange], ); - const onLoadFilterSuggestions = useCallback( - ( - params: any, - tableIdx: number, - filterIdx: number, - config?: { returnOnly?: boolean }, - ) => { - return Promise.resolve(null); - }, - [], - ); - const onDropConcept = useCallback( (concept: DragItemConceptTreeNode) => { const ids = [...node.ids, concept.ids[0]]; @@ -95,13 +90,13 @@ export const EditorV2QueryNodeEditor = ({ [node, onChange], ); - const onSetFilterValue = useCallback( - (tableIdx: number, filterIdx: number, value: any) => { + const setFilterProperties = useCallback( + (tableIdx: number, filterIdx: number, properties: any) => { const tables = [...node.tables]; tables[tableIdx] = { ...tables[tableIdx], filters: tables[tableIdx].filters.map((filter, idx) => - idx === filterIdx ? { ...filter, value } : filter, + idx === filterIdx ? { ...filter, ...properties } : filter, ), }; onChange({ ...node, tables }); @@ -109,6 +104,13 @@ export const EditorV2QueryNodeEditor = ({ [node, onChange], ); + const onSetFilterValue = useCallback( + (tableIdx: number, filterIdx: number, value: any) => { + setFilterProperties(tableIdx, filterIdx, { value }); + }, + [setFilterProperties], + ); + const onSwitchFilterMode = useCallback( (tableIdx: number, filterIdx: number, mode: ModeT) => { const tables = [...node.tables]; @@ -123,6 +125,39 @@ export const EditorV2QueryNodeEditor = ({ [node, onChange], ); + const postPrefixForSuggestions = usePostPrefixForSuggestions(); + const onLoadFilterSuggestions = useCallback( + async ( + params: PostPrefixForSuggestionsParams, + tableIdx: number, + filterIdx: number, + config?: { returnOnly?: boolean }, + ) => { + const suggestions = await postPrefixForSuggestions(params); + + if (!config?.returnOnly) { + const newOptions: SelectOptionT[] = suggestions.values.map( + filterSuggestionToSelectOption, + ); + + const filter = node.tables[tableIdx].filters[filterIdx]; + const options = + params.page === 0 + ? newOptions + : mergeFilterOptions(filter, newOptions); + + if (exists(options)) { + const props = { options, total: suggestions.total }; + + setFilterProperties(tableIdx, filterIdx, props); + } + } + + return suggestions; + }, + [postPrefixForSuggestions, node, setFilterProperties], + ); + const onResetTable = useCallback( (tableIdx: number, config: NodeResetConfig) => { const table = node.tables[tableIdx]; From 81430fd6fda04691f6d17b628eff5a4b44110f46 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 30 May 2023 15:23:14 +0200 Subject: [PATCH 344/679] Feature content infos --- frontend/src/js/app/RightPane.tsx | 1 + frontend/src/js/editor-v2/EditorV2.tsx | 222 +++++++++++++------------ frontend/src/js/editor-v2/TreeNode.tsx | 162 ++++++++++++++++-- frontend/src/localization/de.json | 4 +- frontend/src/localization/en.json | 4 +- 5 files changed, 272 insertions(+), 121 deletions(-) diff --git a/frontend/src/js/app/RightPane.tsx b/frontend/src/js/app/RightPane.tsx index 3013cc98a9..0764ff6aad 100644 --- a/frontend/src/js/app/RightPane.tsx +++ b/frontend/src/js/app/RightPane.tsx @@ -91,6 +91,7 @@ const RightPane = () => { featureExpand featureConnectorRotate featureQueryNodeEdit + featureContentInfos /> ) : ( diff --git a/frontend/src/js/editor-v2/EditorV2.tsx b/frontend/src/js/editor-v2/EditorV2.tsx index 8c925f0799..4584a79180 100644 --- a/frontend/src/js/editor-v2/EditorV2.tsx +++ b/frontend/src/js/editor-v2/EditorV2.tsx @@ -15,6 +15,7 @@ import { useTranslation } from "react-i18next"; import IconButton from "../button/IconButton"; import { nodeIsConceptQueryNode } from "../model/node"; +import { EmptyQueryEditorDropzone } from "../standard-query-editor/EmptyQueryEditorDropzone"; import { DragItemConceptTreeNode, DragItemQuery, @@ -118,12 +119,14 @@ export function EditorV2({ featureExpand, featureConnectorRotate, featureQueryNodeEdit, + featureContentInfos, }: { featureDates: boolean; featureNegate: boolean; featureExpand: boolean; featureConnectorRotate: boolean; featureQueryNodeEdit: boolean; + featureContentInfos: boolean; }) { const { t } = useTranslation(); const { @@ -267,124 +270,126 @@ export function EditorV2({ }} /> )} - - - {featureQueryNodeEdit && - selectedNode?.data && - nodeIsConceptQueryNode(selectedNode.data) && ( + {tree && ( + + + {featureQueryNodeEdit && + selectedNode?.data && + nodeIsConceptQueryNode(selectedNode.data) && ( + + { + e.stopPropagation(); + onOpenQueryNodeEditor(); + }} + > + {t("editorV2.edit")} + + + )} + {featureDates && selectedNode && ( + + { + e.stopPropagation(); + onOpen(); + }} + > + {t("editorV2.dates")} + + + )} + {featureNegate && selectedNode && ( + + { + e.stopPropagation(); + onNegateClick(); + }} + > + {t("editorV2.negate")} + + + )} + {selectedNode?.children && ( + + { + e.stopPropagation(); + onFlip(); + }} + > + {t("editorV2.flip")} + + + )} + {featureConnectorRotate && selectedNode?.children && ( + { + e.stopPropagation(); + onRotateConnector(); + }} + > + {t("editorV2.connector")} + {connection} + + + )} + {canExpand && ( + { e.stopPropagation(); - onOpenQueryNodeEditor(); + onExpand(); }} > - {t("editorV2.edit")} + {t("editorV2.expand")} )} - {featureDates && selectedNode && ( - - { - e.stopPropagation(); - onOpen(); - }} - > - {t("editorV2.dates")} - - - )} - {featureNegate && selectedNode && ( - - { - e.stopPropagation(); - onNegateClick(); - }} - > - {t("editorV2.negate")} - - - )} - {selectedNode?.children && ( - - { - e.stopPropagation(); - onFlip(); - }} - > - {t("editorV2.flip")} - - - )} - {featureConnectorRotate && selectedNode?.children && ( - - { - e.stopPropagation(); - onRotateConnector(); - }} - > - {t("editorV2.connector")} - {connection} - - - )} - {canExpand && ( - - { - e.stopPropagation(); - onExpand(); - }} - > - {t("editorV2.expand")} - - - )} - {selectedNode && ( - - { - e.stopPropagation(); - onDelete(); - }} - > - {t("editorV2.delete")} - - - )} - - - - {t("editorV2.clear")} - - - + {selectedNode && ( + + { + e.stopPropagation(); + onDelete(); + }} + > + {t("editorV2.delete")} + + + )} + + + + {t("editorV2.clear")} + + + + )} { if (!selectedNode || showModal) return; @@ -408,6 +413,7 @@ export function EditorV2({ selectedNode={selectedNode} setSelectedNodeId={setSelectedNodeId} droppable={{ h: true, v: true }} + featureContentInfos={featureContentInfos} /> ) : ( - {() =>
{t("editorV2.initialDropText")}
} + {() => } )} diff --git a/frontend/src/js/editor-v2/TreeNode.tsx b/frontend/src/js/editor-v2/TreeNode.tsx index 810eeb7aa4..c58c23b3fc 100644 --- a/frontend/src/js/editor-v2/TreeNode.tsx +++ b/frontend/src/js/editor-v2/TreeNode.tsx @@ -5,9 +5,11 @@ import { DOMAttributes, memo } from "react"; import { useTranslation } from "react-i18next"; import { DNDType } from "../common/constants/dndTypes"; +import { exists } from "../common/helpers/exists"; import { Icon } from "../icon/FaIcon"; import { nodeIsConceptQueryNode, useActiveState } from "../model/node"; import { getRootNodeLabel } from "../standard-query-editor/helper"; +import { DragItemConceptTreeNode } from "../standard-query-editor/types"; import WithTooltip from "../tooltip/WithTooltip"; import Dropzone, { DropzoneProps } from "../ui-components/Dropzone"; @@ -44,7 +46,7 @@ const Node = styled("div")<{ selected ? `inset 0px 0px 0px 4px ${theme.col.blueGrayVeryLight}` : "none"}; border-radius: ${({ theme }) => theme.borderRadius}; - width: ${({ leaf }) => (leaf ? "180px" : "inherit")}; + width: ${({ leaf }) => (leaf ? "230px" : "inherit")}; background-color: ${({ leaf, theme }) => (leaf ? "white" : theme.col.bg)}; cursor: pointer; display: flex; @@ -96,10 +98,14 @@ const Name = styled("div")` const Description = styled("div")` font-size: ${({ theme }) => theme.font.xs}; color: ${({ theme }) => theme.col.black}; + display: flex; + align-items: center; + gap: 0px 5px; + flex-wrap: wrap; `; const PreviousQueryLabel = styled("p")` - margin: 0 0 4px; + margin: 0; line-height: 1.2; font-size: ${({ theme }) => theme.font.xs}; text-transform: uppercase; @@ -107,8 +113,14 @@ const PreviousQueryLabel = styled("p")` color: ${({ theme }) => theme.col.blueGrayDark}; `; +const ContentContainer = styled("div")` + display: flex; + flex-direction: column; + gap: 4px; +`; + const RootNode = styled("p")` - margin: 0 0 4px; + margin: 0; line-height: 1; text-transform: uppercase; font-weight: 700; @@ -124,11 +136,9 @@ const Dates = styled("div")` font-weight: 400; `; -const Connection = memo(({ connection }: { connection?: ConnectionKind }) => { - const message = useTranslatedConnection(connection); - - return {message}; -}); +const Bold = styled("span")` + font-weight: 400; +`; export function TreeNode({ tree, @@ -138,6 +148,7 @@ export function TreeNode({ selectedNode, setSelectedNodeId, onDoubleClick, + featureContentInfos, }: { tree: Tree; treeParent?: Tree; @@ -149,6 +160,7 @@ export function TreeNode({ selectedNode: Tree | undefined; setSelectedNodeId: (id: Tree["id"] | undefined) => void; onDoubleClick?: DOMAttributes["onDoubleClick"]; + featureContentInfos?: boolean; }) { const gridStyles = getGridStyles(tree); @@ -275,7 +287,7 @@ export function TreeNode({ )} {(!tree.children || tree.data) && ( -
+ {tree.data?.type !== DNDType.CONCEPT_TREE_NODE && ( {t("queryEditor.previousQuery")} @@ -284,9 +296,12 @@ export function TreeNode({ {rootNodeLabel && {rootNodeLabel}} {tree.data?.label && {tree.data.label}} {tree.data && nodeIsConceptQueryNode(tree.data) && ( - {tree.data?.description} + )} -
+ )} {tree.children && ( @@ -304,6 +319,7 @@ export function TreeNode({ <> ); } + +const Value = ({ + value, + isElement, +}: { + value: unknown; + isElement?: boolean; +}) => { + if (typeof value === "string" || typeof value === "number") { + return ( + + {value} + {isElement && ","} + + ); + } else if (typeof value === "boolean") { + return {value ? "" : "false"}; + } else if (value instanceof Array) { + return ( + <> + {value.slice(0, 10).map((v, idx) => ( + <> + + + ))} + {value.length > 10 && {`... +${value.length - 10}`}} + + ); + } else if ( + value instanceof Object && + "label" in value && + typeof value.label === "string" + ) { + return ( + + {value.label} + {isElement && ","} + + ); + } else if (value instanceof Object && "min" in value && "max" in value) { + return ( + + {JSON.stringify(value.min)}-{JSON.stringify(value.max)} + + ); + } else { + return {JSON.stringify(value)}; + } +}; + +const Connection = memo(({ connection }: { connection?: ConnectionKind }) => { + const message = useTranslatedConnection(connection); + + return {message}; +}); + +const SectionHeading = styled("h4")` + font-weight: 400; + color: ${(props) => props.theme.col.blueGrayDark}; + margin: 0; + text-transform: uppercase; + font-size: ${({ theme }) => theme.font.xs}; +`; + +const Appendix = styled("div")` + display: flex; + flex-direction: column; + gap: 6px; + margin-top: 8px; +`; + +const TreeNodeConcept = ({ + node, + featureContentInfos, +}: { + node: DragItemConceptTreeNode; + featureContentInfos?: boolean; +}) => { + const { t } = useTranslation(); + const selectedSelects = [ + ...node.selects, + ...node.tables.flatMap((t) => t.selects), + ].filter((s) => s.selected); + + const filtersWithValues = node.tables.flatMap((t) => + t.filters.filter( + (f) => + exists(f.value) && (!(f.value instanceof Array) || f.value.length > 0), + ), + ); + + const showAppendix = + featureContentInfos && + (selectedSelects.length > 0 || filtersWithValues.length > 0); + + return ( + <> + {node.description && {node.description}} + {showAppendix && ( + + {selectedSelects.length > 0 && ( +
+ {t("editorV2.outputSection")} + + + +
+ )} + {filtersWithValues.length > 0 && ( +
+ {t("editorV2.filtersSection")} + {filtersWithValues.map((f) => ( + + {f.label}: + + + ))} +
+ )} + + )} + + ); +}; diff --git a/frontend/src/localization/de.json b/frontend/src/localization/de.json index 71dcdad502..d34decb337 100644 --- a/frontend/src/localization/de.json +++ b/frontend/src/localization/de.json @@ -529,6 +529,8 @@ "expand": "Expandieren", "edit": "Details", "initialDropText": "Ziehe ein Konzept oder eine Anfrage hier hinein.", - "datesExcluded": "Keine Datumswerte" + "datesExcluded": "Keine Datumswerte", + "outputSection": "Ausgabewerte", + "filtersSection": "Filter" } } diff --git a/frontend/src/localization/en.json b/frontend/src/localization/en.json index 99ce82dd82..88b7453bcd 100644 --- a/frontend/src/localization/en.json +++ b/frontend/src/localization/en.json @@ -528,6 +528,8 @@ "expand": "Expand", "edit": "Details", "initialDropText": "Drop a concept or query here.", - "datesExcluded": "No dates" + "datesExcluded": "No dates", + "outputSection": "Output values", + "filtersSection": "Filters" } } From 2030c515fe1c180b6f5927a56c016d292bbd3a49 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 30 May 2023 15:46:03 +0200 Subject: [PATCH 345/679] Improve theme --- frontend/src/app-theme.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/src/app-theme.ts b/frontend/src/app-theme.ts index 7804351455..d7f0b9dd2c 100644 --- a/frontend/src/app-theme.ts +++ b/frontend/src/app-theme.ts @@ -7,16 +7,11 @@ import spinner from "./images/spinner.png"; export const theme: Theme = { col: { bg: "#fafafa", - bgAlt: "#f4f6f5", black: "#222", gray: "#888", grayMediumLight: "#aaa", grayLight: "#dadada", grayVeryLight: "#eee", - blueGrayDark: "#0C6427", - blueGray: "#72757C", - blueGrayLight: "#52A55C", - blueGrayVeryLight: "#A4E6AC", red: "#b22125", green: "#36971C", orange: "#E9711C", @@ -32,6 +27,11 @@ export const theme: Theme = { "#777", "#fff", ], + bgAlt: "#f4f6f5", + blueGrayDark: "#1f5f30", + blueGray: "#98b099", + blueGrayLight: "#ccd6d0", + blueGrayVeryLight: "#dadedb", }, img: { logo: logo, From b142e58940dcccd733cb923e7d2c3e374d563eca Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 30 May 2023 16:12:05 +0200 Subject: [PATCH 346/679] Fix interaction details --- frontend/src/js/editor-v2/EditorV2.tsx | 4 +++- frontend/src/js/editor-v2/TreeNode.tsx | 16 ++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/frontend/src/js/editor-v2/EditorV2.tsx b/frontend/src/js/editor-v2/EditorV2.tsx index 4584a79180..721edc621d 100644 --- a/frontend/src/js/editor-v2/EditorV2.tsx +++ b/frontend/src/js/editor-v2/EditorV2.tsx @@ -418,10 +418,12 @@ export function EditorV2({ ) : ( { + const id = createId(); setTree({ - id: createId(), + id, data: item as DragItemConceptTreeNode | DragItemQuery, }); + setSelectedNodeId(id); }} acceptedDropTypes={EDITOR_DROP_TYPES} > diff --git a/frontend/src/js/editor-v2/TreeNode.tsx b/frontend/src/js/editor-v2/TreeNode.tsx index c58c23b3fc..b00e9e4d76 100644 --- a/frontend/src/js/editor-v2/TreeNode.tsx +++ b/frontend/src/js/editor-v2/TreeNode.tsx @@ -432,12 +432,20 @@ const Value = ({ {isElement && ","} ); - } else if (value instanceof Object && "min" in value && "max" in value) { + } else if (value instanceof Object) { return ( - - {JSON.stringify(value.min)}-{JSON.stringify(value.max)} - + <> + {Object.entries(value) + .filter(([, v]) => exists(v)) + .map(([k, v]) => ( + <> + {k}: + + ))} + ); + } else if (value === null) { + return ; } else { return {JSON.stringify(value)}; } From 063fa6d925a0b9f47e733fc2b858ea2f8d4527ef Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 30 May 2023 17:35:37 +0200 Subject: [PATCH 347/679] Fix boolean --- frontend/src/js/editor-v2/TreeNode.tsx | 2 +- frontend/src/localization/de.json | 2 +- frontend/src/localization/en.json | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/js/editor-v2/TreeNode.tsx b/frontend/src/js/editor-v2/TreeNode.tsx index b00e9e4d76..521dd68d39 100644 --- a/frontend/src/js/editor-v2/TreeNode.tsx +++ b/frontend/src/js/editor-v2/TreeNode.tsx @@ -409,7 +409,7 @@ const Value = ({ ); } else if (typeof value === "boolean") { - return {value ? "" : "false"}; + return {value ? "✔" : "✗"}; } else if (value instanceof Array) { return ( <> diff --git a/frontend/src/localization/de.json b/frontend/src/localization/de.json index d34decb337..d52333ef34 100644 --- a/frontend/src/localization/de.json +++ b/frontend/src/localization/de.json @@ -517,7 +517,7 @@ "pasted": "Importiert" }, "editorV2": { - "before": "VOR", + "before": "ZEIT", "and": "UND", "or": "ODER", "clear": "Leeren", diff --git a/frontend/src/localization/en.json b/frontend/src/localization/en.json index 88b7453bcd..f0f8c66a10 100644 --- a/frontend/src/localization/en.json +++ b/frontend/src/localization/en.json @@ -516,7 +516,7 @@ "pasted": "Imported" }, "editorV2": { - "before": "BEFORE", + "before": "TIME", "and": "AND", "or": "OR", "clear": "Clear", From 8115c7c321e8becc4a11fc35f4cb40ce22d37b65 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 31 May 2023 16:00:26 +0200 Subject: [PATCH 348/679] Implements getValue for ExternalForm, also move the field into subclasses and make it abstract --- .../conquery/apiv1/forms/ExternalForm.java | 48 +++++++++++++------ .../bakdata/conquery/apiv1/forms/Form.java | 8 +--- .../apiv1/forms/export_form/ExportForm.java | 9 ++++ .../forms/export_form/FullExportForm.java | 10 ++++ .../FormConfigProcessor.java | 20 ++++---- .../query/preview/EntityPreviewForm.java | 2 + .../conquery/api/form/config/TestForm.java | 12 +++++ 7 files changed, 79 insertions(+), 30 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java index 0a8a8e6908..49ec7b6ca2 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java @@ -26,9 +26,9 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.Visitable; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonValue; import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.BeanProperty; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JsonDeserializer; @@ -43,7 +43,9 @@ import lombok.NoArgsConstructor; import lombok.RequiredArgsConstructor; import lombok.Setter; +import lombok.ToString; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.Nullable; @Getter @Setter @@ -51,19 +53,23 @@ @JsonDeserialize(using = Deserializer.class) @RequiredArgsConstructor @Slf4j +@ToString public class ExternalForm extends Form implements SubTyped { @JsonValue + @ToString.Exclude private final ObjectNode node; - private final String subType; - + @Nullable @Override - public String getFormType() { - return CPSTypeIdResolver.createSubTyped(this.getClass().getAnnotation(CPSType.class).id(), getSubType()); + @JsonIgnore + public JsonNode getValues() { + return node; } + private final String subType; + @Override public String getLocalizedTypeLabel() { final JsonNode formTitle = node.get("title"); @@ -72,41 +78,52 @@ public String getLocalizedTypeLabel() { } // Form had no specific title set. Try localized lookup in FormConfig - Locale preferredLocale = I18n.LOCALE.get(); - FormType frontendConfig = FormScanner.FRONTEND_FORM_CONFIGS.get(this.getFormType()); + final Locale preferredLocale = I18n.LOCALE.get(); + final FormType frontendConfig = FormScanner.FRONTEND_FORM_CONFIGS.get(getFormType()); if (frontendConfig == null) { return getSubType(); } - JsonNode titleObj = frontendConfig.getRawConfig().path("title"); + final JsonNode titleObj = frontendConfig.getRawConfig().path("title"); if (!titleObj.isObject()) { log.trace("Expected \"title\" member to be of type Object in {}", frontendConfig); return getSubType(); } - List localesFound = new ArrayList<>(); + final List localesFound = new ArrayList<>(); titleObj.fieldNames().forEachRemaining((lang) -> localesFound.add(new Locale(lang))); + if (localesFound.isEmpty()) { log.trace("Could not extract a locale from the provided FrontendConfig: {}", frontendConfig); return getSubType(); } + Locale chosenLocale = Locale.lookup(Locale.LanguageRange.parse(preferredLocale.getLanguage()), localesFound); + if (chosenLocale == null) { chosenLocale = localesFound.get(0); log.trace("Locale lookup did not return a matching locale. Using the first title encountered: {}", chosenLocale); } - JsonNode title = titleObj.path(chosenLocale.getLanguage()); + + final JsonNode title = titleObj.path(chosenLocale.getLanguage()); + if (!title.isTextual()) { log.trace("Expected a textual node for the localized title. Was: {}", title.getNodeType()); return getSubType(); } - String ret = title.asText(); + + final String ret = title.asText(); log.trace("Extracted localized title. Was: \"{}\"", ret); return ret.isBlank() ? getSubType() : ret; } + @Override + public String getFormType() { + return CPSTypeIdResolver.createSubTyped(getClass().getAnnotation(CPSType.class).id(), getSubType()); + } + @Override public ManagedExecution toManagedExecution(User user, Dataset submittedDataset, MetaStorage storage) { return new ExternalExecution(this, user, submittedDataset, storage); @@ -142,22 +159,25 @@ public static class Deserializer extends JsonDeserializer implemen private String subTypeId; @Override - public ExternalForm deserialize(JsonParser p, DeserializationContext ctxt) throws IOException, JsonProcessingException { + public ExternalForm deserialize(JsonParser p, DeserializationContext ctxt) throws IOException { if (Strings.isNullOrEmpty(subTypeId)) { throw new IllegalStateException("This class needs subtype information for deserialization."); } - ObjectNode tree = p.readValueAsTree(); + + final ObjectNode tree = p.readValueAsTree(); return new ExternalForm(tree, subTypeId); } @Override public JsonDeserializer createContextual(DeserializationContext ctxt, BeanProperty property) throws JsonMappingException { // This is only called once per typeId@SubTypeId - String subTypeId = (String) ctxt.getAttribute(CPSTypeIdResolver.ATTRIBUTE_SUB_TYPE); + final String subTypeId = (String) ctxt.getAttribute(CPSTypeIdResolver.ATTRIBUTE_SUB_TYPE); + if (Strings.isNullOrEmpty(subTypeId)) { throw new IllegalStateException("This class needs subtype information for deserialization."); } + return new Deserializer(subTypeId); } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java index 451c39d333..11bcd5f3e6 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java @@ -15,9 +15,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ClassToInstanceMap; import lombok.EqualsAndHashCode; -import lombok.Getter; import lombok.NonNull; -import lombok.Setter; /** * API representation of a form query. @@ -25,15 +23,13 @@ @EqualsAndHashCode public abstract class Form implements QueryDescription { + /** * Raw form config (basically the raw format of this form), that is used by the backend at the moment to * create a {@link com.bakdata.conquery.models.forms.configs.FormConfig} upon start of this form (see {@link ManagedForm#start()}). */ @Nullable - @Getter - @Setter - @EqualsAndHashCode.Exclude - private JsonNode values; + public abstract JsonNode getValues(); @JsonIgnore public String getFormType() { diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java index 381e4bf70d..acee28d857 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java @@ -37,19 +37,28 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonManagedReference; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableList; import lombok.AccessLevel; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; import lombok.Setter; +import lombok.ToString; @Getter @Setter @CPSType(id = "EXPORT_FORM", base = QueryDescription.class) @EqualsAndHashCode(callSuper = true) +@ToString public class ExportForm extends Form implements InternalForm { + @Getter + @Setter + @EqualsAndHashCode.Exclude + private JsonNode values; + + @NotNull @JsonProperty("queryGroup") private ManagedExecutionId queryGroupId; diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java index 3bfc39a382..29b20e9a82 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java @@ -33,15 +33,25 @@ import com.bakdata.conquery.models.query.Visitable; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableList; +import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.Setter; +import lombok.ToString; @Getter @Setter +@ToString @CPSType(id = "FULL_EXPORT_FORM", base = QueryDescription.class) public class FullExportForm extends Form implements InternalForm { + @Getter + @Setter + @EqualsAndHashCode.Exclude + private JsonNode values; + + @NotNull @JsonProperty("queryGroup") private ManagedExecutionId queryGroupId; diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java index dae18e27c1..2356d2811c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormConfigProcessor.java @@ -72,7 +72,7 @@ public Stream getConfigsByFormType(@NonNull Su // If no specific form type is provided, show all types the subject is permitted to create. // However if a subject queries for specific form types, we will show all matching regardless whether // the form config can be used by the subject again. - Set allowedFormTypes = new HashSet<>(); + final Set allowedFormTypes = new HashSet<>(); for (FormType formType : FormScanner.FRONTEND_FORM_CONFIGS.values()) { if (!subject.isPermitted(formType, Ability.CREATE)) { @@ -86,10 +86,10 @@ public Stream getConfigsByFormType(@NonNull Su final Set formTypesFinal = requestedFormType; - Stream stream = storage.getAllFormConfigs().stream() - .filter(c -> dataset.equals(c.getDataset())) - .filter(c -> formTypesFinal.contains(c.getFormType())) - .filter(c -> subject.isPermitted(c, Ability.READ)); + final Stream stream = storage.getAllFormConfigs().stream() + .filter(c -> dataset.equals(c.getDataset())) + .filter(c -> formTypesFinal.contains(c.getFormType())) + .filter(c -> subject.isPermitted(c, Ability.READ)); return stream.map(c -> c.overview(subject)); @@ -117,7 +117,7 @@ public FormConfig addConfig(Subject subject, Dataset targetDataset, FormConfigAP subject.authorize(namespace.getDataset(), Ability.READ); - FormConfig internalConfig = config.intern(storage.getUser(subject.getId()), targetDataset); + final FormConfig internalConfig = config.intern(storage.getUser(subject.getId()), targetDataset); // Add the config immediately to the submitted dataset addConfigToDataset(internalConfig); @@ -151,14 +151,14 @@ public FormConfigFullRepresentation patchConfig(Subject subject, FormConfig conf * Deletes a configuration from the storage and all permissions, that have this configuration as target. */ public void deleteConfig(Subject subject, FormConfig config) { - User user = storage.getUser(subject.getId()); + final User user = storage.getUser(subject.getId()); user.authorize(config, Ability.DELETE); storage.removeFormConfig(config.getId()); // Delete corresponding permissions (Maybe better to put it into a slow job) for (ConqueryPermission permission : user.getPermissions()) { - WildcardPermission wpermission = (WildcardPermission) permission; + final WildcardPermission wpermission = (WildcardPermission) permission; if (!wpermission.getDomains().contains(FormConfigPermission.DOMAIN.toLowerCase())) { continue; @@ -169,9 +169,9 @@ public void deleteConfig(Subject subject, FormConfig config) { if (!wpermission.getInstances().isEmpty()) { // Create new permission if it was a composite permission - Set instancesCleared = new HashSet<>(wpermission.getInstances()); + final Set instancesCleared = new HashSet<>(wpermission.getInstances()); instancesCleared.remove(config.getId().toString()); - WildcardPermission clearedPermission = + final WildcardPermission clearedPermission = new WildcardPermission(List.of(wpermission.getDomains(), wpermission.getAbilities(), instancesCleared), Instant.now()); user.addPermission(clearedPermission); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewForm.java b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewForm.java index 61aa8a637a..9182dc9ab0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewForm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/preview/EntityPreviewForm.java @@ -43,6 +43,7 @@ import lombok.Getter; import lombok.NonNull; import lombok.RequiredArgsConstructor; +import lombok.ToString; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -60,6 +61,7 @@ @CPSType(id = "ENTITY_PREVIEW", base = QueryDescription.class) @Getter @RequiredArgsConstructor(onConstructor_ = {@JsonCreator}) +@ToString public class EntityPreviewForm extends Form implements InternalForm { public static final String INFOS_QUERY_NAME = "INFOS"; diff --git a/backend/src/test/java/com/bakdata/conquery/api/form/config/TestForm.java b/backend/src/test/java/com/bakdata/conquery/api/form/config/TestForm.java index 97f7d36c88..5af22ad1e6 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/form/config/TestForm.java +++ b/backend/src/test/java/com/bakdata/conquery/api/form/config/TestForm.java @@ -18,6 +18,8 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.Visitable; +import com.fasterxml.jackson.databind.JsonNode; +import org.jetbrains.annotations.Nullable; public abstract class TestForm extends Form implements InternalForm { @@ -53,9 +55,19 @@ public void visit(Consumer visitor) { @CPSType(id = "TEST_FORM_ABS_URL", base = QueryDescription.class) public static class Abs extends TestForm { + @Nullable + @Override + public JsonNode getValues() { + return null; + } } @CPSType(id = "TEST_FORM_REL_URL", base = QueryDescription.class) public static class Rel extends TestForm { + @Nullable + @Override + public JsonNode getValues() { + return null; + } } } From 52c9517ad6a8ad7485c647483663c815ab4e7446 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 31 May 2023 17:14:08 +0200 Subject: [PATCH 349/679] allows not setting queryGroup which will then select ALL entities (CQYES) --- .../apiv1/forms/export_form/AbsoluteMode.java | 8 ++--- .../apiv1/forms/export_form/ExportForm.java | 19 ++++++++++-- .../forms/export_form/FullExportForm.java | 26 +++++++++++++--- .../bakdata/conquery/apiv1/query/CQYes.java | 31 +++++++++++++++++++ 4 files changed, 71 insertions(+), 13 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/AbsoluteMode.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/AbsoluteMode.java index c71dbd788c..2f58701d2e 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/AbsoluteMode.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/AbsoluteMode.java @@ -46,12 +46,8 @@ public Query createSpecializedQuery() { List resolutionsAndAlignments = ExportForm.getResolutionAlignmentMap(getForm().getResolvedResolutions(), getAlignmentHint()); - return new AbsoluteFormQuery( - getForm().getPrerequisite(), - dateRange, - resolvedFeatures, - resolutionsAndAlignments - ); + Query prerequisite = getForm().getPrerequisite(); + return new AbsoluteFormQuery(prerequisite, dateRange, resolvedFeatures, resolutionsAndAlignments); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java index acee28d857..63f37f96b8 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/ExportForm.java @@ -1,11 +1,13 @@ package com.bakdata.conquery.apiv1.forms.export_form; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Consumer; import java.util.stream.Collectors; +import javax.annotation.Nullable; import javax.validation.Valid; import javax.validation.ValidationException; import javax.validation.constraints.NotEmpty; @@ -16,6 +18,8 @@ import com.bakdata.conquery.apiv1.forms.Form; import com.bakdata.conquery.apiv1.forms.InternalForm; import com.bakdata.conquery.apiv1.query.CQElement; +import com.bakdata.conquery.apiv1.query.CQYes; +import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.internationalization.ExportFormC10n; @@ -59,7 +63,7 @@ public class ExportForm extends Form implements InternalForm { private JsonNode values; - @NotNull + @Nullable @JsonProperty("queryGroup") private ManagedExecutionId queryGroupId; @@ -106,19 +110,28 @@ public Map createSubQueries() { @Override public Set collectRequiredQueries() { + if (queryGroupId == null) { + return Collections.emptySet(); + } + return Set.of(queryGroupId); } @Override public void resolve(QueryResolveContext context) { - queryGroup = (ManagedQuery) context.getStorage().getExecution(queryGroupId); + if(queryGroupId != null) { + queryGroup = (ManagedQuery) context.getStorage().getExecution(queryGroupId); + prerequisite = queryGroup.getQuery(); + } + else { + prerequisite = new ConceptQuery(new CQYes()); + } // Apply defaults to user concept ExportForm.DefaultSelectSettable.enable(features); timeMode.resolve(context); - prerequisite = queryGroup.getQuery(); List resolutionsFlat = resolution.stream() .flatMap(ResolutionShortNames::correspondingResolutions) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java index 29b20e9a82..f457c1a04f 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/export_form/FullExportForm.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.apiv1.forms.export_form; import java.time.LocalDate; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -9,12 +10,13 @@ import javax.annotation.Nullable; import javax.validation.Valid; import javax.validation.constraints.NotEmpty; -import javax.validation.constraints.NotNull; import c10n.C10N; import com.bakdata.conquery.ConqueryConstants; import com.bakdata.conquery.apiv1.forms.Form; import com.bakdata.conquery.apiv1.forms.InternalForm; +import com.bakdata.conquery.apiv1.query.CQYes; +import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.apiv1.query.TableExportQuery; @@ -46,13 +48,14 @@ @CPSType(id = "FULL_EXPORT_FORM", base = QueryDescription.class) public class FullExportForm extends Form implements InternalForm { + @Nullable @Getter @Setter @EqualsAndHashCode.Exclude private JsonNode values; - @NotNull + @Nullable @JsonProperty("queryGroup") private ManagedExecutionId queryGroupId; @@ -78,7 +81,16 @@ public Map createSubQueries() { // Forms are sent as an array of standard queries containing AND/OR of CQConcepts, we ignore everything and just convert the CQConcepts into CQUnfiltered for export. - final TableExportQuery exportQuery = new TableExportQuery(queryGroup.getQuery()); + final Query query; + + if (queryGroupId != null) { + query = queryGroup.getQuery(); + } + else { + query = new ConceptQuery(new CQYes()); + } + + final TableExportQuery exportQuery = new TableExportQuery(query); exportQuery.setDateRange(getDateRange()); exportQuery.setTables(tables); @@ -93,12 +105,18 @@ public Map createSubQueries() { @Override public Set collectRequiredQueries() { + if (queryGroupId == null) { + return Collections.emptySet(); + } + return Set.of(queryGroupId); } @Override public void resolve(QueryResolveContext context) { - queryGroup = (ManagedQuery) context.getStorage().getExecution(queryGroupId); + if (queryGroupId != null) { + queryGroup = (ManagedQuery) context.getStorage().getExecution(queryGroupId); + } } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java new file mode 100644 index 0000000000..03aab80a99 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java @@ -0,0 +1,31 @@ +package com.bakdata.conquery.apiv1.query; + +import java.util.Collections; +import java.util.List; + +import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.query.QueryPlanContext; +import com.bakdata.conquery.models.query.QueryResolveContext; +import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; +import com.bakdata.conquery.models.query.queryplan.QPNode; +import com.bakdata.conquery.models.query.queryplan.specific.Leaf; +import com.bakdata.conquery.models.query.resultinfo.ResultInfo; + +@CPSType(id = "YES", base = CQElement.class) +public class CQYes extends CQElement{ + + @Override + public void resolve(QueryResolveContext context) { + + } + + @Override + public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { + return new Leaf(); + } + + @Override + public List getResultInfos() { + return Collections.emptyList(); + } +} From cef7ec5abe7f2747475d0c0b576cbed037f789db Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 1 Jun 2023 10:01:34 +0200 Subject: [PATCH 350/679] adds test for CQYes and implements YesNode, as it needs ALL_IDS_TABLE as marker --- .../bakdata/conquery/apiv1/query/CQYes.java | 13 ++++- .../models/query/queryplan/specific/Yes.java | 57 +++++++++++++++++++ .../tests/query/CQYES/CQYES.test.json | 32 +++++++++++ .../resources/tests/query/CQYES/content.csv | 7 +++ .../resources/tests/query/CQYES/expected.csv | 7 +++ 5 files changed, 113 insertions(+), 3 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Yes.java create mode 100644 backend/src/test/resources/tests/query/CQYES/CQYES.test.json create mode 100644 backend/src/test/resources/tests/query/CQYES/content.csv create mode 100644 backend/src/test/resources/tests/query/CQYES/expected.csv diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java index 03aab80a99..0031abe6bc 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/CQYes.java @@ -4,15 +4,17 @@ import java.util.List; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.QueryPlanContext; import com.bakdata.conquery.models.query.QueryResolveContext; +import com.bakdata.conquery.models.query.RequiredEntities; import com.bakdata.conquery.models.query.queryplan.ConceptQueryPlan; import com.bakdata.conquery.models.query.queryplan.QPNode; -import com.bakdata.conquery.models.query.queryplan.specific.Leaf; +import com.bakdata.conquery.models.query.queryplan.specific.Yes; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; @CPSType(id = "YES", base = CQElement.class) -public class CQYes extends CQElement{ +public class CQYes extends CQElement { @Override public void resolve(QueryResolveContext context) { @@ -21,11 +23,16 @@ public void resolve(QueryResolveContext context) { @Override public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { - return new Leaf(); + return new Yes(context.getDataset().getAllIdsTable()); } @Override public List getResultInfos() { return Collections.emptyList(); } + + @Override + public RequiredEntities collectRequiredEntities(QueryExecutionContext context) { + return new RequiredEntities(context.getBucketManager().getEntities().keySet()); + } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Yes.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Yes.java new file mode 100644 index 0000000000..fd2c228a9b --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/Yes.java @@ -0,0 +1,57 @@ +package com.bakdata.conquery.models.query.queryplan.specific; + +import java.util.Collection; +import java.util.Collections; +import java.util.Set; + +import com.bakdata.conquery.models.common.CDateSet; +import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.events.Bucket; +import com.bakdata.conquery.models.query.QueryExecutionContext; +import com.bakdata.conquery.models.query.entity.Entity; +import com.bakdata.conquery.models.query.queryplan.QPNode; +import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; +import lombok.Data; +import lombok.ToString; + +@ToString +@Data +public class Yes extends QPNode { + + private final Table table; + + @Override + public void init(Entity entity, QueryExecutionContext context) { + super.init(entity, context); + } + + @Override + public void acceptEvent(Bucket bucket, int event) { + + } + + @Override + public void collectRequiredTables(Set
-
+
From cb563b506b173a41ba92a53975b97d20bfc2e6c2 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Fri, 19 May 2023 14:48:55 +0200 Subject: [PATCH 331/679] Implement version 2 of UI dropping between elements --- .../form-components/DropzoneList.tsx | 23 ++++- .../form-concept-group/BetweenElement.tsx | 89 ++++++++++++++++++ .../DropzoneBetweenElements.tsx | 94 ------------------- .../form-concept-group/FormConceptGroup.tsx | 55 +++++------ 4 files changed, 133 insertions(+), 128 deletions(-) create mode 100644 frontend/src/js/external-forms/form-concept-group/BetweenElement.tsx delete mode 100644 frontend/src/js/external-forms/form-concept-group/DropzoneBetweenElements.tsx diff --git a/frontend/src/js/external-forms/form-components/DropzoneList.tsx b/frontend/src/js/external-forms/form-components/DropzoneList.tsx index f4f22a8925..07964fb368 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneList.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneList.tsx @@ -14,6 +14,7 @@ import DropzoneWithFileInput, { } from "../../ui-components/DropzoneWithFileInput"; import Label from "../../ui-components/Label"; import Optional from "../../ui-components/Optional"; +import BetweenElements from "../form-concept-group/BetweenElement"; const ListItem = styled("div")` position: relative; @@ -51,6 +52,10 @@ interface PropsT { ) => void; onDropFile: (file: File) => void; onImportLines: (lines: string[]) => void; + conceptDropzoneText: string; + dropBetween: ( + i: number, + ) => (item: DroppableObject, monitor: DropTargetMonitor) => void; } const DropzoneList = ( @@ -66,6 +71,8 @@ const DropzoneList = ( disallowMultipleColumns, onDrop, onImportLines, + conceptDropzoneText, + dropBetween, }: PropsT, ref: Ref, ) => { @@ -87,10 +94,18 @@ const DropzoneList = ( {items && items.length > 0 && (
{items.map((item, i) => ( - - onDelete(i)} /> - {item} - + <> + + {() => conceptDropzoneText} + + + onDelete(i)} /> + {item} + + ))}
)} diff --git a/frontend/src/js/external-forms/form-concept-group/BetweenElement.tsx b/frontend/src/js/external-forms/form-concept-group/BetweenElement.tsx new file mode 100644 index 0000000000..d83d06d878 --- /dev/null +++ b/frontend/src/js/external-forms/form-concept-group/BetweenElement.tsx @@ -0,0 +1,89 @@ +import styled from "@emotion/styled"; +import { faPlus } from "@fortawesome/free-solid-svg-icons"; +import { ReactNode, useState } from "react"; +import { DropTargetMonitor, useDrop } from "react-dnd"; + +import IconButton from "../../button/IconButton"; +import Dropzone, { + ChildArgs, + PossibleDroppableObject, +} from "../../ui-components/Dropzone"; + +interface Props { + onDrop: (item: DroppableObject, monitor: DropTargetMonitor) => void; + acceptedDropTypes: string[]; + children?: (args: ChildArgs) => ReactNode; +} + +const Root = styled("div")<{ + isOver: boolean; + isDroppable: boolean; +}>` + background-color: ${({ theme, isDroppable, isOver }) => { + if (isOver && isDroppable) return theme.col.grayLight; + if (isDroppable) return theme.col.grayVeryLight; + return "inherit"; + }}; + display: flex; + align-items: center; + width: 100%; + margin-bottom: 1px; + z-index: 2; + position: relative; +`; + +const PlusContainer = styled("div")` + margin-left: 45%; + width: 10%; +`; + +const BetweenElements = ({ + acceptedDropTypes, + children, + onDrop, +}: Props) => { + const [showDropzone, setShowDropzone] = useState(false); + const [{ isOver, isDroppable }, drop] = useDrop({ + accept: acceptedDropTypes, + collect: (monitor) => ({ + isOver: monitor.isOver(), + isDroppable: monitor.canDrop(), + }), + }); + const [{ isOver: isOver2 }, drop2] = useDrop({ + accept: acceptedDropTypes, + collect: (monitor) => ({ + isOver: monitor.isOver(), + }), + }); + + const onDropped = (item: DroppableObject, monitor: DropTargetMonitor) => { + setShowDropzone(false); + onDrop(item, monitor); + }; + + return ( + <> + {!(showDropzone || isOver || isOver2) && ( + + setShowDropzone(true)}> + + + + )} + + {(showDropzone || isOver || isOver2) && ( + // TODO x - to close the dropzone + + {children} + + )} + + ); +}; + +export default BetweenElements; diff --git a/frontend/src/js/external-forms/form-concept-group/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-concept-group/DropzoneBetweenElements.tsx deleted file mode 100644 index 0613fd1d35..0000000000 --- a/frontend/src/js/external-forms/form-concept-group/DropzoneBetweenElements.tsx +++ /dev/null @@ -1,94 +0,0 @@ -import styled from "@emotion/styled"; -import { ReactNode } from "react"; -import { DropTargetMonitor, useDrop } from "react-dnd"; - -import { DNDType } from "../../common/constants/dndTypes"; -import Dropzone, { - ChildArgs, - PossibleDroppableObject, -} from "../../ui-components/Dropzone"; - -interface PropsT { - onDrop: (item: DroppableObject, monitor: DropTargetMonitor) => void; - acceptedDropTypes: string[]; - children?: (args: ChildArgs) => ReactNode; -} - -const DropzoneBetweenElements = < - DroppableObject extends PossibleDroppableObject, ->({ - onDrop, - children, - acceptedDropTypes, -}: PropsT) => { - const Root = styled("div")<{ - isHovered: boolean; - }>` - width: 100%; - left: 0; - top: -17px; - right: 0; - position: absolute; - bottom: 90%; - border-radius: ${({ theme }) => theme.borderRadius}; - `; - - const DropzoneRoot = styled("div")` - width: 100%; - left: 0; - top: -17px; - right: 0; - position: absolute; - bottom: 90%; - z-index: 2; - background-color: ${({ theme }) => theme.col.bg}; - `; - - const [{ isOver, isDroppable }, drop] = useDrop({ - accept: [ - DNDType.FORM_CONFIG, - DNDType.CONCEPT_TREE_NODE, - DNDType.PREVIOUS_QUERY, - DNDType.PREVIOUS_SECONDARY_ID_QUERY, - ], - hover: (_, __) => { - if (!isDroppable) return; - }, - collect: (monitor) => ({ - isOver: monitor.isOver(), - isDroppable: monitor.canDrop(), - }), - }); - - const [{ isOver: isOver2, isDroppable: isDroppable2 }, drop2] = useDrop({ - accept: [ - DNDType.FORM_CONFIG, - DNDType.CONCEPT_TREE_NODE, - DNDType.PREVIOUS_QUERY, - DNDType.PREVIOUS_SECONDARY_ID_QUERY, - ], - hover: (_, __) => { - if (!isDroppable2) return; - }, - collect: (monitor) => ({ - isOver: monitor.isOver(), - isDroppable: monitor.canDrop(), - }), - }); - - return ( - <> - {!isOver && !isOver2 && } - {/* Show when hovered with text and dropzone */} - {(isOver || isOver2) && ( - - - {children} - - - )} - - ); -}; - -export default DropzoneBetweenElements; diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 6d55b3f636..94fcb4af78 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -32,7 +32,6 @@ import { useVisibleConceptListFields, } from "../stateSelectors"; -import DropzoneBetweenElements from "./DropzoneBetweenElements"; import FormConceptCopyModal from "./FormConceptCopyModal"; import FormConceptNode from "./FormConceptNode"; import { @@ -201,6 +200,30 @@ const FormConceptGroup = (props: Props) => { ? t("externalForms.common.concept.copying") : props.attributeDropzoneText } + dropBetween={(i: number) => { + return (item: DragItemConceptTreeNode) => { + if (isMovedObject(item)) { + return props.onChange( + addConcept( + insertValue(props.value, i, newValue), + i, + copyConcept(item), + ), + ); + } + + if (props.isValidConcept && !props.isValidConcept(item)) + return null; + + return props.onChange( + addConcept( + insertValue(props.value, i, newValue), + i, + initializeConcept(item, defaults, tableConfig), + ), + ); + }; + }} acceptedDropTypes={[DNDType.CONCEPT_TREE_NODE]} disallowMultipleColumns={props.disallowMultipleColumns} onDelete={(i) => props.onChange(removeValue(props.value, i))} @@ -238,37 +261,9 @@ const FormConceptGroup = (props: Props) => { ), ); }} + conceptDropzoneText={props.conceptDropzoneText} items={props.value.map((row, i) => ( <> - {!props.disallowMultipleColumns && ( - { - if (isMovedObject(item)) { - return props.onChange( - addConcept( - insertValue(props.value, i, newValue), - i, - copyConcept(item), - ), - ); - } - - if (props.isValidConcept && !props.isValidConcept(item)) - return null; - - return props.onChange( - addConcept( - insertValue(props.value, i, newValue), - i, - initializeConcept(item, defaults, tableConfig), - ), - ); - }} - > - {() => props.conceptDropzoneText} - - )} {props.renderRowPrefix ? props.renderRowPrefix({ From 29b50315cf28b1923667b3991592f2a1268c53b5 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 22 May 2023 11:47:47 +0200 Subject: [PATCH 332/679] Iteration 2 of UI --- .../form-components/DropzoneList.tsx | 2 +- .../form-concept-group/BetweenElement.tsx | 30 +++++++++++-------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneList.tsx b/frontend/src/js/external-forms/form-components/DropzoneList.tsx index 07964fb368..887e48d498 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneList.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneList.tsx @@ -18,7 +18,7 @@ import BetweenElements from "../form-concept-group/BetweenElement"; const ListItem = styled("div")` position: relative; - padding: 5px; + padding: 0px 5px 0px 5px; box-shadow: 0 0 3px 0 rgba(0, 0, 0, 0.1); background-color: white; border-radius: ${({ theme }) => theme.borderRadius}; diff --git a/frontend/src/js/external-forms/form-concept-group/BetweenElement.tsx b/frontend/src/js/external-forms/form-concept-group/BetweenElement.tsx index d83d06d878..a7716cc89e 100644 --- a/frontend/src/js/external-forms/form-concept-group/BetweenElement.tsx +++ b/frontend/src/js/external-forms/form-concept-group/BetweenElement.tsx @@ -3,7 +3,7 @@ import { faPlus } from "@fortawesome/free-solid-svg-icons"; import { ReactNode, useState } from "react"; import { DropTargetMonitor, useDrop } from "react-dnd"; -import IconButton from "../../button/IconButton"; +import FaIcon from "../../icon/FaIcon"; import Dropzone, { ChildArgs, PossibleDroppableObject, @@ -21,20 +21,20 @@ const Root = styled("div")<{ }>` background-color: ${({ theme, isDroppable, isOver }) => { if (isOver && isDroppable) return theme.col.grayLight; - if (isDroppable) return theme.col.grayVeryLight; - return "inherit"; + return isDroppable ? theme.col.grayVeryLight : "inherit"; }}; - display: flex; - align-items: center; width: 100%; - margin-bottom: 1px; - z-index: 2; - position: relative; + text-align: center; `; const PlusContainer = styled("div")` - margin-left: 45%; - width: 10%; + margin: auto; +`; + +const SxFaIcon = styled(FaIcon)` + height: 15px; + color: ${({ theme }) => theme.col.black}; + opacity: 0.75; `; const BetweenElements = ({ @@ -42,6 +42,10 @@ const BetweenElements = ({ children, onDrop, }: Props) => { + const SxDropzone = styled(Dropzone)` + margin: 5px 0 5px 0; + `; + const [showDropzone, setShowDropzone] = useState(false); const [{ isOver, isDroppable }, drop] = useDrop({ accept: acceptedDropTypes, @@ -67,20 +71,20 @@ const BetweenElements = ({ {!(showDropzone || isOver || isOver2) && ( setShowDropzone(true)}> - + )} {(showDropzone || isOver || isOver2) && ( // TODO x - to close the dropzone - {children} - + )} ); From 2010f9495f75452c512f077b21da65871ee21faf Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 22 May 2023 15:05:36 +0200 Subject: [PATCH 333/679] Also unique-ify tab field names --- frontend/src/js/external-forms/FormConfigLoader.tsx | 3 ++- frontend/src/js/external-forms/reducer.ts | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/frontend/src/js/external-forms/FormConfigLoader.tsx b/frontend/src/js/external-forms/FormConfigLoader.tsx index 3d75b8b77f..5d06b9f04b 100644 --- a/frontend/src/js/external-forms/FormConfigLoader.tsx +++ b/frontend/src/js/external-forms/FormConfigLoader.tsx @@ -116,6 +116,7 @@ const FormConfigLoader: FC = ({ if (!formConfig || !formConfigToLoadNext) return; const entries = Object.entries(formConfigToLoadNext.values); + const formFields = collectAllFormFields(formConfig.fields); for (const [fieldname, value] of entries) { // -------------------------- @@ -123,7 +124,7 @@ const FormConfigLoader: FC = ({ // because we changed the SELECT values: // from string, e.g. 'next' // to SelectValueT, e.g. { value: 'next', label: 'Next' } - const field = collectAllFormFields(formConfig.fields).find( + const field = formFields.find( (f): f is Field | Tabs => f.type !== "GROUP" && f.name === getUniqueFieldname(formConfig.type, fieldname), diff --git a/frontend/src/js/external-forms/reducer.ts b/frontend/src/js/external-forms/reducer.ts index e407beb3a2..bb77b5f528 100644 --- a/frontend/src/js/external-forms/reducer.ts +++ b/frontend/src/js/external-forms/reducer.ts @@ -23,6 +23,7 @@ const transformToUniqueFieldnames = ( case "TABS": return { ...field, + name: getUniqueFieldname(formType, field.name), tabs: field.tabs.map((tab) => ({ ...tab, fields: transformToUniqueFieldnames(formType, tab.fields), From 4b1ed62645127b085ffd0911e78e6e9e67966cad Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 22 May 2023 17:52:09 +0200 Subject: [PATCH 334/679] Add query node editing --- frontend/src/js/api/apiHelper.ts | 9 +- frontend/src/js/app/RightPane.tsx | 1 + frontend/src/js/editor-v2/EditorV2.tsx | 82 +++++++- .../js/editor-v2/KeyboardShortcutTooltip.tsx | 6 +- frontend/src/js/editor-v2/TreeNode.tsx | 5 +- frontend/src/js/editor-v2/config.ts | 1 + .../EditorV2QueryNodeEditor.tsx | 182 ++++++++++++++++++ .../query-node-edit/useQueryNodeEditing.ts | 34 ++++ .../js/query-node-editor/ContentColumn.tsx | 14 +- frontend/src/localization/de.json | 2 +- frontend/src/localization/en.json | 2 +- 11 files changed, 314 insertions(+), 24 deletions(-) create mode 100644 frontend/src/js/editor-v2/query-node-edit/EditorV2QueryNodeEditor.tsx create mode 100644 frontend/src/js/editor-v2/query-node-edit/useQueryNodeEditing.ts diff --git a/frontend/src/js/api/apiHelper.ts b/frontend/src/js/api/apiHelper.ts index 7db40ec3ab..815a3020f0 100644 --- a/frontend/src/js/api/apiHelper.ts +++ b/frontend/src/js/api/apiHelper.ts @@ -229,9 +229,7 @@ const transformTreeToApi = (tree: Tree): unknown => { ); } - node = nodeIsConceptQueryNode(tree.data) - ? createQueryConcept(tree.data) - : createSavedQuery(tree.data.id); + node = createQueryConcept(tree.data); } else { switch (tree.children.connection) { case "and": @@ -271,6 +269,11 @@ const transformTreeToApi = (tree: Tree): unknown => { ...dateRestriction, child: node, }; + } else if (negation) { + return { + ...negation, + child: node, + }; } else { return node; } diff --git a/frontend/src/js/app/RightPane.tsx b/frontend/src/js/app/RightPane.tsx index f3533aed26..3013cc98a9 100644 --- a/frontend/src/js/app/RightPane.tsx +++ b/frontend/src/js/app/RightPane.tsx @@ -90,6 +90,7 @@ const RightPane = () => { featureNegate featureExpand featureConnectorRotate + featureQueryNodeEdit /> ) : ( diff --git a/frontend/src/js/editor-v2/EditorV2.tsx b/frontend/src/js/editor-v2/EditorV2.tsx index bcfcbeee24..8c925f0799 100644 --- a/frontend/src/js/editor-v2/EditorV2.tsx +++ b/frontend/src/js/editor-v2/EditorV2.tsx @@ -3,6 +3,7 @@ import { faCalendar, faTrashCan } from "@fortawesome/free-regular-svg-icons"; import { faBan, faCircleNodes, + faEdit, faExpandArrowsAlt, faRefresh, faTrash, @@ -13,6 +14,7 @@ import { useHotkeys } from "react-hotkeys-hook"; import { useTranslation } from "react-i18next"; import IconButton from "../button/IconButton"; +import { nodeIsConceptQueryNode } from "../model/node"; import { DragItemConceptTreeNode, DragItemQuery, @@ -29,6 +31,8 @@ import { DateModal } from "./date-restriction/DateModal"; import { useDateEditing } from "./date-restriction/useDateEditing"; import { useExpandQuery } from "./expand/useExpandQuery"; import { useNegationEditing } from "./negation/useNegationEditing"; +import { EditorV2QueryNodeEditor } from "./query-node-edit/EditorV2QueryNodeEditor"; +import { useQueryNodeEditing } from "./query-node-edit/useQueryNodeEditing"; import { Tree } from "./types"; import { findNodeById, useTranslatedConnection } from "./util"; @@ -113,11 +117,13 @@ export function EditorV2({ featureNegate, featureExpand, featureConnectorRotate, + featureQueryNodeEdit, }: { featureDates: boolean; featureNegate: boolean; featureExpand: boolean; featureConnectorRotate: boolean; + featureQueryNodeEdit: boolean; }) { const { t } = useTranslation(); const { @@ -187,30 +193,54 @@ export function EditorV2({ const { onNegateClick } = useNegationEditing({ enabled: featureNegate, - hotkey: "n", + hotkey: HOTKEYS.negate.keyname, selectedNode, updateTreeNode, }); const { onRotateConnector } = useConnectorEditing({ enabled: featureConnectorRotate, - hotkey: "c", + hotkey: HOTKEYS.rotateConnector.keyname, selectedNode, updateTreeNode, }); + const { + showModal: showQueryNodeEditor, + onOpen: onOpenQueryNodeEditor, + onClose: onCloseQueryNodeEditor, + } = useQueryNodeEditing({ + enabled: featureQueryNodeEdit, + hotkey: HOTKEYS.editQueryNode.keyname, + selectedNode, + }); + const connection = useTranslatedConnection( selectedNode?.children?.connection, ); + const onChangeData = useCallback( + (data: DragItemConceptTreeNode) => { + if (!selectedNode) return; + updateTreeNode(selectedNode.id, (node) => { + node.data = data; + }); + }, + [selectedNode, updateTreeNode], + ); + return ( - { - if (!selectedNode || showModal) return; - setSelectedNodeId(undefined); - }} - > +
+ {showQueryNodeEditor && + selectedNode?.data && + nodeIsConceptQueryNode(selectedNode.data) && ( + + )} {showModal && selectedNode && ( + {featureQueryNodeEdit && + selectedNode?.data && + nodeIsConceptQueryNode(selectedNode.data) && ( + + { + e.stopPropagation(); + onOpenQueryNodeEditor(); + }} + > + {t("editorV2.edit")} + + + )} {featureDates && selectedNode && ( - + { + if (!selectedNode || showModal) return; + setSelectedNodeId(undefined); + }} + > {tree ? ( { + e.stopPropagation(); + if (!selectedNode) return; + if ( + selectedNode?.data && + nodeIsConceptQueryNode(selectedNode.data) + ) { + onOpenQueryNodeEditor(); + } + }} tree={tree} updateTreeNode={updateTreeNode} selectedNode={selectedNode} diff --git a/frontend/src/js/editor-v2/KeyboardShortcutTooltip.tsx b/frontend/src/js/editor-v2/KeyboardShortcutTooltip.tsx index 8d7124de45..059894767f 100644 --- a/frontend/src/js/editor-v2/KeyboardShortcutTooltip.tsx +++ b/frontend/src/js/editor-v2/KeyboardShortcutTooltip.tsx @@ -1,5 +1,5 @@ import styled from "@emotion/styled"; -import { ReactElement } from "react"; +import { Fragment, ReactElement } from "react"; import { useTranslation } from "react-i18next"; import { KeyboardKey } from "../common/components/KeyboardKey"; @@ -35,10 +35,10 @@ export const KeyboardShortcutTooltip = ({ {t("common.shortcut")}:{" "} {keynames.map((keyPart, i) => ( - <> + {keyPart} {i < keynames.length - 1 && "+"} - + ))} diff --git a/frontend/src/js/editor-v2/TreeNode.tsx b/frontend/src/js/editor-v2/TreeNode.tsx index dda372fc5b..9cebacc2f4 100644 --- a/frontend/src/js/editor-v2/TreeNode.tsx +++ b/frontend/src/js/editor-v2/TreeNode.tsx @@ -1,6 +1,6 @@ import styled from "@emotion/styled"; import { createId } from "@paralleldrive/cuid2"; -import { memo } from "react"; +import { DOMAttributes, memo } from "react"; import { useTranslation } from "react-i18next"; import { DNDType } from "../common/constants/dndTypes"; @@ -119,6 +119,7 @@ export function TreeNode({ droppable, selectedNode, setSelectedNodeId, + onDoubleClick, }: { tree: Tree; treeParent?: Tree; @@ -129,6 +130,7 @@ export function TreeNode({ }; selectedNode: Tree | undefined; setSelectedNodeId: (id: Tree["id"] | undefined) => void; + onDoubleClick?: DOMAttributes["onDoubleClick"]; }) { const gridStyles = getGridStyles(tree); @@ -225,6 +227,7 @@ export function TreeNode({ negated={tree.negation} leaf={!tree.children} selected={selectedNode?.id === tree.id} + onDoubleClick={onDoubleClick} onClick={(e) => { e.stopPropagation(); setSelectedNodeId(tree.id); diff --git a/frontend/src/js/editor-v2/config.ts b/frontend/src/js/editor-v2/config.ts index 7fd22a71a3..8938a334f8 100644 --- a/frontend/src/js/editor-v2/config.ts +++ b/frontend/src/js/editor-v2/config.ts @@ -14,4 +14,5 @@ export const HOTKEYS = { flip: { keyname: "f" }, rotateConnector: { keyname: "c" }, reset: { keyname: "shift+backspace" }, + editQueryNode: { keyname: "Enter" }, }; diff --git a/frontend/src/js/editor-v2/query-node-edit/EditorV2QueryNodeEditor.tsx b/frontend/src/js/editor-v2/query-node-edit/EditorV2QueryNodeEditor.tsx new file mode 100644 index 0000000000..8793879912 --- /dev/null +++ b/frontend/src/js/editor-v2/query-node-edit/EditorV2QueryNodeEditor.tsx @@ -0,0 +1,182 @@ +import { useCallback } from "react"; + +import { SelectOptionT } from "../../api/types"; +import { NodeResetConfig } from "../../model/node"; +import { resetSelects } from "../../model/select"; +import { + resetTables, + tableIsEditable, + tableWithDefaults, +} from "../../model/table"; +import QueryNodeEditor from "../../query-node-editor/QueryNodeEditor"; +import { DragItemConceptTreeNode } from "../../standard-query-editor/types"; +import { ModeT } from "../../ui-components/InputRange"; + +export const EditorV2QueryNodeEditor = ({ + node, + onClose, + onChange, +}: { + node: DragItemConceptTreeNode; + onClose: () => void; + onChange: (node: DragItemConceptTreeNode) => void; +}) => { + const showTables = + node.tables.length > 1 && + node.tables.some((table) => tableIsEditable(table)); + + const onUpdateLabel = useCallback( + (label: string) => onChange({ ...node, label }), + [onChange, node], + ); + + const onToggleTable = useCallback( + (tableIdx: number, isExcluded: boolean) => { + const tables = [...node.tables]; + tables[tableIdx] = { ...tables[tableIdx], exclude: isExcluded }; + onChange({ ...node, tables }); + }, + [node, onChange], + ); + + const onLoadFilterSuggestions = useCallback( + ( + params: any, + tableIdx: number, + filterIdx: number, + config?: { returnOnly?: boolean }, + ) => { + return Promise.resolve(null); + }, + [], + ); + + const onDropConcept = useCallback( + (concept: DragItemConceptTreeNode) => { + const ids = [...node.ids, concept.ids[0]]; + onChange({ ...node, ids }); + }, + [node, onChange], + ); + + const onRemoveConcept = useCallback( + (conceptId: string) => { + const ids = node.ids.filter((id) => id !== conceptId); + onChange({ ...node, ids }); + }, + [node, onChange], + ); + + const onSelectSelects = useCallback( + (value: SelectOptionT[]) => { + onChange({ + ...node, + selects: node.selects.map((select) => ({ + ...select, + selected: !!value.find((s) => s.value === select.id), + })), + }); + }, + [node, onChange], + ); + + const onSelectTableSelects = useCallback( + (tableIdx: number, value: SelectOptionT[]) => { + const tables = [...node.tables]; + tables[tableIdx] = { + ...tables[tableIdx], + selects: tables[tableIdx].selects.map((select) => ({ + ...select, + selected: !!value.find((s) => s.value === select.id), + })), + }; + onChange({ ...node, tables }); + }, + [node, onChange], + ); + + const onSetFilterValue = useCallback( + (tableIdx: number, filterIdx: number, value: any) => { + const tables = [...node.tables]; + tables[tableIdx] = { + ...tables[tableIdx], + filters: tables[tableIdx].filters.map((filter, idx) => + idx === filterIdx ? { ...filter, value } : filter, + ), + }; + onChange({ ...node, tables }); + }, + [node, onChange], + ); + + const onSwitchFilterMode = useCallback( + (tableIdx: number, filterIdx: number, mode: ModeT) => { + const tables = [...node.tables]; + tables[tableIdx] = { + ...tables[tableIdx], + filters: tables[tableIdx].filters.map((filter, idx) => + idx === filterIdx ? { ...filter, mode } : filter, + ), + }; + onChange({ ...node, tables }); + }, + [node, onChange], + ); + + const onResetTable = useCallback( + (tableIdx: number, config: NodeResetConfig) => { + const table = node.tables[tableIdx]; + const resetTable = tableWithDefaults(config)(table); + + const tables = [...node.tables]; + tables[tableIdx] = resetTable; + onChange({ ...node, tables }); + }, + [node, onChange], + ); + + const onResetAllSettings = useCallback( + (config: NodeResetConfig) => { + const tables = resetTables(node.tables, config); + const selects = resetSelects(node.selects, config); + onChange({ ...node, tables, selects }); + }, + [node, onChange], + ); + + const onSetDateColumn = useCallback( + (tableIdx: number, value: string) => { + const tables = [...node.tables]; + tables[tableIdx] = { + ...tables[tableIdx], + dateColumn: { + ...tables[tableIdx].dateColumn!, + value, + }, + }; + onChange({ ...node, tables }); + }, + [node, onChange], + ); + + return ( + + ); +}; diff --git a/frontend/src/js/editor-v2/query-node-edit/useQueryNodeEditing.ts b/frontend/src/js/editor-v2/query-node-edit/useQueryNodeEditing.ts new file mode 100644 index 0000000000..a481c05c69 --- /dev/null +++ b/frontend/src/js/editor-v2/query-node-edit/useQueryNodeEditing.ts @@ -0,0 +1,34 @@ +import { useCallback, useState } from "react"; +import { useHotkeys } from "react-hotkeys-hook"; + +import { Tree } from "../types"; + +export const useQueryNodeEditing = ({ + enabled, + hotkey, + selectedNode, +}: { + enabled: boolean; + hotkey: string; + selectedNode: Tree | undefined; +}) => { + const [showModal, setShowModal] = useState(false); + + const onClose = useCallback(() => setShowModal(false), []); + const onOpen = useCallback(() => { + if (!enabled) return; + if (!selectedNode) return; + + setShowModal(true); + }, [enabled, selectedNode]); + + useHotkeys(hotkey, onOpen, [onOpen], { + preventDefault: true, + }); + + return { + showModal, + onClose, + onOpen, + }; +}; diff --git a/frontend/src/js/query-node-editor/ContentColumn.tsx b/frontend/src/js/query-node-editor/ContentColumn.tsx index 58580cc673..667d813f5b 100644 --- a/frontend/src/js/query-node-editor/ContentColumn.tsx +++ b/frontend/src/js/query-node-editor/ContentColumn.tsx @@ -105,12 +105,14 @@ const ContentColumn: FC = ({ {t("queryNodeEditor.properties")} - + {(onToggleSecondaryIdExclude || onToggleTimestamps) && ( + + )} {nodeIsConceptQueryNode(node) && node.selects && ( Date: Tue, 23 May 2023 17:22:25 +0200 Subject: [PATCH 335/679] Fix dates and more --- frontend/src/js/editor-v2/TreeNode.tsx | 204 +++++++++++------- .../editor-v2/date-restriction/DateModal.tsx | 31 ++- .../editor-v2/date-restriction/DateRange.tsx | 30 ++- .../date-restriction/useDateEditing.ts | 2 +- frontend/src/js/icon/FaIcon.tsx | 5 +- .../src/js/ui-components/InputDateRange.tsx | 4 +- frontend/src/localization/de.json | 3 +- frontend/src/localization/en.json | 3 +- 8 files changed, 182 insertions(+), 100 deletions(-) diff --git a/frontend/src/js/editor-v2/TreeNode.tsx b/frontend/src/js/editor-v2/TreeNode.tsx index 9cebacc2f4..1c84aa310a 100644 --- a/frontend/src/js/editor-v2/TreeNode.tsx +++ b/frontend/src/js/editor-v2/TreeNode.tsx @@ -1,9 +1,11 @@ import styled from "@emotion/styled"; +import { faCalendarMinus } from "@fortawesome/free-regular-svg-icons"; import { createId } from "@paralleldrive/cuid2"; import { DOMAttributes, memo } from "react"; import { useTranslation } from "react-i18next"; import { DNDType } from "../common/constants/dndTypes"; +import { Icon } from "../icon/FaIcon"; import { nodeIsConceptQueryNode } from "../model/node"; import { getRootNodeLabel } from "../standard-query-editor/helper"; import Dropzone, { DropzoneProps } from "../ui-components/Dropzone"; @@ -23,13 +25,15 @@ const Node = styled("div")<{ selected?: boolean; negated?: boolean; leaf?: boolean; + isDragging?: boolean; }>` - padding: ${({ leaf }) => (leaf ? "8px 10px" : "20px")}; - border: 1px solid + padding: ${({ leaf, isDragging }) => + leaf ? "8px 10px" : isDragging ? "5px" : "24px"}; + border: 2px solid ${({ negated, theme, selected }) => negated ? "red" : selected ? theme.col.gray : theme.col.grayMediumLight}; box-shadow: ${({ selected, theme }) => - selected ? `inset 0px 0px 0px 1px ${theme.col.gray}` : "none"}; + selected ? `inset 0px 0px 0px 2px ${theme.col.gray}` : "none"}; border-radius: ${({ theme }) => theme.borderRadius}; width: ${({ leaf }) => (leaf ? "180px" : "inherit")}; @@ -56,9 +60,10 @@ function getGridStyles(tree: Tree) { } } -const InvisibleDropzoneContainer = styled(Dropzone)` +const InvisibleDropzoneContainer = styled(Dropzone)<{ bare?: boolean }>` width: 100%; height: 100%; + padding: ${({ bare }) => (bare ? "6px" : "20px")}; `; const InvisibleDropzone = ( @@ -77,10 +82,12 @@ const InvisibleDropzone = ( const Name = styled("div")` font-size: ${({ theme }) => theme.font.sm}; font-weight: 400; + color: ${({ theme }) => theme.col.black}; `; const Description = styled("div")` font-size: ${({ theme }) => theme.font.xs}; + color: ${({ theme }) => theme.col.black}; `; const PreviousQueryLabel = styled("p")` @@ -104,6 +111,9 @@ const RootNode = styled("p")` const Dates = styled("div")` text-align: right; + font-size: ${({ theme }) => theme.font.xs}; + text-transform: uppercase; + font-weight: 400; `; const Connection = memo(({ connection }: { connection?: ConnectionKind }) => { @@ -223,89 +233,119 @@ export function TreeNode({ } /> )} - { - e.stopPropagation(); - setSelectedNodeId(tree.id); - }} + {}} > - {tree.dates?.restriction && ( - - - - )} - {(!tree.children || tree.data) && ( -
- {tree.data?.type !== DNDType.CONCEPT_TREE_NODE && ( - - {t("queryEditor.previousQuery")} - + {({ canDrop }) => ( + { + e.stopPropagation(); + setSelectedNodeId(tree.id); + }} + > + {tree.dates?.restriction && ( + + + )} - {rootNodeLabel && {rootNodeLabel}} - {tree.data?.label && {tree.data.label}} - {tree.data && nodeIsConceptQueryNode(tree.data) && ( - {tree.data?.description} + {tree.dates?.excluded && ( + + + {t("editorV2.datesExcluded")} + )} -
- )} - {tree.children && ( - - onDropAtChildrenIdx({ idx: 0, item })} - /> - {tree.children.items.map((item, i, items) => ( - <> - - {i < items.length - 1 && ( - - onDropAtChildrenIdx({ idx: i + 1, item }) - } - > - {() => ( - - )} - + {(!tree.children || tree.data) && ( +
+ {tree.data?.type !== DNDType.CONCEPT_TREE_NODE && ( + + {t("queryEditor.previousQuery")} + + )} + {rootNodeLabel && {rootNodeLabel}} + {tree.data?.label && {tree.data.label}} + {tree.data && nodeIsConceptQueryNode(tree.data) && ( + {tree.data?.description} )} - - ))} - - onDropAtChildrenIdx({ - idx: tree.children!.items.length, - item, - }) - } - /> - +
+ )} + {tree.children && ( + + onDropAtChildrenIdx({ idx: 0, item })} + > + {() => ( + + )} + + {tree.children.items.map((item, i, items) => ( + <> + + {i < items.length - 1 && ( + + onDropAtChildrenIdx({ idx: i + 1, item }) + } + > + {() => ( + + )} + + )} + + ))} + + onDropAtChildrenIdx({ + idx: tree.children!.items.length, + item, + }) + } + > + {() => ( + + )} + + + )} + )} - + {droppable.h && ( diff --git a/frontend/src/js/editor-v2/date-restriction/DateModal.tsx b/frontend/src/js/editor-v2/date-restriction/DateModal.tsx index 8385011f36..27da9d25a9 100644 --- a/frontend/src/js/editor-v2/date-restriction/DateModal.tsx +++ b/frontend/src/js/editor-v2/date-restriction/DateModal.tsx @@ -1,4 +1,5 @@ import styled from "@emotion/styled"; +import { faCalendarMinus } from "@fortawesome/free-regular-svg-icons"; import { faUndo } from "@fortawesome/free-solid-svg-icons"; import { useCallback, useMemo } from "react"; import { useHotkeys } from "react-hotkeys-hook"; @@ -7,6 +8,7 @@ import { useTranslation } from "react-i18next"; import { DateRangeT } from "../../api/types"; import IconButton from "../../button/IconButton"; import { DateStringMinMax } from "../../common/helpers/dateHelper"; +import { Icon } from "../../icon/FaIcon"; import Modal from "../../modal/Modal"; import InputCheckbox from "../../ui-components/InputCheckbox"; import InputDateRange from "../../ui-components/InputDateRange"; @@ -14,7 +16,16 @@ import InputDateRange from "../../ui-components/InputDateRange"; const Col = styled("div")` display: flex; flex-direction: column; - gap: 20px; + gap: 32px; +`; + +const SectionHeadline = styled("p")` + display: flex; + align-items: center; + gap: 10px; + margin: 0 0 10px; + font-size: ${({ theme }) => theme.font.md}; + font-weight: 400; `; const ResetAll = styled(IconButton)` @@ -74,8 +85,8 @@ export const DateModal = ({ doneButton headline={t("queryGroupModal.explanation")} > -
{headline}
requiredTables) { + requiredTables.add(table); + } + + @Override + public boolean isContained() { + return true; + } + + @Override + public Collection> getDateAggregators() { + return Collections.emptySet(); + } + + @Override + public boolean isOfInterest(Bucket bucket) { + return true; + } + + @Override + public boolean isOfInterest(Entity entity) { + return true; + } +} diff --git a/backend/src/test/resources/tests/query/CQYES/CQYES.test.json b/backend/src/test/resources/tests/query/CQYES/CQYES.test.json new file mode 100644 index 0000000000..23aafee084 --- /dev/null +++ b/backend/src/test/resources/tests/query/CQYES/CQYES.test.json @@ -0,0 +1,32 @@ +{ + "type": "QUERY_TEST", + "label": "CQYES Test", + "expectedCsv": "tests/query/CQYES/expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type" : "YES" + } + }, + "concepts": [ + + ], + "content": { + "tables": [ + { + "csv": "tests/query/CQYES/content.csv", + "name": "test_table", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/query/CQYES/content.csv b/backend/src/test/resources/tests/query/CQYES/content.csv new file mode 100644 index 0000000000..5499484b3d --- /dev/null +++ b/backend/src/test/resources/tests/query/CQYES/content.csv @@ -0,0 +1,7 @@ +pid,datum +4,2013-11-10 +5,2013-11-10 +6,2013-11-10 +1,2012-01-01 +2,2013-11-10 +3,2013-11-10 diff --git a/backend/src/test/resources/tests/query/CQYES/expected.csv b/backend/src/test/resources/tests/query/CQYES/expected.csv new file mode 100644 index 0000000000..a8bf68f3a1 --- /dev/null +++ b/backend/src/test/resources/tests/query/CQYES/expected.csv @@ -0,0 +1,7 @@ +result,dates +4,{} +5,{} +6,{} +1,{} +2,{} +3,{} From 8b6c758bec9ee1173dd8c4861d7e31b5ea84bdf0 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 1 Jun 2023 11:12:33 +0200 Subject: [PATCH 351/679] fixes additionalSemantics being null --- .../datasets/concepts/select/concept/ConceptColumnSelect.java | 3 ++- .../conquery/models/query/resultinfo/SelectResultInfo.java | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java index 9cdcccbd74..f02d2fb086 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/select/concept/ConceptColumnSelect.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.datasets.concepts.select.concept; +import java.util.Collections; import java.util.Set; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; @@ -39,7 +40,7 @@ public Aggregator createAggregator() { @Override public SelectResultInfo getResultInfo(CQConcept cqConcept) { - Set additionalSemantics = null; + Set additionalSemantics = Collections.emptySet(); if (isAsIds()) { additionalSemantics = Set.of(new SemanticType.ConceptColumnT(cqConcept.getConcept())); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java index f427fbdaeb..9b993666b5 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/SelectResultInfo.java @@ -26,6 +26,7 @@ public class SelectResultInfo extends ResultInfo { private final CQConcept cqConcept; @Getter(AccessLevel.PACKAGE) + @NonNull private final Set additionalSemantics; public SelectResultInfo(Select select, CQConcept cqConcept) { From 5d230eef0df42c90c7370432705208794cfa39a1 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 1 Jun 2023 11:38:00 +0200 Subject: [PATCH 352/679] removes NOT_EMPTY validation for frontend, as no query-group is legal --- .../forms/export_form.frontend_conf.json | 653 +++++++++--------- .../table_export_form.frontend_conf.json | 173 +++-- 2 files changed, 412 insertions(+), 414 deletions(-) diff --git a/backend/src/main/resources/com/bakdata/conquery/frontend/forms/export_form.frontend_conf.json b/backend/src/main/resources/com/bakdata/conquery/frontend/forms/export_form.frontend_conf.json index 00c675d710..8516063810 100644 --- a/backend/src/main/resources/com/bakdata/conquery/frontend/forms/export_form.frontend_conf.json +++ b/backend/src/main/resources/com/bakdata/conquery/frontend/forms/export_form.frontend_conf.json @@ -1,351 +1,350 @@ { - "title": { - "en": "Data Export", - "de": "Datenexport" + "title": { + "en": "Data Export", + "de": "Datenexport" + }, + "description": { + "de": "Mit diesem Formular werden Konzept- und Ausgabewerte für jeden Versicherten einer Anfrage einzeln auf einen Beobachtungszeitraum aggregiert. Zusätzlich zum gesamten Zeitraum kann dieser nochmal in Jahre oder Quartale unterteilt werden. Die Daten können dabei in einem absoluten Beobachtungszeitraum oder relativ zu einem mit der Anfrage erstellten Indexdatum analysiert werden. Die Ausgabe kann sowohl als Excel, als auch CSV heruntergeladen werden.", + "en": "With this form, concept and output values for each insured person of a query are aggregated individually to an observation period. In addition to the entire period, this can be subdivided again into years or quarters. The data can be analyzed in an absolute observation period or relative to an index date created with the query. The output can be downloaded as Excel as well as CSV." + }, + "type": "EXPORT_FORM", + "fields": [ + { + "label": { + "en": "Cohort", + "de": "Versichertengruppe" + }, + "style": { + "size": "h1" + }, + "type": "HEADLINE" }, - "description": { - "de": "Mit diesem Formular werden Konzept- und Ausgabewerte für jeden Versicherten einer Anfrage einzeln auf einen Beobachtungszeitraum aggregiert. Zusätzlich zum gesamten Zeitraum kann dieser nochmal in Jahre oder Quartale unterteilt werden. Die Daten können dabei in einem absoluten Beobachtungszeitraum oder relativ zu einem mit der Anfrage erstellten Indexdatum analysiert werden. Die Ausgabe kann sowohl als Excel, als auch CSV heruntergeladen werden.", - "en": "With this form, concept and output values for each insured person of a query are aggregated individually to an observation period. In addition to the entire period, this can be subdivided again into years or quarters. The data can be analyzed in an absolute observation period or relative to an index date created with the query. The output can be downloaded as Excel as well as CSV." - }, - "type": "EXPORT_FORM", - "fields": [ - { - "label": { - "en": "Cohort", - "de": "Versichertengruppe" - }, - "style": { - "size": "h1" - }, - "type": "HEADLINE" - }, + { + "name": "queryGroup", + "type": "RESULT_GROUP", + "label": { + "de": "Versichertengruppe (Anfrage)", + "en": "Cohort (Previous Query)" + }, + "dropzoneLabel": { + "de": "Füge eine Versichertengruppe aus einer bestehenden Anfrage hinzu", + "en": "Add a cohort from a previous query" + }, + "validations": [ + ], + "tooltip": { + "de": "Versichertengruppe (Anfrage) für die Daten ausgegeben werden soll.", + "en": "Cohort whose data is exported" + } + }, + { + "label": { + "de": "Zeitlicher Bezug", + "en": "Time Reference" + }, + "style": { + "size": "h1" + }, + "type": "HEADLINE" + }, + { + "name": "timeMode", + "type": "TABS", + "defaultValue": "ABSOLUTE", + "tabs": [ { - "name": "queryGroup", - "type": "RESULT_GROUP", - "label": { - "de": "Versichertengruppe (Anfrage)", - "en": "Cohort (Previous Query)" - }, - "dropzoneLabel": { - "de": "Füge eine Versichertengruppe aus einer bestehenden Anfrage hinzu", - "en": "Add a cohort from a previous query" - }, - "validations": [ + "name": "ABSOLUTE", + "title": { + "de": "Absolut", + "en": "Absolute" + }, + "fields": [ + { + "name": "dateRange", + "type": "DATE_RANGE", + "label": { + "de": "Beobachtungszeitraum", + "en": "Observation Period" + }, + "validations": [ "NOT_EMPTY" - ], - "tooltip": { - "de": "Versichertengruppe (Anfrage) für die Daten ausgegeben werden soll.", - "en": "Cohort whose data is exported" - } + ] + } + ], + "tooltip": { + "de": "Die Ausgaben beziehen sich auf einen festen absoluten Zeitraum.", + "en": "The output relates to a fixed absolute period." + } }, { - "label": { - "de": "Zeitlicher Bezug", - "en": "Time Reference" - }, - "style": { - "size": "h1" - }, - "type": "HEADLINE" - }, - { - "name": "timeMode", - "type": "TABS", - "defaultValue": "ABSOLUTE", - "tabs": [ + "name": "RELATIVE", + "title": { + "de": "Relativ", + "en": "Relative" + }, + "fields": [ + { + "name": "timeUnit", + "type": "SELECT", + "label": { + "de": "Zeiteinheit des Vor- und Nachbeobachtungszeitraums", + "en": "Time unit of feature and outcome periods" + }, + "defaultValue": "QUARTERS", + "options": [ { - "name": "ABSOLUTE", - "title": { - "de": "Absolut", - "en": "Absolute" - }, - "fields": [ - { - "name": "dateRange", - "type": "DATE_RANGE", - "label": { - "de": "Beobachtungszeitraum", - "en": "Observation Period" - }, - "validations": [ - "NOT_EMPTY" - ] - } - ], - "tooltip": { - "de": "Die Ausgaben beziehen sich auf einen festen absoluten Zeitraum.", - "en": "The output relates to a fixed absolute period." - } + "label": { + "de": "Tage", + "en": "Days" + }, + "value": "DAYS" }, { - "name": "RELATIVE", - "title": { - "de": "Relativ", - "en": "Relative" - }, - "fields": [ - { - "name": "timeUnit", - "type": "SELECT", - "label": { - "de": "Zeiteinheit des Vor- und Nachbeobachtungszeitraums", - "en": "Time unit of feature and outcome periods" - }, - "defaultValue": "QUARTERS", - "options": [ - { - "label": { - "de": "Tage", - "en": "Days" - }, - "value": "DAYS" - }, - { - "label": { - "de": "Quartale", - "en": "Quarters" - }, - "value": "QUARTERS" - } - ], - "validations": [ - "NOT_EMPTY" - ], - "tooltip": { - "de": "Die Zeiteinheit bezieht sich auf die folgenden Eingabefelder, welche den Zeitraum vor und nach dem Indexdatum bestimmen.", - "en": "The time unit refers to the following input fields, which determine the period before and after the index date." - } - }, - { - "name": "timeCountBefore", - "type": "NUMBER", - "defaultValue": 4, - "min": 1, - "label": { - "de": "Zeit davor", - "en": "Units before" - }, - "placeholder": { - "de": "4", - "en": "4" - }, - "pattern": "^(?!-)\\d*$", - "validations": [ - "NOT_EMPTY", - "GREATER_THAN_ZERO" - ], - "tooltip": { - "de": "Anzahl an Zeiteinheiten, die die Größe des Zeitraums vor dem Indexdatum bestimmten.", - "en": "Number of time units that determined the size of the period before the index date." - } - }, - { - "name": "timeCountAfter", - "type": "NUMBER", - "min": 1, - "defaultValue": 4, - "label": { - "de": "Zeit danach", - "en": "Units after" - }, - "placeholder": { - "de": "4", - "en": "4" - }, - "pattern": "^(?!-)\\d*$", - "validations": [ - "NOT_EMPTY", - "GREATER_THAN_ZERO" - ], - "tooltip": { - "de": "Anzahl an Zeiteinheiten, die die Größe des Zeitraums nach dem Indexdatum bestimmten.", - "en": "Number of time units that determined the size of the period after the index date." - } - }, - { - "name": "indexSelector", - "type": "SELECT", - "label": { - "de": "Zeitstempel Indexdatum", - "en": "Index date sampler" - }, - "defaultValue": "EARLIEST", - "options": [ - { - "label": { - "de": "ERSTES", - "en": "First" - }, - "value": "EARLIEST" - }, - { - "label": { - "de": "LETZTES", - "en": "Last" - }, - "value": "LATEST" - }, - { - "label": { - "de": "ZUFÄLLIG", - "en": "Random" - }, - "value": "RANDOM" - } - ], - "validations": [ - "NOT_EMPTY" - ], - "tooltip": { - "de": "Wenn mehr als ein Datumswert pro Person vorliegt, kann hier ausgewählt werden welcher als Indexdatum gewertet werden soll.", - "en": "If there is more than one date value per person, you can select here which one should be evaluated as index date." - } - }, - { - "name": "indexPlacement", - "type": "SELECT", - "label": { - "de": "Zugehörigkeit Indexdatum", - "en": "Index period inclusion" - }, - "defaultValue": "AFTER", - "options": [ - { - "label": { - "de": "VORBEOBACHTUNGSZEITRAUM", - "en": "Feature period" - }, - "value": "BEFORE" - }, - { - "label": { - "de": "NEUTRAL", - "en": "Neutral" - }, - "value": "NEUTRAL" - }, - { - "label": { - "de": "NACHBEOBACHTUNGSZEITRAUM", - "en": "Outcome period" - }, - "value": "AFTER" - } - ], - "validations": [ - "NOT_EMPTY" - ], - "tooltip": { - "de": "Angabe für welchen Zeitraum das Quartal mit dem Indexdatum gewertet wird.", - "en": "Indication which period includes the index period" - } - } - ], - "tooltip": { - "de": "Die Ausgaben beziehen sich auf einen Vor- und Nachbeobachtungszeitraum, abhängig von dem Indexdatum jeder Person in der Versichertengruppe.", - "en": "Outputs are for a pre- and post-observation period, depending on the index period of each person in the cohort." - } + "label": { + "de": "Quartale", + "en": "Quarters" + }, + "value": "QUARTERS" } - ] - }, - { - "label": { - "de": "Datengrundlage und Konzepte", - "en": "Attributes" - }, - "style": { - "size": "h1" - }, - "type": "HEADLINE" - }, - { - "name": "features", - "type": "CONCEPT_LIST", - "label": { - "de": "Konzepte", - "en": "Concepts" - }, - "isTwoDimensional": true, - "conceptDropzoneLabel": { - "de": "Füge ein Konzept oder eine Konzeptliste hinzu", - "en": "Add a concept or a concept list" - }, - "validations": [ - "NOT_EMPTY" - ] - }, - { - "label": { - "de": "Analyse und Ausgabe", - "en": "Analysis and Output" - }, - "style": { - "size": "h1" - }, - "type": "HEADLINE" - }, - { - "name": "resolution", - "type": "SELECT", - "label": { - "de": "Stratifizierung Beobachtungszeitraum", - "en": "Temporal stratification" - }, - "defaultValue": "COMPLETE", - "options": [ - { - "label": { - "de": "Gesamter Zeitraum", - "en": "Total" - }, - "value": "COMPLETE" - }, + ], + "validations": [ + "NOT_EMPTY" + ], + "tooltip": { + "de": "Die Zeiteinheit bezieht sich auf die folgenden Eingabefelder, welche den Zeitraum vor und nach dem Indexdatum bestimmen.", + "en": "The time unit refers to the following input fields, which determine the period before and after the index date." + } + }, + { + "name": "timeCountBefore", + "type": "NUMBER", + "defaultValue": 4, + "min": 1, + "label": { + "de": "Zeit davor", + "en": "Units before" + }, + "placeholder": { + "de": "4", + "en": "4" + }, + "pattern": "^(?!-)\\d*$", + "validations": [ + "NOT_EMPTY", + "GREATER_THAN_ZERO" + ], + "tooltip": { + "de": "Anzahl an Zeiteinheiten, die die Größe des Zeitraums vor dem Indexdatum bestimmten.", + "en": "Number of time units that determined the size of the period before the index date." + } + }, + { + "name": "timeCountAfter", + "type": "NUMBER", + "min": 1, + "defaultValue": 4, + "label": { + "de": "Zeit danach", + "en": "Units after" + }, + "placeholder": { + "de": "4", + "en": "4" + }, + "pattern": "^(?!-)\\d*$", + "validations": [ + "NOT_EMPTY", + "GREATER_THAN_ZERO" + ], + "tooltip": { + "de": "Anzahl an Zeiteinheiten, die die Größe des Zeitraums nach dem Indexdatum bestimmten.", + "en": "Number of time units that determined the size of the period after the index date." + } + }, + { + "name": "indexSelector", + "type": "SELECT", + "label": { + "de": "Zeitstempel Indexdatum", + "en": "Index date sampler" + }, + "defaultValue": "EARLIEST", + "options": [ { - "label": { - "de": "Jahre", - "en": "Years" - }, - "value": "YEARS" + "label": { + "de": "ERSTES", + "en": "First" + }, + "value": "EARLIEST" }, { - "label": { - "de": "Quartale", - "en": "Quarters" - }, - "value": "QUARTERS" + "label": { + "de": "LETZTES", + "en": "Last" + }, + "value": "LATEST" }, { - "label": { - "de": "Jahre und Quartale", - "en": "Years and Quarters" - }, - "value": "YEARS_QUARTERS" - }, + "label": { + "de": "ZUFÄLLIG", + "en": "Random" + }, + "value": "RANDOM" + } + ], + "validations": [ + "NOT_EMPTY" + ], + "tooltip": { + "de": "Wenn mehr als ein Datumswert pro Person vorliegt, kann hier ausgewählt werden welcher als Indexdatum gewertet werden soll.", + "en": "If there is more than one date value per person, you can select here which one should be evaluated as index date." + } + }, + { + "name": "indexPlacement", + "type": "SELECT", + "label": { + "de": "Zugehörigkeit Indexdatum", + "en": "Index period inclusion" + }, + "defaultValue": "AFTER", + "options": [ { - "label": { - "de": "Gesamter Zeitraum und Jahre", - "en": "Total and Years" - }, - "value": "COMPLETE_YEARS" + "label": { + "de": "VORBEOBACHTUNGSZEITRAUM", + "en": "Feature period" + }, + "value": "BEFORE" }, { - "label": { - "de": "Gesamter Zeitraum und Quartale", - "en": "Total and Quarters" - }, - "value": "COMPLETE_QUARTERS" + "label": { + "de": "NEUTRAL", + "en": "Neutral" + }, + "value": "NEUTRAL" }, { - "label": { - "de": "Gesamter Zeitraum, Jahre und Quartale", - "en": "Total, Years and Quarters" - }, - "value": "COMPLETE_YEARS_QUARTERS" + "label": { + "de": "NACHBEOBACHTUNGSZEITRAUM", + "en": "Outcome period" + }, + "value": "AFTER" } - ], - "validations": [ + ], + "validations": [ "NOT_EMPTY" - ] + ], + "tooltip": { + "de": "Angabe für welchen Zeitraum das Quartal mit dem Indexdatum gewertet wird.", + "en": "Indication which period includes the index period" + } + } + ], + "tooltip": { + "de": "Die Ausgaben beziehen sich auf einen Vor- und Nachbeobachtungszeitraum, abhängig von dem Indexdatum jeder Person in der Versichertengruppe.", + "en": "Outputs are for a pre- and post-observation period, depending on the index period of each person in the cohort." + } + } + ] + }, + { + "label": { + "de": "Datengrundlage und Konzepte", + "en": "Attributes" + }, + "style": { + "size": "h1" + }, + "type": "HEADLINE" + }, + { + "name": "features", + "type": "CONCEPT_LIST", + "label": { + "de": "Konzepte", + "en": "Concepts" + }, + "isTwoDimensional": true, + "conceptDropzoneLabel": { + "de": "Füge ein Konzept oder eine Konzeptliste hinzu", + "en": "Add a concept or a concept list" + }, + "validations": [ + "NOT_EMPTY" + ] + }, + { + "label": { + "de": "Analyse und Ausgabe", + "en": "Analysis and Output" + }, + "style": { + "size": "h1" + }, + "type": "HEADLINE" + }, + { + "name": "resolution", + "type": "SELECT", + "label": { + "de": "Stratifizierung Beobachtungszeitraum", + "en": "Temporal stratification" + }, + "defaultValue": "COMPLETE", + "options": [ + { + "label": { + "de": "Gesamter Zeitraum", + "en": "Total" + }, + "value": "COMPLETE" + }, + { + "label": { + "de": "Jahre", + "en": "Years" + }, + "value": "YEARS" + }, + { + "label": { + "de": "Quartale", + "en": "Quarters" + }, + "value": "QUARTERS" + }, + { + "label": { + "de": "Jahre und Quartale", + "en": "Years and Quarters" + }, + "value": "YEARS_QUARTERS" + }, + { + "label": { + "de": "Gesamter Zeitraum und Jahre", + "en": "Total and Years" + }, + "value": "COMPLETE_YEARS" + }, + { + "label": { + "de": "Gesamter Zeitraum und Quartale", + "en": "Total and Quarters" + }, + "value": "COMPLETE_QUARTERS" + }, + { + "label": { + "de": "Gesamter Zeitraum, Jahre und Quartale", + "en": "Total, Years and Quarters" + }, + "value": "COMPLETE_YEARS_QUARTERS" } - ] + ], + "validations": [ + "NOT_EMPTY" + ] + } + ] } \ No newline at end of file diff --git a/backend/src/main/resources/com/bakdata/conquery/frontend/forms/table_export_form.frontend_conf.json b/backend/src/main/resources/com/bakdata/conquery/frontend/forms/table_export_form.frontend_conf.json index 9569be7550..e736367a43 100644 --- a/backend/src/main/resources/com/bakdata/conquery/frontend/forms/table_export_form.frontend_conf.json +++ b/backend/src/main/resources/com/bakdata/conquery/frontend/forms/table_export_form.frontend_conf.json @@ -1,89 +1,88 @@ { - "title": { - "de": "Rohdatenexport", - "en": "Table Export" - }, - "description": { - "en": "This form is used to output the raw tables behind the specified concepts, which are linked to the concepts. The data can be analyzed in an absolute observation period.", - "de": "Mit diesem Formular werden zu den angegebenen Konzepten die dahinterliegenden, rohen Tabellen ausgegeben, welche mit den Konzepten verknüpft sind. Die Daten können dabei in einem absoluten Beobachtungszeitraum analysiert werden. Die Ausgabe kann sowohl als Excel, als auch CSV heruntergeladen werden." - }, - "type": "FULL_EXPORT_FORM", - "fields": [ - { - "label": { - "en": "Cohort", - "de": "Versichertengruppe" - }, - "style": { - "size": "h1" - }, - "type": "HEADLINE" - }, - { - "name": "queryGroup", - "type": "RESULT_GROUP", - "label": { - "de": "Versichertengruppe (Anfrage)", - "en": "Cohort (Previous Query)" - }, - "dropzoneLabel": { - "de": "Füge eine Versichertengruppe aus einer bestehenden Anfrage hinzu.", - "en": "Add a cohort from a previous query" - }, - "validations": [ - "NOT_EMPTY" - ], - "tooltip": { - "de": "Versichertengruppe (Anfrage) für die Daten ausgegeben werden soll.", - "en": "Cohort whose data is exported" - } - }, - { - "label": { - "de": "Zeitlicher Bezug", - "en": "Time Reference" - }, - "style": { - "size": "h1" - }, - "type": "HEADLINE" - }, - { - "name": "dateRange", - "type": "DATE_RANGE", - "label": { - "de": "Beobachtungszeitraum", - "en": "Observation Period" - }, - "validations": [ - "NOT_EMPTY" - ] - }, - { - "label": { - "de": "Datengrundlage und Konzepte", - "en": "Attributes" - }, - "style": { - "size": "h1" - }, - "type": "HEADLINE" - }, - { - "name": "tables", - "type": "CONCEPT_LIST", - "label": { - "de": "Konzepte", - "en": "Concepts" - }, - "isTwoDimensional": false, - "conceptDropzoneLabel": { - "en": "Add a concept or a concept list", - "de": "Füge ein Konzept oder eine Konzeptliste hinzu" - }, - "validations": [ - "NOT_EMPTY" - ] - } - ] + "title": { + "de": "Rohdatenexport", + "en": "Table Export" + }, + "description": { + "en": "This form is used to output the raw tables behind the specified concepts, which are linked to the concepts. The data can be analyzed in an absolute observation period.", + "de": "Mit diesem Formular werden zu den angegebenen Konzepten die dahinterliegenden, rohen Tabellen ausgegeben, welche mit den Konzepten verknüpft sind. Die Daten können dabei in einem absoluten Beobachtungszeitraum analysiert werden. Die Ausgabe kann sowohl als Excel, als auch CSV heruntergeladen werden." + }, + "type": "FULL_EXPORT_FORM", + "fields": [ + { + "label": { + "en": "Cohort", + "de": "Versichertengruppe" + }, + "style": { + "size": "h1" + }, + "type": "HEADLINE" + }, + { + "name": "queryGroup", + "type": "RESULT_GROUP", + "label": { + "de": "Versichertengruppe (Anfrage)", + "en": "Cohort (Previous Query)" + }, + "dropzoneLabel": { + "de": "Füge eine Versichertengruppe aus einer bestehenden Anfrage hinzu.", + "en": "Add a cohort from a previous query" + }, + "validations": [ + ], + "tooltip": { + "de": "Versichertengruppe (Anfrage) für die Daten ausgegeben werden soll.", + "en": "Cohort whose data is exported" + } + }, + { + "label": { + "de": "Zeitlicher Bezug", + "en": "Time Reference" + }, + "style": { + "size": "h1" + }, + "type": "HEADLINE" + }, + { + "name": "dateRange", + "type": "DATE_RANGE", + "label": { + "de": "Beobachtungszeitraum", + "en": "Observation Period" + }, + "validations": [ + "NOT_EMPTY" + ] + }, + { + "label": { + "de": "Datengrundlage und Konzepte", + "en": "Attributes" + }, + "style": { + "size": "h1" + }, + "type": "HEADLINE" + }, + { + "name": "tables", + "type": "CONCEPT_LIST", + "label": { + "de": "Konzepte", + "en": "Concepts" + }, + "isTwoDimensional": false, + "conceptDropzoneLabel": { + "en": "Add a concept or a concept list", + "de": "Füge ein Konzept oder eine Konzeptliste hinzu" + }, + "validations": [ + "NOT_EMPTY" + ] + } + ] } From 2b8f7ed4c67d970c58c6b90134d913aabb920b27 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Fri, 2 Jun 2023 11:56:05 +0200 Subject: [PATCH 353/679] light refactoring and renaming, formatting changes --- .../form-components/DropzoneBetweenElements.tsx | 11 +++++++---- .../external-forms/form-components/DropzoneList.tsx | 10 ++++++---- .../form-concept-group/FormConceptGroup.tsx | 2 +- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index 511857ed7a..d26e4c2215 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -61,14 +61,14 @@ const BetweenElements = ({ const [showDropzone, setShowDropzone] = useState(false); - const [{ isOver, isDroppable }, drop] = useDrop({ + const [{ isOver, isDroppable }, addZoneRef] = useDrop({ accept: acceptedDropTypes, collect: (monitor) => ({ isOver: monitor.isOver(), isDroppable: monitor.canDrop(), }), }); - const [{ isOver: isOver2 }, drop2] = useDrop({ + const [{ isOver: isOver2 }, dropzoneWrapperRef] = useDrop({ accept: acceptedDropTypes, collect: (monitor) => ({ isOver: monitor.isOver(), @@ -84,7 +84,7 @@ const BetweenElements = ({ <> {!(showDropzone || isOver || isOver2) && ( ({ )} {(showDropzone || isOver || isOver2) && ( - setShowDropzone(false)}> + setShowDropzone(false)} + > {() => t("externalForms.default.dropBetweenLabel")} diff --git a/frontend/src/js/external-forms/form-components/DropzoneList.tsx b/frontend/src/js/external-forms/form-components/DropzoneList.tsx index c9d556b3ad..eb5577dc06 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneList.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneList.tsx @@ -52,7 +52,7 @@ interface PropsT { ) => void; onDropFile: (file: File) => void; onImportLines: (lines: string[]) => void; - dropInbetween: ( + dropBetween: ( i: number, ) => (item: DroppableObject, monitor: DropTargetMonitor) => void; } @@ -70,11 +70,13 @@ const DropzoneList = ( disallowMultipleColumns, onDrop, onImportLines, - dropInbetween, + dropBetween, }: PropsT, ref: Ref, ) => { - const SxDropzoneWithFileInput = styled(DropzoneWithFileInput)` + const SxDropzoneWithFileInput = styled( + DropzoneWithFileInput, + )` margin-top: 5px; `; // allow at least one column @@ -99,7 +101,7 @@ const DropzoneList = ( {!disallowMultipleColumns && ( )} diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 7304ef2d31..4bf9a0e73a 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -200,7 +200,7 @@ const FormConceptGroup = (props: Props) => { ? t("externalForms.common.concept.copying") : props.attributeDropzoneText } - dropInbetween={(i: number) => { + dropBetween={(i: number) => { return (item: DragItemConceptTreeNode) => { if (isMovedObject(item)) { return props.onChange( From 339a5d3da7486930a3c45ab4e681cad9ae0c18a9 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 5 Jun 2023 12:43:37 +0200 Subject: [PATCH 354/679] Add error boundary for forms tab --- frontend/src/js/app/RightPane.tsx | 5 +++- frontend/src/js/entity-history/History.tsx | 2 +- .../src/js/error-fallback/ErrorFallback.tsx | 25 ++++++++++++++++--- .../error-fallback/ResetableErrorBoundary.tsx | 25 +++++++++++++++++++ frontend/src/localization/de.json | 9 ++++--- 5 files changed, 57 insertions(+), 9 deletions(-) create mode 100644 frontend/src/js/error-fallback/ResetableErrorBoundary.tsx diff --git a/frontend/src/js/app/RightPane.tsx b/frontend/src/js/app/RightPane.tsx index 801662bf27..1ce9a7b5ad 100644 --- a/frontend/src/js/app/RightPane.tsx +++ b/frontend/src/js/app/RightPane.tsx @@ -3,6 +3,7 @@ import { useMemo } from "react"; import { useTranslation } from "react-i18next"; import { useSelector } from "react-redux"; +import { ResetableErrorBoundary } from "../error-fallback/ResetableErrorBoundary"; import FormsTab from "../external-forms/FormsTab"; import Pane from "../pane/Pane"; import { TabNavigationTab } from "../pane/TabNavigation"; @@ -59,7 +60,9 @@ const RightPane = () => { - + + + ); diff --git a/frontend/src/js/entity-history/History.tsx b/frontend/src/js/entity-history/History.tsx index 983b753deb..91e999409b 100644 --- a/frontend/src/js/entity-history/History.tsx +++ b/frontend/src/js/entity-history/History.tsx @@ -194,7 +194,7 @@ export const History = () => { onLoadFromFile={onLoadFromFile} onResetHistory={onResetEntityStatus} /> - + }>
diff --git a/frontend/src/js/error-fallback/ErrorFallback.tsx b/frontend/src/js/error-fallback/ErrorFallback.tsx index 8e31913064..febcedfd27 100644 --- a/frontend/src/js/error-fallback/ErrorFallback.tsx +++ b/frontend/src/js/error-fallback/ErrorFallback.tsx @@ -29,16 +29,33 @@ const ReloadButton = styled(TransparentButton)` margin-top: 10px; `; -const ErrorFallback = () => { +const ErrorFallback = ({ + allowFullRefresh, + onReset, +}: { + allowFullRefresh?: boolean; + onReset?: () => void; +}) => { const { t } = useTranslation(); return ( {t("error.sorry")} {t("error.description")} - window.location.reload()}> - {t("error.reload")} - + {allowFullRefresh && ( + <> + {t("error.reloadDescription")} + window.location.reload()}> + {t("error.reload")} + + + )} + {onReset && ( + <> + {t("error.resetDescription")} + {t("error.reset")} + + )} ); }; diff --git a/frontend/src/js/error-fallback/ResetableErrorBoundary.tsx b/frontend/src/js/error-fallback/ResetableErrorBoundary.tsx new file mode 100644 index 0000000000..a390b9d593 --- /dev/null +++ b/frontend/src/js/error-fallback/ResetableErrorBoundary.tsx @@ -0,0 +1,25 @@ +import { ReactNode, useCallback, useState } from "react"; +import { ErrorBoundary } from "react-error-boundary"; + +import ErrorFallback from "./ErrorFallback"; + +export const ResetableErrorBoundary = ({ + children, +}: { + children: ReactNode; +}) => { + const [resetKey, setResetKey] = useState(0); + const onReset = useCallback(() => setResetKey((key) => key + 1), []); + + return ( + ( + + )} + > + {children} + + ); +}; diff --git a/frontend/src/localization/de.json b/frontend/src/localization/de.json index 1b5843d915..8b21fe42c1 100644 --- a/frontend/src/localization/de.json +++ b/frontend/src/localization/de.json @@ -2,9 +2,12 @@ "locale": "de", "headline": "Anfragen und Analyse", "error": { - "sorry": "Das hat leider nicht geklappt", - "description": "Versuche es noch einmal. Falls es erneut nicht klappt, bitte hinterlasse uns eine Nachricht, damit wir dieses Problem beheben können.", - "reload": "Seite vollständig neu laden" + "sorry": "Da ist etwas schiefgelaufen!", + "description": "Aber Du hast nichts falsch gemacht. Das Problem liegt auf unserer Seite.", + "reset": "Zurücksetzen", + "resetDescription": "Versuche zurückzusetzen. Falls das Problem weiterhin besteht, bitte kontaktiere uns, damit wir das Problem schneller beheben können.", + "reload": "Seite vollständig neu laden", + "reloadDescription": "Bitte hinterlasse uns eine Nachricht, damit wir dieses Problem beheben können." }, "errorCodes": { "EXAMPLE_ERROR": "Dies ist eine Beispiel-Fehlermeldung", From 3a47f753a4e055ce310f15fb0f14188c312c3a54 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 5 Jun 2023 12:46:27 +0200 Subject: [PATCH 355/679] Add en localization --- frontend/src/localization/en.json | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/frontend/src/localization/en.json b/frontend/src/localization/en.json index 52d4c1f0ef..17e9a3595b 100644 --- a/frontend/src/localization/en.json +++ b/frontend/src/localization/en.json @@ -3,8 +3,11 @@ "headline": "Queries and Analyses", "error": { "sorry": "Sorry, something went wrong here", - "description": "Please try again. If this happens again, please leave us a message. That way, we can fix this issue.", - "reload": "Refresh page" + "description": "But it's not your fault. It's an issue on our side.", + "reset": "Reset", + "resetDescription": "Try to reset. If the issue happens again, please reach out to us. Then we can fix this issue sooner.", + "reload": "Refresh page", + "reloadDescription": "If this happens again, please leave us a message. That way, we can fix this issue sooner." }, "errorCodes": { "EXAMPLE_ERROR": "This is an example error", From ef1f49d6637871f307aa835f349628f3a6d4233d Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 5 Jun 2023 15:48:56 +0200 Subject: [PATCH 356/679] Remove unused imports --- frontend/src/js/standard-query-editor/QueryNode.tsx | 3 --- 1 file changed, 3 deletions(-) diff --git a/frontend/src/js/standard-query-editor/QueryNode.tsx b/frontend/src/js/standard-query-editor/QueryNode.tsx index 353405fdd2..358c9f3e46 100644 --- a/frontend/src/js/standard-query-editor/QueryNode.tsx +++ b/frontend/src/js/standard-query-editor/QueryNode.tsx @@ -1,7 +1,6 @@ import styled from "@emotion/styled"; import { memo, useCallback, useRef } from "react"; import { useDrag } from "react-dnd"; -import { useTranslation } from "react-i18next"; import { useSelector } from "react-redux"; import type { QueryT } from "../api/types"; @@ -9,8 +8,6 @@ import { getWidthAndHeight } from "../app/DndProvider"; import type { StateT } from "../app/reducers"; import { getConceptById } from "../concept-trees/globalTreeStoreHelper"; import { - nodeHasNonDefaultSettings, - nodeHasFilterValues, nodeIsConceptQueryNode, canNodeBeDropped, useActiveState, From ecbd28cbb32283d60b0337cb3575e1d66c347d9e Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 5 Jun 2023 15:53:32 +0200 Subject: [PATCH 357/679] Add keys --- frontend/src/js/editor-v2/TreeNode.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/js/editor-v2/TreeNode.tsx b/frontend/src/js/editor-v2/TreeNode.tsx index 521dd68d39..efabf83689 100644 --- a/frontend/src/js/editor-v2/TreeNode.tsx +++ b/frontend/src/js/editor-v2/TreeNode.tsx @@ -415,7 +415,7 @@ const Value = ({ <> {value.slice(0, 10).map((v, idx) => ( <> - + ))} {value.length > 10 && {`... +${value.length - 10}`}} @@ -458,7 +458,7 @@ const Connection = memo(({ connection }: { connection?: ConnectionKind }) => { }); const SectionHeading = styled("h4")` - font-weight: 400; + font-weight: 700; color: ${(props) => props.theme.col.blueGrayDark}; margin: 0; text-transform: uppercase; From 61ce2a87fc0a3cbd17fcc34cf62160eb685c0f3e Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 6 Jun 2023 14:12:37 +0200 Subject: [PATCH 358/679] Start modeling time connection --- frontend/src/js/app/RightPane.tsx | 1 + frontend/src/js/editor-v2/EditorV2.tsx | 8 +- frontend/src/js/editor-v2/TimeConnection.tsx | 69 ++++++++ frontend/src/js/editor-v2/TreeNode.tsx | 152 +----------------- frontend/src/js/editor-v2/TreeNodeConcept.tsx | 145 +++++++++++++++++ .../connector-update/useConnectorRotation.ts | 41 ++++- frontend/src/js/editor-v2/types.ts | 31 +++- frontend/src/js/editor-v2/util.ts | 110 +++++++++++-- frontend/src/localization/de.json | 16 +- frontend/src/localization/en.json | 16 +- 10 files changed, 412 insertions(+), 177 deletions(-) create mode 100644 frontend/src/js/editor-v2/TimeConnection.tsx create mode 100644 frontend/src/js/editor-v2/TreeNodeConcept.tsx diff --git a/frontend/src/js/app/RightPane.tsx b/frontend/src/js/app/RightPane.tsx index 0764ff6aad..ad50815737 100644 --- a/frontend/src/js/app/RightPane.tsx +++ b/frontend/src/js/app/RightPane.tsx @@ -92,6 +92,7 @@ const RightPane = () => { featureConnectorRotate featureQueryNodeEdit featureContentInfos + featureTimebasedQueries /> ) : ( diff --git a/frontend/src/js/editor-v2/EditorV2.tsx b/frontend/src/js/editor-v2/EditorV2.tsx index 721edc621d..1c81a6b264 100644 --- a/frontend/src/js/editor-v2/EditorV2.tsx +++ b/frontend/src/js/editor-v2/EditorV2.tsx @@ -35,7 +35,7 @@ import { useNegationEditing } from "./negation/useNegationEditing"; import { EditorV2QueryNodeEditor } from "./query-node-edit/EditorV2QueryNodeEditor"; import { useQueryNodeEditing } from "./query-node-edit/useQueryNodeEditing"; import { Tree } from "./types"; -import { findNodeById, useTranslatedConnection } from "./util"; +import { findNodeById, useGetTranslatedConnection } from "./util"; const Root = styled("div")` flex-grow: 1; @@ -120,6 +120,7 @@ export function EditorV2({ featureConnectorRotate, featureQueryNodeEdit, featureContentInfos, + featureTimebasedQueries, }: { featureDates: boolean; featureNegate: boolean; @@ -127,6 +128,7 @@ export function EditorV2({ featureConnectorRotate: boolean; featureQueryNodeEdit: boolean; featureContentInfos: boolean; + featureTimebasedQueries: boolean; }) { const { t } = useTranslation(); const { @@ -203,6 +205,7 @@ export function EditorV2({ const { onRotateConnector } = useConnectorEditing({ enabled: featureConnectorRotate, + timebasedQueriesEnabled: featureTimebasedQueries, hotkey: HOTKEYS.rotateConnector.keyname, selectedNode, updateTreeNode, @@ -218,7 +221,8 @@ export function EditorV2({ selectedNode, }); - const connection = useTranslatedConnection( + const getTranslatedConnection = useGetTranslatedConnection(); + const connection = getTranslatedConnection( selectedNode?.children?.connection, ); diff --git a/frontend/src/js/editor-v2/TimeConnection.tsx b/frontend/src/js/editor-v2/TimeConnection.tsx new file mode 100644 index 0000000000..e233ee511c --- /dev/null +++ b/frontend/src/js/editor-v2/TimeConnection.tsx @@ -0,0 +1,69 @@ +import styled from "@emotion/styled"; +import { memo } from "react"; +import { useTranslation } from "react-i18next"; + +import { TreeChildrenTime } from "./types"; +import { + useGetNodeLabel, + useGetTranslatedTimestamp, + useTranslatedInterval, + useTranslatedOperator, +} from "./util"; + +const TimeConnectionContainer = styled("div")` + display: flex; + align-items: center; + gap: 5px; + font-size: ${({ theme }) => theme.font.xs}; +`; + +const ConceptName = styled("span")` + font-weight: bold; + color: ${({ theme }) => theme.col.blueGrayDark}; +`; +const Timestamp = styled("span")` + font-weight: bold; + color: ${({ theme }) => theme.col.palette[6]}; +`; +const Interval = styled("span")` + font-weight: bold; + color: ${({ theme }) => theme.col.orange}; +`; +const Operator = styled("span")` + font-weight: bold; + color: ${({ theme }) => theme.col.green}; +`; + +export const TimeConnection = memo( + ({ conditions }: { conditions: TreeChildrenTime }) => { + const { t } = useTranslation(); + const getNodeLabel = useGetNodeLabel(); + const getTranslatedTimestamp = useGetTranslatedTimestamp(); + + const aTimestamp = getTranslatedTimestamp(conditions.timestamps[0]); + const bTimestamp = getTranslatedTimestamp(conditions.timestamps[1]); + const a = getNodeLabel(conditions.items[0]); + const b = getNodeLabel(conditions.items[1]); + const operator = useTranslatedOperator(conditions.operator); + const interval = useTranslatedInterval(conditions.interval); + + return ( +
+ + {aTimestamp} + {t("editorV2.dateRangeFrom")} + {a} + + + {interval} + {operator} + + + {bTimestamp} + {t("editorV2.dateRangeFrom")} + {b} + +
+ ); + }, +); diff --git a/frontend/src/js/editor-v2/TreeNode.tsx b/frontend/src/js/editor-v2/TreeNode.tsx index efabf83689..9d1fb34c15 100644 --- a/frontend/src/js/editor-v2/TreeNode.tsx +++ b/frontend/src/js/editor-v2/TreeNode.tsx @@ -5,19 +5,19 @@ import { DOMAttributes, memo } from "react"; import { useTranslation } from "react-i18next"; import { DNDType } from "../common/constants/dndTypes"; -import { exists } from "../common/helpers/exists"; import { Icon } from "../icon/FaIcon"; import { nodeIsConceptQueryNode, useActiveState } from "../model/node"; import { getRootNodeLabel } from "../standard-query-editor/helper"; -import { DragItemConceptTreeNode } from "../standard-query-editor/types"; import WithTooltip from "../tooltip/WithTooltip"; import Dropzone, { DropzoneProps } from "../ui-components/Dropzone"; import { Connector, Grid } from "./EditorLayout"; +import { TimeConnection } from "./TimeConnection"; +import { TreeNodeConcept } from "./TreeNodeConcept"; import { EDITOR_DROP_TYPES } from "./config"; import { DateRange } from "./date-restriction/DateRange"; import { ConnectionKind, Tree } from "./types"; -import { useTranslatedConnection } from "./util"; +import { useGetTranslatedConnection } from "./util"; const NodeContainer = styled("div")` display: grid; @@ -95,15 +95,6 @@ const Name = styled("div")` color: ${({ theme }) => theme.col.black}; `; -const Description = styled("div")` - font-size: ${({ theme }) => theme.font.xs}; - color: ${({ theme }) => theme.col.black}; - display: flex; - align-items: center; - gap: 0px 5px; - flex-wrap: wrap; -`; - const PreviousQueryLabel = styled("p")` margin: 0; line-height: 1.2; @@ -136,10 +127,6 @@ const Dates = styled("div")` font-weight: 400; `; -const Bold = styled("span")` - font-weight: 400; -`; - export function TreeNode({ tree, treeParent, @@ -275,6 +262,9 @@ export function TreeNode({ setSelectedNodeId(tree.id); }} > + {tree.children && tree.children.connection === "time" && ( + + )} {tree.dates?.restriction && ( @@ -394,134 +384,8 @@ export function TreeNode({ ); } -const Value = ({ - value, - isElement, -}: { - value: unknown; - isElement?: boolean; -}) => { - if (typeof value === "string" || typeof value === "number") { - return ( - - {value} - {isElement && ","} - - ); - } else if (typeof value === "boolean") { - return {value ? "✔" : "✗"}; - } else if (value instanceof Array) { - return ( - <> - {value.slice(0, 10).map((v, idx) => ( - <> - - - ))} - {value.length > 10 && {`... +${value.length - 10}`}} - - ); - } else if ( - value instanceof Object && - "label" in value && - typeof value.label === "string" - ) { - return ( - - {value.label} - {isElement && ","} - - ); - } else if (value instanceof Object) { - return ( - <> - {Object.entries(value) - .filter(([, v]) => exists(v)) - .map(([k, v]) => ( - <> - {k}: - - ))} - - ); - } else if (value === null) { - return ; - } else { - return {JSON.stringify(value)}; - } -}; - const Connection = memo(({ connection }: { connection?: ConnectionKind }) => { - const message = useTranslatedConnection(connection); + const getTranslatedConnection = useGetTranslatedConnection(); - return {message}; + return {getTranslatedConnection(connection)}; }); - -const SectionHeading = styled("h4")` - font-weight: 700; - color: ${(props) => props.theme.col.blueGrayDark}; - margin: 0; - text-transform: uppercase; - font-size: ${({ theme }) => theme.font.xs}; -`; - -const Appendix = styled("div")` - display: flex; - flex-direction: column; - gap: 6px; - margin-top: 8px; -`; - -const TreeNodeConcept = ({ - node, - featureContentInfos, -}: { - node: DragItemConceptTreeNode; - featureContentInfos?: boolean; -}) => { - const { t } = useTranslation(); - const selectedSelects = [ - ...node.selects, - ...node.tables.flatMap((t) => t.selects), - ].filter((s) => s.selected); - - const filtersWithValues = node.tables.flatMap((t) => - t.filters.filter( - (f) => - exists(f.value) && (!(f.value instanceof Array) || f.value.length > 0), - ), - ); - - const showAppendix = - featureContentInfos && - (selectedSelects.length > 0 || filtersWithValues.length > 0); - - return ( - <> - {node.description && {node.description}} - {showAppendix && ( - - {selectedSelects.length > 0 && ( -
- {t("editorV2.outputSection")} - - - -
- )} - {filtersWithValues.length > 0 && ( -
- {t("editorV2.filtersSection")} - {filtersWithValues.map((f) => ( - - {f.label}: - - - ))} -
- )} - - )} - - ); -}; diff --git a/frontend/src/js/editor-v2/TreeNodeConcept.tsx b/frontend/src/js/editor-v2/TreeNodeConcept.tsx new file mode 100644 index 0000000000..72a81a71b0 --- /dev/null +++ b/frontend/src/js/editor-v2/TreeNodeConcept.tsx @@ -0,0 +1,145 @@ +import styled from "@emotion/styled"; +import { Fragment } from "react"; +import { useTranslation } from "react-i18next"; + +import { exists } from "../common/helpers/exists"; +import { DragItemConceptTreeNode } from "../standard-query-editor/types"; + +const Bold = styled("span")` + font-weight: 400; +`; + +const SectionHeading = styled("h4")` + font-weight: 700; + color: ${(props) => props.theme.col.blueGrayDark}; + margin: 0; + text-transform: uppercase; + font-size: ${({ theme }) => theme.font.xs}; +`; + +const Appendix = styled("div")` + display: flex; + flex-direction: column; + gap: 6px; + margin-top: 8px; +`; + +const Description = styled("div")` + font-size: ${({ theme }) => theme.font.xs}; + color: ${({ theme }) => theme.col.black}; + display: flex; + align-items: center; + gap: 0px 5px; + flex-wrap: wrap; +`; + +export const TreeNodeConcept = ({ + node, + featureContentInfos, +}: { + node: DragItemConceptTreeNode; + featureContentInfos?: boolean; +}) => { + const { t } = useTranslation(); + const selectedSelects = [ + ...node.selects, + ...node.tables.flatMap((t) => t.selects), + ].filter((s) => s.selected); + + const filtersWithValues = node.tables.flatMap((t) => + t.filters.filter( + (f) => + exists(f.value) && (!(f.value instanceof Array) || f.value.length > 0), + ), + ); + + const showAppendix = + featureContentInfos && + (selectedSelects.length > 0 || filtersWithValues.length > 0); + + return ( + <> + {node.description && {node.description}} + {showAppendix && ( + + {selectedSelects.length > 0 && ( +
+ {t("editorV2.outputSection")} + + + +
+ )} + {filtersWithValues.length > 0 && ( +
+ {t("editorV2.filtersSection")} + {filtersWithValues.map((f) => ( + + {f.label}: + + + ))} +
+ )} + + )} + + ); +}; + +const Value = ({ + value, + isElement, +}: { + value: unknown; + isElement?: boolean; +}) => { + if (typeof value === "string" || typeof value === "number") { + return ( + + {value} + {isElement && ","} + + ); + } else if (typeof value === "boolean") { + return {value ? "✔" : "✗"}; + } else if (value instanceof Array) { + return ( + <> + {value.slice(0, 10).map((v, idx) => ( + <> + + + ))} + {value.length > 10 && {`... +${value.length - 10}`}} + + ); + } else if ( + value instanceof Object && + "label" in value && + typeof value.label === "string" + ) { + return ( + + {value.label} + {isElement && ","} + + ); + } else if (value instanceof Object) { + return ( + <> + {Object.entries(value) + .filter(([, v]) => exists(v)) + .map(([k, v]) => ( + + {k}: + + ))} + + ); + } else if (value === null) { + return ; + } else { + return {JSON.stringify(value)}; + } +}; diff --git a/frontend/src/js/editor-v2/connector-update/useConnectorRotation.ts b/frontend/src/js/editor-v2/connector-update/useConnectorRotation.ts index 34b956702d..cc44406fc6 100644 --- a/frontend/src/js/editor-v2/connector-update/useConnectorRotation.ts +++ b/frontend/src/js/editor-v2/connector-update/useConnectorRotation.ts @@ -1,22 +1,49 @@ import { useCallback } from "react"; import { useHotkeys } from "react-hotkeys-hook"; -import { Tree } from "../types"; +import { ConnectionKind, Tree, TreeChildren } from "../types"; -const CONNECTORS = ["and", "or", "before"] as const; +const CONNECTIONS = ["and", "or", "time"] as ConnectionKind[]; -const getNextConnector = (connector: (typeof CONNECTORS)[number]) => { - const index = CONNECTORS.indexOf(connector); - return CONNECTORS[(index + 1) % CONNECTORS.length]; +const getNextConnector = ( + children: TreeChildren, + timebasedQueriesEnabled: boolean, +) => { + const allowedConnectors = timebasedQueriesEnabled + ? CONNECTIONS + : CONNECTIONS.filter((c) => c !== "time"); + + const index = allowedConnectors.indexOf(children.connection); + + const nextConnector = + allowedConnectors[(index + 1) % allowedConnectors.length]; + + if (nextConnector !== "time") { + return { + items: children.items, + direction: children.direction, + connection: nextConnector, + }; + } else { + return { + items: children.items, + direction: children.direction, + connection: "time" as const, + timestamps: children.items.map(() => "some" as const), + operator: "before" as const, + }; + } }; export const useConnectorEditing = ({ enabled, + timebasedQueriesEnabled, hotkey, selectedNode, updateTreeNode, }: { enabled: boolean; + timebasedQueriesEnabled: boolean; hotkey: string; selectedNode: Tree | undefined; updateTreeNode: (id: string, update: (node: Tree) => void) => void; @@ -27,9 +54,9 @@ export const useConnectorEditing = ({ updateTreeNode(selectedNode.id, (node) => { if (!node.children) return; - node.children.connection = getNextConnector(node.children.connection); + node.children = getNextConnector(node.children, timebasedQueriesEnabled); }); - }, [enabled, selectedNode, updateTreeNode]); + }, [enabled, selectedNode, updateTreeNode, timebasedQueriesEnabled]); useHotkeys(hotkey, onRotateConnector, [onRotateConnector]); diff --git a/frontend/src/js/editor-v2/types.ts b/frontend/src/js/editor-v2/types.ts index d4f7a82a11..c18470186d 100644 --- a/frontend/src/js/editor-v2/types.ts +++ b/frontend/src/js/editor-v2/types.ts @@ -4,7 +4,7 @@ import { DragItemQuery, } from "../standard-query-editor/types"; -export type ConnectionKind = "and" | "or" | "before"; +export type ConnectionKind = "and" | "or" | "time"; export type DirectionKind = "horizontal" | "vertical"; export interface Tree { @@ -16,12 +16,33 @@ export interface Tree { excluded?: boolean; }; data?: DragItemQuery | DragItemConceptTreeNode; - children?: { - connection: ConnectionKind; - direction: DirectionKind; - items: Tree[]; + children?: TreeChildren; +} + +export interface TreeChildrenBase { + direction: DirectionKind; + items: Tree[]; +} + +export interface TreeChildrenAnd extends TreeChildrenBase { + connection: "and"; +} +export interface TreeChildrenOr extends TreeChildrenBase { + connection: "or"; +} + +export type TimeTimestamp = "some" | "earliest" | "latest"; +export type TimeOperator = "before" | "after" | "while"; +export interface TreeChildrenTime extends TreeChildrenBase { + connection: "time"; + operator: TimeOperator; + timestamps: TimeTimestamp[]; // items.length + interval?: { + min?: number; + max?: number; }; } +export type TreeChildren = TreeChildrenAnd | TreeChildrenOr | TreeChildrenTime; export interface EditorV2Query { tree?: Tree; diff --git a/frontend/src/js/editor-v2/util.ts b/frontend/src/js/editor-v2/util.ts index e014f5327e..c133585556 100644 --- a/frontend/src/js/editor-v2/util.ts +++ b/frontend/src/js/editor-v2/util.ts @@ -1,7 +1,7 @@ -import { useMemo } from "react"; +import { useCallback } from "react"; import { useTranslation } from "react-i18next"; -import { ConnectionKind, Tree } from "./types"; +import { ConnectionKind, Tree, TreeChildrenTime } from "./types"; export const findNodeById = (tree: Tree, id: string): Tree | undefined => { if (tree.id === id) { @@ -18,20 +18,100 @@ export const findNodeById = (tree: Tree, id: string): Tree | undefined => { return undefined; }; -export const useTranslatedConnection = ( - connection: ConnectionKind | undefined, +const getNodeLabel = ( + node: Tree, + getTranslatedConnection: ReturnType, +): string => { + if (node.data?.label) { + return node.data.label; + } else if (node.children) { + return node.children.items + .map((n) => getNodeLabel(n, getTranslatedConnection)) + .join(" " + getTranslatedConnection(node.children.connection) + " "); + } else { + return ""; + } +}; + +export const useGetNodeLabel = (): ((node: Tree) => string) => { + const getTranslatedConnection = useGetTranslatedConnection(); + + return useCallback( + (node: Tree) => getNodeLabel(node, getTranslatedConnection), + [getTranslatedConnection], + ); +}; + +export const useGetTranslatedConnection = () => { + const { t } = useTranslation(); + + return useCallback( + (connection: ConnectionKind | undefined) => { + if (connection === "and") { + return t("editorV2.and"); + } else if (connection === "or") { + return t("editorV2.or"); + } else if (connection === "time") { + return t("editorV2.time"); + } else { + return ""; + } + }, + [t], + ); +}; + +export const useGetTranslatedTimestamp = () => { + const { t } = useTranslation(); + + return useCallback( + (timestamp: "every" | "some" | "earliest" | "latest") => { + if (timestamp === "every") { + return t("editorV2.every"); + } else if (timestamp === "some") { + return t("editorV2.some"); + } else if (timestamp === "earliest") { + return t("editorV2.earliest"); + } else if (timestamp === "latest") { + return t("editorV2.latest"); + } else { + return ""; + } + }, + [t], + ); +}; + +export const useTranslatedOperator = ( + operator: "before" | "after" | "while", ) => { const { t } = useTranslation(); - return useMemo(() => { - if (connection === "and") { - return t("editorV2.and"); - } else if (connection === "or") { - return t("editorV2.or"); - } else if (connection === "before") { - return t("editorV2.before"); - } else { - return ""; - } - }, [t, connection]); + if (operator === "before") { + return t("editorV2.before"); + } else if (operator === "after") { + return t("editorV2.after"); + } else if (operator === "while") { + return t("editorV2.while"); + } else { + return ""; + } +}; + +export const useTranslatedInterval = ( + interval: TreeChildrenTime["interval"], +) => { + const { t } = useTranslation(); + + if (!interval) return t("editorV2.intervalSome"); + + const { min, max } = interval; + + if (!min && !max) return t("editorV2.intervalSome"); + if (min && !max) return t("editorV2.intervalMinDays", { days: min }); + if (!min && max) return t("editorV2.intervalMaxDays", { days: max }); + if (min && max) + return t("editorV2.intervalMinMaxDays", { minDays: min, maxDays: max }); + + return t("editorV2.intervalSome"); }; diff --git a/frontend/src/localization/de.json b/frontend/src/localization/de.json index d52333ef34..8dc28f47c1 100644 --- a/frontend/src/localization/de.json +++ b/frontend/src/localization/de.json @@ -517,7 +517,7 @@ "pasted": "Importiert" }, "editorV2": { - "before": "ZEIT", + "time": "ZEIT", "and": "UND", "or": "ODER", "clear": "Leeren", @@ -531,6 +531,18 @@ "initialDropText": "Ziehe ein Konzept oder eine Anfrage hier hinein.", "datesExcluded": "Keine Datumswerte", "outputSection": "Ausgabewerte", - "filtersSection": "Filter" + "filtersSection": "Filter", + "every": "Jeder", + "some": "Irgend ein", + "earliest": "Der früheste", + "latest": "Der späteste", + "dateRangeFrom": "Zeitraum aus", + "intervalSome": "irgendwann", + "intervalMinDays": "mindestens {{days}} Tage", + "intervalMaxDays": "höchstens {{days}} Tage", + "intervalMinMaxDays": "zwischen {{minDays}} und {{maxDays}} Tagen", + "before": "zeitlich vor", + "after": "zeitlich nach", + "while": "zeitlich während" } } diff --git a/frontend/src/localization/en.json b/frontend/src/localization/en.json index f0f8c66a10..7f0bee2429 100644 --- a/frontend/src/localization/en.json +++ b/frontend/src/localization/en.json @@ -516,7 +516,7 @@ "pasted": "Imported" }, "editorV2": { - "before": "TIME", + "time": "TIME", "and": "AND", "or": "OR", "clear": "Clear", @@ -530,6 +530,18 @@ "initialDropText": "Drop a concept or query here.", "datesExcluded": "No dates", "outputSection": "Output values", - "filtersSection": "Filters" + "filtersSection": "Filters", + "every": "Every", + "some": "Some", + "earliest": "The earliest", + "latest": "The latest", + "dateRangeFrom": "date range from", + "intervalSome": "some time", + "intervalMinDays": "at least {{days}} days", + "intervalMaxDays": "at most {{days}} days", + "intervalMinMaxDays": "between {{minDays}} and {{maxDays}} days", + "before": "before", + "after": "after", + "while": "while" } } From 3abe5c403dd2f8246194936d3c808963ddf7e5e5 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 6 Jun 2023 16:23:32 +0200 Subject: [PATCH 359/679] Allow editing time connection --- frontend/src/js/editor-v2/EditorV2.tsx | 43 +++- frontend/src/js/editor-v2/TreeNode.tsx | 2 +- frontend/src/js/editor-v2/config.ts | 1 + .../connector-update/useConnectorRotation.ts | 2 +- .../{ => time-connection}/TimeConnection.tsx | 30 +-- .../time-connection/TimeConnectionModal.tsx | 203 ++++++++++++++++++ .../useTimeConnectionEditing.ts | 34 +++ frontend/src/js/editor-v2/types.ts | 6 +- frontend/src/js/editor-v2/util.ts | 2 - frontend/src/js/ui-components/BaseInput.tsx | 7 +- frontend/src/localization/de.json | 3 + frontend/src/localization/en.json | 3 + 12 files changed, 314 insertions(+), 22 deletions(-) rename frontend/src/js/editor-v2/{ => time-connection}/TimeConnection.tsx (81%) create mode 100644 frontend/src/js/editor-v2/time-connection/TimeConnectionModal.tsx create mode 100644 frontend/src/js/editor-v2/time-connection/useTimeConnectionEditing.ts diff --git a/frontend/src/js/editor-v2/EditorV2.tsx b/frontend/src/js/editor-v2/EditorV2.tsx index 1c81a6b264..1935b1a668 100644 --- a/frontend/src/js/editor-v2/EditorV2.tsx +++ b/frontend/src/js/editor-v2/EditorV2.tsx @@ -5,6 +5,7 @@ import { faCircleNodes, faEdit, faExpandArrowsAlt, + faHourglass, faRefresh, faTrash, } from "@fortawesome/free-solid-svg-icons"; @@ -34,7 +35,9 @@ import { useExpandQuery } from "./expand/useExpandQuery"; import { useNegationEditing } from "./negation/useNegationEditing"; import { EditorV2QueryNodeEditor } from "./query-node-edit/EditorV2QueryNodeEditor"; import { useQueryNodeEditing } from "./query-node-edit/useQueryNodeEditing"; -import { Tree } from "./types"; +import { TimeConnectionModal } from "./time-connection/TimeConnectionModal"; +import { useTimeConnectionEditing } from "./time-connection/useTimeConnectionEditing"; +import { Tree, TreeChildrenTime } from "./types"; import { findNodeById, useGetTranslatedConnection } from "./util"; const Root = styled("div")` @@ -211,6 +214,16 @@ export function EditorV2({ updateTreeNode, }); + const { + showModal: showTimeModal, + onOpen: onOpenTimeModal, + onClose: onCloseTimeModal, + } = useTimeConnectionEditing({ + enabled: featureTimebasedQueries, + hotkey: HOTKEYS.editTimeConnection.keyname, + selectedNode, + }); + const { showModal: showQueryNodeEditor, onOpen: onOpenQueryNodeEditor, @@ -274,6 +287,17 @@ export function EditorV2({ }} /> )} + {showTimeModal && selectedNode && ( + { + updateTreeNode(selectedNode.id, (node) => { + node.children = nodeChildren; + }); + }} + /> + )} {tree && ( @@ -353,11 +377,26 @@ export function EditorV2({ onRotateConnector(); }} > - {t("editorV2.connector")} {connection} )} + {selectedNode?.children?.connection === "time" && ( + + { + e.stopPropagation(); + onOpenTimeModal(); + }} + > + {t("editorV2.timeConnection")} + + + )} {canExpand && ( "some" as const), + timestamps: children.items.map(() => "every" as const), operator: "before" as const, }; } diff --git a/frontend/src/js/editor-v2/TimeConnection.tsx b/frontend/src/js/editor-v2/time-connection/TimeConnection.tsx similarity index 81% rename from frontend/src/js/editor-v2/TimeConnection.tsx rename to frontend/src/js/editor-v2/time-connection/TimeConnection.tsx index e233ee511c..2e6e18176a 100644 --- a/frontend/src/js/editor-v2/TimeConnection.tsx +++ b/frontend/src/js/editor-v2/time-connection/TimeConnection.tsx @@ -2,15 +2,21 @@ import styled from "@emotion/styled"; import { memo } from "react"; import { useTranslation } from "react-i18next"; -import { TreeChildrenTime } from "./types"; +import { TreeChildrenTime } from "../types"; import { useGetNodeLabel, useGetTranslatedTimestamp, useTranslatedInterval, useTranslatedOperator, -} from "./util"; +} from "../util"; -const TimeConnectionContainer = styled("div")` +const Container = styled("div")` + margin: 0 auto; + display: inline-flex; + flex-direction: column; +`; + +const Row = styled("div")` display: flex; align-items: center; gap: 5px; @@ -48,22 +54,22 @@ export const TimeConnection = memo( const interval = useTranslatedInterval(conditions.interval); return ( -
- + + {aTimestamp} {t("editorV2.dateRangeFrom")} {a} - - - {interval} + + + {conditions.operator !== "while" && {interval}} {operator} - - + + {bTimestamp} {t("editorV2.dateRangeFrom")} {b} - -
+ + ); }, ); diff --git a/frontend/src/js/editor-v2/time-connection/TimeConnectionModal.tsx b/frontend/src/js/editor-v2/time-connection/TimeConnectionModal.tsx new file mode 100644 index 0000000000..d3a7a0f6d3 --- /dev/null +++ b/frontend/src/js/editor-v2/time-connection/TimeConnectionModal.tsx @@ -0,0 +1,203 @@ +import styled from "@emotion/styled"; +import { memo, useEffect, useMemo, useRef, useState } from "react"; +import { useTranslation } from "react-i18next"; + +import { exists } from "../../common/helpers/exists"; +import Modal from "../../modal/Modal"; +import BaseInput from "../../ui-components/BaseInput"; +import InputSelect from "../../ui-components/InputSelect/InputSelect"; +import { TimeOperator, TimeTimestamp, TreeChildrenTime } from "../types"; +import { useGetNodeLabel } from "../util"; + +const Content = styled("div")` + display: flex; + flex-direction: column; + gap: 15px; + min-width: 350px; +`; + +const Row = styled("div")` + display: flex; + align-items: center; + gap: 15px; +`; + +const SxBaseInput = styled(BaseInput)` + width: 100px; +`; + +const SxInputSelect = styled(InputSelect)<{ disabled?: boolean }>` + min-width: 150px; + flex-basis: 0; + opacity: ${({ disabled }) => (disabled ? 0.5 : 1)}; +`; + +const DateRangeFrom = styled("span")` + white-space: nowrap; +`; + +const ConceptName = styled("span")` + white-space: nowrap; + font-weight: bold; + color: ${({ theme }) => theme.col.blueGrayDark}; + flex-grow: 1; +`; + +export const TimeConnectionModal = memo( + ({ + conditions, + onChange, + onClose, + }: { + conditions: TreeChildrenTime; + onChange: (conditions: TreeChildrenTime) => void; + onClose: () => void; + }) => { + const conditionsRef = useRef(conditions); + conditionsRef.current = conditions; + + const onChangeRef = useRef(onChange); + onChangeRef.current = onChange; + + const { t } = useTranslation(); + const TIMESTAMP_OPTIONS = useMemo( + () => [ + { value: "every", label: t("editorV2.every") }, + { value: "some", label: t("editorV2.some") }, + { value: "latest", label: t("editorV2.latest") }, + { value: "earliest", label: t("editorV2.earliest") }, + ], + [t], + ); + const OPERATOR_OPTIONS = useMemo( + () => [ + { value: "before", label: t("editorV2.before") }, + { value: "after", label: t("editorV2.after") }, + { value: "while", label: t("editorV2.while") }, + ], + [t], + ); + + const INTERVAL_OPTIONS = useMemo( + () => [ + { value: "some", label: t("editorV2.intervalSome") }, + { value: "dayInterval", label: t("editorV2.dayInterval") }, + ], + [t], + ); + + const [aTimestamp, setATimestamp] = useState(conditions.timestamps[0]); + const [bTimestamp, setBTimestamp] = useState(conditions.timestamps[1]); + const [operator, setOperator] = useState(conditions.operator); + const [interval, setInterval] = useState(conditions.interval); + + const getNodeLabel = useGetNodeLabel(); + const a = getNodeLabel(conditions.items[0]); + const b = getNodeLabel(conditions.items[1]); + + useEffect(() => { + onChangeRef.current({ + ...conditionsRef.current, + timestamps: [aTimestamp, bTimestamp], + }); + }, [aTimestamp, bTimestamp]); + + useEffect(() => { + onChangeRef.current({ + ...conditionsRef.current, + operator, + }); + }, [operator]); + + useEffect(() => { + onChangeRef.current({ + ...conditionsRef.current, + interval, + }); + }, [interval]); + + return ( + + + + o.value === aTimestamp)!} + onChange={(opt) => { + if (opt) { + setATimestamp(opt.value as TimeTimestamp); + } + }} + /> + {t("editorV2.dateRangeFrom")} + {a} + + + { + setInterval({ min: val as number, max: interval?.max || null }); + }} + /> + – + { + setInterval({ max: val as number, min: interval?.min || null }); + }} + /> + { + if (opt?.value === "some") { + setInterval(undefined); + } else { + setInterval({ min: 0, max: 0 }); + } + }} + /> + o.value === operator)!} + onChange={(opt) => { + if (opt) { + setOperator(opt.value as TimeOperator); + if (opt.value === "while") { + setInterval(undefined); + } + } + }} + /> + + + o.value === bTimestamp)!} + onChange={(opt) => { + if (opt) { + setBTimestamp(opt.value as TimeTimestamp); + } + }} + /> + {t("editorV2.dateRangeFrom")} + {b} + + + + ); + }, +); diff --git a/frontend/src/js/editor-v2/time-connection/useTimeConnectionEditing.ts b/frontend/src/js/editor-v2/time-connection/useTimeConnectionEditing.ts new file mode 100644 index 0000000000..d39e4afd65 --- /dev/null +++ b/frontend/src/js/editor-v2/time-connection/useTimeConnectionEditing.ts @@ -0,0 +1,34 @@ +import { useState, useCallback } from "react"; +import { useHotkeys } from "react-hotkeys-hook"; + +import { Tree } from "../types"; + +export const useTimeConnectionEditing = ({ + enabled, + hotkey, + selectedNode, +}: { + enabled: boolean; + hotkey: string; + selectedNode: Tree | undefined; +}) => { + const [showModal, setShowModal] = useState(false); + + const onClose = useCallback(() => setShowModal(false), []); + const onOpen = useCallback(() => { + if (!enabled) return; + if (!selectedNode) return; + + setShowModal(true); + }, [enabled, selectedNode]); + + useHotkeys(hotkey, onOpen, [onOpen], { + preventDefault: true, + }); + + return { + showModal, + onClose, + onOpen, + }; +}; diff --git a/frontend/src/js/editor-v2/types.ts b/frontend/src/js/editor-v2/types.ts index c18470186d..9e1b1fe979 100644 --- a/frontend/src/js/editor-v2/types.ts +++ b/frontend/src/js/editor-v2/types.ts @@ -31,15 +31,15 @@ export interface TreeChildrenOr extends TreeChildrenBase { connection: "or"; } -export type TimeTimestamp = "some" | "earliest" | "latest"; +export type TimeTimestamp = "every" | "some" | "earliest" | "latest"; export type TimeOperator = "before" | "after" | "while"; export interface TreeChildrenTime extends TreeChildrenBase { connection: "time"; operator: TimeOperator; timestamps: TimeTimestamp[]; // items.length interval?: { - min?: number; - max?: number; + min: number | null; + max: number | null; }; } export type TreeChildren = TreeChildrenAnd | TreeChildrenOr | TreeChildrenTime; diff --git a/frontend/src/js/editor-v2/util.ts b/frontend/src/js/editor-v2/util.ts index c133585556..209b6f0cb8 100644 --- a/frontend/src/js/editor-v2/util.ts +++ b/frontend/src/js/editor-v2/util.ts @@ -93,8 +93,6 @@ export const useTranslatedOperator = ( return t("editorV2.after"); } else if (operator === "while") { return t("editorV2.while"); - } else { - return ""; } }; diff --git a/frontend/src/js/ui-components/BaseInput.tsx b/frontend/src/js/ui-components/BaseInput.tsx index 48416e4c29..79da15ae5f 100644 --- a/frontend/src/js/ui-components/BaseInput.tsx +++ b/frontend/src/js/ui-components/BaseInput.tsx @@ -20,9 +20,10 @@ const Root = styled("div")` position: relative; `; -const Input = styled("input")<{ large?: boolean }>` +const Input = styled("input")<{ large?: boolean; disabled?: boolean }>` outline: 0; width: 100%; + opacity: ${({ disabled }) => (disabled ? 0.5 : 1)}; border: 1px solid ${({ theme }) => theme.col.grayMediumLight}; padding: ${({ large }) => @@ -86,6 +87,7 @@ export interface Props { large?: boolean; inputProps?: InputProps; currencyConfig?: CurrencyConfigT; + disabled?: boolean; onFocus?: (e: FocusEvent) => void; onBlur?: (e: FocusEvent) => void; onClick?: (e: React.MouseEvent) => void; @@ -131,6 +133,7 @@ const BaseInput = forwardRef( valid, invalid, invalidText, + disabled, }, ref, ) => { @@ -179,6 +182,7 @@ const BaseInput = forwardRef( }} value={exists(value) ? value : ""} large={large} + disabled={disabled} onFocus={onFocus} onBlur={onBlur} onClick={onClick} @@ -204,6 +208,7 @@ const BaseInput = forwardRef( tiny icon={faTimes} tabIndex={-1} + disabled={disabled} title={t("common.clearValue")} aria-label={t("common.clearValue")} onClick={() => onChange(null)} diff --git a/frontend/src/localization/de.json b/frontend/src/localization/de.json index 8dc28f47c1..2fc7701c55 100644 --- a/frontend/src/localization/de.json +++ b/frontend/src/localization/de.json @@ -525,6 +525,8 @@ "dates": "Datum", "negate": "Nicht", "connector": "Verknüpfung", + "timeConnection": "Zeitverknüpfung", + "editTimeConnection": "Zeitverknüpfung bearbeiten", "delete": "Löschen", "expand": "Expandieren", "edit": "Details", @@ -537,6 +539,7 @@ "earliest": "Der früheste", "latest": "Der späteste", "dateRangeFrom": "Zeitraum aus", + "dayInterval": "Tage", "intervalSome": "irgendwann", "intervalMinDays": "mindestens {{days}} Tage", "intervalMaxDays": "höchstens {{days}} Tage", diff --git a/frontend/src/localization/en.json b/frontend/src/localization/en.json index 7f0bee2429..f24924e49e 100644 --- a/frontend/src/localization/en.json +++ b/frontend/src/localization/en.json @@ -524,6 +524,8 @@ "dates": "Dates", "negate": "Negate", "connector": "Connector", + "timeConnection": "Time connection", + "editTimeConnection": "Edit time connection", "delete": "Delete", "expand": "Expand", "edit": "Details", @@ -536,6 +538,7 @@ "earliest": "The earliest", "latest": "The latest", "dateRangeFrom": "date range from", + "dayInterval": "Days", "intervalSome": "some time", "intervalMinDays": "at least {{days}} days", "intervalMaxDays": "at most {{days}} days", From ed5939fefb9a065e88c58a9f2419552f8974dcb4 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 6 Jun 2023 16:35:03 +0200 Subject: [PATCH 360/679] Add missing translation --- frontend/src/localization/en.json | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/localization/en.json b/frontend/src/localization/en.json index f24924e49e..aa01d74e19 100644 --- a/frontend/src/localization/en.json +++ b/frontend/src/localization/en.json @@ -444,6 +444,7 @@ "tabQueryEditor": "In the Editor, a query may be built and sent.", "tabTimebasedEditor": "In the Timebased Editor, a time-based query may be built and sent. That means, previous queries may be combined using time based relations, such as 'before' and 'after'.", "tabFormEditor": "The Form Editor allows for further analysis and statistics of previous queries.", + "tabEditorV2": "An extended editor that allows advanced queries.", "datasetSelector": "Select the dataset – concept trees, previous queries and forms will be loaded.", "excludeTimestamps": "If selected, will avoid using the date values from this concept within a query.", "excludeFromSecondaryId": "If selected, will avoid using this concept in the analysis layer, should an analysis layer be selected.", From 80b3b1eb99583bbddb603485279376fa83bdee9b Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 6 Jun 2023 16:41:47 +0200 Subject: [PATCH 361/679] Transform to timebased api (WIP) --- frontend/src/js/api/apiHelper.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/frontend/src/js/api/apiHelper.ts b/frontend/src/js/api/apiHelper.ts index 815a3020f0..5320441c71 100644 --- a/frontend/src/js/api/apiHelper.ts +++ b/frontend/src/js/api/apiHelper.ts @@ -238,17 +238,20 @@ const transformTreeToApi = (tree: Tree): unknown => { case "or": node = createOr(tree.children.items.map(transformTreeToApi)); break; - case "before": + case "time": node = { - type: "BEFORE", - // TODO: - // ...days, + type: "BEFORE", // SHOULD BE: tree.children.operator, + days: { + ...(tree.children.interval || {}), + }, + // TODO: improve this to be more flexible with the "preceding" and "index" keys + // based on the operator, which would be "before" | "after" | "while" preceding: { - sampler: "EARLIEST", + sampler: "EARLIEST", // SHOULD BE: tree.children.timestamps[0], child: transformTreeToApi(tree.children.items[0]), }, index: { - sampler: "EARLIEST", + sampler: "EARLIEST", // SHOULD BE: tree.children.timestamps[1] child: transformTreeToApi(tree.children.items[1]), }, }; From 0bc81237e8edd07e939cf6821a37ee3749861140 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 6 Jun 2023 16:48:52 +0200 Subject: [PATCH 362/679] Fix hotkeys --- frontend/src/js/editor-v2/EditorV2.tsx | 7 ++++--- frontend/src/js/editor-v2/config.ts | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/frontend/src/js/editor-v2/EditorV2.tsx b/frontend/src/js/editor-v2/EditorV2.tsx index 1935b1a668..3e4101d620 100644 --- a/frontend/src/js/editor-v2/EditorV2.tsx +++ b/frontend/src/js/editor-v2/EditorV2.tsx @@ -179,8 +179,7 @@ export function EditorV2({ } }, [selectedNode, setTree, updateTreeNode]); - useHotkeys(HOTKEYS.delete[0].keyname, onDelete, [onDelete]); - useHotkeys(HOTKEYS.delete[1].keyname, onDelete, [onDelete]); + useHotkeys(HOTKEYS.delete.keyname, onDelete, [onDelete]); useHotkeys(HOTKEYS.flip.keyname, onFlip, [onFlip]); useHotkeys(HOTKEYS.reset.keyname, onReset, [onReset]); @@ -412,7 +411,9 @@ export function EditorV2({ )} {selectedNode && ( - + Date: Wed, 7 Jun 2023 17:14:00 +0200 Subject: [PATCH 363/679] use HTTPHealthCheck instead of custom impl doing the same thing --- .../external/form/ExternalFormBackendApi.java | 18 ++++++++---------- .../form/ExternalFormBackendHealthCheck.java | 15 --------------- 2 files changed, 8 insertions(+), 25 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendHealthCheck.java diff --git a/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java b/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java index d62be767e6..42e3fe35bd 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java +++ b/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendApi.java @@ -21,6 +21,7 @@ import com.bakdata.conquery.models.datasets.Dataset; import com.codahale.metrics.health.HealthCheck; import com.fasterxml.jackson.databind.node.ObjectNode; +import io.dropwizard.health.check.http.HttpHealthCheck; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -96,7 +97,8 @@ public ExternalTaskState postForm(ExternalForm form, User originalUser, User ser .header(HTTP_HEADER_CQ_AUTHENTICATION, serviceUserToken) .header(HTTP_HEADER_CQ_AUTHENTICATION_ORIGINAL, originalUserToken); - return request.post(Entity.entity(form, MediaType.APPLICATION_JSON_TYPE), ExternalTaskState.class); + ExternalTaskState post = request.post(Entity.entity(form.getExternalApiPayload(), MediaType.APPLICATION_JSON_TYPE), ExternalTaskState.class); + return post; } public ExternalTaskState getFormState(UUID externalId) { @@ -113,14 +115,10 @@ public Response getResult(final URI resultURL) { } - public HealthCheck.Result checkHealth() { - log.trace("Checking health from: {}", getHealthTarget); - try { - getHealthTarget.request(MediaType.APPLICATION_JSON_TYPE).get(Void.class); - return HealthCheck.Result.healthy(); - } - catch (Exception e) { - return HealthCheck.Result.unhealthy(e.getMessage()); - } + public HealthCheck createHealthCheck() { + return new HttpHealthCheck( + getHealthTarget.getUri().toString(), client + ); } + } diff --git a/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendHealthCheck.java b/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendHealthCheck.java deleted file mode 100644 index c0d87ea4cd..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/io/external/form/ExternalFormBackendHealthCheck.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.bakdata.conquery.io.external.form; - -import com.codahale.metrics.health.HealthCheck; -import lombok.RequiredArgsConstructor; - -@RequiredArgsConstructor -public class ExternalFormBackendHealthCheck extends HealthCheck { - - private final ExternalFormBackendApi externalApi; - - @Override - protected Result check() throws Exception { - return externalApi.checkHealth(); - } -} From 649ea9a39ea4dc16265518ceaa0287f536154242 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 7 Jun 2023 17:14:36 +0200 Subject: [PATCH 364/679] properly format apiPayload for ExternalForm --- .../conquery/apiv1/forms/ExternalForm.java | 13 +- .../models/config/FormBackendConfig.java | 11 +- .../integration/common/IntegrationUtils.java | 117 +++++++++--------- .../tests/ExternalFormBackendTest.java | 44 +++---- 4 files changed, 88 insertions(+), 97 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java index 49ec7b6ca2..cc5fc0cc8b 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java @@ -37,6 +37,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.deser.ContextualDeserializer; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; import com.google.common.base.Strings; import lombok.AllArgsConstructor; import lombok.Getter; @@ -60,16 +61,22 @@ public class ExternalForm extends Form implements SubTyped { @JsonValue @ToString.Exclude private final ObjectNode node; + private final String subType; + + public JsonNode getExternalApiPayload() { + return ((ObjectNode) node.deepCopy() + .without("values")) + .set("type", new TextNode(subType)); + + } @Nullable @Override @JsonIgnore public JsonNode getValues() { - return node; + return node.get("values"); } - private final String subType; - @Override public String getLocalizedTypeLabel() { final JsonNode formTitle = node.get("title"); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java index da0e4f9ae9..d592a8f691 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FormBackendConfig.java @@ -17,7 +17,6 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.cps.CPSTypeIdResolver; import com.bakdata.conquery.io.external.form.ExternalFormBackendApi; -import com.bakdata.conquery.io.external.form.ExternalFormBackendHealthCheck; import com.bakdata.conquery.io.external.form.ExternalFormMixin; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; @@ -94,7 +93,9 @@ public void initialize(ManagerNode managerNode) { client.register(new JacksonMessageBodyProvider(om)); // Register health check - managerNode.getEnvironment().healthChecks().register(getId(), new ExternalFormBackendHealthCheck(createApi())); + final ExternalFormBackendApi externalApi = createApi(); + + managerNode.getEnvironment().healthChecks().register(getId(), externalApi.createHealthCheck()); // Register form configuration provider managerNode.getFormScanner().registerFrontendFormConfigProvider(this::registerFormConfigs); @@ -121,11 +122,11 @@ public boolean supportsFormType(String formType) { * @param formConfigs Collection to add received form configs to. */ private void registerFormConfigs(ImmutableCollection.Builder formConfigs) { - Set supportedFormTypes = new HashSet<>(); + final Set supportedFormTypes = new HashSet<>(); for (ObjectNode formConfig : createApi().getFormConfigs()) { final String subType = formConfig.get("type").asText(); - String formType = createSubTypedId(subType); + final String formType = createSubTypedId(subType); // Override type with our subtype formConfig.set("type", new TextNode(formType)); @@ -155,7 +156,7 @@ public User createServiceUser(User originalUser, Dataset dataset) { // the actual user and download permissions. final User serviceUser = - managerNode.getAuthController().flatCopyUser(originalUser, String.format("%s_%s", this.getClass().getSimpleName().toLowerCase(), getId())); + managerNode.getAuthController().flatCopyUser(originalUser, String.format("%s_%s", getClass().getSimpleName().toLowerCase(), getId())); // The user is able to read the dataset, ensure that the service user can download results serviceUser.addPermission(dataset.createPermission(Ability.DOWNLOAD.asSet())); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java b/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java index be2ccc74e5..5f34d01a92 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/common/IntegrationUtils.java @@ -44,10 +44,10 @@ public static void importPermissionConstellation(MetaStorage storage, Role[] rol } for (RequiredUser rUser : rUsers) { - User user = rUser.getUser(); + final User user = rUser.getUser(); storage.addUser(user); - RoleId[] rolesInjected = rUser.getRolesInjected(); + final RoleId[] rolesInjected = rUser.getRolesInjected(); for (RoleId mandatorId : rolesInjected) { user.addRole(storage.getRole(mandatorId)); @@ -60,45 +60,6 @@ public static Query parseQuery(StandaloneSupport support, JsonNode rawQuery) thr return ConqueryTestSpec.parseSubTree(support, rawQuery, Query.class); } - - private static URI getPostQueryURI(StandaloneSupport conquery) { - return HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), DatasetQueryResource.class, "postQuery") - .buildFromMap(Map.of( - "dataset", conquery.getDataset().getId() - )); - } - - private static JsonNode getRawExecutionStatus(String id, StandaloneSupport conquery, User user) { - final URI queryStatusURI = getQueryStatusURI(conquery, id); - // We try at most 5 times, queryStatus waits for 10s, we therefore don't need to timeout here. - // Query getQueryStatus until it is no longer running. - for (int trial = 0; trial < 5; trial++) { - log.debug("Trying to get Query result"); - - JsonNode execStatusRaw = - conquery.getClient() - .target(queryStatusURI) - .request(MediaType.APPLICATION_JSON_TYPE) - .header("Authorization", "Bearer " + conquery.getAuthorizationController().getConqueryTokenRealm().createTokenForUser(user.getId())) - .get(JsonNode.class); - - String status = execStatusRaw.get(ExecutionStatus.Fields.status).asText(); - - if (!ExecutionState.RUNNING.name().equals(status)) { - return execStatusRaw; - } - } - - throw new IllegalStateException("Query was running too long."); - } - - private static URI getQueryStatusURI(StandaloneSupport conquery, String id) { - return HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), QueryResource.class, "getStatus") - .buildFromMap(Map.of( - "query", id, "dataset", conquery.getDataset().getId() - )); - } - /** * Send a query onto the conquery instance and assert the result's size. * @@ -112,15 +73,15 @@ public static ManagedExecutionId assertQueryResult(StandaloneSupport conquery, O .createTokenForUser(user.getId()); // Submit Query - Response response = conquery.getClient() - .target(postQueryURI) - .request(MediaType.APPLICATION_JSON_TYPE) - .header("Authorization", "Bearer " + userToken) - .post(Entity.entity(query, MediaType.APPLICATION_JSON_TYPE)); + final Response response = conquery.getClient() + .target(postQueryURI) + .request(MediaType.APPLICATION_JSON_TYPE) + .header("Authorization", "Bearer " + userToken) + .post(Entity.entity(query, MediaType.APPLICATION_JSON_TYPE)); assertThat(response.getStatusInfo().getStatusCode()).as("Result of %s", postQueryURI) - .isEqualTo(expectedResponseCode); + .isEqualTo(expectedResponseCode); if (expectedState == ExecutionState.FAILED && !response.getStatusInfo().getFamily().equals(Response.Status.Family.SUCCESSFUL)) { return null; @@ -132,14 +93,14 @@ public static ManagedExecutionId assertQueryResult(StandaloneSupport conquery, O // TODO implement this properly: ExecutionStatus status = response.readEntity(ExecutionStatus.Full.class); - JsonNode execStatusRaw = getRawExecutionStatus(id, conquery, user); + final JsonNode execStatusRaw = getRawExecutionStatus(id, conquery, user); - String status = execStatusRaw.get(ExecutionStatus.Fields.status).asText(); - long numberOfResults = execStatusRaw.get(ExecutionStatus.Fields.numberOfResults).asLong(0); + final String status = execStatusRaw.get(ExecutionStatus.Fields.status).asText(); + final long numberOfResults = execStatusRaw.get(ExecutionStatus.Fields.numberOfResults).asLong(0); assertThat(status).isEqualTo(expectedState.name()); - if (expectedState == ExecutionState.DONE && expectedSize != -1) { + if (expectedState == ExecutionState.DONE && expectedSize != -1) { assertThat(numberOfResults) .describedAs("Query results") .isEqualTo(expectedSize); @@ -148,22 +109,60 @@ public static ManagedExecutionId assertQueryResult(StandaloneSupport conquery, O return ManagedExecutionId.Parser.INSTANCE.parse(id); } + private static URI getPostQueryURI(StandaloneSupport conquery) { + return HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), DatasetQueryResource.class, "postQuery") + .buildFromMap(Map.of( + "dataset", conquery.getDataset().getId() + )); + } + + private static JsonNode getRawExecutionStatus(String id, StandaloneSupport conquery, User user) { + final URI queryStatusURI = getQueryStatusURI(conquery, id); + // We try at most 5 times, queryStatus waits for 10s, we therefore don't need to timeout here. + // Query getQueryStatus until it is no longer running. + for (int trial = 0; trial < 5; trial++) { + log.debug("Trying to get Query result"); + + final JsonNode execStatusRaw = + conquery.getClient() + .target(queryStatusURI) + .request(MediaType.APPLICATION_JSON_TYPE) + .header("Authorization", "Bearer " + conquery.getAuthorizationController().getConqueryTokenRealm().createTokenForUser(user.getId())) + .get(JsonNode.class); + + final String status = execStatusRaw.get(ExecutionStatus.Fields.status).asText(); + + if (!ExecutionState.RUNNING.name().equals(status)) { + return execStatusRaw; + } + } + + throw new IllegalStateException("Query was running too long."); + } + + private static URI getQueryStatusURI(StandaloneSupport conquery, String id) { + return HierarchyHelper.hierarchicalPath(conquery.defaultApiURIBuilder(), QueryResource.class, "getStatus") + .buildFromMap(Map.of( + "query", id, "dataset", conquery.getDataset().getId() + )); + } + public static FullExecutionStatus getExecutionStatus(StandaloneSupport conquery, ManagedExecutionId executionId, User user, int expectedResponseCode) { final URI queryStatusURI = getQueryStatusURI(conquery, executionId.toString()); final String userToken = conquery.getAuthorizationController() - .getConqueryTokenRealm() - .createTokenForUser(user.getId()); + .getConqueryTokenRealm() + .createTokenForUser(user.getId()); - Response response = conquery.getClient() - .target(queryStatusURI) - .request(MediaType.APPLICATION_JSON_TYPE) - .header("Authorization", "Bearer " + userToken) - .get(); + final Response response = conquery.getClient() + .target(queryStatusURI) + .request(MediaType.APPLICATION_JSON_TYPE) + .header("Authorization", "Bearer " + userToken) + .get(); assertThat(response.getStatusInfo().getStatusCode()).as("Result of %s", queryStatusURI) - .isEqualTo(expectedResponseCode); + .isEqualTo(expectedResponseCode); return response.readEntity(FullExecutionStatus.class); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java index 58ed118406..3931d12eeb 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java @@ -1,7 +1,6 @@ package com.bakdata.conquery.integration.tests; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.Assert.fail; import static org.mockserver.model.HttpRequest.request; import java.io.File; @@ -34,6 +33,7 @@ import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; import org.mockserver.integration.ClientAndServer; +import org.mockserver.mock.Expectation; import org.mockserver.mock.OpenAPIExpectation; import org.mockserver.model.HttpResponse; import org.mockserver.model.StringBody; @@ -56,7 +56,8 @@ public void execute(String name, TestConquery testConquery) throws Exception { .getEnvironment() .healthChecks() .runHealthCheck(FORM_BACKEND_ID) - .isHealthy()).describedAs("Checking health of form backend").isTrue(); + .isHealthy()) + .describedAs("Checking health of form backend").isTrue(); log.info("Get external form configs"); final FormScanner formScanner = testConquery.getStandaloneCommand().getManager().getFormScanner(); @@ -76,11 +77,8 @@ public void execute(String name, TestConquery testConquery) throws Exception { // Generate asset urls and check them in the status - final UriBuilder apiUriBuilder = testConquery.getSupport(name) - .defaultApiURIBuilder(); - final ManagedExecution storedExecution = testConquery.getSupport(name) - .getMetaStorage() - .getExecution(managedExecutionId); + final UriBuilder apiUriBuilder = testConquery.getSupport(name).defaultApiURIBuilder(); + final ManagedExecution storedExecution = testConquery.getSupport(name).getMetaStorage().getExecution(managedExecutionId); final URI downloadURLasset1 = ResultExternalResource.getDownloadURL(apiUriBuilder.clone(), (ManagedExecution & ExternalResult) storedExecution, executionStatus.getResultUrls() @@ -94,18 +92,12 @@ public void execute(String name, TestConquery testConquery) throws Exception { assertThat(executionStatus.getStatus()).isEqualTo(ExecutionState.DONE); - assertThat(executionStatus.getResultUrls()).containsExactly( - new ResultAsset("Result", downloadURLasset1), - new ResultAsset("Another Result", downloadURLasset2) - ); + assertThat(executionStatus.getResultUrls()).containsExactly(new ResultAsset("Result", downloadURLasset1), new ResultAsset("Another Result", downloadURLasset2)); log.info("Download Result"); final String response = - support.getClient() - .target(executionStatus.getResultUrls().get(0).url()) - .request(javax.ws.rs.core.MediaType.TEXT_PLAIN_TYPE) - .get(String.class); + support.getClient().target(executionStatus.getResultUrls().get(0).url()).request(javax.ws.rs.core.MediaType.TEXT_PLAIN_TYPE).get(String.class); assertThat(response).isEqualTo("Hello"); @@ -139,6 +131,7 @@ public ConqueryConfig overrideConfig(ConqueryConfig conf, File workdir) { return conf.withStorage(storageConfig.withDirectory(storageDir)); } + @SneakyThrows @NotNull private URI createFormServer() throws IOException { log.info("Starting mock form backend server"); @@ -146,26 +139,17 @@ private URI createFormServer() throws IOException { final URI baseURI = URI.create(String.format("http://127.0.0.1:%d", formBackend.getPort())); - formBackend.upsert(OpenAPIExpectation.openAPIExpectation("/com/bakdata/conquery/external/openapi-form-backend.yaml")); + Expectation[] expectations = formBackend.upsert(OpenAPIExpectation.openAPIExpectation("/com/bakdata/conquery/external/openapi-form-backend.yaml")); + // Result request matcher - formBackend.when(request("/result.txt")) - .respond(HttpResponse.response() - .withBody(StringBody.exact("Hello") - ) - ); + formBackend.when(request("/result.txt")).respond(HttpResponse.response().withBody(StringBody.exact("Hello"))); + // Trap: Log failed request formBackend.when(request()).respond(httpRequest -> { - log.error( - "{} on {}\n\t Headers: {}\n\tBody {}", - httpRequest.getMethod(), - httpRequest.getPath(), - httpRequest.getHeaderList(), - httpRequest.getBodyAsString() - ); - fail("Trapped because request did not match. See log."); - throw new Error("Received unmappable request"); + log.error("{} on {}\n\t Headers: {}\n\tBody {}", httpRequest.getMethod(), httpRequest.getPath(), httpRequest.getHeaderList(), httpRequest.getBodyAsString()); + return HttpResponse.notFoundResponse(); }); return baseURI; } From aefc9859830655d0844078ed988bacaa0734ffbb Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 13 Jun 2023 11:10:51 +0200 Subject: [PATCH 365/679] print conceptElementId with dataset --- .../bakdata/conquery/apiv1/query/TableExportQuery.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java index 8ab94340d9..63a798787c 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java @@ -143,7 +143,7 @@ public void resolve(QueryResolveContext context) { query.resolve(context); // First is dates, second is source id - AtomicInteger currentPosition = new AtomicInteger(2); + final AtomicInteger currentPosition = new AtomicInteger(2); final Map secondaryIdPositions = calculateSecondaryIdPositions(currentPosition); @@ -153,7 +153,7 @@ public void resolve(QueryResolveContext context) { } private Map calculateSecondaryIdPositions(AtomicInteger currentPosition) { - Map secondaryIdPositions = new HashMap<>(); + final Map secondaryIdPositions = new HashMap<>(); // SecondaryIds are pulled to the front and grouped over all tables tables.stream() @@ -303,12 +303,12 @@ public static String printValue(Concept concept, Object rawValue, PrintSettings final TreeConcept tree = (TreeConcept) concept; - int localId = (int) rawValue; + final int localId = (int) rawValue; final ConceptTreeNode node = tree.getElementByLocalId(localId); if (!printSettings.isPrettyPrint()) { - return node.getId().toStringWithoutDataset(); + return node.getId().toString(); } if (node.getDescription() == null) { From 19675b35ccadffaa591a57144b47ed09738411b4 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Tue, 13 Jun 2023 12:05:58 +0200 Subject: [PATCH 366/679] add colored labels and increase size of download text --- frontend/src/app-theme.ts | 6 +++++ frontend/src/js/button/DownloadButton.tsx | 27 ++++++++++++++++------- frontend/src/js/button/IconButton.tsx | 14 +++++++++--- frontend/src/react-app-env.d.ts | 6 +++++ 4 files changed, 42 insertions(+), 11 deletions(-) diff --git a/frontend/src/app-theme.ts b/frontend/src/app-theme.ts index d7f0b9dd2c..f73533cd18 100644 --- a/frontend/src/app-theme.ts +++ b/frontend/src/app-theme.ts @@ -27,6 +27,12 @@ export const theme: Theme = { "#777", "#fff", ], + files: { + csv: "#5aa86f", + pdf: "#c9181e", + zip: "#e5c527", + xlsx: "#3B843C", + }, bgAlt: "#f4f6f5", blueGrayDark: "#1f5f30", blueGray: "#98b099", diff --git a/frontend/src/js/button/DownloadButton.tsx b/frontend/src/js/button/DownloadButton.tsx index 6fb2d32473..bc487beea1 100644 --- a/frontend/src/js/button/DownloadButton.tsx +++ b/frontend/src/js/button/DownloadButton.tsx @@ -10,6 +10,7 @@ import { } from "@fortawesome/free-solid-svg-icons"; import { ReactNode, useContext, forwardRef } from "react"; +import { theme } from "../../app-theme"; import { ResultUrlWithLabel } from "../api/types"; import { AuthTokenContext } from "../authorization/AuthTokenProvider"; import { getEnding } from "../query-runner/DownloadResultsDropdownButton"; @@ -24,13 +25,19 @@ const Link = styled("a")` line-height: 1; `; -const fileTypeToIcon: Record = { - ZIP: faFileArchive, - XLSX: faFileExcel, - PDF: faFilePdf, - CSV: faFileCsv, +interface FileIcon { + icon: IconProp; + color?: string; +} + +const fileTypeToIcon: Record = { + ZIP: { icon: faFileArchive, color: theme.col.files.zip }, + XLSX: { icon: faFileExcel, color: theme.col.files.xlsx }, + PDF: { icon: faFilePdf, color: theme.col.files.pdf }, + CSV: { icon: faFileCsv, color: theme.col.files.csv }, }; -function getFileIcon(url: string): IconProp { + +function getFileInfo(url: string): FileIcon { // Forms if (url.includes(".")) { const ext = getEnding(url); @@ -38,7 +45,7 @@ function getFileIcon(url: string): IconProp { return fileTypeToIcon[ext]; } } - return faFileDownload; + return { icon: faFileDownload }; } interface Props extends Omit { @@ -60,12 +67,16 @@ const DownloadButton = forwardRef( authToken, )}&charset=ISO_8859_1`; + const fileInfo = getFileInfo(resultUrl.url); + return ( {children} diff --git a/frontend/src/js/button/IconButton.tsx b/frontend/src/js/button/IconButton.tsx index 2de24b10fb..fef7a99ed2 100644 --- a/frontend/src/js/button/IconButton.tsx +++ b/frontend/src/js/button/IconButton.tsx @@ -11,11 +11,14 @@ interface StyledFaIconProps extends FaIconPropsT { red?: boolean; secondary?: boolean; hasChildren: boolean; + iconColor?: string; } const SxFaIcon = styled(FaIcon)` - color: ${({ theme, active, red, secondary, light }) => - red + color: ${({ theme, active, red, secondary, light, iconColor }) => + iconColor + ? iconColor + : red ? theme.col.red : active ? theme.col.blueGrayDark @@ -42,6 +45,7 @@ const SxBasicButton = styled(BasicButton)<{ tight?: boolean; bgHover?: boolean; red?: boolean; + large?: boolean; }>` background-color: transparent; color: ${({ theme, active, secondary, red }) => @@ -62,7 +66,7 @@ const SxBasicButton = styled(BasicButton)<{ display: inline-flex; align-items: center; gap: ${({ tight }) => (tight ? "5px" : "10px")}; - + font-size: ${({ theme, large }) => (large ? theme.font.md : theme.font.sm)}; &:hover { opacity: 1; @@ -98,6 +102,7 @@ export interface IconButtonPropsT extends BasicButtonProps { light?: boolean; fixedIconWidth?: number; bgHover?: boolean; + iconColor?: string; } // A button that is prefixed by an icon @@ -117,6 +122,7 @@ const IconButton = forwardRef( light, fixedIconWidth, bgHover, + iconColor, ...restProps }, ref, @@ -135,6 +141,7 @@ const IconButton = forwardRef( tight={tight} small={small} light={light} + iconColor={iconColor} {...iconProps} /> ); @@ -169,6 +176,7 @@ const IconButton = forwardRef( red={red} {...restProps} ref={ref} + large={large} > {iconElement} {children && {children}} diff --git a/frontend/src/react-app-env.d.ts b/frontend/src/react-app-env.d.ts index 9cf631a06b..f2bf4b86c6 100644 --- a/frontend/src/react-app-env.d.ts +++ b/frontend/src/react-app-env.d.ts @@ -31,6 +31,12 @@ declare module "@emotion/react" { green: string; orange: string; palette: string[]; + files: { + csv: string; + pdf: string; + zip: string; + xlsx: string; + }; }; img: { logo: string; From 4b0f6dd2772feb64b0e8a9846d657cebf706e191 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Tue, 13 Jun 2023 12:08:12 +0200 Subject: [PATCH 367/679] fix linter error --- frontend/src/js/button/IconButton.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/frontend/src/js/button/IconButton.tsx b/frontend/src/js/button/IconButton.tsx index fef7a99ed2..2b7f5e076a 100644 --- a/frontend/src/js/button/IconButton.tsx +++ b/frontend/src/js/button/IconButton.tsx @@ -166,6 +166,7 @@ const IconButton = forwardRef( secondary, light, fixedIconWidth, + iconColor, ]); return ( Date: Tue, 13 Jun 2023 12:16:59 +0200 Subject: [PATCH 368/679] Support concept columns in time stratified infos --- frontend/src/js/api/types.ts | 14 ++- frontend/src/js/entity-history/Timeline.tsx | 47 +++++---- .../js/entity-history/timeline/Quarter.tsx | 2 +- .../js/entity-history/timeline/YearHead.tsx | 97 ++++++++++++++++--- frontend/src/localization/de.json | 8 +- 5 files changed, 124 insertions(+), 44 deletions(-) diff --git a/frontend/src/js/api/types.ts b/frontend/src/js/api/types.ts index 55d19c2008..82eb4a1d50 100644 --- a/frontend/src/js/api/types.ts +++ b/frontend/src/js/api/types.ts @@ -326,7 +326,8 @@ export type ColumnDescriptionKind = | "MONEY" | "DATE" | "DATE_RANGE" - | "LIST[DATE_RANGE]"; + | "LIST[DATE_RANGE]" + | "LIST[STRING]"; export interface ColumnDescriptionSemanticColumn { type: "COLUMN"; @@ -384,6 +385,7 @@ export interface ColumnDescription { // `label` matches column name in CSV // So it's more of an id, TODO: rename this to 'id', label: string; + description: string | null; type: ColumnDescriptionKind; semantics: ColumnDescriptionSemantic[]; @@ -564,13 +566,9 @@ export interface TimeStratifiedInfo { totals: { [label: string]: number | string[]; }; - columns: { - label: string; // Matches `label` with `year.values` and `year.quarters[].values` - defaultLabel: string; // Probably not used by us - description: string | null; - type: ColumnDescriptionKind; // Relevant to show e.g. € for money - semantics: ColumnDescriptionSemantic[]; // Probably not used by us - }[]; + // `columns[].label` matches with + // `year.values` and `year.quarters[].values` + columns: ColumnDescription[]; years: TimeStratifiedInfoYear[]; } diff --git a/frontend/src/js/entity-history/Timeline.tsx b/frontend/src/js/entity-history/Timeline.tsx index c8e9b67501..ee6a19fce8 100644 --- a/frontend/src/js/entity-history/Timeline.tsx +++ b/frontend/src/js/entity-history/Timeline.tsx @@ -31,14 +31,20 @@ import { const Root = styled("div")` overflow-y: auto; -webkit-overflow-scrolling: touch; - padding: 0 20px 0 10px; + padding: 0 20px 20px 10px; display: inline-grid; grid-template-columns: 200px auto; grid-auto-rows: minmax(min-content, max-content); - gap: 12px 4px; + gap: 20px 4px; width: 100%; `; +const Divider = styled("div")` + grid-column: 1 / span 2; + height: 1px; + background: ${({ theme }) => theme.col.grayLight}; +`; + const SxEntityCard = styled(EntityCard)` grid-column: span 2; `; @@ -94,23 +100,26 @@ const Timeline = ({ infos={currentEntityInfos} timeStratifiedInfos={currentEntityTimeStratifiedInfos} /> - {eventsByQuarterWithGroups.map(({ year, quarterwiseData }) => ( - + {eventsByQuarterWithGroups.map(({ year, quarterwiseData }, i) => ( + <> + + {i < eventsByQuarterWithGroups.length - 1 && } + ))} ); diff --git a/frontend/src/js/entity-history/timeline/Quarter.tsx b/frontend/src/js/entity-history/timeline/Quarter.tsx index 1042b68c03..369f03b2c3 100644 --- a/frontend/src/js/entity-history/timeline/Quarter.tsx +++ b/frontend/src/js/entity-history/timeline/Quarter.tsx @@ -55,7 +55,7 @@ const InlineGrid = styled("div")` cursor: pointer; border: 1px solid transparent; border-radius: ${({ theme }) => theme.borderRadius}; - padding: 5px; + padding: 6px 10px; &:hover { border: 1px solid ${({ theme }) => theme.col.blueGray}; } diff --git a/frontend/src/js/entity-history/timeline/YearHead.tsx b/frontend/src/js/entity-history/timeline/YearHead.tsx index 94ad76ebd0..e6b93b8a04 100644 --- a/frontend/src/js/entity-history/timeline/YearHead.tsx +++ b/frontend/src/js/entity-history/timeline/YearHead.tsx @@ -4,12 +4,18 @@ import { Fragment, memo } from "react"; import { useTranslation } from "react-i18next"; import { useSelector } from "react-redux"; -import { TimeStratifiedInfo } from "../../api/types"; +import { + ColumnDescriptionSemanticConceptColumn, + TimeStratifiedInfo, +} from "../../api/types"; import { StateT } from "../../app/reducers"; +import { exists } from "../../common/helpers/exists"; +import { getConceptById } from "../../concept-trees/globalTreeStoreHelper"; import FaIcon from "../../icon/FaIcon"; +import WithTooltip from "../../tooltip/WithTooltip"; import { SmallHeading } from "./SmallHeading"; -import { getColumnType } from "./util"; +import { isConceptColumn, isMoneyColumn } from "./util"; const Root = styled("div")` font-size: ${({ theme }) => theme.font.xs}; @@ -20,7 +26,7 @@ const StickyWrap = styled("div")` position: sticky; top: 0; left: 0; - padding: 5px; + padding: 6px 10px; cursor: pointer; display: grid; grid-template-columns: 16px 1fr; @@ -44,6 +50,22 @@ const Grid = styled("div")` gap: 0px 10px; `; +const ConceptRow = styled("div")` + grid-column: span 2; + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 2px; +`; + +const ConceptBubble = styled("span")` + padding: 0 2px; + border-radius: ${({ theme }) => theme.borderRadius}; + color: ${({ theme }) => theme.col.black}; + border: 1px solid ${({ theme }) => theme.col.blueGrayLight}; + font-size: ${({ theme }) => theme.font.tiny}; +`; + const Value = styled("div")` font-size: ${({ theme }) => theme.font.tiny}; font-weight: 400; @@ -102,20 +124,71 @@ const TimeStratifiedInfos = ({ info.columns.findIndex((c) => c.label === l2), ) .map(([label, value]) => { - const columnType = getColumnType(info, label); - const valueFormatted = - typeof value === "number" - ? Math.round(value) - : value instanceof Array - ? value.join(", ") - : value; + const column = info.columns.find((c) => c.label === label); + + if (!column) { + return null; + } + + if (isConceptColumn(column)) { + const semantic = column.semantics.find( + (s): s is ColumnDescriptionSemanticConceptColumn => + s.type === "CONCEPT_COLUMN", + ); + + if (value instanceof Array) { + const concepts = value + .map((v) => + getConceptById( + // TODO: should be just v + semantic?.concept.split(".")[0] + "." + v, + semantic!.concept, + ), + ) + .filter(exists); + + return ( + + + + {concepts.map((concept) => ( + + {concept.label} + + ))} + + + ); + } + // else if (typeof value === "string") { + // const concept = getConceptById(semantic!.concept, value); + + // if (concept) valueFormatted = concept.label; + // } + } + + let valueFormatted: string | number | string[] = value; + if (typeof value === "number") { + valueFormatted = Math.round(value); + } else if (value instanceof Array) { + valueFormatted = value.join(", "); + } return ( - + {valueFormatted} - {columnType === "MONEY" ? " " + currencyUnit : ""} + {isMoneyColumn(column) ? " " + currencyUnit : ""} ); diff --git a/frontend/src/localization/de.json b/frontend/src/localization/de.json index 3c09170e1f..6aa4c25e4a 100644 --- a/frontend/src/localization/de.json +++ b/frontend/src/localization/de.json @@ -469,13 +469,13 @@ "backButtonWarning": "Achtung: Wenn Du zurück gehst, werden Änderungen an der Liste verworfen. Lade die Liste vorher herunter, um sie zu speichern.", "history": "Historie", "marked": "mit Status", - "events_one": "Datenpunkt", - "events_other": "Datenpunkte", + "events_one": "Eintrag", + "events_other": "Einträge", "downloadButtonLabel": "Liste mit Status-Einträgen herunterladen", "nextButtonLabel": "Weiterblättern", "prevButtonLabel": "Zurückblättern", "downloadEntityData": "Einzelhistorie herunterladen", - "differencesTooltip": "Unterschiede aus den einzelnen Datenpunkten", + "differencesTooltip": "Unterschiede aus den einzelnen Einträgen", "closeAll": "Alle schließen", "openAll": "Alle aufklappen", "dates": "Datumswerte", @@ -488,7 +488,7 @@ "detail": { "summary": "Zusammenfassung", "detail": "Gruppiert", - "full": "Alle Einzeldatenpunkte" + "full": "Alle Einzeleinträge" }, "content": { "money": "Geldbeträge", From 47ea7f7a16699ca6884732c2c049e4611ca0f0f1 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 13 Jun 2023 12:18:10 +0200 Subject: [PATCH 369/679] Fix broken date columns --- frontend/src/js/entity-history/Timeline.tsx | 7 +++---- frontend/src/js/entity-history/timeline/GroupedContent.tsx | 3 ++- frontend/src/js/entity-history/timeline/util.ts | 3 +++ 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/frontend/src/js/entity-history/Timeline.tsx b/frontend/src/js/entity-history/Timeline.tsx index ee6a19fce8..2394f746d7 100644 --- a/frontend/src/js/entity-history/Timeline.tsx +++ b/frontend/src/js/entity-history/Timeline.tsx @@ -1,5 +1,5 @@ import styled from "@emotion/styled"; -import { memo, useMemo } from "react"; +import { Fragment, memo, useMemo } from "react"; import { useSelector } from "react-redux"; import { @@ -101,9 +101,8 @@ const Timeline = ({ timeStratifiedInfos={currentEntityTimeStratifiedInfos} /> {eventsByQuarterWithGroups.map(({ year, quarterwiseData }, i) => ( - <> + {i < eventsByQuarterWithGroups.length - 1 && } - + ))} ); diff --git a/frontend/src/js/entity-history/timeline/GroupedContent.tsx b/frontend/src/js/entity-history/timeline/GroupedContent.tsx index b6248455d7..7a4414f4b9 100644 --- a/frontend/src/js/entity-history/timeline/GroupedContent.tsx +++ b/frontend/src/js/entity-history/timeline/GroupedContent.tsx @@ -18,6 +18,7 @@ import ConceptName from "./ConceptName"; import { TinyLabel } from "./TinyLabel"; import { isConceptColumn, + isDateColumn, isMoneyColumn, isSecondaryIdColumn, isVisibleColumn, @@ -154,7 +155,7 @@ const Cell = memo( datasetId: DatasetT["id"]; rootConceptIdsByColumn: Record; }) => { - if (!columnDescription) { + if (isDateColumn(columnDescription)) { return cell.from === cell.to ? ( {formatHistoryDayRange(cell.from)} ) : ( diff --git a/frontend/src/js/entity-history/timeline/util.ts b/frontend/src/js/entity-history/timeline/util.ts index eef88d1c25..422ed415cd 100644 --- a/frontend/src/js/entity-history/timeline/util.ts +++ b/frontend/src/js/entity-history/timeline/util.ts @@ -3,6 +3,9 @@ import { ColumnDescription, TimeStratifiedInfo } from "../../api/types"; export const isIdColumn = (columnDescription: ColumnDescription) => columnDescription.semantics.some((s) => s.type === "ID"); +export const isDateColumn = (columnDescription: ColumnDescription) => + columnDescription.semantics.some((s) => s.type === "EVENT_DATE"); + export const isGroupableColumn = (columnDescription: ColumnDescription) => columnDescription.semantics.some((s) => s.type === "GROUP"); From b834c859b1b7df516d801d251f86c5dd6c6d0d8a Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 13 Jun 2023 11:27:34 +0200 Subject: [PATCH 370/679] fixes format for full id with TableExportQuery --- .../integration/tests/EntityExportTest.java | 6 +++--- .../CONCEPT_VALUES_RESOLVED.test.json | 8 ++++---- .../CONCEPT_COLUMN_SELECTS/expected_resolved.csv | 8 ++++---- .../query/TABLE_EXPORT/RAW_TABLE_EXPORT.test.json | 2 +- .../query/TABLE_EXPORT/TABLE_EXPORT.test.json | 2 +- .../tests/query/TABLE_EXPORT/expected.csv | 14 +++++++------- 6 files changed, 20 insertions(+), 20 deletions(-) diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/EntityExportTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/EntityExportTest.java index ab862e3de4..309141f5c2 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/EntityExportTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/EntityExportTest.java @@ -245,9 +245,9 @@ public void execute(String name, TestConquery testConquery) throws Exception { assertThat(resultLines.readEntity(String.class).lines().collect(Collectors.toList())) .containsExactlyInAnyOrder( "result,dates,source,secondaryid,table1 column,table2 column", - "1,{2013-11-10/2013-11-10},table1,External: oneone,tree1.child_a,", - "1,{2012-01-01/2012-01-01},table2,2222,,tree2", - "1,{2010-07-15/2010-07-15},table2,External: threethree,,tree2" + "1,{2013-11-10/2013-11-10},table1,External: oneone,EntityExportTest.tree1.child_a,", + "1,{2012-01-01/2012-01-01},table2,2222,,EntityExportTest.tree2", + "1,{2010-07-15/2010-07-15},table2,External: threethree,,EntityExportTest.tree2" ); } diff --git a/backend/src/test/resources/tests/aggregator/CONCEPT_COLUMN_SELECTS/CONCEPT_VALUES_RESOLVED.test.json b/backend/src/test/resources/tests/aggregator/CONCEPT_COLUMN_SELECTS/CONCEPT_VALUES_RESOLVED.test.json index fc85a046d6..a98482ec1c 100644 --- a/backend/src/test/resources/tests/aggregator/CONCEPT_COLUMN_SELECTS/CONCEPT_VALUES_RESOLVED.test.json +++ b/backend/src/test/resources/tests/aggregator/CONCEPT_COLUMN_SELECTS/CONCEPT_VALUES_RESOLVED.test.json @@ -6,7 +6,7 @@ "type": "CONCEPT_QUERY", "root": { "type": "CONCEPT", - "selects" : [ + "selects": [ "tree.select" ], "ids": [ @@ -28,9 +28,9 @@ "type": "TREE", "selects": [ { - "type" : "CONCEPT_VALUES", - "name" : "select", - "asIds" : true + "type": "CONCEPT_VALUES", + "name": "select", + "asIds": true } ], "connectors": [ diff --git a/backend/src/test/resources/tests/aggregator/CONCEPT_COLUMN_SELECTS/expected_resolved.csv b/backend/src/test/resources/tests/aggregator/CONCEPT_COLUMN_SELECTS/expected_resolved.csv index 36a1e899b1..d99355deeb 100644 --- a/backend/src/test/resources/tests/aggregator/CONCEPT_COLUMN_SELECTS/expected_resolved.csv +++ b/backend/src/test/resources/tests/aggregator/CONCEPT_COLUMN_SELECTS/expected_resolved.csv @@ -1,5 +1,5 @@ result,dates,tree select -1,{2012-01-01/2012-01-02},"{tree.test_child2,tree.test_child1}" -2,{2010-07-15/2010-07-15},{tree.test_child2} -3,{2013-11-10/2013-11-10},{tree.test_child1} -4,{2012-11-11/2012-11-11},{tree.test_child2} \ No newline at end of file +1,{2012-01-01/2012-01-02},"{CONCEPT_VALUES$20$28Resolved$29$20Test.tree.test_child2,CONCEPT_VALUES$20$28Resolved$29$20Test.tree.test_child1}" +2,{2010-07-15/2010-07-15},{CONCEPT_VALUES$20$28Resolved$29$20Test.tree.test_child2} +3,{2013-11-10/2013-11-10},{CONCEPT_VALUES$20$28Resolved$29$20Test.tree.test_child1} +4,{2012-11-11/2012-11-11},{CONCEPT_VALUES$20$28Resolved$29$20Test.tree.test_child2} \ No newline at end of file diff --git a/backend/src/test/resources/tests/query/TABLE_EXPORT/RAW_TABLE_EXPORT.test.json b/backend/src/test/resources/tests/query/TABLE_EXPORT/RAW_TABLE_EXPORT.test.json index c3c321cdc0..bf49129fca 100644 --- a/backend/src/test/resources/tests/query/TABLE_EXPORT/RAW_TABLE_EXPORT.test.json +++ b/backend/src/test/resources/tests/query/TABLE_EXPORT/RAW_TABLE_EXPORT.test.json @@ -1,6 +1,6 @@ { "type": "QUERY_TEST", - "label": "TABLE_EXPORT Test", + "label": "RAW_TABLE_EXPORT Test", "expectedCsv": "tests/query/TABLE_EXPORT/raw_expected.csv", "query": { "type": "TABLE_EXPORT", diff --git a/backend/src/test/resources/tests/query/TABLE_EXPORT/TABLE_EXPORT.test.json b/backend/src/test/resources/tests/query/TABLE_EXPORT/TABLE_EXPORT.test.json index 7107f3904a..4b78d2cf0b 100644 --- a/backend/src/test/resources/tests/query/TABLE_EXPORT/TABLE_EXPORT.test.json +++ b/backend/src/test/resources/tests/query/TABLE_EXPORT/TABLE_EXPORT.test.json @@ -34,7 +34,7 @@ "min": "2000-01-01", "max": "2020-12-31" }, - "rawConceptValues" : false, + "rawConceptValues": false, "query": { "type": "CONCEPT_QUERY", "root": { diff --git a/backend/src/test/resources/tests/query/TABLE_EXPORT/expected.csv b/backend/src/test/resources/tests/query/TABLE_EXPORT/expected.csv index 9dab1a4faa..88e9196f5d 100644 --- a/backend/src/test/resources/tests/query/TABLE_EXPORT/expected.csv +++ b/backend/src/test/resources/tests/query/TABLE_EXPORT/expected.csv @@ -1,8 +1,8 @@ result,dates,source,SecondaryId,table1 value,table1 code,table2 value,table2 code -a,{2020-06-30/2020-06-30},table2,,,,13.0,concept.a_child -a,{2014-06-30/2015-06-30},table1,f_a1,1.0,concept.a_child,, -a,{2016-06-30/2016-06-30},table1,f_a1,1.0,concept.a_child,, -a,{2014-06-30/2015-06-30},table1,f_a2,1.0,concept.a_child,, -a,{2010-06-30/2010-06-30},table1,,1.0,concept.a_child,, -b,{2015-02-03/2015-06-30},table1,f_b1,1.0,concept,, -a,{2020-06-30/2020-06-30},table2,,,,13.0,concept \ No newline at end of file +a,{2020-06-30/2020-06-30},table2,,,,13.0,TABLE_EXPORT$20Test.concept.a_child +a,{2014-06-30/2015-06-30},table1,f_a1,1.0,TABLE_EXPORT$20Test.concept.a_child,, +a,{2016-06-30/2016-06-30},table1,f_a1,1.0,TABLE_EXPORT$20Test.concept.a_child,, +a,{2014-06-30/2015-06-30},table1,f_a2,1.0,TABLE_EXPORT$20Test.concept.a_child,, +a,{2010-06-30/2010-06-30},table1,,1.0,TABLE_EXPORT$20Test.concept.a_child,, +b,{2015-02-03/2015-06-30},table1,f_b1,1.0,TABLE_EXPORT$20Test.concept,, +a,{2020-06-30/2020-06-30},table2,,,,13.0,TABLE_EXPORT$20Test.concept \ No newline at end of file From 828785be02d2fedc821c1f0cb480b4830c9b2f5f Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Tue, 13 Jun 2023 12:59:58 +0200 Subject: [PATCH 371/679] change csv color --- frontend/src/app-theme.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/app-theme.ts b/frontend/src/app-theme.ts index f73533cd18..63f6825b57 100644 --- a/frontend/src/app-theme.ts +++ b/frontend/src/app-theme.ts @@ -28,7 +28,7 @@ export const theme: Theme = { "#fff", ], files: { - csv: "#5aa86f", + csv: "#279e47", pdf: "#c9181e", zip: "#e5c527", xlsx: "#3B843C", From 7612b00fd55cfff7c655f8979c5375c162a80622 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 13 Jun 2023 13:35:47 +0200 Subject: [PATCH 372/679] Remove unnecessary fn --- frontend/src/js/entity-history/EntityCard.tsx | 9 ++------- frontend/src/js/entity-history/timeline/util.ts | 9 +-------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/frontend/src/js/entity-history/EntityCard.tsx b/frontend/src/js/entity-history/EntityCard.tsx index d91f481be7..04a78393bc 100644 --- a/frontend/src/js/entity-history/EntityCard.tsx +++ b/frontend/src/js/entity-history/EntityCard.tsx @@ -9,7 +9,7 @@ import { exists } from "../common/helpers/exists"; import EntityInfos from "./EntityInfos"; import { TimeStratifiedChart } from "./TimeStratifiedChart"; -import { getColumnType } from "./timeline/util"; +import { getColumnType, isMoneyColumn } from "./timeline/util"; const Container = styled("div")` display: grid; @@ -60,11 +60,6 @@ const Table = ({ return ( {timeStratifiedInfo.columns.map((column) => { - const columnType = getColumnType( - timeStratifiedInfo, - column.label, - ); - const label = column.label; const value = timeStratifiedInfo.totals[column.label]; @@ -81,7 +76,7 @@ const Table = ({ - {columnType === "MONEY" && typeof value === "number" ? ( + {isMoneyColumn(column) && typeof value === "number" ? ( columnDescription.semantics.some((s) => s.type === "ID"); @@ -21,10 +21,3 @@ export const isMoneyColumn = (columnDescription: ColumnDescription) => export const isSecondaryIdColumn = (columnDescription: ColumnDescription) => columnDescription.semantics.some((s) => s.type === "SECONDARY_ID"); - -export const getColumnType = ( - timeStratifiedInfo: TimeStratifiedInfo, - label: string, -) => { - return timeStratifiedInfo.columns.find((c) => c.label === label)?.type; -}; From b536b3f0c3a1223821e881a49182b02669d830b3 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 13 Jun 2023 13:48:36 +0200 Subject: [PATCH 373/679] Remove dataset / concept id concatenation --- frontend/src/js/entity-history/Timeline.tsx | 5 ---- .../entity-history/timeline/ConceptName.tsx | 20 +++----------- .../js/entity-history/timeline/EventCard.tsx | 26 +++++++------------ .../timeline/GroupedContent.tsx | 7 ----- .../js/entity-history/timeline/Quarter.tsx | 5 ---- .../src/js/entity-history/timeline/Year.tsx | 4 --- .../js/entity-history/timeline/YearHead.tsx | 13 ++-------- 7 files changed, 16 insertions(+), 64 deletions(-) diff --git a/frontend/src/js/entity-history/Timeline.tsx b/frontend/src/js/entity-history/Timeline.tsx index 2394f746d7..42515cc6e5 100644 --- a/frontend/src/js/entity-history/Timeline.tsx +++ b/frontend/src/js/entity-history/Timeline.tsx @@ -11,7 +11,6 @@ import { TimeStratifiedInfo, } from "../api/types"; import type { StateT } from "../app/reducers"; -import { useDatasetId } from "../dataset/selectors"; import { ContentFilterValue } from "./ContentControl"; import type { DetailLevel } from "./DetailControl"; @@ -72,7 +71,6 @@ const Timeline = ({ toggleOpenYear, toggleOpenQuarter, }: Props) => { - const datasetId = useDatasetId(); const data = useSelector( (state) => state.entityHistory.currentEntityData, ); @@ -88,8 +86,6 @@ const Timeline = ({ secondaryIds: columnBuckets.secondaryIds, }); - if (!datasetId) return null; - if (eventsByQuarterWithGroups.length === 0) { return ; } @@ -104,7 +100,6 @@ const Timeline = ({ { - // TODO: refactor. It's very implicit that the id is - // somehow containing the datasetId. - if (!datasetId) return null; - - const fullConceptId = `${datasetId}.${conceptId}`; - const concept = getConceptById(fullConceptId, rootConceptId); +const ConceptName = ({ className, title, rootConceptId, conceptId }: Props) => { + const concept = getConceptById(conceptId, rootConceptId); if (!concept) { return ( @@ -55,7 +43,7 @@ const ConceptName = ({ ); - if (fullConceptId === rootConceptId) { + if (conceptId === rootConceptId) { return (
{conceptName} diff --git a/frontend/src/js/entity-history/timeline/EventCard.tsx b/frontend/src/js/entity-history/timeline/EventCard.tsx index 60e3b85868..11bdb01202 100644 --- a/frontend/src/js/entity-history/timeline/EventCard.tsx +++ b/frontend/src/js/entity-history/timeline/EventCard.tsx @@ -11,7 +11,6 @@ import type { ColumnDescription, ConceptIdT, CurrencyConfigT, - DatasetT, } from "../../api/types"; import { exists } from "../../common/helpers/exists"; import FaIcon from "../../icon/FaIcon"; @@ -93,29 +92,25 @@ const Bullet = styled("div")` flex-shrink: 0; `; -interface Props { - row: EntityEvent; - columns: Record; - columnBuckets: ColumnBuckets; - datasetId: DatasetT["id"]; - contentFilter: ContentFilterValue; - currencyConfig: CurrencyConfigT; - rootConceptIdsByColumn: Record; - groupedRows?: EntityEvent[]; - groupedRowsKeysWithDifferentValues?: string[]; -} - const EventCard = ({ row, columns, columnBuckets, - datasetId, currencyConfig, contentFilter, rootConceptIdsByColumn, groupedRows, groupedRowsKeysWithDifferentValues, -}: Props) => { +}: { + row: EntityEvent; + columns: Record; + columnBuckets: ColumnBuckets; + contentFilter: ContentFilterValue; + currencyConfig: CurrencyConfigT; + rootConceptIdsByColumn: Record; + groupedRows?: EntityEvent[]; + groupedRowsKeysWithDifferentValues?: string[]; +}) => { const { t } = useTranslation(); const applicableGroupableIds = columnBuckets.groupableIds.filter( @@ -168,7 +163,6 @@ const EventCard = ({ )} {groupedRowsKeysWithDifferentValues && groupedRows && ( ; groupedRows: EntityEvent[]; groupedRowsKeysWithDifferentValues: string[]; @@ -65,7 +63,6 @@ interface Props { } const GroupedContent = ({ - datasetId, columns, groupedRows, groupedRowsKeysWithDifferentValues, @@ -115,7 +112,6 @@ const GroupedContent = ({ differencesKeys.map((key) => ( ; }) => { if (isDateColumn(columnDescription)) { @@ -170,7 +164,6 @@ const Cell = memo( ); diff --git a/frontend/src/js/entity-history/timeline/Quarter.tsx b/frontend/src/js/entity-history/timeline/Quarter.tsx index 369f03b2c3..178897552c 100644 --- a/frontend/src/js/entity-history/timeline/Quarter.tsx +++ b/frontend/src/js/entity-history/timeline/Quarter.tsx @@ -7,7 +7,6 @@ import { ColumnDescription, ConceptIdT, CurrencyConfigT, - DatasetT, } from "../../api/types"; import FaIcon from "../../icon/FaIcon"; import { ContentFilterValue } from "../ContentControl"; @@ -90,7 +89,6 @@ const Quarter = ({ currencyConfig, rootConceptIdsByColumn, contentFilter, - datasetId, }: { year: number; quarter: number; @@ -100,7 +98,6 @@ const Quarter = ({ detailLevel: DetailLevel; toggleOpenQuarter: (year: number, quarter: number) => void; differences: string[][]; - datasetId: DatasetT["id"]; columns: Record; columnBuckets: ColumnBuckets; contentFilter: ContentFilterValue; @@ -151,7 +148,6 @@ const Quarter = ({ key={`${index}-${evtIdx}`} columns={columns} columnBuckets={columnBuckets} - datasetId={datasetId} contentFilter={contentFilter} rootConceptIdsByColumn={rootConceptIdsByColumn} row={evt} @@ -174,7 +170,6 @@ const Quarter = ({ key={index} columns={columns} columnBuckets={columnBuckets} - datasetId={datasetId} contentFilter={contentFilter} rootConceptIdsByColumn={rootConceptIdsByColumn} row={firstRowWithoutDifferences} diff --git a/frontend/src/js/entity-history/timeline/Year.tsx b/frontend/src/js/entity-history/timeline/Year.tsx index ccc43d5250..4ed30bbc67 100644 --- a/frontend/src/js/entity-history/timeline/Year.tsx +++ b/frontend/src/js/entity-history/timeline/Year.tsx @@ -5,7 +5,6 @@ import { ColumnDescription, ConceptIdT, CurrencyConfigT, - DatasetT, TimeStratifiedInfo, } from "../../api/types"; import { ContentFilterValue } from "../ContentControl"; @@ -22,7 +21,6 @@ const YearGroup = styled("div")` `; const Year = ({ - datasetId, year, getIsOpen, toggleOpenYear, @@ -36,7 +34,6 @@ const Year = ({ rootConceptIdsByColumn, timeStratifiedInfos, }: { - datasetId: DatasetT["id"]; year: number; getIsOpen: (year: number, quarter?: number) => boolean; toggleOpenYear: (year: number) => void; @@ -78,7 +75,6 @@ const Year = ({ - getConceptById( - // TODO: should be just v - semantic?.concept.split(".")[0] + "." + v, - semantic!.concept, - ), - ) + .map((v) => getConceptById(v, semantic!.concept)) .filter(exists); return ( @@ -169,11 +163,8 @@ const TimeStratifiedInfos = ({ ); } - // else if (typeof value === "string") { - // const concept = getConceptById(semantic!.concept, value); - // if (concept) valueFormatted = concept.label; - // } + // TOOD: Potentially support single-value concepts } let valueFormatted: string | number | string[] = value; From 96b12a3aff31c46e1399b1bf87660fec2980ef7b Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 13 Jun 2023 13:52:47 +0200 Subject: [PATCH 374/679] Fix unused import --- frontend/src/js/entity-history/EntityCard.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/js/entity-history/EntityCard.tsx b/frontend/src/js/entity-history/EntityCard.tsx index 04a78393bc..2e26f237d2 100644 --- a/frontend/src/js/entity-history/EntityCard.tsx +++ b/frontend/src/js/entity-history/EntityCard.tsx @@ -9,7 +9,7 @@ import { exists } from "../common/helpers/exists"; import EntityInfos from "./EntityInfos"; import { TimeStratifiedChart } from "./TimeStratifiedChart"; -import { getColumnType, isMoneyColumn } from "./timeline/util"; +import { isMoneyColumn } from "./timeline/util"; const Container = styled("div")` display: grid; From 04bcd01f0cc0a3a1fb45aab1e9a10466d6ba4746 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Tue, 13 Jun 2023 14:16:06 +0200 Subject: [PATCH 375/679] fix spelling --- frontend/src/localization/de.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/localization/de.json b/frontend/src/localization/de.json index a4f1f256bb..549e4029e5 100644 --- a/frontend/src/localization/de.json +++ b/frontend/src/localization/de.json @@ -274,7 +274,7 @@ "default": { "conceptDropzoneLabel": "Füge ein Konzept oder eine Konzeptliste hinzu", "conceptColumnDropzoneLabel": "Füge ein Konzept oder eine Konzeptliste hinzu", - "dropBetweenLabel": "Füge ein Konzept oder eine Konzeptliste hinzu. Clicke hier um abzubrechen" + "dropBetweenLabel": "Füge ein Konzept oder eine Konzeptliste hinzu. Klicke hier um abzubrechen" }, "copyModal": { "headline": "Kopieren von Konzepten aus anderem Feld", From 40deac363097a42407e73068f61ac2e6e93385d5 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 13 Jun 2023 15:53:24 +0200 Subject: [PATCH 376/679] move observationPeriodStart to FrontendConfig and map it on FrontendConfiguration level --- .../apiv1/frontend/FrontendConfiguration.java | 4 +++- .../apiv1/frontend/FrontendPreviewConfig.java | 3 --- .../conquery/models/config/FrontendConfig.java | 14 ++++++++------ .../conquery/models/datasets/PreviewConfig.java | 6 ------ .../conquery/resources/api/ConceptsProcessor.java | 1 - .../conquery/resources/api/ConfigResource.java | 3 ++- .../integration/tests/EntityExportTest.java | 8 -------- 7 files changed, 13 insertions(+), 26 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendConfiguration.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendConfiguration.java index 8aa6ce070a..57fcfa27ce 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendConfiguration.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendConfiguration.java @@ -1,6 +1,7 @@ package com.bakdata.conquery.apiv1.frontend; import java.net.URL; +import java.time.LocalDate; import com.bakdata.conquery.models.config.FrontendConfig; import com.bakdata.conquery.models.config.IdColumnConfig; @@ -19,6 +20,7 @@ public record FrontendConfiguration( FrontendConfig.CurrencyConfig currency, IdColumnConfig queryUpload, URL manualUrl, - String contactEmail + String contactEmail, + LocalDate observationPeriodStart ) { } diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendPreviewConfig.java b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendPreviewConfig.java index 6fe30b9994..9769140747 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendPreviewConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/frontend/FrontendPreviewConfig.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.apiv1.frontend; -import java.time.LocalDate; import java.util.Collection; import java.util.List; @@ -19,8 +18,6 @@ public static class Labelled { private final String label; } - private final LocalDate observationPeriodMin; - private final Collection all; @JsonProperty("default") private final Collection defaultConnectors; diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index e50fb7008f..66f6e7b293 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -2,6 +2,7 @@ import java.net.URI; import java.net.URL; +import java.time.LocalDate; import javax.annotation.Nullable; import javax.validation.Valid; @@ -10,27 +11,28 @@ import com.bakdata.conquery.models.forms.frontendconfiguration.FormScanner; import com.fasterxml.jackson.annotation.JsonAlias; -import groovy.transform.ToString; import lombok.AllArgsConstructor; import lombok.Data; -import lombok.Getter; import lombok.NoArgsConstructor; -import lombok.Setter; import lombok.With; import lombok.extern.slf4j.Slf4j; -@ToString -@Getter -@Setter @NoArgsConstructor @AllArgsConstructor @Slf4j @With +@Data public class FrontendConfig { @Valid @NotNull private CurrencyConfig currency = new CurrencyConfig(); + /** + * Default start-date for EntityPreview and DatePicker. + */ + @NotNull + private LocalDate observationStart; + /** * The url that points a manual. This is also used by the {@link FormScanner} * as the base url for forms that specify a relative url. Internally {@link URI#resolve(URI)} diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java index c08a23458c..9deccb3249 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.models.datasets; -import java.time.LocalDate; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -46,11 +45,6 @@ @NoArgsConstructor public class PreviewConfig { - /** - * Default start-date for EntityPreview, end date will always be LocalDate.now() - */ - @NotNull - private LocalDate observationStart; /** * Selects to be used in {@link com.bakdata.conquery.apiv1.QueryProcessor#getSingleEntityExport(Subject, UriBuilder, String, String, List, Dataset, Range)}. diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java index c59be20ec6..a3ff393d83 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java @@ -139,7 +139,6 @@ public FrontendPreviewConfig getEntityPreviewFrontendConfig(Dataset dataset) { // Connectors only act as bridge to table for the fronted, but also provide ConceptColumnT semantic return new FrontendPreviewConfig( - previewConfig.getObservationStart(), previewConfig.getAllConnectors() .stream() .map(id -> new FrontendPreviewConfig.Labelled(id.toString(), namespace.getCentralRegistry().resolve(id).getTable().getLabel())) diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java index 8f1f695329..0c2432c762 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java @@ -37,7 +37,8 @@ public FrontendConfiguration getFrontendConfig() { frontendConfig.getCurrency(), idColumns, frontendConfig.getManualUrl(), - frontendConfig.getContactEmail() + frontendConfig.getContactEmail(), + frontendConfig.getObservationStart() ); } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/EntityExportTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/EntityExportTest.java index ab862e3de4..6f21b6f6c4 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/EntityExportTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/EntityExportTest.java @@ -13,7 +13,6 @@ import java.util.Set; import java.util.stream.Collectors; -import javax.servlet.http.HttpServletRequest; import javax.ws.rs.client.Entity; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -24,7 +23,6 @@ import com.bakdata.conquery.integration.common.RequiredData; import com.bakdata.conquery.integration.json.JsonIntegrationTest; import com.bakdata.conquery.integration.json.QueryTest; -import com.bakdata.conquery.models.auth.entities.Subject; import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.PreviewConfig; @@ -42,7 +40,6 @@ import com.bakdata.conquery.resources.admin.rest.AdminDatasetResource; import com.bakdata.conquery.resources.api.DatasetQueryResource; import com.bakdata.conquery.resources.api.EntityPreviewRequest; -import com.bakdata.conquery.resources.api.QueryResource; import com.bakdata.conquery.resources.hierarchies.HierarchyHelper; import com.bakdata.conquery.util.support.StandaloneSupport; import com.bakdata.conquery.util.support.TestConquery; @@ -50,9 +47,6 @@ import lombok.extern.slf4j.Slf4j; import org.assertj.core.description.LazyTextDescription; -/** - * Adapted from {@link com.bakdata.conquery.integration.tests.deletion.ImportDeletionTest}, tests {@link QueryResource#getEntityData(Subject, QueryResource.EntityPreview, HttpServletRequest)}. - */ @Slf4j public class EntityExportTest implements ProgrammaticIntegrationTest { @@ -100,8 +94,6 @@ public void execute(String name, TestConquery testConquery) throws Exception { final PreviewConfig previewConfig = new PreviewConfig(); - previewConfig.setObservationStart(LocalDate.of(2010,1,1)); - previewConfig.setInfoCardSelects(List.of( new PreviewConfig.InfoCardSelect("Age", SelectId.Parser.INSTANCE.parsePrefixed(dataset.getName(), "tree1.connector.age"), null), new PreviewConfig.InfoCardSelect("Values", valuesSelectId, null) From 41a89962780e404e73bd0e357a9b2d907ec6977e Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 13 Jun 2023 16:17:33 +0200 Subject: [PATCH 377/679] Fix unused import --- frontend/src/js/entity-history/timeline/ConceptName.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/js/entity-history/timeline/ConceptName.tsx b/frontend/src/js/entity-history/timeline/ConceptName.tsx index d3324e6b09..d49facb336 100644 --- a/frontend/src/js/entity-history/timeline/ConceptName.tsx +++ b/frontend/src/js/entity-history/timeline/ConceptName.tsx @@ -2,7 +2,7 @@ import styled from "@emotion/styled"; import { faFolder } from "@fortawesome/free-solid-svg-icons"; import { memo } from "react"; -import { ConceptIdT, DatasetT } from "../../api/types"; +import { ConceptIdT } from "../../api/types"; import { getConceptById } from "../../concept-trees/globalTreeStoreHelper"; import FaIcon from "../../icon/FaIcon"; From b9d4768fed36107be1510fedccbe35c97024a476 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 13 Jun 2023 16:33:20 +0200 Subject: [PATCH 378/679] adds default to FrontendConfig#observationStart --- .../com/bakdata/conquery/models/config/FrontendConfig.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index 66f6e7b293..599b78a222 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -3,6 +3,7 @@ import java.net.URI; import java.net.URL; import java.time.LocalDate; +import java.time.temporal.ChronoUnit; import javax.annotation.Nullable; import javax.validation.Valid; @@ -31,7 +32,7 @@ public class FrontendConfig { * Default start-date for EntityPreview and DatePicker. */ @NotNull - private LocalDate observationStart; + private LocalDate observationStart = LocalDate.now().minus(10, ChronoUnit.YEARS); /** * The url that points a manual. This is also used by the {@link FormScanner} From 68570e64de983f0eb691b24da9370b715c02a478 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 13 Jun 2023 17:02:57 +0200 Subject: [PATCH 379/679] Use observation period start from frontend config --- frontend/src/js/api/types.ts | 2 +- frontend/src/js/entity-history/actions.ts | 10 +++++++++- frontend/src/js/entity-history/reducer.ts | 3 --- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/frontend/src/js/api/types.ts b/frontend/src/js/api/types.ts index 82eb4a1d50..ef0fc4d964 100644 --- a/frontend/src/js/api/types.ts +++ b/frontend/src/js/api/types.ts @@ -296,6 +296,7 @@ export interface GetFrontendConfigResponseT { queryUpload: QueryUploadConfigT; manualUrl?: string; contactEmail?: string; + observationPeriodStart?: string; // yyyy-mm-dd format, start of the data } export type GetConceptResponseT = Record; @@ -535,7 +536,6 @@ export interface HistorySources { export type GetEntityHistoryDefaultParamsResponse = HistorySources & { searchConcept: string | null; // concept id searchFilters?: string[]; // allowlisted filter ids within the searchConcept - observationPeriodMin: string; // yyyy-MM-dd }; export interface EntityInfo { diff --git a/frontend/src/js/entity-history/actions.ts b/frontend/src/js/entity-history/actions.ts index fb7b8d18f6..2f72407754 100644 --- a/frontend/src/js/entity-history/actions.ts +++ b/frontend/src/js/entity-history/actions.ts @@ -1,3 +1,5 @@ +import startOfYear from "date-fns/startOfYear"; +import subYears from "date-fns/subYears"; import { useCallback } from "react"; import { useTranslation } from "react-i18next"; import { useDispatch, useSelector } from "react-redux"; @@ -180,6 +182,12 @@ export function useUpdateHistorySession() { StateT, StateT["entityHistory"]["defaultParams"] >((state) => state.entityHistory.defaultParams); + const observationPeriodMin = useSelector((state) => { + return ( + state.startup.config.observationPeriodStart || + formatStdDate(subYears(startOfYear(new Date()), 1)) + ); + }); return useCallback( async ({ @@ -203,7 +211,7 @@ export function useUpdateHistorySession() { entityId, defaultEntityHistoryParams.sources, { - min: defaultEntityHistoryParams.observationPeriodMin, + min: observationPeriodMin, max: formatStdDate(new Date()), }, ); diff --git a/frontend/src/js/entity-history/reducer.ts b/frontend/src/js/entity-history/reducer.ts index 11a577b2ee..f6d59e45d5 100644 --- a/frontend/src/js/entity-history/reducer.ts +++ b/frontend/src/js/entity-history/reducer.ts @@ -35,7 +35,6 @@ export interface EntityId { export type EntityHistoryStateT = { defaultParams: { - observationPeriodMin: string; sources: HistorySources; searchConcept: string | null; searchFilters: string[]; @@ -57,7 +56,6 @@ export type EntityHistoryStateT = { const initialState: EntityHistoryStateT = { defaultParams: { - observationPeriodMin: "2020-01-01", sources: { all: [], default: [] }, searchConcept: null, searchFilters: [], @@ -86,7 +84,6 @@ export default function reducer( return { ...state, defaultParams: { - observationPeriodMin: action.payload.observationPeriodMin, sources: { all: action.payload.all, default: action.payload.default }, searchConcept: action.payload.searchConcept, searchFilters: action.payload.searchFilters || [], From ddda5aa1f04ca9293b7fe3bf2a0b7ec1d3b30141 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 13 Jun 2023 17:14:37 +0200 Subject: [PATCH 380/679] Use the observation period start for date picker --- frontend/src/js/common/helpers/dateHelper.ts | 2 ++ frontend/src/js/entity-history/actions.ts | 1 + .../ui-components/InputDate/CustomHeader.tsx | 20 +++++++++++++------ 3 files changed, 17 insertions(+), 6 deletions(-) diff --git a/frontend/src/js/common/helpers/dateHelper.ts b/frontend/src/js/common/helpers/dateHelper.ts index c5fc13fc8c..f7d7a566ad 100644 --- a/frontend/src/js/common/helpers/dateHelper.ts +++ b/frontend/src/js/common/helpers/dateHelper.ts @@ -216,11 +216,13 @@ export function getFirstAndLastDateOfRange(dateRangeStr: string): { export function useMonthName(date: Date): string { const locale = useDateLocale(); + return format(date, "LLLL", { locale }); } export function useMonthNames(): string[] { const locale = useDateLocale(); + return [...Array(12).keys()].map((month) => { const date = new Date(); date.setMonth(month); diff --git a/frontend/src/js/entity-history/actions.ts b/frontend/src/js/entity-history/actions.ts index 2f72407754..a8e416ce0a 100644 --- a/frontend/src/js/entity-history/actions.ts +++ b/frontend/src/js/entity-history/actions.ts @@ -278,6 +278,7 @@ export function useUpdateHistorySession() { dispatch, getAuthorizedUrl, getEntityHistory, + observationPeriodMin, ], ); } diff --git a/frontend/src/js/ui-components/InputDate/CustomHeader.tsx b/frontend/src/js/ui-components/InputDate/CustomHeader.tsx index ca1269e39f..83b0b594c0 100644 --- a/frontend/src/js/ui-components/InputDate/CustomHeader.tsx +++ b/frontend/src/js/ui-components/InputDate/CustomHeader.tsx @@ -5,8 +5,10 @@ import { } from "@fortawesome/free-solid-svg-icons"; import { useState } from "react"; import { ReactDatePickerCustomHeaderProps } from "react-datepicker"; +import { useSelector } from "react-redux"; import { SelectOptionT } from "../../api/types"; +import { StateT } from "../../app/reducers"; import IconButton from "../../button/IconButton"; import { TransparentButton } from "../../button/TransparentButton"; import { useMonthName, useMonthNames } from "../../common/helpers/dateHelper"; @@ -79,13 +81,19 @@ const YearMonthSelect = ({ ReactDatePickerCustomHeaderProps, "date" | "changeYear" | "changeMonth" >) => { - const yearSelectionSpan = 10; - const yearOptions: SelectOptionT[] = [...Array(yearSelectionSpan).keys()] + const numLastYearsToShow = useSelector((state) => { + if (state.startup.config.observationPeriodStart) { + return ( + new Date().getFullYear() - + new Date(state.startup.config.observationPeriodStart).getFullYear() + ); + } else { + return 10; + } + }); + const yearOptions: SelectOptionT[] = [...Array(numLastYearsToShow).keys()] .map((n) => new Date().getFullYear() - n) - .map((year) => ({ - label: String(year), - value: year, - })) + .map((year) => ({ label: String(year), value: year })) .reverse(); const monthNames = useMonthNames(); From 3bc01dee4afb8ce7823003f10090b70fcfcbf0aa Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Tue, 13 Jun 2023 17:40:14 +0200 Subject: [PATCH 381/679] change colors and rename variables --- frontend/src/app-theme.ts | 10 +++++----- frontend/src/js/button/DownloadButton.tsx | 15 ++++++++------- frontend/src/react-app-env.d.ts | 2 +- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/frontend/src/app-theme.ts b/frontend/src/app-theme.ts index 63f6825b57..572c8b3fd2 100644 --- a/frontend/src/app-theme.ts +++ b/frontend/src/app-theme.ts @@ -27,11 +27,11 @@ export const theme: Theme = { "#777", "#fff", ], - files: { - csv: "#279e47", - pdf: "#c9181e", - zip: "#e5c527", - xlsx: "#3B843C", + fileTypes: { + csv: "#1c8e3b", + pdf: "#b50a10", + zip: "#c1a515", + xlsx: "#2b602c", }, bgAlt: "#f4f6f5", blueGrayDark: "#1f5f30", diff --git a/frontend/src/js/button/DownloadButton.tsx b/frontend/src/js/button/DownloadButton.tsx index bc487beea1..6d5df34a08 100644 --- a/frontend/src/js/button/DownloadButton.tsx +++ b/frontend/src/js/button/DownloadButton.tsx @@ -30,19 +30,20 @@ interface FileIcon { color?: string; } -const fileTypeToIcon: Record = { - ZIP: { icon: faFileArchive, color: theme.col.files.zip }, - XLSX: { icon: faFileExcel, color: theme.col.files.xlsx }, - PDF: { icon: faFilePdf, color: theme.col.files.pdf }, - CSV: { icon: faFileCsv, color: theme.col.files.csv }, +const fileTypeToFileIcon: Record = { + ZIP: { icon: faFileArchive, color: theme.col.fileTypes.zip }, + XLSX: { icon: faFileExcel, color: theme.col.fileTypes.xlsx }, + PDF: { icon: faFilePdf, color: theme.col.fileTypes.pdf }, + CSV: { icon: faFileCsv, color: theme.col.fileTypes.csv }, }; function getFileInfo(url: string): FileIcon { // Forms + if (url.includes(".")) { const ext = getEnding(url); - if (ext in fileTypeToIcon) { - return fileTypeToIcon[ext]; + if (ext in fileTypeToFileIcon) { + return fileTypeToFileIcon[ext]; } } return { icon: faFileDownload }; diff --git a/frontend/src/react-app-env.d.ts b/frontend/src/react-app-env.d.ts index f2bf4b86c6..16604e8489 100644 --- a/frontend/src/react-app-env.d.ts +++ b/frontend/src/react-app-env.d.ts @@ -31,7 +31,7 @@ declare module "@emotion/react" { green: string; orange: string; palette: string[]; - files: { + fileTypes: { csv: string; pdf: string; zip: string; From 8ed55c246e83c93dd6fa21fe431fae0b0a6a59e8 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Fri, 16 Jun 2023 11:42:49 +0200 Subject: [PATCH 382/679] implement v3 --- .../DropzoneBetweenElements.tsx | 75 +++++-------------- .../form-components/DropzoneList.tsx | 1 - .../form-concept-group/FormConceptGroup.tsx | 5 +- 3 files changed, 22 insertions(+), 59 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index d26e4c2215..b21c644f9d 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -1,11 +1,6 @@ import styled from "@emotion/styled"; -import { faPlus } from "@fortawesome/free-solid-svg-icons"; -import { useState } from "react"; import { DropTargetMonitor, useDrop } from "react-dnd"; -import { useTranslation } from "react-i18next"; - -import FaIcon from "../../icon/FaIcon"; -import Dropzone, { +import { PossibleDroppableObject, } from "../../ui-components/Dropzone"; @@ -20,49 +15,28 @@ const Root = styled("div")<{ isDroppable: boolean; isFirstElement: boolean; }>` - background-color: ${({ theme, isDroppable, isOver }) => { - if (isOver && isDroppable) return theme.col.grayLight; - return isDroppable ? theme.col.grayVeryLight : "inherit"; - }}; - margin-top: ${({ isFirstElement }) => (isFirstElement ? "5px" : "0px")}; width: 100%; - text-align: center; -`; - -const PlusContainer = styled("div")` - margin: 0; - display: flex; - justify-content: center; - align-items: center; - height: 21px; + left: 0; + top: -17px; + height: 40px; + right: 0; + position: relative; + border-radius: ${({ theme }) => theme.borderRadius}; `; const DropzoneContainer = styled("div")` overflow: hidden; - height: 54px; -`; - -const SxFaIcon = styled(FaIcon)` - height: 12px; - color: ${({ theme }) => theme.col.black}; - opacity: 0.75; + height: 20px; `; const BetweenElements = ({ acceptedDropTypes, - onDrop: onDropCallback, + onDrop, isFirstElement, }: Props) => { - const { t } = useTranslation(); - - const SxDropzone = styled(Dropzone)` - margin: 5px 0 5px 0; - `; - - const [showDropzone, setShowDropzone] = useState(false); - const [{ isOver, isDroppable }, addZoneRef] = useDrop({ accept: acceptedDropTypes, + drop: onDrop, collect: (monitor) => ({ isOver: monitor.isOver(), isDroppable: monitor.canDrop(), @@ -74,37 +48,24 @@ const BetweenElements = ({ isOver: monitor.isOver(), }), }); - - const onDrop = (item: DroppableObject, monitor: DropTargetMonitor) => { - setShowDropzone(false); - onDropCallback(item, monitor); - }; + console.log(isOver, isDroppable); return ( <> - {!(showDropzone || isOver || isOver2) && ( - setShowDropzone(true)}> - - + > - )} - {(showDropzone || isOver || isOver2) && ( - setShowDropzone(false)} - > - - {() => t("externalForms.default.dropBetweenLabel")} - - - )} + {( isOver || isOver2) && ( + + + )} ); }; diff --git a/frontend/src/js/external-forms/form-components/DropzoneList.tsx b/frontend/src/js/external-forms/form-components/DropzoneList.tsx index eb5577dc06..76b6f4fc2a 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneList.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneList.tsx @@ -14,7 +14,6 @@ import DropzoneWithFileInput, { } from "../../ui-components/DropzoneWithFileInput"; import Label from "../../ui-components/Label"; import Optional from "../../ui-components/Optional"; - import DropzoneBetweenElements from "./DropzoneBetweenElements"; const ListItem = styled("div")` diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 4bf9a0e73a..3f16486173 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -24,6 +24,7 @@ import ToggleButton from "../../ui-components/ToggleButton"; import UploadConceptListModal from "../../upload-concept-list-modal/UploadConceptListModal"; import type { ConceptListDefaults as ConceptListDefaultsType } from "../config-types"; import { Description } from "../form-components/Description"; +import DropzoneBetweenElements from "../form-components/DropzoneBetweenElements"; import DropzoneList from "../form-components/DropzoneList"; import DynamicInputGroup from "../form-components/DynamicInputGroup"; import FormQueryNodeEditor from "../form-query-node-editor/FormQueryNodeEditor"; @@ -87,7 +88,9 @@ interface Props { }) => ReactNode; } -const DropzoneListItem = styled("div")``; +const DropzoneListItem = styled("div")` + margin-top: -30px; +`; const Row = styled("div")` display: flex; align-items: center; From 46bf6ec78a959606692f20891fe9dc8809419013 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Fri, 16 Jun 2023 11:43:09 +0200 Subject: [PATCH 383/679] format --- .../DropzoneBetweenElements.tsx | 27 ++++++++----------- .../form-components/DropzoneList.tsx | 1 + 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index b21c644f9d..9a24ff75bc 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -1,8 +1,7 @@ import styled from "@emotion/styled"; import { DropTargetMonitor, useDrop } from "react-dnd"; -import { - PossibleDroppableObject, -} from "../../ui-components/Dropzone"; + +import { PossibleDroppableObject } from "../../ui-components/Dropzone"; interface Props { onDrop: (item: DroppableObject, monitor: DropTargetMonitor) => void; @@ -52,20 +51,16 @@ const BetweenElements = ({ return ( <> - - + - {( isOver || isOver2) && ( - - - )} + {(isOver || isOver2) && ( + + )} ); }; diff --git a/frontend/src/js/external-forms/form-components/DropzoneList.tsx b/frontend/src/js/external-forms/form-components/DropzoneList.tsx index 76b6f4fc2a..eb5577dc06 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneList.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneList.tsx @@ -14,6 +14,7 @@ import DropzoneWithFileInput, { } from "../../ui-components/DropzoneWithFileInput"; import Label from "../../ui-components/Label"; import Optional from "../../ui-components/Optional"; + import DropzoneBetweenElements from "./DropzoneBetweenElements"; const ListItem = styled("div")` From 05e56f07d1eeeb890966058047deb4bea987f676 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 19 Jun 2023 10:01:32 +0200 Subject: [PATCH 384/679] added reordering, changed method of where the dropzone is --- .../DropzoneBetweenElements.tsx | 12 +-------- .../form-concept-group/FormConceptGroup.tsx | 26 +++++++++++++------ frontend/src/localization/de.json | 3 +-- frontend/src/localization/en.json | 3 +-- 4 files changed, 21 insertions(+), 23 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index 9a24ff75bc..eb8a7a1c0e 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -41,14 +41,6 @@ const BetweenElements = ({ isDroppable: monitor.canDrop(), }), }); - const [{ isOver: isOver2 }, dropzoneWrapperRef] = useDrop({ - accept: acceptedDropTypes, - collect: (monitor) => ({ - isOver: monitor.isOver(), - }), - }); - console.log(isOver, isDroppable); - return ( <> ({ isFirstElement={isFirstElement} > - {(isOver || isOver2) && ( - - )} + {isOver && } ); }; diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 3f16486173..8c65aec38e 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -24,7 +24,6 @@ import ToggleButton from "../../ui-components/ToggleButton"; import UploadConceptListModal from "../../upload-concept-list-modal/UploadConceptListModal"; import type { ConceptListDefaults as ConceptListDefaultsType } from "../config-types"; import { Description } from "../form-components/Description"; -import DropzoneBetweenElements from "../form-components/DropzoneBetweenElements"; import DropzoneList from "../form-components/DropzoneList"; import DynamicInputGroup from "../form-components/DynamicInputGroup"; import FormQueryNodeEditor from "../form-query-node-editor/FormQueryNodeEditor"; @@ -205,19 +204,30 @@ const FormConceptGroup = (props: Props) => { } dropBetween={(i: number) => { return (item: DragItemConceptTreeNode) => { + if (props.isValidConcept && !props.isValidConcept(item)) + return null; + if (isMovedObject(item)) { + let removed = + props.value[item.dragContext.movedFromAndIdx].concepts + .length === 1 + ? removeValue(props.value, item.dragContext.movedFromAndIdx) + : removeConcept( + props.value, + item.dragContext.movedFromAndIdx, + item.dragContext.movedFromOrIdx, + ); + let insertIndex = + i > item.dragContext.movedFromAndIdx ? i - 1 : i; return props.onChange( addConcept( - insertValue(props.value, i, newValue), - i, + insertValue(removed, insertIndex, newValue), + insertIndex, copyConcept(item), ), ); } - if (props.isValidConcept && !props.isValidConcept(item)) - return null; - return props.onChange( addConcept( insertValue(props.value, i, newValue), @@ -244,6 +254,8 @@ const FormConceptGroup = (props: Props) => { return; } + if (props.isValidConcept && !props.isValidConcept(item)) return; + if (isMovedObject(item)) { return props.onChange( addConcept( @@ -254,8 +266,6 @@ const FormConceptGroup = (props: Props) => { ); } - if (props.isValidConcept && !props.isValidConcept(item)) return; - return props.onChange( addConcept( addValue(props.value, newValue), diff --git a/frontend/src/localization/de.json b/frontend/src/localization/de.json index 549e4029e5..1b5843d915 100644 --- a/frontend/src/localization/de.json +++ b/frontend/src/localization/de.json @@ -273,8 +273,7 @@ }, "default": { "conceptDropzoneLabel": "Füge ein Konzept oder eine Konzeptliste hinzu", - "conceptColumnDropzoneLabel": "Füge ein Konzept oder eine Konzeptliste hinzu", - "dropBetweenLabel": "Füge ein Konzept oder eine Konzeptliste hinzu. Klicke hier um abzubrechen" + "conceptColumnDropzoneLabel": "Füge ein Konzept oder eine Konzeptliste hinzu" }, "copyModal": { "headline": "Kopieren von Konzepten aus anderem Feld", diff --git a/frontend/src/localization/en.json b/frontend/src/localization/en.json index efa70d3c78..52d4c1f0ef 100644 --- a/frontend/src/localization/en.json +++ b/frontend/src/localization/en.json @@ -274,8 +274,7 @@ }, "default": { "conceptDropzoneLabel": "Add a concept or a concept list", - "conceptColumnDropzoneLabel": "Add a concept or a concept list", - "dropBetweenLabel": "Add a concept or a concept list. Click here to cancel" + "conceptColumnDropzoneLabel": "Add a concept or a concept list" }, "copyModal": { "headline": "Copy concepts form another field", From 8de2c071bc80d307e09227bd3e009c623490f3c0 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 19 Jun 2023 11:33:54 +0200 Subject: [PATCH 385/679] Reset results on dataset select --- frontend/src/js/dataset/actions.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/frontend/src/js/dataset/actions.ts b/frontend/src/js/dataset/actions.ts index 3e7287e561..caa4aee6f3 100644 --- a/frontend/src/js/dataset/actions.ts +++ b/frontend/src/js/dataset/actions.ts @@ -9,8 +9,12 @@ import { StateT } from "../app/reducers"; import { ErrorObject } from "../common/actions/genericActions"; import { exists } from "../common/helpers/exists"; import { useLoadTrees } from "../concept-trees/actions"; -import { useLoadDefaultHistoryParams } from "../entity-history/actions"; +import { + resetHistory, + useLoadDefaultHistoryParams, +} from "../entity-history/actions"; import { useLoadQueries } from "../previous-queries/list/actions"; +import { queryResultReset } from "../query-runner/actions"; import { setMessage } from "../snack-message/actions"; import { SnackMessageType } from "../snack-message/reducer"; import { clearQuery, loadSavedQuery } from "../standard-query-editor/actions"; @@ -109,6 +113,12 @@ export const useSelectDataset = () => { dispatch(selectDatasetInput({ id: datasetId })); + dispatch(resetHistory()); + dispatch(queryResultReset({ queryType: "standard" })); + dispatch(queryResultReset({ queryType: "timebased" })); + dispatch(queryResultReset({ queryType: "editorV2" })); + dispatch(queryResultReset({ queryType: "externalForms" })); + // To allow loading trees to check whether they should abort or not setDatasetId(datasetId); From e3429bfc4ae62dc0922773528d1d0ed5d9f64dc0 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 19 Jun 2023 12:13:54 +0200 Subject: [PATCH 386/679] Avoid importing theme directly, use hook instead --- frontend/src/js/button/DownloadButton.tsx | 25 ++++++++++++++--------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/frontend/src/js/button/DownloadButton.tsx b/frontend/src/js/button/DownloadButton.tsx index 6d5df34a08..0862be59eb 100644 --- a/frontend/src/js/button/DownloadButton.tsx +++ b/frontend/src/js/button/DownloadButton.tsx @@ -1,3 +1,4 @@ +import { useTheme } from "@emotion/react"; import styled from "@emotion/styled"; import { IconProp } from "@fortawesome/fontawesome-svg-core"; import { @@ -8,9 +9,8 @@ import { faFileExcel, faFilePdf, } from "@fortawesome/free-solid-svg-icons"; -import { ReactNode, useContext, forwardRef } from "react"; +import { ReactNode, useContext, forwardRef, useMemo } from "react"; -import { theme } from "../../app-theme"; import { ResultUrlWithLabel } from "../api/types"; import { AuthTokenContext } from "../authorization/AuthTokenProvider"; import { getEnding } from "../query-runner/DownloadResultsDropdownButton"; @@ -30,22 +30,27 @@ interface FileIcon { color?: string; } -const fileTypeToFileIcon: Record = { - ZIP: { icon: faFileArchive, color: theme.col.fileTypes.zip }, - XLSX: { icon: faFileExcel, color: theme.col.fileTypes.xlsx }, - PDF: { icon: faFilePdf, color: theme.col.fileTypes.pdf }, - CSV: { icon: faFileCsv, color: theme.col.fileTypes.csv }, -}; +function useFileIcon(url: string): FileIcon { + const theme = useTheme(); -function getFileInfo(url: string): FileIcon { - // Forms + const fileTypeToFileIcon: Record = useMemo( + () => ({ + ZIP: { icon: faFileArchive, color: theme.col.fileTypes.zip }, + XLSX: { icon: faFileExcel, color: theme.col.fileTypes.xlsx }, + PDF: { icon: faFilePdf, color: theme.col.fileTypes.pdf }, + CSV: { icon: faFileCsv, color: theme.col.fileTypes.csv }, + }), + [theme], + ); if (url.includes(".")) { const ext = getEnding(url); + if (ext in fileTypeToFileIcon) { return fileTypeToFileIcon[ext]; } } + return { icon: faFileDownload }; } From 70c4dc0b0b7054d2e30fcbfee5b44ae12eeb2571 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 19 Jun 2023 12:14:18 +0200 Subject: [PATCH 387/679] Iterate file type colors based on best practices + ChatGPT suggestions --- frontend/src/app-theme.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/app-theme.ts b/frontend/src/app-theme.ts index 572c8b3fd2..fb8cfc19c4 100644 --- a/frontend/src/app-theme.ts +++ b/frontend/src/app-theme.ts @@ -28,10 +28,10 @@ export const theme: Theme = { "#fff", ], fileTypes: { - csv: "#1c8e3b", - pdf: "#b50a10", - zip: "#c1a515", - xlsx: "#2b602c", + csv: "#007BFF", + pdf: "#d73a49", + zip: "#6f42c1", + xlsx: "#28a745", }, bgAlt: "#f4f6f5", blueGrayDark: "#1f5f30", From 5b3dc08c1ad97e65f3a879e1914bf7eb27ad2264 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 19 Jun 2023 12:14:38 +0200 Subject: [PATCH 388/679] Simplify props passing --- frontend/src/js/button/DownloadButton.tsx | 8 ++++---- frontend/src/js/button/IconButton.tsx | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/src/js/button/DownloadButton.tsx b/frontend/src/js/button/DownloadButton.tsx index 0862be59eb..2868f260bf 100644 --- a/frontend/src/js/button/DownloadButton.tsx +++ b/frontend/src/js/button/DownloadButton.tsx @@ -73,16 +73,16 @@ const DownloadButton = forwardRef( authToken, )}&charset=ISO_8859_1`; - const fileInfo = getFileInfo(resultUrl.url); + const { icon, color } = useFileIcon(resultUrl.url); return ( {children} diff --git a/frontend/src/js/button/IconButton.tsx b/frontend/src/js/button/IconButton.tsx index 2b7f5e076a..772e52b4a2 100644 --- a/frontend/src/js/button/IconButton.tsx +++ b/frontend/src/js/button/IconButton.tsx @@ -175,9 +175,9 @@ const IconButton = forwardRef( tight={tight} bgHover={bgHover} red={red} + large={large} {...restProps} ref={ref} - large={large} > {iconElement} {children && {children}} From f05722ca95f16a54f5008ada24e98ea6dac9e6a5 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 19 Jun 2023 13:09:47 +0200 Subject: [PATCH 389/679] Add tabbable money column charts --- frontend/src/js/entity-history/EntityCard.tsx | 92 ++----------------- .../TabbableTimeStratifiedCharts.tsx | 42 +++++++++ .../js/entity-history/TimeStratifiedChart.tsx | 17 ++-- 3 files changed, 59 insertions(+), 92 deletions(-) create mode 100644 frontend/src/js/entity-history/TabbableTimeStratifiedCharts.tsx diff --git a/frontend/src/js/entity-history/EntityCard.tsx b/frontend/src/js/entity-history/EntityCard.tsx index 2e26f237d2..1654920b5e 100644 --- a/frontend/src/js/entity-history/EntityCard.tsx +++ b/frontend/src/js/entity-history/EntityCard.tsx @@ -1,24 +1,20 @@ import styled from "@emotion/styled"; -import { Fragment } from "react"; -import { NumericFormat } from "react-number-format"; -import { useSelector } from "react-redux"; -import { CurrencyConfigT, EntityInfo, TimeStratifiedInfo } from "../api/types"; -import { StateT } from "../app/reducers"; -import { exists } from "../common/helpers/exists"; +import { EntityInfo, TimeStratifiedInfo } from "../api/types"; import EntityInfos from "./EntityInfos"; -import { TimeStratifiedChart } from "./TimeStratifiedChart"; +import { TabbableTimeStratifiedCharts } from "./TabbableTimeStratifiedCharts"; import { isMoneyColumn } from "./timeline/util"; const Container = styled("div")` display: grid; grid-template-columns: 1fr 1fr; gap: 10px; - padding: 20px; + padding: 20px 24px; background-color: ${({ theme }) => theme.col.bg}; border-radius: ${({ theme }) => theme.borderRadius}; border: 1px solid ${({ theme }) => theme.col.grayLight}; + align-items: center; `; const Centered = styled("div")` @@ -28,76 +24,6 @@ const Centered = styled("div")` gap: 10px; `; -const Grid = styled("div")` - display: grid; - gap: 0 20px; - grid-template-columns: auto auto; -`; - -const Label = styled("div")` - font-size: ${({ theme }) => theme.font.xs}; -`; - -const Value = styled("div")` - font-size: ${({ theme }) => theme.font.sm}; - font-weight: 400; - justify-self: end; -`; - -// @ts-ignore EVALUATE IF WE WANT TO SHOW THIS TABLE WITH FUTURE DATA -// eslint-disable-next-line @typescript-eslint/no-unused-vars -const Table = ({ - timeStratifiedInfos, -}: { - timeStratifiedInfos: TimeStratifiedInfo[]; -}) => { - const currencyConfig = useSelector( - (state) => state.startup.config.currency, - ); - return ( - <> - {timeStratifiedInfos.map((timeStratifiedInfo) => { - return ( - - {timeStratifiedInfo.columns.map((column) => { - const label = column.label; - const value = timeStratifiedInfo.totals[column.label]; - - if (!exists(value)) return <>; - - const valueFormatted = - typeof value === "number" - ? Math.round(value) - : value instanceof Array - ? value.join(", ") - : value; - - return ( - - - - {isMoneyColumn(column) && typeof value === "number" ? ( - - ) : ( - valueFormatted - )} - - - ); - })} - - ); - })} - - ); -}; - export const EntityCard = ({ className, infos, @@ -107,16 +33,16 @@ export const EntityCard = ({ infos: EntityInfo[]; timeStratifiedInfos: TimeStratifiedInfo[]; }) => { + const infosWithOnlyMoneyColumns = timeStratifiedInfos.filter((info) => + info.columns.every(isMoneyColumn), + ); + return ( - {/* TODO: EVALUATE IF WE WANT TO SHOW THIS TABLE WITH FUTURE DATA -
*/} - + ); }; diff --git a/frontend/src/js/entity-history/TabbableTimeStratifiedCharts.tsx b/frontend/src/js/entity-history/TabbableTimeStratifiedCharts.tsx new file mode 100644 index 0000000000..49c248d070 --- /dev/null +++ b/frontend/src/js/entity-history/TabbableTimeStratifiedCharts.tsx @@ -0,0 +1,42 @@ +import styled from "@emotion/styled"; +import { useState, useMemo } from "react"; + +import { TimeStratifiedInfo } from "../api/types"; +import SmallTabNavigation from "../small-tab-navigation/SmallTabNavigation"; + +import { TimeStratifiedChart } from "./TimeStratifiedChart"; + +const Container = styled("div")` + display: flex; + flex-direction: column; + align-items: flex-end; +`; + +export const TabbableTimeStratifiedCharts = ({ + infos, +}: { + infos: TimeStratifiedInfo[]; +}) => { + const [activeTab, setActiveTab] = useState(infos[0].label); + const options = useMemo(() => { + return infos.map((info) => ({ + value: info.label, + label: () => info.label, + })); + }, [infos]); + + const activeInfos = useMemo(() => { + return infos.find((info) => info.label === activeTab); + }, [infos, activeTab]); + + return ( + + + {activeInfos && } + + ); +}; diff --git a/frontend/src/js/entity-history/TimeStratifiedChart.tsx b/frontend/src/js/entity-history/TimeStratifiedChart.tsx index 8bfdb576d9..2bd08588f2 100644 --- a/frontend/src/js/entity-history/TimeStratifiedChart.tsx +++ b/frontend/src/js/entity-history/TimeStratifiedChart.tsx @@ -19,7 +19,7 @@ import { exists } from "../common/helpers/exists"; ChartJS.register(CategoryScale, LinearScale, BarElement, Tooltip); const ChartContainer = styled("div")` - height: 185px; + height: 190px; width: 100%; display: flex; justify-content: flex-end; @@ -65,16 +65,15 @@ const useFormatCurrency = () => { }; export const TimeStratifiedChart = ({ - timeStratifiedInfos, + timeStratifiedInfo, }: { - timeStratifiedInfos: TimeStratifiedInfo[]; + timeStratifiedInfo: TimeStratifiedInfo; }) => { const theme = useTheme(); - const infosToVisualize = timeStratifiedInfos[0]; - const labels = infosToVisualize.columns.map((col) => col.label); + const labels = timeStratifiedInfo.columns.map((col) => col.label); const datasets = useMemo(() => { - const sortedYears = [...infosToVisualize.years].sort( + const sortedYears = [...timeStratifiedInfo.years].sort( (a, b) => b.year - a.year, ); @@ -87,7 +86,7 @@ export const TimeStratifiedChart = ({ )}, ${interpolateDecreasingOpacity(i)})`, }; }); - }, [theme, infosToVisualize, labels]); + }, [theme, timeStratifiedInfo, labels]); const data = { labels, @@ -101,7 +100,7 @@ export const TimeStratifiedChart = ({ plugins: { title: { display: true, - text: infosToVisualize.label, + text: timeStratifiedInfo.label, }, tooltip: { usePointStyle: true, @@ -159,7 +158,7 @@ export const TimeStratifiedChart = ({ }, }, }; - }, [infosToVisualize, labels, formatCurrency]); + }, [timeStratifiedInfo, labels, formatCurrency]); return ( From 9af584733c77688c130327229bd7a7dffb10e173 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 19 Jun 2023 15:39:29 +0200 Subject: [PATCH 390/679] Blur history entity data by default --- frontend/src/js/entity-history/EntityCard.tsx | 4 +- .../src/js/entity-history/EntityInfos.tsx | 13 ++++-- frontend/src/js/entity-history/History.tsx | 10 +++++ frontend/src/js/entity-history/Timeline.tsx | 27 +++++++------ .../js/entity-history/VisibilityControl.tsx | 40 +++++++++++++++++++ frontend/src/localization/de.json | 1 + frontend/src/localization/en.json | 1 + 7 files changed, 79 insertions(+), 17 deletions(-) create mode 100644 frontend/src/js/entity-history/VisibilityControl.tsx diff --git a/frontend/src/js/entity-history/EntityCard.tsx b/frontend/src/js/entity-history/EntityCard.tsx index 1654920b5e..f4633d1468 100644 --- a/frontend/src/js/entity-history/EntityCard.tsx +++ b/frontend/src/js/entity-history/EntityCard.tsx @@ -25,10 +25,12 @@ const Centered = styled("div")` `; export const EntityCard = ({ + blurred, className, infos, timeStratifiedInfos, }: { + blurred?: boolean; className?: string; infos: EntityInfo[]; timeStratifiedInfos: TimeStratifiedInfo[]; @@ -40,7 +42,7 @@ export const EntityCard = ({ return ( - + diff --git a/frontend/src/js/entity-history/EntityInfos.tsx b/frontend/src/js/entity-history/EntityInfos.tsx index 7c8786b704..d425d3032a 100644 --- a/frontend/src/js/entity-history/EntityInfos.tsx +++ b/frontend/src/js/entity-history/EntityInfos.tsx @@ -12,18 +12,25 @@ const Grid = styled("div")` const Label = styled("div")` font-size: ${({ theme }) => theme.font.xs}; `; -const Value = styled("div")` +const Value = styled("div")<{ blurred?: boolean }>` font-size: ${({ theme }) => theme.font.sm}; font-weight: 400; + ${({ blurred }) => blurred && "filter: blur(6px);"} `; -const EntityInfos = ({ infos }: { infos: EntityInfo[] }) => { +const EntityInfos = ({ + infos, + blurred, +}: { + infos: EntityInfo[]; + blurred?: boolean; +}) => { return ( {infos.map((info) => ( - {info.value} + {info.value} ))} diff --git a/frontend/src/js/entity-history/History.tsx b/frontend/src/js/entity-history/History.tsx index 91e999409b..e134ca0b21 100644 --- a/frontend/src/js/entity-history/History.tsx +++ b/frontend/src/js/entity-history/History.tsx @@ -25,6 +25,7 @@ import type { LoadingPayload } from "./LoadHistoryDropzone"; import { Navigation } from "./Navigation"; import SourcesControl from "./SourcesControl"; import Timeline from "./Timeline"; +import VisibilityControl from "./VisibilityControl"; import { useUpdateHistorySession } from "./actions"; import { EntityId } from "./reducer"; @@ -120,6 +121,10 @@ export const History = () => { (state) => state.entityHistory.resultUrls, ); + const [blurred, setBlurred] = useState(true); + const toggleBlurred = useCallback(() => setBlurred((v) => !v), []); + useHotkeys("p", toggleBlurred, [toggleBlurred]); + const [showAdvancedControls, setShowAdvancedControls] = useState(false); useHotkeys("shift+alt+h", () => { @@ -216,6 +221,10 @@ export const History = () => { + {showAdvancedControls && ( { ; - contentFilter: ContentFilterValue; - getIsOpen: (year: number, quarter?: number) => boolean; - toggleOpenYear: (year: number) => void; - toggleOpenQuarter: (year: number, quarter: number) => void; -} - const Timeline = ({ className, currentEntityInfos, @@ -70,7 +58,19 @@ const Timeline = ({ getIsOpen, toggleOpenYear, toggleOpenQuarter, -}: Props) => { + blurred, +}: { + className?: string; + currentEntityInfos: EntityInfo[]; + currentEntityTimeStratifiedInfos: TimeStratifiedInfo[]; + detailLevel: DetailLevel; + sources: Set; + contentFilter: ContentFilterValue; + getIsOpen: (year: number, quarter?: number) => boolean; + toggleOpenYear: (year: number) => void; + toggleOpenQuarter: (year: number, quarter: number) => void; + blurred?: boolean; +}) => { const data = useSelector( (state) => state.entityHistory.currentEntityData, ); @@ -93,6 +93,7 @@ const Timeline = ({ return ( diff --git a/frontend/src/js/entity-history/VisibilityControl.tsx b/frontend/src/js/entity-history/VisibilityControl.tsx new file mode 100644 index 0000000000..f1bfed7ce6 --- /dev/null +++ b/frontend/src/js/entity-history/VisibilityControl.tsx @@ -0,0 +1,40 @@ +import styled from "@emotion/styled"; +import { faEye, faEyeSlash } from "@fortawesome/free-regular-svg-icons"; +import { memo } from "react"; +import { useTranslation } from "react-i18next"; + +import IconButton from "../button/IconButton"; +import WithTooltip from "../tooltip/WithTooltip"; + +const Root = styled("div")` + display: flex; + flex-direction: column; + align-items: center; +`; + +const SxIconButton = styled(IconButton)` + padding: 8px 10px; +`; + +const VisibilityControl = ({ + blurred, + toggleBlurred, +}: { + blurred?: boolean; + toggleBlurred: () => void; +}) => { + const { t } = useTranslation(); + + return ( + + + + + + ); +}; + +export default memo(VisibilityControl); diff --git a/frontend/src/localization/de.json b/frontend/src/localization/de.json index 6aa4c25e4a..da050656ef 100644 --- a/frontend/src/localization/de.json +++ b/frontend/src/localization/de.json @@ -459,6 +459,7 @@ "queryNodeDetails": "Detail-Einstellungen bearbeiten" }, "history": { + "blurred": "Daten-Sichtbarkeit", "emptyTimeline": { "headline": "Historie", "description": "Hier werden Informationen chronologisch dargestellt.", diff --git a/frontend/src/localization/en.json b/frontend/src/localization/en.json index aa5462a49a..007499910f 100644 --- a/frontend/src/localization/en.json +++ b/frontend/src/localization/en.json @@ -459,6 +459,7 @@ "queryNodeDetails": "Detail-Einstellungen bearbeiten" }, "history": { + "blurred": "Data visibility", "emptyTimeline": { "headline": "History", "description": "Exploring individual events chronologically.", From 8b6edd70e9ad6ecb8a98f3c18088ae47916f7500 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Mon, 19 Jun 2023 16:33:14 +0200 Subject: [PATCH 391/679] fixes rasining an NPE when unknown form was submitted --- .../conquery/apiv1/forms/ExternalForm.java | 2 +- .../bakdata/conquery/apiv1/forms/Form.java | 10 +- .../frontendconfiguration/FormProcessor.java | 4 +- .../frontendconfiguration/FormScanner.java | 121 ++++++++++-------- 4 files changed, 81 insertions(+), 56 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java index 49ec7b6ca2..6416184df7 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/ExternalForm.java @@ -79,7 +79,7 @@ public String getLocalizedTypeLabel() { // Form had no specific title set. Try localized lookup in FormConfig final Locale preferredLocale = I18n.LOCALE.get(); - final FormType frontendConfig = FormScanner.FRONTEND_FORM_CONFIGS.get(getFormType()); + final FormType frontendConfig = FormScanner.resolveFormType(getFormType()); if (frontendConfig == null) { return getSubType(); diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java index 11bcd5f3e6..c9844ce5c4 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/forms/Form.java @@ -8,7 +8,9 @@ import com.bakdata.conquery.models.auth.entities.Subject; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.forms.frontendconfiguration.FormScanner; +import com.bakdata.conquery.models.forms.frontendconfiguration.FormType; import com.bakdata.conquery.models.forms.managed.ManagedForm; import com.bakdata.conquery.models.query.visitor.QueryVisitor; import com.fasterxml.jackson.annotation.JsonIgnore; @@ -42,7 +44,13 @@ public String getFormType() { public void authorize(Subject subject, Dataset submittedDataset, @NonNull ClassToInstanceMap visitors, MetaStorage storage) { QueryDescription.super.authorize(subject, submittedDataset, visitors, storage); // Check if subject is allowed to create this form - subject.authorize(FormScanner.FRONTEND_FORM_CONFIGS.get(getFormType()), Ability.CREATE); + final FormType formType = FormScanner.resolveFormType(getFormType()); + + if (formType == null) { + throw new ConqueryError.ExecutionCreationErrorUnspecified(); + } + + subject.authorize(formType, Ability.CREATE); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormProcessor.java b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormProcessor.java index 1a202425aa..df03966ba2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormProcessor.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.models.forms.frontendconfiguration; -import static com.bakdata.conquery.models.forms.frontendconfiguration.FormScanner.FRONTEND_FORM_CONFIGS; - import java.util.ArrayList; import java.util.Collection; import java.util.List; @@ -15,7 +13,7 @@ public class FormProcessor { public Collection getFormsForUser(Subject subject) { List allowedForms = new ArrayList<>(); - for (FormType formMapping : FRONTEND_FORM_CONFIGS.values()) { + for (FormType formMapping : FormScanner.getAllFormTypes()) { if (!subject.isPermitted(formMapping, Ability.CREATE)) { continue; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormScanner.java b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormScanner.java index a5ca29059f..ee88e86290 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormScanner.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormScanner.java @@ -9,6 +9,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Consumer; import com.bakdata.conquery.apiv1.forms.Form; @@ -36,15 +37,13 @@ public class FormScanner extends Task { public static final String MANUAL_URL_KEY = "manualUrl"; public static Map FRONTEND_FORM_CONFIGS = Collections.emptyMap(); - - private Consumer> providerChain = QueryUtils.getNoOpEntryPoint(); - /** * The config is used to look up the base url for manuals see {@link FrontendConfig#getManualUrl()}. * If the url was changed (e.g. using {@link AdminProcessor#executeScript(String)}) an execution of this * task accounts the change. */ private final ConqueryConfig config; + private Consumer> providerChain = QueryUtils.getNoOpEntryPoint(); public FormScanner(ConqueryConfig config) { super("form-scanner"); @@ -52,68 +51,48 @@ public FormScanner(ConqueryConfig config) { registerFrontendFormConfigProvider(ResourceFormConfigProvider::accept); } - private static Map> findBackendMappingClasses() { - Builder> backendClasses = ImmutableMap.builder(); - // Gather form implementations first - for (Class subclass : CPSTypeIdResolver.SCAN_RESULT.getSubclasses(Form.class.getName()).loadClasses()) { - if (Modifier.isAbstract(subclass.getModifiers())) { - continue; - } - CPSType[] cpsAnnotations = subclass.getAnnotationsByType(CPSType.class); - - if (cpsAnnotations.length == 0) { - log.warn("Implemented Form {} has no CPSType annotation", subclass); - continue; - } - for (CPSType cpsType : cpsAnnotations) { - backendClasses.put(cpsType.id(), (Class) subclass); - } - } - return backendClasses.build(); + public synchronized void registerFrontendFormConfigProvider(Consumer> provider) { + providerChain = providerChain.andThen(provider); } - public synchronized void registerFrontendFormConfigProvider(Consumer> provider){ - providerChain = providerChain.andThen(provider); + public static FormType resolveFormType(String formType) { + return FRONTEND_FORM_CONFIGS.get(formType); } - /** - * Frontend form configurations can be provided from different sources. - * Each source must register a provider with {@link FormScanner#registerFrontendFormConfigProvider(Consumer)} beforehand. - */ - @SneakyThrows - private List findFrontendFormConfigs() { + public static Set getAllFormTypes() { + return Set.copyOf(FRONTEND_FORM_CONFIGS.values()); + } - ImmutableList.Builder frontendConfigs = ImmutableList.builder(); - try { - providerChain.accept(frontendConfigs); - } catch (Exception e) { - log.error("Unable to collect all frontend form configurations.", e); - } - return frontendConfigs.build(); + @Override + public void execute(Map> parameters, PrintWriter output) throws Exception { + FRONTEND_FORM_CONFIGS = generateFEFormConfigMap(); } private Map generateFEFormConfigMap() { // Collect backend implementations for specific forms - Map> forms = findBackendMappingClasses(); + final Map> forms = findBackendMappingClasses(); // Collect frontend form configurations for the specific forms - List frontendConfigs = findFrontendFormConfigs(); + final List frontendConfigs = findFrontendFormConfigs(); // Match frontend form configurations to backend implementations final ImmutableMap.Builder result = ImmutableMap.builderWithExpectedSize(frontendConfigs.size()); for (FormFrontendConfigInformation configInfo : frontendConfigs) { - ObjectNode configTree = configInfo.getConfigTree(); - JsonNode type = configTree.get("type"); + + final ObjectNode configTree = configInfo.getConfigTree(); + final JsonNode type = configTree.get("type"); + if (!validTypeId(type)) { log.warn("Found invalid type id in {}. Was: {}", configInfo.getOrigin(), type); continue; } // Extract complete type information (type@subtype) and type information - String fullTypeIdentifier = type.asText(); - String typeIdentifier = CPSTypeIdResolver.truncateSubTypeInformation(fullTypeIdentifier); + final String fullTypeIdentifier = type.asText(); + final String typeIdentifier = CPSTypeIdResolver.truncateSubTypeInformation(fullTypeIdentifier); + if (!forms.containsKey(typeIdentifier)) { log.error("Frontend form config {} (type = {}) does not map to a backend class.", configInfo, type); continue; @@ -139,14 +118,19 @@ private Map generateFEFormConfigMap() { return URI.create(manualUrl.textValue()); }); + + final URL manualBaseUrl = config.getFrontend().getManualUrl(); + if (manualBaseUrl != null && manualURL != null) { final TextNode manualNode = relativizeManualUrl(fullTypeIdentifier, manualURL, manualBaseUrl); + if (manualNode == null) { log.warn("Manual url relativization did not succeed for {}. Skipping registration.", fullTypeIdentifier); continue; } + configTree.set(MANUAL_URL_KEY, manualNode); } @@ -158,6 +142,50 @@ private Map generateFEFormConfigMap() { return result.build(); } + private static Map> findBackendMappingClasses() { + final Builder> backendClasses = ImmutableMap.builder(); + // Gather form implementations first + for (Class subclass : CPSTypeIdResolver.SCAN_RESULT.getSubclasses(Form.class.getName()).loadClasses()) { + if (Modifier.isAbstract(subclass.getModifiers())) { + continue; + } + + final CPSType[] cpsAnnotations = subclass.getAnnotationsByType(CPSType.class); + + if (cpsAnnotations.length == 0) { + log.warn("Implemented Form {} has no CPSType annotation", subclass); + continue; + } + + for (CPSType cpsType : cpsAnnotations) { + backendClasses.put(cpsType.id(), (Class) subclass); + } + } + return backendClasses.build(); + } + + /** + * Frontend form configurations can be provided from different sources. + * Each source must register a provider with {@link FormScanner#registerFrontendFormConfigProvider(Consumer)} beforehand. + */ + @SneakyThrows + private List findFrontendFormConfigs() { + + final ImmutableList.Builder frontendConfigs = ImmutableList.builder(); + + try { + providerChain.accept(frontendConfigs); + } + catch (Exception e) { + log.error("Unable to collect all frontend form configurations.", e); + } + return frontendConfigs.build(); + } + + private static boolean validTypeId(JsonNode node) { + return node != null && node.isTextual() && !node.asText().isEmpty(); + } + private TextNode relativizeManualUrl(@NonNull String formTypeIdentifier, @NonNull URI manualUri, @NonNull URL manualBaseUrl) { try { @@ -182,13 +210,4 @@ private TextNode relativizeManualUrl(@NonNull String formTypeIdentifier, @NonNul } } - private static boolean validTypeId(JsonNode node) { - return node != null && node.isTextual() && !node.asText().isEmpty(); - } - - @Override - public void execute(Map> parameters, PrintWriter output) throws Exception { - FRONTEND_FORM_CONFIGS = generateFEFormConfigMap(); - } - } From ff69232adf6a80073b09988f9368a750347d20e0 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 20 Jun 2023 11:21:34 +0200 Subject: [PATCH 392/679] Support default time aggregation settings --- frontend/src/js/api/types.ts | 1 + .../src/js/concept-trees/ConceptTreeNode.tsx | 57 +++++-------------- frontend/src/js/model/node.ts | 11 +++- .../src/js/standard-query-editor/helper.ts | 2 +- 4 files changed, 26 insertions(+), 45 deletions(-) diff --git a/frontend/src/js/api/types.ts b/frontend/src/js/api/types.ts index ef0fc4d964..2820fcc75e 100644 --- a/frontend/src/js/api/types.ts +++ b/frontend/src/js/api/types.ts @@ -163,6 +163,7 @@ export interface ConceptBaseT { description?: string; // Empty array: key not defined additionalInfos?: InfoT[]; // Empty array: key not defined dateRange?: DateRangeT; + excludeFromTimeAggregation?: boolean; // To default-exclude some concepts from time aggregation } export type ConceptStructT = ConceptBaseT; diff --git a/frontend/src/js/concept-trees/ConceptTreeNode.tsx b/frontend/src/js/concept-trees/ConceptTreeNode.tsx index ffe51d382a..5095f42d27 100644 --- a/frontend/src/js/concept-trees/ConceptTreeNode.tsx +++ b/frontend/src/js/concept-trees/ConceptTreeNode.tsx @@ -1,13 +1,6 @@ import styled from "@emotion/styled"; -import { FC } from "react"; - -import type { - ConceptIdT, - InfoT, - DateRangeT, - ConceptT, - ConceptElementT, -} from "../api/types"; + +import type { ConceptIdT, ConceptT, ConceptElementT } from "../api/types"; import { useOpenableConcept } from "../concept-trees-open/useOpenableConcept"; import { resetSelects } from "../model/select"; import { resetTables } from "../model/table"; @@ -22,44 +15,18 @@ const Root = styled("div")` font-size: ${({ theme }) => theme.font.sm}; `; -// Concept data that is necessary to display tree nodes. Includes additional infos -// for the tooltip as well as the id of the corresponding tree -interface TreeNodeData { - label: string; - description?: string; - active?: boolean; - matchingEntries: number | null; - matchingEntities: number | null; - dateRange?: DateRangeT; - additionalInfos?: InfoT[]; - children?: ConceptIdT[]; -} - -interface PropsT { - rootConceptId: ConceptIdT; - conceptId: ConceptIdT; - data: TreeNodeData; - depth: number; - search: SearchT; -} - -const selectTreeNodeData = (concept: ConceptT) => ({ - label: concept.label, - description: concept.description, - active: concept.active, - matchingEntries: concept.matchingEntries, - matchingEntities: concept.matchingEntities, - dateRange: concept.dateRange, - additionalInfos: concept.additionalInfos, - children: concept.children, -}); - -const ConceptTreeNode: FC = ({ +const ConceptTreeNode = ({ data, rootConceptId, conceptId, depth, search, +}: { + rootConceptId: ConceptIdT; + conceptId: ConceptIdT; + data: ConceptT; + depth: number; + search: SearchT; }) => { const { open, onToggleOpen } = useOpenableConcept({ conceptId, @@ -121,6 +88,10 @@ const ConceptTreeNode: FC = ({ matchingEntities: data.matchingEntities, dateRange: data.dateRange, + excludeTimestamps: + root.excludeFromTimeAggregation || + data.excludeFromTimeAggregation, + tree: rootConceptId, }; }} @@ -140,7 +111,7 @@ const ConceptTreeNode: FC = ({ key={childId} rootConceptId={rootConceptId} conceptId={childId} - data={selectTreeNodeData(child)} + data={child} depth={depth + 1} search={search} /> diff --git a/frontend/src/js/model/node.ts b/frontend/src/js/model/node.ts index 4cf533d58e..2f3d01a8fb 100644 --- a/frontend/src/js/model/node.ts +++ b/frontend/src/js/model/node.ts @@ -11,6 +11,7 @@ import { useTranslation } from "react-i18next"; import { ConceptElementT, ConceptT } from "../api/types"; import { DNDType } from "../common/constants/dndTypes"; +import { getConceptById } from "../concept-trees/globalTreeStoreHelper"; import type { ConceptQueryNodeType, DragItemConceptTreeNode, @@ -55,8 +56,16 @@ export const nodeHasEmptySettings = (node: StandardQueryNodeT) => { export const nodeHasFilterValues = (node: StandardQueryNodeT) => nodeIsConceptQueryNode(node) && tablesHaveFilterValues(node.tables); +const nodeHasNonDefaultExcludeTimestamps = (node: StandardQueryNodeT) => { + if (!nodeIsConceptQueryNode(node)) return node.excludeTimestamps; + + const root = getConceptById(node.tree, node.tree); + + return node.excludeTimestamps !== root?.excludeFromTimeAggregation; +}; + export const nodeHasNonDefaultSettings = (node: StandardQueryNodeT) => - node.excludeTimestamps || + nodeHasNonDefaultExcludeTimestamps(node) || node.excludeFromSecondaryId || (nodeIsConceptQueryNode(node) && (objectHasNonDefaultSelects(node) || diff --git a/frontend/src/js/standard-query-editor/helper.ts b/frontend/src/js/standard-query-editor/helper.ts index 06efce90db..bb2c20a483 100644 --- a/frontend/src/js/standard-query-editor/helper.ts +++ b/frontend/src/js/standard-query-editor/helper.ts @@ -7,7 +7,7 @@ export function getRootNodeLabel(node: StandardQueryNodeT) { if (!nodeIsConceptQueryNode(node) || !node.ids || !node.tree) return null; const nodeIsRootNode = node.ids.includes(node.tree); - const root = getConceptById(node.tree); + const root = getConceptById(node.tree, node.tree); if (nodeIsRootNode) { const noRootOrSameLabel = !root || root.label === node.label; From 63c8f5433bbb4f344f6947053ae92885be97b7cd Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 20 Jun 2023 11:37:35 +0200 Subject: [PATCH 393/679] adds nullable annotation --- .../models/forms/frontendconfiguration/FormScanner.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormScanner.java b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormScanner.java index ee88e86290..c6f2ffc4e0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormScanner.java +++ b/backend/src/main/java/com/bakdata/conquery/models/forms/frontendconfiguration/FormScanner.java @@ -12,6 +12,8 @@ import java.util.Set; import java.util.function.Consumer; +import javax.annotation.Nullable; + import com.bakdata.conquery.apiv1.forms.Form; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.cps.CPSTypeIdResolver; @@ -55,6 +57,8 @@ public synchronized void registerFrontendFormConfigProvider(Consumer Date: Tue, 20 Jun 2023 13:33:04 +0200 Subject: [PATCH 394/679] improve dragging - expand drop zone --- .../DropzoneBetweenElements.tsx | 40 ++++++++++++------- .../form-components/DropzoneList.tsx | 1 - 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index eb8a7a1c0e..af7d5798b1 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -1,4 +1,5 @@ import styled from "@emotion/styled"; +import { useState } from "react"; import { DropTargetMonitor, useDrop } from "react-dnd"; import { PossibleDroppableObject } from "../../ui-components/Dropzone"; @@ -6,51 +7,60 @@ import { PossibleDroppableObject } from "../../ui-components/Dropzone"; interface Props { onDrop: (item: DroppableObject, monitor: DropTargetMonitor) => void; acceptedDropTypes: string[]; - isFirstElement: boolean; } const Root = styled("div")<{ - isOver: boolean; - isDroppable: boolean; - isFirstElement: boolean; + height: number; }>` width: 100%; left: 0; - top: -17px; - height: 40px; + top: -15px; + height: ${({ height }) => height + 40}px; right: 0; position: relative; border-radius: ${({ theme }) => theme.borderRadius}; `; -const DropzoneContainer = styled("div")` +const DropzoneContainer = styled("div")<{ + height: number; +}>` overflow: hidden; - height: 20px; + margin-top: ${({ height }) => -height}px; + display: block; + height: ${({ height }) => height}px; `; const BetweenElements = ({ acceptedDropTypes, onDrop, - isFirstElement, }: Props) => { - const [{ isOver, isDroppable }, addZoneRef] = useDrop({ + const [height, setHeight] = useState(40); + + const [{ isOver }, addZoneRef] = useDrop({ accept: acceptedDropTypes, drop: onDrop, + hover(item) { + if (item.type === "CONCEPT_TREE_NODE") { + return setHeight(item.dragContext.height); + } + return setHeight(0); + }, + collect: (monitor) => ({ isOver: monitor.isOver(), isDroppable: monitor.canDrop(), }), }); + return ( <> - - {isOver && } + {isOver && ( + + )} ); }; diff --git a/frontend/src/js/external-forms/form-components/DropzoneList.tsx b/frontend/src/js/external-forms/form-components/DropzoneList.tsx index eb5577dc06..cd1c91e2af 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneList.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneList.tsx @@ -102,7 +102,6 @@ const DropzoneList = ( )} From 2c9e60c0185b4c77237682b4bf768a60deb1bdbe Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Tue, 20 Jun 2023 13:35:20 +0200 Subject: [PATCH 395/679] format --- .../form-components/DropzoneBetweenElements.tsx | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index af7d5798b1..8a6ff2945c 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -54,13 +54,8 @@ const BetweenElements = ({ return ( <> - - {isOver && ( - - )} + + {isOver && } ); }; From d780c4b4b2b947394ced7bad76b4f6230d08d71b Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 20 Jun 2023 16:19:33 +0200 Subject: [PATCH 396/679] don't merge CONCEPT_COLUMN into SECONDARY_ID --- .../conquery/apiv1/query/TableExportQuery.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java index 63a798787c..1b9d71b4f1 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java @@ -2,6 +2,7 @@ import java.time.LocalDate; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; @@ -172,6 +173,15 @@ private Map calculateSecondaryIdPositions(Atomi private static Map calculateColumnPositions(AtomicInteger currentPosition, List tables, Map secondaryIdPositions) { final Map positions = new HashMap<>(); + // We need to know if a column is a concept column so we can prioritize it if it is also a SecondaryId + final Set conceptColumns = tables.stream() + .map(CQConcept::getTables) + .flatMap(Collection::stream) + .map(CQTable::getConnector) + .map(Connector::getColumn) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + for (CQConcept concept : tables) { for (CQTable table : concept.getTables()) { @@ -188,7 +198,8 @@ private static Map calculateColumnPositions(AtomicInteger curre continue; } - if (column.getSecondaryId() != null) { + // We want to have ConceptColumns separate here. + if (column.getSecondaryId() != null && !conceptColumns.contains(column)) { positions.putIfAbsent(column, secondaryIdPositions.get(column.getSecondaryId())); continue; } From d368ef018aefa3f438ed2e9e11b7f4ad866018d3 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 20 Jun 2023 16:55:42 +0200 Subject: [PATCH 397/679] Iterate blurification --- frontend/.env.e2e | 2 ++ frontend/.env.example | 5 +++- frontend/Dockerfile | 1 + frontend/scripts/replace-env-at-runtime.sh | 1 + .../src/js/entity-history/EntityHeader.tsx | 25 ++++++++++--------- .../src/js/entity-history/EntityIdsList.tsx | 21 ++++++++++------ frontend/src/js/entity-history/History.tsx | 5 +++- frontend/src/js/entity-history/Navigation.tsx | 3 +++ frontend/src/js/environment/index.ts | 4 +++ 9 files changed, 45 insertions(+), 22 deletions(-) diff --git a/frontend/.env.e2e b/frontend/.env.e2e index 1e786c501c..f85d08b732 100644 --- a/frontend/.env.e2e +++ b/frontend/.env.e2e @@ -18,3 +18,5 @@ REACT_APP_IDP_ENABLE=false REACT_APP_IDP_URL=http://localhost:8080/auth REACT_APP_IDP_REALM=Myrealm REACT_APP_IDP_CLIENT_ID=frontend + +REACT_APP_BLURRED_ENTITY_DEFAULT=true \ No newline at end of file diff --git a/frontend/.env.example b/frontend/.env.example index 40734bdcc7..fd6878cafc 100644 --- a/frontend/.env.example +++ b/frontend/.env.example @@ -22,4 +22,7 @@ REACT_APP_IDP_URL=http://localhost:8080/auth # NOTE: You will need to create this realm in keycloak REACT_APP_IDP_REALM=Myrealm # NOTE: You will need to create this client in keycloak -REACT_APP_IDP_CLIENT_ID=frontend \ No newline at end of file +REACT_APP_IDP_CLIENT_ID=frontend + +# Application features +REACT_APP_BLURRED_ENTITY_DEFAULT=true \ No newline at end of file diff --git a/frontend/Dockerfile b/frontend/Dockerfile index d1d6e8ceb8..3245718e0b 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -24,6 +24,7 @@ ENV REACT_APP_IDP_ENABLE=$REACT_APP_IDP_ENABLE ENV REACT_APP_IDP_URL=$REACT_APP_IDP_URL ENV REACT_APP_IDP_REALM=$REACT_APP_IDP_REALM ENV REACT_APP_IDP_CLIENT_ID=$REACT_APP_IDP_CLIENT_ID +ENV REACT_APP_BLURRED_ENTITY_DEFAULT=$REACT_APP_BLURRED_ENTITY_DEFAULT # Copy the build artifacts from the builder phase COPY --from=builder /app/dist /usr/share/nginx/html diff --git a/frontend/scripts/replace-env-at-runtime.sh b/frontend/scripts/replace-env-at-runtime.sh index 5b359112f0..df87e7f85b 100755 --- a/frontend/scripts/replace-env-at-runtime.sh +++ b/frontend/scripts/replace-env-at-runtime.sh @@ -25,6 +25,7 @@ ENVSTRING="${ENVSTRING}REACT_APP_IDP_ENABLE: \"${REACT_APP_IDP_ENABLE:-null}\"," ENVSTRING="${ENVSTRING}REACT_APP_IDP_URL: \"${REACT_APP_IDP_URL:-null}\"," ENVSTRING="${ENVSTRING}REACT_APP_IDP_REALM: \"${REACT_APP_IDP_REALM:-null}\"," ENVSTRING="${ENVSTRING}REACT_APP_IDP_CLIENT_ID: \"${REACT_APP_IDP_CLIENT_ID:-null}\"," +ENVSTRING="${ENVSTRING}REACT_APP_BLURRED_ENTITY_DEFAULT: \"${REACT_APP_BLURRED_ENTITY_DEFAULT:-null}\"," # Replace the marker diff --git a/frontend/src/js/entity-history/EntityHeader.tsx b/frontend/src/js/entity-history/EntityHeader.tsx index d3caa665cf..2012de8306 100644 --- a/frontend/src/js/entity-history/EntityHeader.tsx +++ b/frontend/src/js/entity-history/EntityHeader.tsx @@ -29,9 +29,10 @@ const Buttons = styled("div")` gap: 5px; `; -const SxHeading3 = styled(Heading3)` +const SxHeading3 = styled(Heading3)<{ blurred?: boolean }>` flex-shrink: 0; margin: 0; + ${({ blurred }) => blurred && "filter: blur(6px);"} `; const Subtitle = styled("div")` font-size: ${({ theme }) => theme.font.xs}; @@ -47,23 +48,23 @@ const Avatar = styled(SxHeading3)` font-weight: 300; `; -interface Props { - className?: string; - currentEntityIndex: number; - currentEntityId: EntityId; - status: SelectOptionT[]; - setStatus: (value: SelectOptionT[]) => void; - entityStatusOptions: SelectOptionT[]; -} - export const EntityHeader = ({ + blurred, className, currentEntityIndex, currentEntityId, status, setStatus, entityStatusOptions, -}: Props) => { +}: { + blurred?: boolean; + className?: string; + currentEntityIndex: number; + currentEntityId: EntityId; + status: SelectOptionT[]; + setStatus: (value: SelectOptionT[]) => void; + entityStatusOptions: SelectOptionT[]; +}) => { const totalEvents = useSelector( (state) => state.entityHistory.currentEntityData.length, ); @@ -87,7 +88,7 @@ export const EntityHeader = ({
#{currentEntityIndex + 1} - {currentEntityId.id} + {currentEntityId.id} {totalEvents} {t("history.events", { count: totalEvents })} diff --git a/frontend/src/js/entity-history/EntityIdsList.tsx b/frontend/src/js/entity-history/EntityIdsList.tsx index 1d7da5ae22..316db42c52 100644 --- a/frontend/src/js/entity-history/EntityIdsList.tsx +++ b/frontend/src/js/entity-history/EntityIdsList.tsx @@ -54,19 +54,23 @@ const Gray = styled("span")` color: ${({ theme }) => theme.col.gray}; `; -interface Props { - currentEntityId: EntityId | null; - entityIds: EntityId[]; - updateHistorySession: ReturnType; - entityIdsStatus: EntityIdsStatus; -} +const Blurred = styled("span")<{ blurred?: boolean }>` + ${({ blurred }) => blurred && "filter: blur(6px);"} +`; export const EntityIdsList = ({ + blurred, currentEntityId, entityIds, entityIdsStatus, updateHistorySession, -}: Props) => { +}: { + blurred?: boolean; + currentEntityId: EntityId | null; + entityIds: EntityId[]; + updateHistorySession: ReturnType; + entityIdsStatus: EntityIdsStatus; +}) => { const numberWidth = useMemo(() => { const magnitude = Math.ceil(Math.log(entityIds.length) / Math.log(10)); @@ -85,7 +89,8 @@ export const EntityIdsList = ({ > #{index + 1} - {entityId.id} ({entityId.kind}) + {entityId.id}{" "} + ({entityId.kind}) {entityIdsStatus[entityId.id] && diff --git a/frontend/src/js/entity-history/History.tsx b/frontend/src/js/entity-history/History.tsx index e134ca0b21..a083589bc0 100644 --- a/frontend/src/js/entity-history/History.tsx +++ b/frontend/src/js/entity-history/History.tsx @@ -14,6 +14,7 @@ import type { TimeStratifiedInfo, } from "../api/types"; import type { StateT } from "../app/reducers"; +import { isEntityBlurredByDefault } from "../environment"; import ErrorFallback from "../error-fallback/ErrorFallback"; import DownloadResultsDropdownButton from "../query-runner/DownloadResultsDropdownButton"; @@ -121,7 +122,7 @@ export const History = () => { (state) => state.entityHistory.resultUrls, ); - const [blurred, setBlurred] = useState(true); + const [blurred, setBlurred] = useState(isEntityBlurredByDefault); const toggleBlurred = useCallback(() => setBlurred((v) => !v), []); useHotkeys("p", toggleBlurred, [toggleBlurred]); @@ -190,6 +191,7 @@ export const History = () => { defaultSize="400px" > { {currentEntityId && ( )} Date: Tue, 20 Jun 2023 17:36:29 +0200 Subject: [PATCH 398/679] Increase truncation len, round money to 0 decimals --- .../js/entity-history/TimeStratifiedChart.tsx | 45 +++++++------------ 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/frontend/src/js/entity-history/TimeStratifiedChart.tsx b/frontend/src/js/entity-history/TimeStratifiedChart.tsx index 2bd08588f2..4eeb6c951f 100644 --- a/frontend/src/js/entity-history/TimeStratifiedChart.tsx +++ b/frontend/src/js/entity-history/TimeStratifiedChart.tsx @@ -8,14 +8,14 @@ import { Tooltip, ChartOptions, } from "chart.js"; -import { useCallback, useMemo } from "react"; +import { useMemo } from "react"; import { Bar } from "react-chartjs-2"; -import { useSelector } from "react-redux"; -import { CurrencyConfigT, TimeStratifiedInfo } from "../api/types"; -import { StateT } from "../app/reducers"; +import { TimeStratifiedInfo } from "../api/types"; import { exists } from "../common/helpers/exists"; +const TRUNCATE_X_AXIS_LABELS_LEN = 18; + ChartJS.register(CategoryScale, LinearScale, BarElement, Tooltip); const ChartContainer = styled("div")` @@ -42,27 +42,13 @@ function interpolateDecreasingOpacity(index: number) { return Math.min(1, 1 / (index + 0.3)); } -const useFormatCurrency = () => { - const currencyConfig = useSelector( - (state) => state.startup.config.currency, - ); - - const formatCurrency = useCallback( - (value: number) => { - return value.toLocaleString("de-DE", { - style: "currency", - currency: "EUR", - minimumFractionDigits: currencyConfig.decimalScale, - maximumFractionDigits: currencyConfig.decimalScale, - }); - }, - [currencyConfig], - ); - - return { - formatCurrency, - }; -}; +const formatCurrency = (value: number) => + value.toLocaleString("de-DE", { + style: "currency", + currency: "EUR", + minimumFractionDigits: 0, + maximumFractionDigits: 0, + }); export const TimeStratifiedChart = ({ timeStratifiedInfo, @@ -93,8 +79,6 @@ export const TimeStratifiedChart = ({ datasets, }; - const { formatCurrency } = useFormatCurrency(); - const options: ChartOptions<"bar"> = useMemo(() => { return { plugins: { @@ -144,8 +128,9 @@ export const TimeStratifiedChart = ({ x: { ticks: { callback: (idx: any) => { - return labels[idx].length > 12 - ? labels[idx].substring(0, 9) + "..." + return labels[idx].length > TRUNCATE_X_AXIS_LABELS_LEN + ? labels[idx].substring(0, TRUNCATE_X_AXIS_LABELS_LEN - 3) + + "..." : labels[idx]; }, }, @@ -158,7 +143,7 @@ export const TimeStratifiedChart = ({ }, }, }; - }, [timeStratifiedInfo, labels, formatCurrency]); + }, [timeStratifiedInfo, labels]); return ( From ac0d699130e60d8943cb1a02dd0a691320a10ed7 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 21 Jun 2023 16:07:05 +0200 Subject: [PATCH 399/679] fixes not applying same logic to result infos --- .../apiv1/query/TableExportQuery.java | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java index 1b9d71b4f1..993cee86fc 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java @@ -148,9 +148,18 @@ public void resolve(QueryResolveContext context) { final Map secondaryIdPositions = calculateSecondaryIdPositions(currentPosition); - positions = calculateColumnPositions(currentPosition, tables, secondaryIdPositions); + // We need to know if a column is a concept column so we can prioritize it if it is also a SecondaryId + final Set conceptColumns = tables.stream() + .map(CQConcept::getTables) + .flatMap(Collection::stream) + .map(CQTable::getConnector) + .map(Connector::getColumn) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + + positions = calculateColumnPositions(currentPosition, tables, secondaryIdPositions, conceptColumns); - resultInfos = createResultInfos(secondaryIdPositions); + resultInfos = createResultInfos(secondaryIdPositions, conceptColumns); } private Map calculateSecondaryIdPositions(AtomicInteger currentPosition) { @@ -170,17 +179,9 @@ private Map calculateSecondaryIdPositions(Atomi return secondaryIdPositions; } - private static Map calculateColumnPositions(AtomicInteger currentPosition, List tables, Map secondaryIdPositions) { + private static Map calculateColumnPositions(AtomicInteger currentPosition, List tables, Map secondaryIdPositions, Set conceptColumns) { final Map positions = new HashMap<>(); - // We need to know if a column is a concept column so we can prioritize it if it is also a SecondaryId - final Set conceptColumns = tables.stream() - .map(CQConcept::getTables) - .flatMap(Collection::stream) - .map(CQTable::getConnector) - .map(Connector::getColumn) - .filter(Objects::nonNull) - .collect(Collectors.toSet()); for (CQConcept concept : tables) { for (CQTable table : concept.getTables()) { @@ -212,7 +213,7 @@ private static Map calculateColumnPositions(AtomicInteger curre return positions; } - private List createResultInfos(Map secondaryIdPositions) { + private List createResultInfos(Map secondaryIdPositions, Set conceptColumns) { final int size = positions.values().stream().mapToInt(i -> i).max().getAsInt() + 1; @@ -263,7 +264,7 @@ private List createResultInfos(Map } // SecondaryIds and date columns are pulled to the front, thus already covered. - if (column.getSecondaryId() != null) { + if (column.getSecondaryId() != null && !conceptColumns.contains(column)) { infos[secondaryIdPositions.get(column.getSecondaryId())].getSemantics() .add(new SemanticType.ColumnT(column)); continue; From 272927c80a28ab4b2726b4647a56f1e457cf91f0 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 22 Jun 2023 10:00:39 +0200 Subject: [PATCH 400/679] log NoSuchElementException when caught in REST context --- .../bakdata/conquery/io/jetty/NoSuchElementExceptionMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/jetty/NoSuchElementExceptionMapper.java b/backend/src/main/java/com/bakdata/conquery/io/jetty/NoSuchElementExceptionMapper.java index 531786b0c5..f751f9f31f 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/jetty/NoSuchElementExceptionMapper.java +++ b/backend/src/main/java/com/bakdata/conquery/io/jetty/NoSuchElementExceptionMapper.java @@ -12,7 +12,7 @@ public class NoSuchElementExceptionMapper implements ExceptionMapper { @Override public Response toResponse(NoSuchElementException exception) { - log.trace("Mapping exception:", exception); + log.warn("Uncaught NoSuchElementException", exception); return Response.status(Response.Status.NOT_FOUND).type(MediaType.APPLICATION_JSON_TYPE).entity(exception.getMessage()).build(); } } From bbde558b1265b5f533b89db616c1da61474f22eb Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Fri, 23 Jun 2023 11:43:14 +0200 Subject: [PATCH 401/679] introduce showColoredIcon and apply it to results dropdown --- frontend/src/js/button/DownloadButton.tsx | 5 +++-- .../src/js/query-runner/DownloadResultsDropdownButton.tsx | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/frontend/src/js/button/DownloadButton.tsx b/frontend/src/js/button/DownloadButton.tsx index 2868f260bf..5c382da729 100644 --- a/frontend/src/js/button/DownloadButton.tsx +++ b/frontend/src/js/button/DownloadButton.tsx @@ -60,11 +60,12 @@ interface Props extends Omit { children?: ReactNode; simpleIcon?: boolean; onClick?: () => void; + showColoredIcon?: boolean; } const DownloadButton = forwardRef( ( - { simpleIcon, resultUrl, className, children, onClick, ...restProps }, + { simpleIcon, resultUrl, className, children, onClick, showColoredIcon, ...restProps }, ref, ) => { const { authToken } = useContext(AuthTokenContext); @@ -82,7 +83,7 @@ const DownloadButton = forwardRef( large icon={simpleIcon ? faDownload : icon} onClick={onClick} - iconColor={color} + iconColor={showColoredIcon ? color : undefined} > {children} diff --git a/frontend/src/js/query-runner/DownloadResultsDropdownButton.tsx b/frontend/src/js/query-runner/DownloadResultsDropdownButton.tsx index a59e91f7ed..ca5fa21e89 100644 --- a/frontend/src/js/query-runner/DownloadResultsDropdownButton.tsx +++ b/frontend/src/js/query-runner/DownloadResultsDropdownButton.tsx @@ -124,6 +124,7 @@ const DownloadResultsDropdownButton = ({ resultUrl={resultUrl} onClick={() => setFileChoice({ label: resultUrl.label, ending })} bgHover + showColoredIcon > {truncate(resultUrl.label)} @@ -137,7 +138,7 @@ const DownloadResultsDropdownButton = ({ {!tiny && ( <> - + {truncChosenLabel} From c6585d839ea3c195c4c7af54ece6c3187f5d5ec3 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Fri, 23 Jun 2023 11:44:03 +0200 Subject: [PATCH 402/679] format --- frontend/src/js/button/DownloadButton.tsx | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/frontend/src/js/button/DownloadButton.tsx b/frontend/src/js/button/DownloadButton.tsx index 5c382da729..fb1a2d6872 100644 --- a/frontend/src/js/button/DownloadButton.tsx +++ b/frontend/src/js/button/DownloadButton.tsx @@ -65,7 +65,15 @@ interface Props extends Omit { const DownloadButton = forwardRef( ( - { simpleIcon, resultUrl, className, children, onClick, showColoredIcon, ...restProps }, + { + simpleIcon, + resultUrl, + className, + children, + onClick, + showColoredIcon, + ...restProps + }, ref, ) => { const { authToken } = useContext(AuthTokenContext); @@ -83,7 +91,7 @@ const DownloadButton = forwardRef( large icon={simpleIcon ? faDownload : icon} onClick={onClick} - iconColor={showColoredIcon ? color : undefined} + iconColor={showColoredIcon ? color : undefined} > {children} From 9526f4851f572055257bddffcefa52b10741070d Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 27 Jun 2023 10:13:39 +0200 Subject: [PATCH 403/679] Remove env variable again --- frontend/.env.e2e | 4 +--- frontend/.env.example | 3 --- frontend/Dockerfile | 1 - frontend/scripts/replace-env-at-runtime.sh | 1 - frontend/src/js/entity-history/History.tsx | 3 +-- frontend/src/js/environment/index.ts | 4 ---- 6 files changed, 2 insertions(+), 14 deletions(-) diff --git a/frontend/.env.e2e b/frontend/.env.e2e index f85d08b732..1e5b246a4f 100644 --- a/frontend/.env.e2e +++ b/frontend/.env.e2e @@ -17,6 +17,4 @@ REACT_APP_LANG=de REACT_APP_IDP_ENABLE=false REACT_APP_IDP_URL=http://localhost:8080/auth REACT_APP_IDP_REALM=Myrealm -REACT_APP_IDP_CLIENT_ID=frontend - -REACT_APP_BLURRED_ENTITY_DEFAULT=true \ No newline at end of file +REACT_APP_IDP_CLIENT_ID=frontend \ No newline at end of file diff --git a/frontend/.env.example b/frontend/.env.example index fd6878cafc..f3763b5599 100644 --- a/frontend/.env.example +++ b/frontend/.env.example @@ -23,6 +23,3 @@ REACT_APP_IDP_URL=http://localhost:8080/auth REACT_APP_IDP_REALM=Myrealm # NOTE: You will need to create this client in keycloak REACT_APP_IDP_CLIENT_ID=frontend - -# Application features -REACT_APP_BLURRED_ENTITY_DEFAULT=true \ No newline at end of file diff --git a/frontend/Dockerfile b/frontend/Dockerfile index 3245718e0b..d1d6e8ceb8 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -24,7 +24,6 @@ ENV REACT_APP_IDP_ENABLE=$REACT_APP_IDP_ENABLE ENV REACT_APP_IDP_URL=$REACT_APP_IDP_URL ENV REACT_APP_IDP_REALM=$REACT_APP_IDP_REALM ENV REACT_APP_IDP_CLIENT_ID=$REACT_APP_IDP_CLIENT_ID -ENV REACT_APP_BLURRED_ENTITY_DEFAULT=$REACT_APP_BLURRED_ENTITY_DEFAULT # Copy the build artifacts from the builder phase COPY --from=builder /app/dist /usr/share/nginx/html diff --git a/frontend/scripts/replace-env-at-runtime.sh b/frontend/scripts/replace-env-at-runtime.sh index df87e7f85b..5b359112f0 100755 --- a/frontend/scripts/replace-env-at-runtime.sh +++ b/frontend/scripts/replace-env-at-runtime.sh @@ -25,7 +25,6 @@ ENVSTRING="${ENVSTRING}REACT_APP_IDP_ENABLE: \"${REACT_APP_IDP_ENABLE:-null}\"," ENVSTRING="${ENVSTRING}REACT_APP_IDP_URL: \"${REACT_APP_IDP_URL:-null}\"," ENVSTRING="${ENVSTRING}REACT_APP_IDP_REALM: \"${REACT_APP_IDP_REALM:-null}\"," ENVSTRING="${ENVSTRING}REACT_APP_IDP_CLIENT_ID: \"${REACT_APP_IDP_CLIENT_ID:-null}\"," -ENVSTRING="${ENVSTRING}REACT_APP_BLURRED_ENTITY_DEFAULT: \"${REACT_APP_BLURRED_ENTITY_DEFAULT:-null}\"," # Replace the marker diff --git a/frontend/src/js/entity-history/History.tsx b/frontend/src/js/entity-history/History.tsx index a083589bc0..94da6be0ec 100644 --- a/frontend/src/js/entity-history/History.tsx +++ b/frontend/src/js/entity-history/History.tsx @@ -14,7 +14,6 @@ import type { TimeStratifiedInfo, } from "../api/types"; import type { StateT } from "../app/reducers"; -import { isEntityBlurredByDefault } from "../environment"; import ErrorFallback from "../error-fallback/ErrorFallback"; import DownloadResultsDropdownButton from "../query-runner/DownloadResultsDropdownButton"; @@ -122,7 +121,7 @@ export const History = () => { (state) => state.entityHistory.resultUrls, ); - const [blurred, setBlurred] = useState(isEntityBlurredByDefault); + const [blurred, setBlurred] = useState(false); const toggleBlurred = useCallback(() => setBlurred((v) => !v), []); useHotkeys("p", toggleBlurred, [toggleBlurred]); diff --git a/frontend/src/js/environment/index.ts b/frontend/src/js/environment/index.ts index 63df1b96d2..9328fbbe10 100644 --- a/frontend/src/js/environment/index.ts +++ b/frontend/src/js/environment/index.ts @@ -28,9 +28,6 @@ const idpRealmEnv = const idpClientIdEnv = runtimeVar("REACT_APP_IDP_CLIENT_ID") || import.meta.env.REACT_APP_IDP_CLIENT_ID; -const entityBlurredByDefaultEnv = - runtimeVar("REACT_APP_BLURRED_ENTITY_DEFAULT") || - import.meta.env.REACT_APP_BLURRED_ENTITY_DEFAULT; export const isProduction = isProductionEnv === "production" || true; export const language = languageEnv === "de" ? "de" : "en"; @@ -41,7 +38,6 @@ export const basename = basenameEnv || ""; export const idpUrl = idpUrlEnv || ""; export const idpRealm = idpRealmEnv || ""; export const idpClientId = idpClientIdEnv || ""; -export const isEntityBlurredByDefault = entityBlurredByDefaultEnv === "true"; export interface CustomEnvironment { getExternalSupportedErrorMessage?: ( From a5113d16c3580e4223eb54835261d1a389fc8378 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Tue, 27 Jun 2023 10:42:27 +0200 Subject: [PATCH 404/679] Dropping works, removing when dragging to a new position works --- .../form-concept-group/FormConceptGroup.tsx | 47 ++++++++++++------- .../form-concept-group/FormConceptNode.tsx | 3 ++ .../src/js/standard-query-editor/types.ts | 1 + 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 8c65aec38e..90aec6f41a 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -59,6 +59,7 @@ import { } from "./formConceptGroupState"; import { useCopyModal } from "./useCopyModal"; import { useUploadConceptListModal } from "./useUploadConceptListModal"; +import { useFormContext } from "react-hook-form"; interface Props { formType: string; @@ -178,6 +179,8 @@ const FormConceptGroup = (props: Props) => { : null; }, [editedFormQueryNodePosition, props.value]); + const { getValues } = useFormContext(); + return (
*/ @@ -208,20 +211,15 @@ const FormConceptGroup = (props: Props) => { return null; if (isMovedObject(item)) { - let removed = - props.value[item.dragContext.movedFromAndIdx].concepts - .length === 1 - ? removeValue(props.value, item.dragContext.movedFromAndIdx) - : removeConcept( - props.value, - item.dragContext.movedFromAndIdx, - item.dragContext.movedFromOrIdx, - ); + if (exists(item.dragContext.deleteInOrigin)){ + item.dragContext.deleteInOrigin(); + } + let insertIndex = i > item.dragContext.movedFromAndIdx ? i - 1 : i; return props.onChange( addConcept( - insertValue(removed, insertIndex, newValue), + insertValue(getValues(props.fieldName), insertIndex, newValue), insertIndex, copyConcept(item), ), @@ -257,10 +255,14 @@ const FormConceptGroup = (props: Props) => { if (props.isValidConcept && !props.isValidConcept(item)) return; if (isMovedObject(item)) { + if (exists(item.dragContext.deleteInOrigin)){ + item.dragContext.deleteInOrigin(); + } + const updatedValue = getValues(props.fieldName); return props.onChange( addConcept( - addValue(props.value, newValue), - props.value.length, + addValue(updatedValue, newValue), + updatedValue.length, copyConcept(item), ), ); @@ -338,6 +340,13 @@ const FormConceptGroup = (props: Props) => { conceptIdx: j, }) } + deleteInForm={() => { + return props.onChange( + props.value[i].concepts.length === 1 + ? removeValue(props.value, i) + : removeConcept(props.value, i, j) + ); + }} expand={{ onClick: () => props.onChange( @@ -372,16 +381,16 @@ const FormConceptGroup = (props: Props) => { return; } + + if (props.isValidConcept && !props.isValidConcept(item)) + return null; if (isMovedObject(item)) { return props.onChange( - setConcept(props.value, i, j, copyConcept(item)), + setConcept(getValues(props.fieldName), i, j, copyConcept(item)), ); } - if (props.isValidConcept && !props.isValidConcept(item)) - return null; - return props.onChange( setConcept( props.value, @@ -438,9 +447,13 @@ const FormConceptGroup = (props: Props) => { ); }} onDropConcept={(concept) => { + if (isMovedObject(concept) && exists(concept.dragContext.deleteInOrigin)) { + concept.dragContext.deleteInOrigin(); + } + const { valueIdx, conceptIdx } = editedFormQueryNodePosition; props.onChange( - setConceptProperties(props.value, valueIdx, conceptIdx, { + setConceptProperties(getValues(props.fieldName), valueIdx, conceptIdx, { ids: [...concept.ids, ...editedNode.ids], }), ); diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx index b4e118c9a2..dedfa2fd7a 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx @@ -81,6 +81,7 @@ interface PropsT { expandable: boolean; active: boolean; }; + deleteInForm: () => void; } // generalized node to handle concepts queried in forms @@ -92,6 +93,7 @@ const FormConceptNode: FC = ({ hasNonDefaultSettings, hasFilterValues, expand, + deleteInForm, }) => { const { t } = useTranslation(); const rootNodeLabel = getRootNodeLabel(conceptNode); @@ -113,6 +115,7 @@ const FormConceptNode: FC = ({ dragContext: { ...item.dragContext, ...getWidthAndHeight(ref), + deleteInOrigin: deleteInForm, }, }), }); diff --git a/frontend/src/js/standard-query-editor/types.ts b/frontend/src/js/standard-query-editor/types.ts index 17af92884f..0d22e6910c 100644 --- a/frontend/src/js/standard-query-editor/types.ts +++ b/frontend/src/js/standard-query-editor/types.ts @@ -56,6 +56,7 @@ export interface DragContext { height: number; movedFromAndIdx?: number; movedFromOrIdx?: number; + deleteInOrigin?: () => void; } export interface DragItemQuery extends PreviousQueryQueryNodeType { From 03a2dcc67388e86eb4f4104c47b37015d31f8e74 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Tue, 27 Jun 2023 10:43:13 +0200 Subject: [PATCH 405/679] fix formatting --- .../form-concept-group/FormConceptGroup.tsx | 45 +++++++++++++------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 90aec6f41a..885801c081 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -1,5 +1,6 @@ import styled from "@emotion/styled"; import { ReactNode, useEffect, useState, useRef, useMemo } from "react"; +import { useFormContext } from "react-hook-form"; import { useTranslation } from "react-i18next"; import { usePostPrefixForSuggestions } from "../../api/api"; @@ -59,7 +60,6 @@ import { } from "./formConceptGroupState"; import { useCopyModal } from "./useCopyModal"; import { useUploadConceptListModal } from "./useUploadConceptListModal"; -import { useFormContext } from "react-hook-form"; interface Props { formType: string; @@ -211,15 +211,19 @@ const FormConceptGroup = (props: Props) => { return null; if (isMovedObject(item)) { - if (exists(item.dragContext.deleteInOrigin)){ + if (exists(item.dragContext.deleteInOrigin)) { item.dragContext.deleteInOrigin(); } - + let insertIndex = i > item.dragContext.movedFromAndIdx ? i - 1 : i; return props.onChange( addConcept( - insertValue(getValues(props.fieldName), insertIndex, newValue), + insertValue( + getValues(props.fieldName), + insertIndex, + newValue, + ), insertIndex, copyConcept(item), ), @@ -255,7 +259,7 @@ const FormConceptGroup = (props: Props) => { if (props.isValidConcept && !props.isValidConcept(item)) return; if (isMovedObject(item)) { - if (exists(item.dragContext.deleteInOrigin)){ + if (exists(item.dragContext.deleteInOrigin)) { item.dragContext.deleteInOrigin(); } const updatedValue = getValues(props.fieldName); @@ -342,9 +346,9 @@ const FormConceptGroup = (props: Props) => { } deleteInForm={() => { return props.onChange( - props.value[i].concepts.length === 1 - ? removeValue(props.value, i) - : removeConcept(props.value, i, j) + props.value[i].concepts.length === 1 + ? removeValue(props.value, i) + : removeConcept(props.value, i, j), ); }} expand={{ @@ -381,13 +385,18 @@ const FormConceptGroup = (props: Props) => { return; } - + if (props.isValidConcept && !props.isValidConcept(item)) return null; if (isMovedObject(item)) { return props.onChange( - setConcept(getValues(props.fieldName), i, j, copyConcept(item)), + setConcept( + getValues(props.fieldName), + i, + j, + copyConcept(item), + ), ); } @@ -447,15 +456,23 @@ const FormConceptGroup = (props: Props) => { ); }} onDropConcept={(concept) => { - if (isMovedObject(concept) && exists(concept.dragContext.deleteInOrigin)) { + if ( + isMovedObject(concept) && + exists(concept.dragContext.deleteInOrigin) + ) { concept.dragContext.deleteInOrigin(); } const { valueIdx, conceptIdx } = editedFormQueryNodePosition; props.onChange( - setConceptProperties(getValues(props.fieldName), valueIdx, conceptIdx, { - ids: [...concept.ids, ...editedNode.ids], - }), + setConceptProperties( + getValues(props.fieldName), + valueIdx, + conceptIdx, + { + ids: [...concept.ids, ...editedNode.ids], + }, + ), ); }} onRemoveConcept={(conceptId) => { From 832ceeb9183e8d506e6bdebe407249391b2f6f85 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 27 Jun 2023 10:48:12 +0200 Subject: [PATCH 406/679] Show history loading indication when it takes longer than 300ms --- .../src/js/entity-history/EntityIdsList.tsx | 24 +++++++++++----- frontend/src/js/entity-history/History.tsx | 2 +- frontend/src/js/entity-history/Navigation.tsx | 3 +- frontend/src/js/entity-history/actions.ts | 28 +++++++++++++++++-- 4 files changed, 45 insertions(+), 12 deletions(-) diff --git a/frontend/src/js/entity-history/EntityIdsList.tsx b/frontend/src/js/entity-history/EntityIdsList.tsx index 1d7da5ae22..1609c71654 100644 --- a/frontend/src/js/entity-history/EntityIdsList.tsx +++ b/frontend/src/js/entity-history/EntityIdsList.tsx @@ -1,7 +1,10 @@ import styled from "@emotion/styled"; +import { faSpinner } from "@fortawesome/free-solid-svg-icons"; import { useMemo } from "react"; import ReactList from "react-list"; +import FaIcon from "../icon/FaIcon"; + import type { EntityIdsStatus } from "./History"; import { useUpdateHistorySession } from "./actions"; import { EntityId } from "./reducer"; @@ -54,19 +57,25 @@ const Gray = styled("span")` color: ${({ theme }) => theme.col.gray}; `; -interface Props { - currentEntityId: EntityId | null; - entityIds: EntityId[]; - updateHistorySession: ReturnType; - entityIdsStatus: EntityIdsStatus; -} +const SxFaIcon = styled(FaIcon)` + margin: 3px 6px; +`; export const EntityIdsList = ({ currentEntityId, entityIds, entityIdsStatus, updateHistorySession, -}: Props) => { + loadingId, +}: { + currentEntityId: EntityId | null; + entityIds: EntityId[]; + updateHistorySession: ReturnType< + typeof useUpdateHistorySession + >["updateHistorySession"]; + entityIdsStatus: EntityIdsStatus; + loadingId?: string; +}) => { const numberWidth = useMemo(() => { const magnitude = Math.ceil(Math.log(entityIds.length) / Math.log(10)); @@ -87,6 +96,7 @@ export const EntityIdsList = ({ {entityId.id} ({entityId.kind}) + {loadingId === entityId.id && } {entityIdsStatus[entityId.id] && entityIdsStatus[entityId.id].map((val) => ( diff --git a/frontend/src/js/entity-history/History.tsx b/frontend/src/js/entity-history/History.tsx index e134ca0b21..85ed15c9cb 100644 --- a/frontend/src/js/entity-history/History.tsx +++ b/frontend/src/js/entity-history/History.tsx @@ -132,7 +132,7 @@ export const History = () => { }); const [detailLevel, setDetailLevel] = useState("summary"); - const updateHistorySession = useUpdateHistorySession(); + const { updateHistorySession } = useUpdateHistorySession(); const { options, sourcesSet, sourcesFilter, setSourcesFilter } = useSourcesControl(); diff --git a/frontend/src/js/entity-history/Navigation.tsx b/frontend/src/js/entity-history/Navigation.tsx index f125a93589..4631ba22ad 100644 --- a/frontend/src/js/entity-history/Navigation.tsx +++ b/frontend/src/js/entity-history/Navigation.tsx @@ -105,7 +105,7 @@ export const Navigation = memo( }) => { const { t } = useTranslation(); const dispatch = useDispatch(); - const updateHistorySession = useUpdateHistorySession(); + const { loadingId, updateHistorySession } = useUpdateHistorySession(); const onCloseHistory = useCallback(() => { dispatch(closeHistory()); }, [dispatch]); @@ -204,6 +204,7 @@ export const Navigation = memo( entityIds={entityIds} updateHistorySession={updateHistorySession} entityIdsStatus={entityIdsStatus} + loadingId={loadingId} /> {!empty && ( diff --git a/frontend/src/js/entity-history/actions.ts b/frontend/src/js/entity-history/actions.ts index a8e416ce0a..1ccb5c82e5 100644 --- a/frontend/src/js/entity-history/actions.ts +++ b/frontend/src/js/entity-history/actions.ts @@ -1,6 +1,6 @@ import startOfYear from "date-fns/startOfYear"; import subYears from "date-fns/subYears"; -import { useCallback } from "react"; +import { useCallback, useRef, useState } from "react"; import { useTranslation } from "react-i18next"; import { useDispatch, useSelector } from "react-redux"; import { ActionType, createAction, createAsyncAction } from "typesafe-actions"; @@ -119,7 +119,7 @@ function getPreferredIdColumns(columns: ColumnDescription[]) { export function useNewHistorySession() { const dispatch = useDispatch(); const loadPreviewData = useLoadPreviewData(); - const updateHistorySession = useUpdateHistorySession(); + const { updateHistorySession } = useUpdateHistorySession(); return async (url: string, columns: ColumnDescription[], label: string) => { dispatch(loadHistoryData.request()); @@ -171,6 +171,8 @@ export function useNewHistorySession() { }; } +const SHOW_LOADING_DELAY = 300; + export function useUpdateHistorySession() { const dispatch = useDispatch(); const datasetId = useDatasetId(); @@ -178,6 +180,9 @@ export function useUpdateHistorySession() { const getAuthorizedUrl = useGetAuthorizedUrl(); const { t } = useTranslation(); + const loadingIdTimeout = useRef(); + const [loadingId, setLoadingId] = useState(); + const defaultEntityHistoryParams = useSelector< StateT, StateT["entityHistory"]["defaultParams"] @@ -189,7 +194,7 @@ export function useUpdateHistorySession() { ); }); - return useCallback( + const updateHistorySession = useCallback( async ({ entityId, entityIds, @@ -202,6 +207,13 @@ export function useUpdateHistorySession() { }) => { if (!datasetId) return; + if (loadingIdTimeout.current) { + clearTimeout(loadingIdTimeout.current); + } + loadingIdTimeout.current = setTimeout(() => { + setLoadingId(entityId.id); + }, SHOW_LOADING_DELAY); + try { dispatch(loadHistoryData.request()); @@ -270,6 +282,11 @@ export function useUpdateHistorySession() { }), ); } + + if (loadingIdTimeout.current) { + clearTimeout(loadingIdTimeout.current); + } + setLoadingId(undefined); }, [ t, @@ -281,6 +298,11 @@ export function useUpdateHistorySession() { observationPeriodMin, ], ); + + return { + loadingId, + updateHistorySession, + }; } const transformEntityData = (data: { [key: string]: any }[]): EntityEvent[] => { From 2e6ad30d274c4a77fdfe0676966414d525cfeab0 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 27 Jun 2023 15:08:27 +0200 Subject: [PATCH 407/679] cleanup ShutdownTask --- .../resources/admin/ShutdownTask.java | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ShutdownTask.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ShutdownTask.java index e87cff242b..0965112c78 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ShutdownTask.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ShutdownTask.java @@ -25,24 +25,25 @@ public void serverStarted(Server server) { @Override public void execute(Map> parameters, PrintWriter output) throws Exception { + log.info("Received Shutdown command"); + if(server == null) { output.print("Server not yet started"); + return; } - else { - output.print("Shutting down"); - log.info("Received Shutdown command"); - //this must be done in an extra step or the shutdown will wait for this request to be resolved - new Thread("shutdown waiter thread") { - @Override - public void run() { - try { - server.stop(); - } catch (Exception e) { - log.error("Failed while shutting down", e); - } + + output.print("Shutting down"); + //this must be done in an extra step or the shutdown will wait for this request to be resolved + new Thread("shutdown waiter thread") { + @Override + public void run() { + try { + server.stop(); + } catch (Exception e) { + log.error("Failed while shutting down", e); } - }.start(); - } + } + }.start(); } } From c73bd293170525a0f1ed1f3498c0aa4d0665eb4d Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 27 Jun 2023 15:18:45 +0200 Subject: [PATCH 408/679] adds Task to reload MetaStorage from disk --- .../conquery/commands/ManagerNode.java | 4 +- .../conquery/tasks/ReloadMetaStorageTask.java | 56 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/tasks/ReloadMetaStorageTask.java diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java index 543e7723f6..7d7d2176ef 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java @@ -46,6 +46,7 @@ import com.bakdata.conquery.resources.unprotected.AuthServlet; import com.bakdata.conquery.tasks.PermissionCleanupTask; import com.bakdata.conquery.tasks.QueryCleanupTask; +import com.bakdata.conquery.tasks.ReloadMetaStorageTask; import com.bakdata.conquery.tasks.ReportConsistencyTask; import com.bakdata.conquery.util.io.ConqueryMDC; import com.fasterxml.jackson.databind.DeserializationConfig; @@ -194,8 +195,9 @@ public void run(ConqueryConfig config, Environment environment) throws Interrupt ))); environment.admin().addTask(new PermissionCleanupTask(storage)); environment.admin().addTask(new ReportConsistencyTask(datasetRegistry)); + environment.admin().addTask(new ReloadMetaStorageTask(storage)); - ShutdownTask shutdown = new ShutdownTask(); + final ShutdownTask shutdown = new ShutdownTask(); environment.admin().addTask(shutdown); environment.lifecycle().addServerLifecycleListener(shutdown); } diff --git a/backend/src/main/java/com/bakdata/conquery/tasks/ReloadMetaStorageTask.java b/backend/src/main/java/com/bakdata/conquery/tasks/ReloadMetaStorageTask.java new file mode 100644 index 0000000000..fb5d423dc2 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/tasks/ReloadMetaStorageTask.java @@ -0,0 +1,56 @@ +package com.bakdata.conquery.tasks; + +import java.io.PrintWriter; +import java.util.List; +import java.util.Map; + +import com.bakdata.conquery.io.storage.MetaStorage; +import com.google.common.base.Stopwatch; +import io.dropwizard.servlets.tasks.Task; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class ReloadMetaStorageTask extends Task { + + private final MetaStorage storage; + + public ReloadMetaStorageTask(MetaStorage storage) { + super("reload-meta-storage"); + this.storage = storage; + } + + @Override + public void execute(Map> parameters, PrintWriter output) throws Exception { + final Stopwatch timer = Stopwatch.createStarted(); + + output.println("BEGIN reloading MetaStorage."); + + { + final int allUsers = storage.getAllUsers().size(); + final int allExecutions = storage.getAllExecutions().size(); + final int allFormConfigs = storage.getAllFormConfigs().size(); + final int allGroups = storage.getAllGroups().size(); + final int allRoles = storage.getAllRoles().size(); + + log.debug("BEFORE: Have {} Users, {} Groups, {} Roles, {} Executions, {} FormConfigs.", + allUsers, allGroups, allRoles, allExecutions, allFormConfigs); + } + + storage.loadData(); + output.println("DONE reloading MetaStorage within %s.".formatted(timer.elapsed())); + + { + final int allUsers = storage.getAllUsers().size(); + final int allExecutions = storage.getAllExecutions().size(); + final int allFormConfigs = storage.getAllFormConfigs().size(); + final int allGroups = storage.getAllGroups().size(); + final int allRoles = storage.getAllRoles().size(); + + log.debug("AFTER: Have {} Users, {} Groups, {} Roles, {} Executions, {} FormConfigs.", + allUsers, allGroups, allRoles, allExecutions, allFormConfigs); + } + + + + } +} From 96466ef9c4c5f07b6d3026733c60a8d4865af15f Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 27 Jun 2023 15:30:40 +0200 Subject: [PATCH 409/679] Increase year head size, format currency --- .../js/entity-history/TimeStratifiedChart.tsx | 10 +---- frontend/src/js/entity-history/Timeline.tsx | 2 +- .../entity-history/timeline/SmallHeading.tsx | 1 + .../js/entity-history/timeline/YearHead.tsx | 37 ++++++++++--------- .../src/js/entity-history/timeline/util.ts | 10 +++++ 5 files changed, 34 insertions(+), 26 deletions(-) diff --git a/frontend/src/js/entity-history/TimeStratifiedChart.tsx b/frontend/src/js/entity-history/TimeStratifiedChart.tsx index 4eeb6c951f..718d4d1c4c 100644 --- a/frontend/src/js/entity-history/TimeStratifiedChart.tsx +++ b/frontend/src/js/entity-history/TimeStratifiedChart.tsx @@ -14,6 +14,8 @@ import { Bar } from "react-chartjs-2"; import { TimeStratifiedInfo } from "../api/types"; import { exists } from "../common/helpers/exists"; +import { formatCurrency } from "./timeline/util"; + const TRUNCATE_X_AXIS_LABELS_LEN = 18; ChartJS.register(CategoryScale, LinearScale, BarElement, Tooltip); @@ -42,14 +44,6 @@ function interpolateDecreasingOpacity(index: number) { return Math.min(1, 1 / (index + 0.3)); } -const formatCurrency = (value: number) => - value.toLocaleString("de-DE", { - style: "currency", - currency: "EUR", - minimumFractionDigits: 0, - maximumFractionDigits: 0, - }); - export const TimeStratifiedChart = ({ timeStratifiedInfo, }: { diff --git a/frontend/src/js/entity-history/Timeline.tsx b/frontend/src/js/entity-history/Timeline.tsx index 518180033e..84c60d327e 100644 --- a/frontend/src/js/entity-history/Timeline.tsx +++ b/frontend/src/js/entity-history/Timeline.tsx @@ -32,7 +32,7 @@ const Root = styled("div")` -webkit-overflow-scrolling: touch; padding: 0 20px 20px 10px; display: inline-grid; - grid-template-columns: 200px auto; + grid-template-columns: 280px auto; grid-auto-rows: minmax(min-content, max-content); gap: 20px 4px; width: 100%; diff --git a/frontend/src/js/entity-history/timeline/SmallHeading.tsx b/frontend/src/js/entity-history/timeline/SmallHeading.tsx index e72a52de23..51d5a8a06f 100644 --- a/frontend/src/js/entity-history/timeline/SmallHeading.tsx +++ b/frontend/src/js/entity-history/timeline/SmallHeading.tsx @@ -6,4 +6,5 @@ export const SmallHeading = styled(Heading4)` flex-shrink: 0; margin: 0; color: ${({ theme }) => theme.col.black}; + font-size: ${({ theme }) => theme.font.md}; `; diff --git a/frontend/src/js/entity-history/timeline/YearHead.tsx b/frontend/src/js/entity-history/timeline/YearHead.tsx index f05bd5a59c..1d35d4660c 100644 --- a/frontend/src/js/entity-history/timeline/YearHead.tsx +++ b/frontend/src/js/entity-history/timeline/YearHead.tsx @@ -2,24 +2,21 @@ import styled from "@emotion/styled"; import { faCaretDown, faCaretRight } from "@fortawesome/free-solid-svg-icons"; import { Fragment, memo } from "react"; import { useTranslation } from "react-i18next"; -import { useSelector } from "react-redux"; import { ColumnDescriptionSemanticConceptColumn, TimeStratifiedInfo, } from "../../api/types"; -import { StateT } from "../../app/reducers"; import { exists } from "../../common/helpers/exists"; import { getConceptById } from "../../concept-trees/globalTreeStoreHelper"; import FaIcon from "../../icon/FaIcon"; import WithTooltip from "../../tooltip/WithTooltip"; import { SmallHeading } from "./SmallHeading"; -import { isConceptColumn, isMoneyColumn } from "./util"; +import { formatCurrency, isConceptColumn, isMoneyColumn } from "./util"; const Root = styled("div")` font-size: ${({ theme }) => theme.font.xs}; - color: ${({ theme }) => theme.col.gray}; padding: 0 10px 0 0; `; const StickyWrap = styled("div")` @@ -55,19 +52,20 @@ const ConceptRow = styled("div")` display: flex; flex-wrap: wrap; align-items: center; - gap: 2px; + gap: 4px; `; const ConceptBubble = styled("span")` - padding: 0 2px; + padding: 0 3px; border-radius: ${({ theme }) => theme.borderRadius}; color: ${({ theme }) => theme.col.black}; - border: 1px solid ${({ theme }) => theme.col.blueGrayLight}; - font-size: ${({ theme }) => theme.font.tiny}; + border: 1px solid ${({ theme }) => theme.col.gray}; + background-color: white; + font-size: ${({ theme }) => theme.font.sm}; `; const Value = styled("div")` - font-size: ${({ theme }) => theme.font.tiny}; + font-size: ${({ theme }) => theme.font.sm}; font-weight: 400; justify-self: end; white-space: nowrap; @@ -78,7 +76,7 @@ const Value = styled("div")` `; const Label = styled("div")` - font-size: ${({ theme }) => theme.font.tiny}; + font-size: ${({ theme }) => theme.font.sm}; max-width: 100%; white-space: nowrap; overflow: hidden; @@ -92,10 +90,6 @@ const TimeStratifiedInfos = ({ year: number; timeStratifiedInfos: TimeStratifiedInfo[]; }) => { - const currencyUnit = useSelector( - (state) => state.startup.config.currency.unit, - ); - const infos = timeStratifiedInfos .map((info) => { return { @@ -139,7 +133,15 @@ const TimeStratifiedInfos = ({ if (value instanceof Array) { const concepts = value .map((v) => getConceptById(v, semantic!.concept)) - .filter(exists); + .filter(exists) + .sort((c1, c2) => { + const n1 = Number(c1.label); + const n2 = Number(c2.label); + if (!isNaN(n1) && !isNaN(n2)) { + return n1 - n2; + } + return c1.label.localeCompare(c2.label); + }); return ( @@ -169,7 +171,9 @@ const TimeStratifiedInfos = ({ let valueFormatted: string | number | string[] = value; if (typeof value === "number") { - valueFormatted = Math.round(value); + valueFormatted = isMoneyColumn(column) + ? formatCurrency(value) + : Math.round(value); } else if (value instanceof Array) { valueFormatted = value.join(", "); } @@ -179,7 +183,6 @@ const TimeStratifiedInfos = ({ {valueFormatted} - {isMoneyColumn(column) ? " " + currencyUnit : ""} ); diff --git a/frontend/src/js/entity-history/timeline/util.ts b/frontend/src/js/entity-history/timeline/util.ts index bcbac7f1ae..b3f7db5013 100644 --- a/frontend/src/js/entity-history/timeline/util.ts +++ b/frontend/src/js/entity-history/timeline/util.ts @@ -21,3 +21,13 @@ export const isMoneyColumn = (columnDescription: ColumnDescription) => export const isSecondaryIdColumn = (columnDescription: ColumnDescription) => columnDescription.semantics.some((s) => s.type === "SECONDARY_ID"); + +export const formatCurrency = (value: number) => + value.toLocaleString(navigator.language, { + style: "currency", + + currency: "EUR", + unitDisplay: "short", + minimumFractionDigits: 0, + maximumFractionDigits: 0, + }); From fed99f14a6c957ac9ae2096352b405b711e683fc Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 27 Jun 2023 15:33:51 +0200 Subject: [PATCH 410/679] Also increase size of entity card labels --- frontend/src/js/entity-history/EntityInfos.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/js/entity-history/EntityInfos.tsx b/frontend/src/js/entity-history/EntityInfos.tsx index d425d3032a..8b7e4a12b3 100644 --- a/frontend/src/js/entity-history/EntityInfos.tsx +++ b/frontend/src/js/entity-history/EntityInfos.tsx @@ -10,7 +10,7 @@ const Grid = styled("div")` place-items: center start; `; const Label = styled("div")` - font-size: ${({ theme }) => theme.font.xs}; + font-size: ${({ theme }) => theme.font.sm}; `; const Value = styled("div")<{ blurred?: boolean }>` font-size: ${({ theme }) => theme.font.sm}; From ac59b7d200ff42a0ef00476741b68b2be45dc88e Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 27 Jun 2023 16:46:03 +0200 Subject: [PATCH 411/679] Start iterating towards displaying concept data --- frontend/src/js/entity-history/EntityCard.tsx | 9 +-- .../TabbableTimeStratifiedCharts.tsx | 42 ------------ .../TabbableTimeStratifiedInfos.tsx | 68 +++++++++++++++++++ 3 files changed, 70 insertions(+), 49 deletions(-) delete mode 100644 frontend/src/js/entity-history/TabbableTimeStratifiedCharts.tsx create mode 100644 frontend/src/js/entity-history/TabbableTimeStratifiedInfos.tsx diff --git a/frontend/src/js/entity-history/EntityCard.tsx b/frontend/src/js/entity-history/EntityCard.tsx index f4633d1468..d20e7ee7fc 100644 --- a/frontend/src/js/entity-history/EntityCard.tsx +++ b/frontend/src/js/entity-history/EntityCard.tsx @@ -3,8 +3,7 @@ import styled from "@emotion/styled"; import { EntityInfo, TimeStratifiedInfo } from "../api/types"; import EntityInfos from "./EntityInfos"; -import { TabbableTimeStratifiedCharts } from "./TabbableTimeStratifiedCharts"; -import { isMoneyColumn } from "./timeline/util"; +import { TabbableTimeStratifiedInfos } from "./TabbableTimeStratifiedInfos"; const Container = styled("div")` display: grid; @@ -35,16 +34,12 @@ export const EntityCard = ({ infos: EntityInfo[]; timeStratifiedInfos: TimeStratifiedInfo[]; }) => { - const infosWithOnlyMoneyColumns = timeStratifiedInfos.filter((info) => - info.columns.every(isMoneyColumn), - ); - return ( - + ); }; diff --git a/frontend/src/js/entity-history/TabbableTimeStratifiedCharts.tsx b/frontend/src/js/entity-history/TabbableTimeStratifiedCharts.tsx deleted file mode 100644 index 49c248d070..0000000000 --- a/frontend/src/js/entity-history/TabbableTimeStratifiedCharts.tsx +++ /dev/null @@ -1,42 +0,0 @@ -import styled from "@emotion/styled"; -import { useState, useMemo } from "react"; - -import { TimeStratifiedInfo } from "../api/types"; -import SmallTabNavigation from "../small-tab-navigation/SmallTabNavigation"; - -import { TimeStratifiedChart } from "./TimeStratifiedChart"; - -const Container = styled("div")` - display: flex; - flex-direction: column; - align-items: flex-end; -`; - -export const TabbableTimeStratifiedCharts = ({ - infos, -}: { - infos: TimeStratifiedInfo[]; -}) => { - const [activeTab, setActiveTab] = useState(infos[0].label); - const options = useMemo(() => { - return infos.map((info) => ({ - value: info.label, - label: () => info.label, - })); - }, [infos]); - - const activeInfos = useMemo(() => { - return infos.find((info) => info.label === activeTab); - }, [infos, activeTab]); - - return ( - - - {activeInfos && } - - ); -}; diff --git a/frontend/src/js/entity-history/TabbableTimeStratifiedInfos.tsx b/frontend/src/js/entity-history/TabbableTimeStratifiedInfos.tsx new file mode 100644 index 0000000000..bdbe9ed57f --- /dev/null +++ b/frontend/src/js/entity-history/TabbableTimeStratifiedInfos.tsx @@ -0,0 +1,68 @@ +import styled from "@emotion/styled"; +import { useState, useMemo } from "react"; + +import { TimeStratifiedInfo } from "../api/types"; +import SmallTabNavigation from "../small-tab-navigation/SmallTabNavigation"; + +import { TimeStratifiedChart } from "./TimeStratifiedChart"; +import { isConceptColumn, isMoneyColumn } from "./timeline/util"; + +const Container = styled("div")` + display: flex; + flex-direction: column; + align-items: flex-end; +`; + +export const TabbableTimeStratifiedInfos = ({ + infos, +}: { + infos: TimeStratifiedInfo[]; +}) => { + const [activeTab, setActiveTab] = useState(infos[0].label); + const options = useMemo(() => { + return infos.map((info) => ({ + value: info.label, + label: () => info.label, + })); + }, [infos]); + + const { data, type } = useMemo(() => { + let infoType = "money"; + let infoData = infos.find((info) => info.label === activeTab); + + if (infoData?.columns.some((c) => !isMoneyColumn(c))) { + const columns = infoData?.columns.filter(isMoneyColumn); + + infoData = { + ...infoData, + totals: Object.fromEntries( + Object.entries(infoData?.totals).filter(([k]) => + columns?.map((c) => c.label).includes(k), + ), + ), + columns: columns ?? [], + }; + } else if (infoData?.columns.some(isConceptColumn)) { + // TODO: Handle concept data + infoType = "concept"; + } + + return { data: infoData, type: infoType }; + }, [infos, activeTab]); + + console.log(data); + + return ( + + + {data && type === "money" && ( + + )} + {data && type === "concept" &&
Concept
} + + ); +}; From a226d86b76ed456e9ebaa0cd32a5cdfd2a5a32a8 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 27 Jun 2023 17:34:49 +0200 Subject: [PATCH 412/679] Iterate further towards concept charts --- .../src/js/entity-history/ConceptBubble.ts | 10 ++ .../TabbableTimeStratifiedInfos.tsx | 7 +- .../TimeStratifiedConceptChart.tsx | 91 +++++++++++++++++++ .../js/entity-history/timeline/YearHead.tsx | 10 +- 4 files changed, 106 insertions(+), 12 deletions(-) create mode 100644 frontend/src/js/entity-history/ConceptBubble.ts create mode 100644 frontend/src/js/entity-history/TimeStratifiedConceptChart.tsx diff --git a/frontend/src/js/entity-history/ConceptBubble.ts b/frontend/src/js/entity-history/ConceptBubble.ts new file mode 100644 index 0000000000..9c1ff3539a --- /dev/null +++ b/frontend/src/js/entity-history/ConceptBubble.ts @@ -0,0 +1,10 @@ +import styled from "@emotion/styled"; + +export const ConceptBubble = styled("span")` + padding: 0 3px; + border-radius: ${({ theme }) => theme.borderRadius}; + color: ${({ theme }) => theme.col.black}; + border: 1px solid ${({ theme }) => theme.col.gray}; + background-color: white; + font-size: ${({ theme }) => theme.font.sm}; +`; diff --git a/frontend/src/js/entity-history/TabbableTimeStratifiedInfos.tsx b/frontend/src/js/entity-history/TabbableTimeStratifiedInfos.tsx index bdbe9ed57f..a6ccde9de1 100644 --- a/frontend/src/js/entity-history/TabbableTimeStratifiedInfos.tsx +++ b/frontend/src/js/entity-history/TabbableTimeStratifiedInfos.tsx @@ -5,6 +5,7 @@ import { TimeStratifiedInfo } from "../api/types"; import SmallTabNavigation from "../small-tab-navigation/SmallTabNavigation"; import { TimeStratifiedChart } from "./TimeStratifiedChart"; +import { TimeStratifiedConceptChart } from "./TimeStratifiedConceptChart"; import { isConceptColumn, isMoneyColumn } from "./timeline/util"; const Container = styled("div")` @@ -59,10 +60,10 @@ export const TabbableTimeStratifiedInfos = ({ selectedTab={activeTab} onSelectTab={setActiveTab} /> - {data && type === "money" && ( + {/* {data && type === "money" && ( - )} - {data && type === "concept" &&
Concept
} + )} */} + {data && } ); }; diff --git a/frontend/src/js/entity-history/TimeStratifiedConceptChart.tsx b/frontend/src/js/entity-history/TimeStratifiedConceptChart.tsx new file mode 100644 index 0000000000..d44b5bd1f0 --- /dev/null +++ b/frontend/src/js/entity-history/TimeStratifiedConceptChart.tsx @@ -0,0 +1,91 @@ +import styled from "@emotion/styled"; + +import { + ColumnDescriptionSemanticConceptColumn, + TimeStratifiedInfo, +} from "../api/types"; +import { getConceptById } from "../concept-trees/globalTreeStoreHelper"; +import WithTooltip from "../tooltip/WithTooltip"; + +import { ConceptBubble } from "./ConceptBubble"; + +const Container = styled("div")` + display: grid; + place-items: center; + gap: 0 3px; + padding: 10px; +`; + +const BubbleYes = styled("div")` + width: 10px; + height: 10px; + border-radius: ${({ theme }) => theme.borderRadius}; + background-color: ${({ theme }) => theme.col.blueGray}; +`; +const BubbleNo = styled("div")` + width: 10px; + height: 10px; + border-radius: ${({ theme }) => theme.borderRadius}; + background-color: ${({ theme }) => theme.col.grayLight}; +`; + +export const TimeStratifiedConceptChart = ({ + timeStratifiedInfo, +}: { + timeStratifiedInfo: TimeStratifiedInfo; +}) => { + const conceptColumn = timeStratifiedInfo.columns.at(-1); + + if (!conceptColumn) return null; + + const conceptSemantic = conceptColumn.semantics.find( + (s): s is ColumnDescriptionSemanticConceptColumn => + s.type === "CONCEPT_COLUMN", + ); + + if (!conceptSemantic) return null; + + const years = timeStratifiedInfo.years.map((y) => y.year); + console.log(timeStratifiedInfo.years, conceptColumn); + const valuesPerYear = timeStratifiedInfo.years.map((y) => + ((y.values[Object.keys(y.values)[0]] as string[]) || []).map( + (conceptId) => getConceptById(conceptId, conceptSemantic?.concept)!, + ), + ); + + const allValues = [ + ...new Set( + valuesPerYear + .flatMap((v) => v) + .sort((a, b) => { + const nA = Number(a?.label); + const nB = Number(b?.label); + if (!isNaN(nA) && !isNaN(nB)) return nA - nB; + return a?.label.localeCompare(b?.label!); + }), + ), + ]; + + return ( + +
+ {allValues.map((val) => ( + + {val.label} + + ))} + {years.map((y, i) => ( + <> +
{y}
+ {allValues.map((val) => + valuesPerYear[i].includes(val) ? : , + )} + + ))} + + ); +}; diff --git a/frontend/src/js/entity-history/timeline/YearHead.tsx b/frontend/src/js/entity-history/timeline/YearHead.tsx index 1d35d4660c..6b6871628f 100644 --- a/frontend/src/js/entity-history/timeline/YearHead.tsx +++ b/frontend/src/js/entity-history/timeline/YearHead.tsx @@ -11,6 +11,7 @@ import { exists } from "../../common/helpers/exists"; import { getConceptById } from "../../concept-trees/globalTreeStoreHelper"; import FaIcon from "../../icon/FaIcon"; import WithTooltip from "../../tooltip/WithTooltip"; +import { ConceptBubble } from "../ConceptBubble"; import { SmallHeading } from "./SmallHeading"; import { formatCurrency, isConceptColumn, isMoneyColumn } from "./util"; @@ -55,15 +56,6 @@ const ConceptRow = styled("div")` gap: 4px; `; -const ConceptBubble = styled("span")` - padding: 0 3px; - border-radius: ${({ theme }) => theme.borderRadius}; - color: ${({ theme }) => theme.col.black}; - border: 1px solid ${({ theme }) => theme.col.gray}; - background-color: white; - font-size: ${({ theme }) => theme.font.sm}; -`; - const Value = styled("div")` font-size: ${({ theme }) => theme.font.sm}; font-weight: 400; From 963d19d83f76a9e3b5435d8b95b020cd35711821 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 27 Jun 2023 18:32:39 +0200 Subject: [PATCH 413/679] Fix empty timeline --- frontend/src/js/entity-history/Timeline.tsx | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/frontend/src/js/entity-history/Timeline.tsx b/frontend/src/js/entity-history/Timeline.tsx index 518180033e..89e62f0aaf 100644 --- a/frontend/src/js/entity-history/Timeline.tsx +++ b/frontend/src/js/entity-history/Timeline.tsx @@ -86,10 +86,6 @@ const Timeline = ({ secondaryIds: columnBuckets.secondaryIds, }); - if (eventsByQuarterWithGroups.length === 0) { - return ; - } - return ( + {eventsByQuarterWithGroups.length === 0 && } {eventsByQuarterWithGroups.map(({ year, quarterwiseData }, i) => ( Date: Wed, 28 Jun 2023 10:51:44 +0200 Subject: [PATCH 414/679] Re-add charts --- .../src/js/entity-history/TabbableTimeStratifiedInfos.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/js/entity-history/TabbableTimeStratifiedInfos.tsx b/frontend/src/js/entity-history/TabbableTimeStratifiedInfos.tsx index a6ccde9de1..c3e2914132 100644 --- a/frontend/src/js/entity-history/TabbableTimeStratifiedInfos.tsx +++ b/frontend/src/js/entity-history/TabbableTimeStratifiedInfos.tsx @@ -60,9 +60,9 @@ export const TabbableTimeStratifiedInfos = ({ selectedTab={activeTab} onSelectTab={setActiveTab} /> - {/* {data && type === "money" && ( + {data && type === "money" && ( - )} */} + )} {data && } ); From 5a5d70f1fecfb3cb8ada08a28771e021fc3f8a2a Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Wed, 28 Jun 2023 11:13:03 +0200 Subject: [PATCH 415/679] Fix layout --- frontend/src/js/entity-history/EntityCard.tsx | 4 +++- frontend/src/js/entity-history/Timeline.tsx | 9 +++++++-- .../entity-history/timeline/TimelineEmptyPlaceholder.tsx | 8 ++++++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/frontend/src/js/entity-history/EntityCard.tsx b/frontend/src/js/entity-history/EntityCard.tsx index d20e7ee7fc..6026552573 100644 --- a/frontend/src/js/entity-history/EntityCard.tsx +++ b/frontend/src/js/entity-history/EntityCard.tsx @@ -39,7 +39,9 @@ export const EntityCard = ({ - + {timeStratifiedInfos.length === 0 && ( + + )} ); }; diff --git a/frontend/src/js/entity-history/Timeline.tsx b/frontend/src/js/entity-history/Timeline.tsx index 858e383b3e..e8eb61319d 100644 --- a/frontend/src/js/entity-history/Timeline.tsx +++ b/frontend/src/js/entity-history/Timeline.tsx @@ -33,7 +33,7 @@ const Root = styled("div")` padding: 0 20px 20px 10px; display: inline-grid; grid-template-columns: 280px auto; - grid-auto-rows: minmax(min-content, max-content); + grid-auto-rows: minmax(min-content, max-content) 1fr; gap: 20px 4px; width: 100%; `; @@ -48,6 +48,11 @@ const SxEntityCard = styled(EntityCard)` grid-column: span 2; `; +const SxTimelineEmptyPlaceholder = styled(TimelineEmptyPlaceholder)` + grid-column: span 2; + height: 100%; +`; + const Timeline = ({ className, currentEntityInfos, @@ -93,7 +98,7 @@ const Timeline = ({ infos={currentEntityInfos} timeStratifiedInfos={currentEntityTimeStratifiedInfos} /> - {eventsByQuarterWithGroups.length === 0 && } + {eventsByQuarterWithGroups.length === 0 && } {eventsByQuarterWithGroups.map(({ year, quarterwiseData }, i) => ( { +export const TimelineEmptyPlaceholder = ({ + className, +}: { + className?: string; +}) => { const { t } = useTranslation(); const ids = useSelector( @@ -58,7 +62,7 @@ export const TimelineEmptyPlaceholder = () => { ); return ( - +
From 8e7384744ccc69e39f0e5f20a845b74cb7ba47cc Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 28 Jun 2023 11:57:34 +0200 Subject: [PATCH 416/679] always list ValdityDates even if there is no choice for the users --- .../datasets/concepts/FrontEndConceptBuilder.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java index c88586659b..fee4d75bda 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java @@ -185,12 +185,11 @@ public static FrontendTable createTable(Connector con) { .collect(Collectors.toSet())) .build(); - if (con.getValidityDates().size() > 1) { - result.setDateColumn(new FrontendValidityDate(con.getValidityDatesDescription(), null, con.getValidityDates() - .stream() - .map(vd -> new FrontendValue(vd.getId() - .toString(), vd.getLabel())) - .collect(Collectors.toList()))); + if (!con.getValidityDates().isEmpty()) { + result.setDateColumn(new FrontendValidityDate(con.getValidityDatesDescription(), null, + con.getValidityDates().stream() + .map(vd -> new FrontendValue(vd.getId().toString(), vd.getLabel())) + .collect(Collectors.toList()))); if (!result.getDateColumn().getOptions().isEmpty()) { result.getDateColumn().setDefaultValue(result.getDateColumn().getOptions().get(0).getValue()); From be6e2fab073f03b40b49f54bb03b0ce69b9fc852 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Wed, 28 Jun 2023 11:59:22 +0200 Subject: [PATCH 417/679] Fix yearhead and charts rendering --- frontend/src/js/entity-history/EntityCard.tsx | 2 +- frontend/src/js/entity-history/TabbableTimeStratifiedInfos.tsx | 2 -- frontend/src/js/entity-history/timeline/YearHead.tsx | 3 --- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/frontend/src/js/entity-history/EntityCard.tsx b/frontend/src/js/entity-history/EntityCard.tsx index 6026552573..b9d4f86bab 100644 --- a/frontend/src/js/entity-history/EntityCard.tsx +++ b/frontend/src/js/entity-history/EntityCard.tsx @@ -39,7 +39,7 @@ export const EntityCard = ({ - {timeStratifiedInfos.length === 0 && ( + {timeStratifiedInfos.length > 0 && ( )} diff --git a/frontend/src/js/entity-history/TabbableTimeStratifiedInfos.tsx b/frontend/src/js/entity-history/TabbableTimeStratifiedInfos.tsx index c3e2914132..d23f424625 100644 --- a/frontend/src/js/entity-history/TabbableTimeStratifiedInfos.tsx +++ b/frontend/src/js/entity-history/TabbableTimeStratifiedInfos.tsx @@ -51,8 +51,6 @@ export const TabbableTimeStratifiedInfos = ({ return { data: infoData, type: infoType }; }, [infos, activeTab]); - console.log(data); - return ( theme.font.sm}; font-weight: 400; justify-self: end; - white-space: nowrap; - overflow: hidden; - text-overflow: ellipsis; width: 100%; text-align: right; `; From af2f5cd59a95b1e9bb8d312cb24cebdc5beb48ea Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 28 Jun 2023 15:45:11 +0200 Subject: [PATCH 418/679] validate uniqueness constraint for timeStratifiedInfos globally --- .../models/datasets/PreviewConfig.java | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java index 9deccb3249..4d84e6150e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.datasets; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Optional; @@ -114,17 +115,18 @@ public record InfoCardSelect(@NotNull String label, SelectId select, String desc * Defines a group of selects that will be evaluated per quarter and year in the requested period of the entity-preview. */ public record TimeStratifiedSelects(@NotNull String label, String description, @NotEmpty List selects){ - @ValidationMethod(message = "Selects may be referenced only once.") - @JsonIgnore - public boolean isSelectsUnique() { - return selects().stream().map(InfoCardSelect::select).distinct().count() == selects().size(); - } + } - @ValidationMethod(message = "Labels must be unique.") - @JsonIgnore - public boolean isLabelsUnique() { - return selects().stream().map(InfoCardSelect::label).distinct().count() == selects().size(); - } + @ValidationMethod(message = "Selects may be referenced only once.") + @JsonIgnore + public boolean isSelectsUnique() { + return timeStratifiedSelects.stream().map(TimeStratifiedSelects::selects).flatMap(Collection::stream).map(InfoCardSelect::select).distinct().count() == timeStratifiedSelects.stream().map(TimeStratifiedSelects::selects).flatMap(Collection::stream).count(); + } + + @ValidationMethod(message = "Labels must be unique.") + @JsonIgnore + public boolean isLabelsUnique() { + return timeStratifiedSelects.stream().map(TimeStratifiedSelects::selects).flatMap(Collection::stream).map(InfoCardSelect::label).distinct().count() == timeStratifiedSelects.stream().map(TimeStratifiedSelects::selects).flatMap(Collection::stream).count(); } @JsonIgnore From e976aa990619199f15a90adccb229c9142b1a4e7 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Wed, 28 Jun 2023 16:39:20 +0200 Subject: [PATCH 419/679] Iterate editor v2 time condition --- frontend/src/js/api/apiHelper.ts | 11 +++- frontend/src/js/editor-v2/EditorLayout.ts | 1 + frontend/src/js/editor-v2/EditorV2.tsx | 61 ++++++++++--------- frontend/src/js/editor-v2/TreeNode.tsx | 50 ++++++++++++--- .../connector-update/useConnectorRotation.ts | 2 +- .../time-connection/TimeConnection.tsx | 13 +++- .../time-connection/TimeConnectionModal.tsx | 25 +++++--- frontend/src/js/editor-v2/util.ts | 10 +-- .../src/js/external-forms/FormsNavigation.tsx | 4 +- .../upload/CSVColumnPicker.tsx | 4 +- .../src/js/query-node-editor/ConceptEntry.tsx | 4 +- .../QueryClearButton.tsx | 4 +- .../TimebasedQueryClearButton.tsx | 4 +- frontend/src/localization/de.json | 3 +- frontend/src/localization/en.json | 3 +- 15 files changed, 132 insertions(+), 67 deletions(-) diff --git a/frontend/src/js/api/apiHelper.ts b/frontend/src/js/api/apiHelper.ts index 5320441c71..a20af07de0 100644 --- a/frontend/src/js/api/apiHelper.ts +++ b/frontend/src/js/api/apiHelper.ts @@ -242,7 +242,16 @@ const transformTreeToApi = (tree: Tree): unknown => { node = { type: "BEFORE", // SHOULD BE: tree.children.operator, days: { - ...(tree.children.interval || {}), + min: tree.children.interval + ? tree.children.interval.min === null + ? 1 + : tree.children.interval.max + : undefined, + max: tree.children.interval + ? tree.children.interval.max === null + ? undefined + : tree.children.interval.max + : undefined, }, // TODO: improve this to be more flexible with the "preceding" and "index" keys // based on the operator, which would be "before" | "after" | "while" diff --git a/frontend/src/js/editor-v2/EditorLayout.ts b/frontend/src/js/editor-v2/EditorLayout.ts index c6f993e92d..3cb8d5902e 100644 --- a/frontend/src/js/editor-v2/EditorLayout.ts +++ b/frontend/src/js/editor-v2/EditorLayout.ts @@ -14,6 +14,7 @@ export const Connector = styled("span")` text-transform: uppercase; font-size: ${({ theme }) => theme.font.sm}; color: black; + user-select: none; border-radius: ${({ theme }) => theme.borderRadius}; padding: 0px 5px; diff --git a/frontend/src/js/editor-v2/EditorV2.tsx b/frontend/src/js/editor-v2/EditorV2.tsx index 3e4101d620..67765fe649 100644 --- a/frontend/src/js/editor-v2/EditorV2.tsx +++ b/frontend/src/js/editor-v2/EditorV2.tsx @@ -21,6 +21,8 @@ import { DragItemConceptTreeNode, DragItemQuery, } from "../standard-query-editor/types"; +import { ConfirmableTooltip } from "../tooltip/ConfirmableTooltip"; +import WithTooltip from "../tooltip/WithTooltip"; import Dropzone from "../ui-components/Dropzone"; import { Connector, Grid } from "./EditorLayout"; @@ -350,20 +352,6 @@ export function EditorV2({ )} - {selectedNode?.children && ( - - { - e.stopPropagation(); - onFlip(); - }} - > - {t("editorV2.flip")} - - - )} {featureConnectorRotate && selectedNode?.children && ( )} + + + {selectedNode?.children && ( + + { + e.stopPropagation(); + onFlip(); + }} + > + {t("editorV2.flip")} + + + )} {selectedNode && ( )} + + + + + - - - {t("editorV2.clear")} - - )} {tree ? ( { - e.stopPropagation(); - if (!selectedNode) return; - if ( - selectedNode?.data && - nodeIsConceptQueryNode(selectedNode.data) - ) { - onOpenQueryNodeEditor(); - } - }} tree={tree} updateTreeNode={updateTreeNode} selectedNode={selectedNode} setSelectedNodeId={setSelectedNodeId} droppable={{ h: true, v: true }} featureContentInfos={featureContentInfos} + onOpenQueryNodeEditor={onOpenQueryNodeEditor} + onOpenTimeModal={onOpenTimeModal} + onRotateConnector={onRotateConnector} /> ) : ( void; - onDoubleClick?: DOMAttributes["onDoubleClick"]; featureContentInfos?: boolean; + onOpenQueryNodeEditor?: () => void; + onOpenTimeModal?: () => void; + onRotateConnector?: () => void; }) { const gridStyles = getGridStyles(tree); @@ -256,14 +260,25 @@ export function TreeNode({ negated={tree.negation} leaf={!tree.children} selected={selectedNode?.id === tree.id} - onDoubleClick={onDoubleClick} + onDoubleClick={(e) => { + if (tree.data && nodeIsConceptQueryNode(tree.data)) { + e.stopPropagation(); + onOpenQueryNodeEditor?.(); + } + }} onClick={(e) => { e.stopPropagation(); setSelectedNodeId(tree.id); }} > {tree.children && tree.children.connection === "time" && ( - + { + e.stopPropagation(); + onOpenTimeModal?.(); + }} + /> )} {tree.dates?.restriction && ( @@ -315,6 +330,9 @@ export function TreeNode({ updateTreeNode={updateTreeNode} selectedNode={selectedNode} setSelectedNodeId={setSelectedNodeId} + onOpenQueryNodeEditor={onOpenQueryNodeEditor} + onOpenTimeModal={onOpenTimeModal} + onRotateConnector={onRotateConnector} droppable={{ h: !item.children && @@ -337,6 +355,10 @@ export function TreeNode({ > {() => ( { + e.stopPropagation(); + onRotateConnector?.(); + }} connection={tree.children?.connection} /> )} @@ -384,8 +406,20 @@ export function TreeNode({ ); } -const Connection = memo(({ connection }: { connection?: ConnectionKind }) => { - const getTranslatedConnection = useGetTranslatedConnection(); +const Connection = memo( + ({ + connection, + onDoubleClick, + }: { + connection?: ConnectionKind; + onDoubleClick?: DOMAttributes["onDoubleClick"]; + }) => { + const getTranslatedConnection = useGetTranslatedConnection(); - return {getTranslatedConnection(connection)}; -}); + return ( + + {getTranslatedConnection(connection)} + + ); + }, +); diff --git a/frontend/src/js/editor-v2/connector-update/useConnectorRotation.ts b/frontend/src/js/editor-v2/connector-update/useConnectorRotation.ts index d74490249d..cc44406fc6 100644 --- a/frontend/src/js/editor-v2/connector-update/useConnectorRotation.ts +++ b/frontend/src/js/editor-v2/connector-update/useConnectorRotation.ts @@ -29,7 +29,7 @@ const getNextConnector = ( items: children.items, direction: children.direction, connection: "time" as const, - timestamps: children.items.map(() => "every" as const), + timestamps: children.items.map(() => "some" as const), operator: "before" as const, }; } diff --git a/frontend/src/js/editor-v2/time-connection/TimeConnection.tsx b/frontend/src/js/editor-v2/time-connection/TimeConnection.tsx index 2e6e18176a..f1f3f0e3ed 100644 --- a/frontend/src/js/editor-v2/time-connection/TimeConnection.tsx +++ b/frontend/src/js/editor-v2/time-connection/TimeConnection.tsx @@ -1,5 +1,5 @@ import styled from "@emotion/styled"; -import { memo } from "react"; +import { DOMAttributes, memo } from "react"; import { useTranslation } from "react-i18next"; import { TreeChildrenTime } from "../types"; @@ -14,6 +14,7 @@ const Container = styled("div")` margin: 0 auto; display: inline-flex; flex-direction: column; + user-select: none; `; const Row = styled("div")` @@ -41,7 +42,13 @@ const Operator = styled("span")` `; export const TimeConnection = memo( - ({ conditions }: { conditions: TreeChildrenTime }) => { + ({ + conditions, + onDoubleClick, + }: { + conditions: TreeChildrenTime; + onDoubleClick: DOMAttributes["onDoubleClick"]; + }) => { const { t } = useTranslation(); const getNodeLabel = useGetNodeLabel(); const getTranslatedTimestamp = useGetTranslatedTimestamp(); @@ -54,7 +61,7 @@ export const TimeConnection = memo( const interval = useTranslatedInterval(conditions.interval); return ( - + {aTimestamp} {t("editorV2.dateRangeFrom")} diff --git a/frontend/src/js/editor-v2/time-connection/TimeConnectionModal.tsx b/frontend/src/js/editor-v2/time-connection/TimeConnectionModal.tsx index d3a7a0f6d3..65cf034484 100644 --- a/frontend/src/js/editor-v2/time-connection/TimeConnectionModal.tsx +++ b/frontend/src/js/editor-v2/time-connection/TimeConnectionModal.tsx @@ -62,10 +62,10 @@ export const TimeConnectionModal = memo( const { t } = useTranslation(); const TIMESTAMP_OPTIONS = useMemo( () => [ - { value: "every", label: t("editorV2.every") }, { value: "some", label: t("editorV2.some") }, { value: "latest", label: t("editorV2.latest") }, { value: "earliest", label: t("editorV2.earliest") }, + { value: "every", label: t("editorV2.every") }, ], [t], ); @@ -89,7 +89,7 @@ export const TimeConnectionModal = memo( const [aTimestamp, setATimestamp] = useState(conditions.timestamps[0]); const [bTimestamp, setBTimestamp] = useState(conditions.timestamps[1]); const [operator, setOperator] = useState(conditions.operator); - const [interval, setInterval] = useState(conditions.interval); + const [interval, setTheInterval] = useState(conditions.interval); const getNodeLabel = useGetNodeLabel(); const a = getNodeLabel(conditions.items[0]); @@ -135,27 +135,33 @@ export const TimeConnectionModal = memo( { - setInterval({ min: val as number, max: interval?.max || null }); + setTheInterval({ + min: val as number, + max: interval ? interval.max : null, + }); }} /> – { - setInterval({ max: val as number, min: interval?.min || null }); + setTheInterval({ + max: val as number | null, + min: interval ? interval.min : null, + }); }} /> { if (opt?.value === "some") { - setInterval(undefined); + setTheInterval(undefined); } else { - setInterval({ min: 0, max: 0 }); + setTheInterval({ min: 1, max: null }); } }} /> @@ -177,7 +183,8 @@ export const TimeConnectionModal = memo( if (opt) { setOperator(opt.value as TimeOperator); if (opt.value === "while") { - setInterval(undefined); + // Timeout to avoid race condition on effect update above + setTimeout(() => setTheInterval(undefined), 10); } } }} diff --git a/frontend/src/js/editor-v2/util.ts b/frontend/src/js/editor-v2/util.ts index 209b6f0cb8..df43e0ecdc 100644 --- a/frontend/src/js/editor-v2/util.ts +++ b/frontend/src/js/editor-v2/util.ts @@ -105,10 +105,12 @@ export const useTranslatedInterval = ( const { min, max } = interval; - if (!min && !max) return t("editorV2.intervalSome"); - if (min && !max) return t("editorV2.intervalMinDays", { days: min }); - if (!min && max) return t("editorV2.intervalMaxDays", { days: max }); - if (min && max) + if (min === null && max === null) return t("editorV2.intervalSome"); + if (min !== null && max === null) + return t("editorV2.intervalMinDays", { days: min }); + if (min === null && max !== null) + return t("editorV2.intervalMaxDays", { days: max }); + if (min !== null && max !== null) return t("editorV2.intervalMinMaxDays", { minDays: min, maxDays: max }); return t("editorV2.intervalSome"); diff --git a/frontend/src/js/external-forms/FormsNavigation.tsx b/frontend/src/js/external-forms/FormsNavigation.tsx index 0e9832423b..6da71d6a0a 100644 --- a/frontend/src/js/external-forms/FormsNavigation.tsx +++ b/frontend/src/js/external-forms/FormsNavigation.tsx @@ -1,5 +1,5 @@ import styled from "@emotion/styled"; -import { faTrashAlt } from "@fortawesome/free-regular-svg-icons"; +import { faTrash } from "@fortawesome/free-solid-svg-icons"; import { useTranslation } from "react-i18next"; import { useSelector, useDispatch } from "react-redux"; @@ -89,7 +89,7 @@ const FormsNavigation = ({ onReset }: { onReset: () => void }) => { confirmationText={t("externalForms.common.clearConfirm")} > - + diff --git a/frontend/src/js/previous-queries/upload/CSVColumnPicker.tsx b/frontend/src/js/previous-queries/upload/CSVColumnPicker.tsx index a330e6220f..f53f1f6cb9 100644 --- a/frontend/src/js/previous-queries/upload/CSVColumnPicker.tsx +++ b/frontend/src/js/previous-queries/upload/CSVColumnPicker.tsx @@ -1,5 +1,5 @@ import styled from "@emotion/styled"; -import { faTrashAlt } from "@fortawesome/free-regular-svg-icons"; +import { faTrash } from "@fortawesome/free-solid-svg-icons"; import { faCheckCircle, faDownload, @@ -282,7 +282,7 @@ const CSVColumnPicker: FC = ({ {csv.length} Zeilen
- + {csv.length > 0 && ( diff --git a/frontend/src/js/query-node-editor/ConceptEntry.tsx b/frontend/src/js/query-node-editor/ConceptEntry.tsx index 310881ebff..07a41c396b 100644 --- a/frontend/src/js/query-node-editor/ConceptEntry.tsx +++ b/frontend/src/js/query-node-editor/ConceptEntry.tsx @@ -1,5 +1,5 @@ import styled from "@emotion/styled"; -import { faTrashAlt } from "@fortawesome/free-regular-svg-icons"; +import { faTrashCan } from "@fortawesome/free-regular-svg-icons"; import { useTranslation } from "react-i18next"; import type { ConceptIdT, ConceptT } from "../api/types"; @@ -77,7 +77,7 @@ const ConceptEntry = ({ onRemoveConcept(conceptId)} tiny - icon={faTrashAlt} + icon={faTrashCan} /> )} diff --git a/frontend/src/js/standard-query-editor/QueryClearButton.tsx b/frontend/src/js/standard-query-editor/QueryClearButton.tsx index 97aa4c7575..9cf8ef0ef2 100644 --- a/frontend/src/js/standard-query-editor/QueryClearButton.tsx +++ b/frontend/src/js/standard-query-editor/QueryClearButton.tsx @@ -1,4 +1,4 @@ -import { faTrashAlt } from "@fortawesome/free-regular-svg-icons"; +import { faTrash } from "@fortawesome/free-solid-svg-icons"; import { FC } from "react"; import { useTranslation } from "react-i18next"; import { useDispatch } from "react-redux"; @@ -25,7 +25,7 @@ const QueryClearButton: FC = ({ className }) => { onConfirm={onClearQuery} > - +
diff --git a/frontend/src/js/timebased-query-editor/TimebasedQueryClearButton.tsx b/frontend/src/js/timebased-query-editor/TimebasedQueryClearButton.tsx index af1e4e7577..e1b4be4db7 100644 --- a/frontend/src/js/timebased-query-editor/TimebasedQueryClearButton.tsx +++ b/frontend/src/js/timebased-query-editor/TimebasedQueryClearButton.tsx @@ -1,5 +1,5 @@ import styled from "@emotion/styled"; -import { faTrashAlt } from "@fortawesome/free-regular-svg-icons"; +import { faTrash } from "@fortawesome/free-solid-svg-icons"; import { useTranslation } from "react-i18next"; import { useDispatch, useSelector } from "react-redux"; @@ -30,7 +30,7 @@ const TimebasedQueryClearButton = () => { {t("common.clear")} diff --git a/frontend/src/localization/de.json b/frontend/src/localization/de.json index da050656ef..4d429dc251 100644 --- a/frontend/src/localization/de.json +++ b/frontend/src/localization/de.json @@ -524,7 +524,8 @@ "time": "ZEIT", "and": "UND", "or": "ODER", - "clear": "Leeren", + "clear": "Editor vollständig zurücksetzen", + "clearConfirm": "Jetzt zurücksetzen", "flip": "Drehen", "dates": "Datum", "negate": "Nicht", diff --git a/frontend/src/localization/en.json b/frontend/src/localization/en.json index 007499910f..69aed6ea75 100644 --- a/frontend/src/localization/en.json +++ b/frontend/src/localization/en.json @@ -524,7 +524,8 @@ "time": "TIME", "and": "AND", "or": "OR", - "clear": "Clear", + "clear": "Reset editor completely", + "clearConfirm": "Reset now", "flip": "Flip", "dates": "Dates", "negate": "Negate", From fda70d5ab3b35344e9e675f79968ca613de5c8ca Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Wed, 28 Jun 2023 17:02:08 +0200 Subject: [PATCH 420/679] Update palette --- frontend/src/app-theme.ts | 8 ++++---- .../src/js/editor-v2/time-connection/TimeConnection.tsx | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/frontend/src/app-theme.ts b/frontend/src/app-theme.ts index fb8cfc19c4..4435b90328 100644 --- a/frontend/src/app-theme.ts +++ b/frontend/src/app-theme.ts @@ -16,13 +16,13 @@ export const theme: Theme = { green: "#36971C", orange: "#E9711C", palette: [ - "#f9c74f", - "#f8961e", "#277da1", - "#90be6d", "#43aa8b", - "#f94144", "#5e60ce", + "#f9c74f", + "#90be6d", + "#f8961e", + "#f94144", "#aaa", "#777", "#fff", diff --git a/frontend/src/js/editor-v2/time-connection/TimeConnection.tsx b/frontend/src/js/editor-v2/time-connection/TimeConnection.tsx index f1f3f0e3ed..c57748319c 100644 --- a/frontend/src/js/editor-v2/time-connection/TimeConnection.tsx +++ b/frontend/src/js/editor-v2/time-connection/TimeConnection.tsx @@ -21,7 +21,7 @@ const Row = styled("div")` display: flex; align-items: center; gap: 5px; - font-size: ${({ theme }) => theme.font.xs}; + font-size: ${({ theme }) => theme.font.sm}; `; const ConceptName = styled("span")` @@ -30,15 +30,15 @@ const ConceptName = styled("span")` `; const Timestamp = styled("span")` font-weight: bold; - color: ${({ theme }) => theme.col.palette[6]}; + color: ${({ theme }) => theme.col.palette[0]}; `; const Interval = styled("span")` font-weight: bold; - color: ${({ theme }) => theme.col.orange}; + color: ${({ theme }) => theme.col.palette[1]}; `; const Operator = styled("span")` font-weight: bold; - color: ${({ theme }) => theme.col.green}; + color: ${({ theme }) => theme.col.palette[2]}; `; export const TimeConnection = memo( From d6932776db30e58597a89a002042b0f1c10032a2 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Wed, 28 Jun 2023 17:33:58 +0200 Subject: [PATCH 421/679] Fix time stratified concepts chart --- frontend/src/js/editor-v2/EditorV2.tsx | 2 +- .../src/js/entity-history/TabbableTimeStratifiedInfos.tsx | 6 ++++-- .../src/js/entity-history/TimeStratifiedConceptChart.tsx | 7 +++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/frontend/src/js/editor-v2/EditorV2.tsx b/frontend/src/js/editor-v2/EditorV2.tsx index 67765fe649..4eff775dd8 100644 --- a/frontend/src/js/editor-v2/EditorV2.tsx +++ b/frontend/src/js/editor-v2/EditorV2.tsx @@ -432,7 +432,7 @@ export function EditorV2({ )} info.label === activeTab); - if (infoData?.columns.some((c) => !isMoneyColumn(c))) { + if (infoData?.columns.some((c) => isMoneyColumn(c))) { const columns = infoData?.columns.filter(isMoneyColumn); infoData = { @@ -61,7 +61,9 @@ export const TabbableTimeStratifiedInfos = ({ {data && type === "money" && ( )} - {data && } + {data && type === "concept" && ( + + )} ); }; diff --git a/frontend/src/js/entity-history/TimeStratifiedConceptChart.tsx b/frontend/src/js/entity-history/TimeStratifiedConceptChart.tsx index d44b5bd1f0..8fb7a71cd8 100644 --- a/frontend/src/js/entity-history/TimeStratifiedConceptChart.tsx +++ b/frontend/src/js/entity-history/TimeStratifiedConceptChart.tsx @@ -29,6 +29,10 @@ const BubbleNo = styled("div")` background-color: ${({ theme }) => theme.col.grayLight}; `; +const Year = styled("div")` + font-size: ${({ theme }) => theme.font.sm}; +`; + export const TimeStratifiedConceptChart = ({ timeStratifiedInfo, }: { @@ -46,7 +50,6 @@ export const TimeStratifiedConceptChart = ({ if (!conceptSemantic) return null; const years = timeStratifiedInfo.years.map((y) => y.year); - console.log(timeStratifiedInfo.years, conceptColumn); const valuesPerYear = timeStratifiedInfo.years.map((y) => ((y.values[Object.keys(y.values)[0]] as string[]) || []).map( (conceptId) => getConceptById(conceptId, conceptSemantic?.concept)!, @@ -80,7 +83,7 @@ export const TimeStratifiedConceptChart = ({ ))} {years.map((y, i) => ( <> -
{y}
+ {y} {allValues.map((val) => valuesPerYear[i].includes(val) ? : , )} From 1e1d54a9f6cc1f00ef344d7b080063353e9ac5ee Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 29 Jun 2023 13:34:56 +0200 Subject: [PATCH 422/679] log ConqueryError at WARN level --- .../models/query/ExecutionManager.java | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java index ea2e844612..933fe3daab 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java @@ -14,6 +14,7 @@ import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.execution.InternalExecution; import com.bakdata.conquery.models.execution.ManagedExecution; @@ -34,13 +35,6 @@ public class ExecutionManager { private final MetaStorage storage; - private final Cache>> executionResults = - CacheBuilder.newBuilder() - .softValues() - .removalListener(this::executionRemoved) - .build(); - - /** * Manage state of evicted Queries, setting them to NEW. */ @@ -56,7 +50,11 @@ private void executionRemoved(RemovalNotification> r log.warn("Evicted Results for Query[{}] (Reason: {})", executionId, removalNotification.getCause()); storage.getExecution(executionId).reset(); - } + } private final Cache>> executionResults = + CacheBuilder.newBuilder() + .softValues() + .removalListener(this::executionRemoved) + .build(); public ManagedExecution runQuery(Namespace namespace, QueryDescription query, User user, Dataset submittedDataset, ConqueryConfig config, boolean system) { final ManagedExecution execution = createExecution(query, user, submittedDataset, system); @@ -70,14 +68,18 @@ public ManagedExecution createExecution(QueryDescription query, User user, Datas } public void execute(Namespace namespace, ManagedExecution execution, ConqueryConfig config) { - // Initialize the query / create subqueries try { execution.initExecutable(namespace, config); } catch (Exception e) { - log.error("Failed to initialize Query[{}]", execution.getId(), e); + // ConqueryErrors are usually user input errors so no need to log them at level=ERROR + if (e instanceof ConqueryError) { + log.warn("Failed to initialize Query[{}]", execution.getId(), e); + } + else { + log.error("Failed to initialize Query[{}]", execution.getId(), e); + } - //TODO we don't want to store completely faulty queries but is that right like this? storage.removeExecution(execution.getId()); throw e; } @@ -96,7 +98,6 @@ public void execute(Namespace namespace, ManagedExecution execution, ConqueryCon } } - public ManagedExecution createQuery(QueryDescription query, UUID queryId, User user, Dataset submittedDataset, boolean system) { // Transform the submitted query into an initialized execution ManagedExecution managed = query.toManagedExecution(user, submittedDataset, storage); From 026a115b0b462a92e41117fcaf6e6c76b33e2011 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 29 Jun 2023 13:39:41 +0200 Subject: [PATCH 423/679] adds descriptions for optional CQYes populations --- .../conquery/frontend/forms/export_form.frontend_conf.json | 4 ++-- .../frontend/forms/table_export_form.frontend_conf.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/src/main/resources/com/bakdata/conquery/frontend/forms/export_form.frontend_conf.json b/backend/src/main/resources/com/bakdata/conquery/frontend/forms/export_form.frontend_conf.json index 8516063810..cf71a70858 100644 --- a/backend/src/main/resources/com/bakdata/conquery/frontend/forms/export_form.frontend_conf.json +++ b/backend/src/main/resources/com/bakdata/conquery/frontend/forms/export_form.frontend_conf.json @@ -27,8 +27,8 @@ "en": "Cohort (Previous Query)" }, "dropzoneLabel": { - "de": "Füge eine Versichertengruppe aus einer bestehenden Anfrage hinzu", - "en": "Add a cohort from a previous query" + "de": "Füge eine Versichertengruppe aus einer bestehenden Anfrage hinzu. Ist das Feld leer wird die Gesamtpopulation verwendet.", + "en": "Add a cohort from a previous query. When no population is provided, the entire dataset's population is used." }, "validations": [ ], diff --git a/backend/src/main/resources/com/bakdata/conquery/frontend/forms/table_export_form.frontend_conf.json b/backend/src/main/resources/com/bakdata/conquery/frontend/forms/table_export_form.frontend_conf.json index e736367a43..cb51887547 100644 --- a/backend/src/main/resources/com/bakdata/conquery/frontend/forms/table_export_form.frontend_conf.json +++ b/backend/src/main/resources/com/bakdata/conquery/frontend/forms/table_export_form.frontend_conf.json @@ -27,8 +27,8 @@ "en": "Cohort (Previous Query)" }, "dropzoneLabel": { - "de": "Füge eine Versichertengruppe aus einer bestehenden Anfrage hinzu.", - "en": "Add a cohort from a previous query" + "de": "Füge eine Versichertengruppe aus einer bestehenden Anfrage hinzu. Ist das Feld leer wird die Gesamtpopulation verwendet.", + "en": "Add a cohort from a previous query. When no population is provided, the entire dataset's population is used." }, "validations": [ ], From 8c1ad749a8f6b7c04655e045dc8e45f940c715a2 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 29 Jun 2023 14:13:33 +0200 Subject: [PATCH 424/679] use years as observationPeriod --- .../bakdata/conquery/models/config/FrontendConfig.java | 9 ++++----- .../bakdata/conquery/resources/api/ConfigResource.java | 4 +++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index 599b78a222..1bbfcddd56 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -2,12 +2,11 @@ import java.net.URI; import java.net.URL; -import java.time.LocalDate; -import java.time.temporal.ChronoUnit; import javax.annotation.Nullable; import javax.validation.Valid; import javax.validation.constraints.Email; +import javax.validation.constraints.Min; import javax.validation.constraints.NotNull; import com.bakdata.conquery.models.forms.frontendconfiguration.FormScanner; @@ -29,10 +28,10 @@ public class FrontendConfig { private CurrencyConfig currency = new CurrencyConfig(); /** - * Default start-date for EntityPreview and DatePicker. + * Years to include in entity preview. */ - @NotNull - private LocalDate observationStart = LocalDate.now().minus(10, ChronoUnit.YEARS); + @Min(0) + private int observationPeriodYears = 6; /** * The url that points a manual. This is also used by the {@link FormScanner} diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java index 0c2432c762..deeb7e5d21 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConfigResource.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.resources.api; +import java.time.Year; + import javax.inject.Inject; import javax.ws.rs.GET; import javax.ws.rs.Path; @@ -38,7 +40,7 @@ public FrontendConfiguration getFrontendConfig() { idColumns, frontendConfig.getManualUrl(), frontendConfig.getContactEmail(), - frontendConfig.getObservationStart() + Year.now().minusYears(frontendConfig.getObservationPeriodYears()).atDay(1) ); } From eb162241d4d285991e40bb331f2e2890c3cdd8ac Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Mon, 3 Jul 2023 14:56:11 +0200 Subject: [PATCH 425/679] limit SELECT FilterValue size to < 20000 as big values cause memory issues --- .../query/concept/filter/FilterValue.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java index 681899eeed..f3d2a1de35 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java @@ -19,6 +19,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.FilterId; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.queryplan.filter.FilterNode; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonProcessingException; @@ -27,6 +28,7 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.deser.std.StdDeserializer; import com.fasterxml.jackson.databind.exc.InvalidTypeIdException; +import io.dropwizard.validation.ValidationMethod; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.NoArgsConstructor; @@ -44,6 +46,10 @@ @EqualsAndHashCode @ToString(of = "value") public abstract class FilterValue { + /** + * Very large SELECT FilterValues can cause issues, so we just limit it to large but not gigantic quantities. + */ + private static final int MAX_NUMBER_FILTER_VALUES = 20_000; @NotNull @Nonnull @NsIdRef @@ -68,6 +74,12 @@ public static class CQMultiSelectFilter extends FilterValue { public CQMultiSelectFilter(@NsIdRef Filter filter, String[] value) { super(filter, value); } + + @ValidationMethod(message = "Too many values selected.") + @JsonIgnore + public boolean isSaneAmountOfFilterValues() { + return getValue().length < MAX_NUMBER_FILTER_VALUES; + } } @NoArgsConstructor @@ -77,6 +89,12 @@ public static class CQBigMultiSelectFilter extends FilterValue { public CQBigMultiSelectFilter(@NsIdRef Filter filter, String[] value) { super(filter, value); } + + @ValidationMethod(message = "Too many values selected.") + @JsonIgnore + public boolean isSaneAmountOfFilterValues() { + return getValue().length < MAX_NUMBER_FILTER_VALUES; + } } @NoArgsConstructor From a5a404523933f4c1a398c955d465a17a9f07317d Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 4 Jul 2023 10:18:29 +0200 Subject: [PATCH 426/679] use workStealingPool to read SerializingStore --- .../xodus/stores/SerializingStore.java | 92 +++++++++++-------- 1 file changed, 54 insertions(+), 38 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index 6cf3fcddbd..d0b18f34ae 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -7,6 +7,9 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collection; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.function.Supplier; @@ -28,6 +31,7 @@ import jetbrains.exodus.ByteIterable; import lombok.Data; import lombok.NonNull; +import lombok.SneakyThrows; import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; @@ -180,52 +184,64 @@ public VALUE get(KEY key) { * Depending on the {@link XodusStoreFactory} corrupt entries may be dump to a file and/or removed from the store. * These entries are not submitted to the consumer. */ + @SneakyThrows @Override public IterationStatistic forEach(StoreEntryConsumer consumer) { final IterationStatistic result = new IterationStatistic(); final ArrayList unreadables = new ArrayList<>(); + final ExecutorService executorService = Executors.newWorkStealingPool(10); + store.forEach((k, v) -> { - result.incrTotalProcessed(); - - // Try to read the key first - final KEY key = getDeserializedAndDumpFailed( - k, - this::readKey, - () -> new String(k.getBytesUnsafe()), - v, - "Could not parse key [{}]" - ); - if (key == null) { - unreadables.add(k); - result.incrFailedKeys(); - return; - } + executorService.submit(() -> { + + result.incrTotalProcessed(); + + // Try to read the key first + final KEY key = getDeserializedAndDumpFailed( + k, + this::readKey, + () -> new String(k.getBytesUnsafe()), + v, + "Could not parse key [{}]" + ); + if (key == null) { + unreadables.add(k); + result.incrFailedKeys(); + return; + } + + // Try to read the value + final VALUE value = getDeserializedAndDumpFailed( + v, + this::readValue, + key::toString, + v, + "Could not parse value for key [{}]" + ); + + if (value == null) { + unreadables.add(k); + result.incrFailedValues(); + return; + } + + // Apply the consumer to key and value + try { + consumer.accept(key, value, v.getLength()); + } + catch (Exception e) { + log.warn("Unable to apply for-each consumer on key[{}]", key, e); + } + }); + }); - // Try to read the value - final VALUE value = getDeserializedAndDumpFailed( - v, - this::readValue, - key::toString, - v, - "Could not parse value for key [{}]" - ); - - if (value == null) { - unreadables.add(k); - result.incrFailedValues(); - return; - } + executorService.shutdown(); - // Apply the consumer to key and value - try { - consumer.accept(key, value, v.getLength()); - } - catch (Exception e) { - log.warn("Unable to apply for-each consumer on key[{}]", key, e); - } + while (executorService.awaitTermination(1, TimeUnit.MINUTES)){ + log.debug("Still waiting for {} to load.", this); + } - }); // Print some statistics final int total = result.getTotalProcessed(); log.debug( @@ -254,7 +270,7 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { * @param deserializer The concrete deserializer to use. * @param onFailKeyStringSupplier When deserilization failed and dump is enabled this is used in the dump file name. * @param onFailOrigValue Will be the dumpfile content rendered as a json. - * @param onFailWarnMsgFmt The warn message that will be logged on failure. + * @param onFailWarnMsgFmt The warning message that will be logged on failure. * @return The deserialized value */ private TYPE getDeserializedAndDumpFailed(ByteIterable serial, Function deserializer, Supplier onFailKeyStringSupplier, ByteIterable onFailOrigValue, String onFailWarnMsgFmt) { From cd6dc5b2e80b9477486edebcf5542f5ad1b242cc Mon Sep 17 00:00:00 2001 From: Torben Meyer Date: Tue, 4 Jul 2023 10:35:13 +0200 Subject: [PATCH 427/679] Decouple distributed execution (#3059) * Decouple distributed execution This PR introduces a decoupling of the manager and the distributed execution. This is done mainly by introducing interfaces for the following classes and moving everything related to remote calls into a separate implementation: - `Namespace` - `DatasetRegistry` - `AdminDatasetProcessor` Whenever an existing class requires access to the remote calls, it either has a hard coupling to the distributed implementation of the interface or there is an interface or method that can safely result in a no-op (e.g., `getShardInformation` returns an empty list). * Alow serialization of SqlManagedQuery --- .../java/com/bakdata/conquery/Conquery.java | 18 +- .../bakdata/conquery/apiv1/MeProcessor.java | 3 +- .../conquery/apiv1/QueryProcessor.java | 8 +- .../conquery/commands/ManagerNode.java | 173 ++--------- .../conquery/commands/StandaloneCommand.java | 10 +- .../conquery/io/storage/MetaStorage.java | 3 +- .../conquery/mode/DelegateManager.java | 48 +++ .../bakdata/conquery/mode/ImportHandler.java | 19 ++ .../mode/InternalObjectMapperCreator.java | 68 +++++ .../com/bakdata/conquery/mode/Manager.java | 31 ++ .../conquery/mode/ManagerProvider.java | 42 +++ .../conquery/mode/NamespaceHandler.java | 54 ++++ .../conquery/mode/NamespaceSetupData.java | 23 ++ .../conquery/mode/StorageListener.java | 27 ++ .../cluster/ClusterConnectionManager.java | 122 ++++++++ .../mode/cluster/ClusterImportHandler.java | 93 ++++++ .../conquery/mode/cluster/ClusterManager.java | 33 ++ .../mode/cluster/ClusterManagerProvider.java | 56 ++++ .../mode/cluster/ClusterNamespaceHandler.java | 56 ++++ .../conquery/mode/cluster/ClusterState.java | 35 +++ .../mode/cluster/ClusterStorageListener.java | 79 +++++ .../mode/local/FailingImportHandler.java | 29 ++ .../mode/local/LocalManagerProvider.java | 40 +++ .../mode/local/LocalNamespaceHandler.java | 42 +++ .../mode/local/LocalStorageListener.java | 40 +++ .../models/auth/AuthorizationHelper.java | 3 +- .../models/config/ConqueryConfig.java | 5 + .../models/config/SqlConnectorConfig.java | 9 + .../models/datasets/PreviewConfig.java | 3 +- .../conquery/models/jobs/ImportJob.java | 32 +- .../messages/namespaces/NamespaceMessage.java | 4 +- .../specific/CollectQueryResult.java | 4 +- .../specific/ReportConsistency.java | 6 +- .../specific/UpdateElementMatchingStats.java | 4 +- .../network/NetworkMessageContext.java | 10 +- .../network/specific/AddShardNode.java | 2 +- .../network/specific/ForwardToNamespace.java | 4 +- .../network/specific/RegisterWorker.java | 6 +- .../network/specific/RemoveShardNode.java | 2 +- .../specific/UpdateJobManagerStatus.java | 5 +- .../query/DistributedExecutionManager.java | 184 ++++++++++++ .../models/query/ExecutionManager.java | 147 +-------- .../conquery/models/query/ManagedQuery.java | 90 +----- .../conquery/models/query/entity/Entity.java | 2 +- .../models/worker/DatasetRegistry.java | 98 ++---- .../models/worker/DistributedNamespace.java | 39 +++ .../models/worker/LocalNamespace.java | 141 +++++++++ .../conquery/models/worker/Namespace.java | 284 ++---------------- .../conquery/models/worker/WorkerHandler.java | 155 ++++++++++ .../resources/admin/AdminServlet.java | 8 +- .../admin/rest/AdminDatasetProcessor.java | 90 ++---- .../admin/rest/AdminDatasetsResource.java | 3 +- .../resources/admin/rest/AdminProcessor.java | 6 +- .../resources/admin/rest/AdminResource.java | 7 +- .../resources/admin/rest/UIProcessor.java | 5 +- .../resources/admin/ui/model/UIContext.java | 29 +- .../resources/api/ConceptsProcessor.java | 2 +- .../sql/conquery/SqlExecutionManager.java | 45 +++ .../sql/conquery/SqlManagedQuery.java | 78 +++++ .../conquery/sql/conquery/package-info.java | 4 + .../conquery/tasks/ReportConsistencyTask.java | 12 +- .../com/bakdata/conquery/util/QueryUtils.java | 87 ++++++ .../resources/admin/ui/index.html.ftl | 4 +- .../admin/ui/templates/template.html.ftl | 4 +- .../api/StoredQueriesProcessorTest.java | 4 +- .../api/form/config/FormConfigTest.java | 3 +- .../conquery/integration/IntegrationTest.java | 2 +- .../integration/json/JsonIntegrationTest.java | 3 +- .../integration/tests/AdminEndpointTest.java | 2 +- .../tests/AdminUIEndpointTest.java | 2 +- .../tests/ConceptPermissionTest.java | 4 +- .../tests/ExternalFormBackendTest.java | 4 +- .../tests/MetadataCollectionTest.java | 12 +- .../integration/tests/RestartTest.java | 5 +- .../io/AbstractSerializationTest.java | 14 +- .../io/jackson/serializer/IdRefrenceTest.java | 3 +- .../models/execution/DefaultLabelTest.java | 3 +- .../models/query/DefaultColumnNameTest.java | 3 +- .../util/support/StandaloneSupport.java | 14 +- .../conquery/util/support/TestConquery.java | 33 +- 80 files changed, 1978 insertions(+), 908 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/DelegateManager.java create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/ImportHandler.java create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/Manager.java create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/NamespaceSetupData.java create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/StorageListener.java create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionManager.java create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterImportHandler.java create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManager.java create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterState.java create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterStorageListener.java create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/local/FailingImportHandler.java create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java create mode 100644 backend/src/main/java/com/bakdata/conquery/mode/local/LocalStorageListener.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/query/DistributedExecutionManager.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java create mode 100644 backend/src/main/java/com/bakdata/conquery/models/worker/WorkerHandler.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlManagedQuery.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conquery/package-info.java diff --git a/backend/src/main/java/com/bakdata/conquery/Conquery.java b/backend/src/main/java/com/bakdata/conquery/Conquery.java index 1f681a3d22..2ddccdb5f3 100644 --- a/backend/src/main/java/com/bakdata/conquery/Conquery.java +++ b/backend/src/main/java/com/bakdata/conquery/Conquery.java @@ -12,6 +12,10 @@ import com.bakdata.conquery.commands.StandaloneCommand; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.MutableInjectableValues; +import com.bakdata.conquery.mode.Manager; +import com.bakdata.conquery.mode.ManagerProvider; +import com.bakdata.conquery.mode.cluster.ClusterManagerProvider; +import com.bakdata.conquery.mode.local.LocalManagerProvider; import com.bakdata.conquery.models.config.ConqueryConfig; import com.fasterxml.jackson.databind.ObjectMapper; import io.dropwizard.Application; @@ -34,7 +38,7 @@ public class Conquery extends Application { private final String name; @Setter - private ManagerNode manager; + private ManagerNode managerNode; public Conquery() { this("Conquery"); @@ -90,10 +94,16 @@ protected Level bootstrapLogLevel() { @Override public void run(ConqueryConfig configuration, Environment environment) throws Exception { - if (manager == null) { - manager = new ManagerNode(); + ManagerProvider provider = configuration.getSqlConnectorConfig().isEnabled() ? + new LocalManagerProvider() : new ClusterManagerProvider(); + run(provider.provideManager(configuration, environment)); + } + + public void run(Manager manager) throws InterruptedException { + if (managerNode == null) { + managerNode = new ManagerNode(); } - manager.run(configuration, environment); + managerNode.run(manager); } public static void main(String... args) throws Exception { diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/MeProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/MeProcessor.java index 1c631e0ccc..43ccea532a 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/MeProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/MeProcessor.java @@ -15,6 +15,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.resources.api.MeResource; import lombok.AllArgsConstructor; import lombok.Builder; @@ -34,7 +35,7 @@ public class MeProcessor { @Inject private MetaStorage storage; @Inject - private DatasetRegistry datasetRegistry; + private DatasetRegistry datasetRegistry; /** * Generates a summary of a user. It contains its name, the groups it belongs to and its permissions on a dataset. diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index 4a9e91d0f9..af35a0955a 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -60,7 +60,6 @@ import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; -import com.bakdata.conquery.models.messages.namespaces.specific.CancelQuery; import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.SingleTableResult; @@ -86,7 +85,7 @@ public class QueryProcessor { @Inject - private DatasetRegistry datasetRegistry; + private DatasetRegistry datasetRegistry; @Inject private MetaStorage storage; @Inject @@ -305,11 +304,10 @@ public void cancel(Subject subject, Dataset dataset, ManagedExecution query) { log.info("User[{}] cancelled Query[{}]", subject.getId(), query.getId()); - final Namespace namespace = datasetRegistry.get(dataset.getId()); - query.reset(); - namespace.sendToAll(new CancelQuery(query.getId())); + final ExecutionManager executionManager = datasetRegistry.get(dataset.getId()).getExecutionManager(); + executionManager.cancelQuery(dataset, query); } public void patchQuery(Subject subject, ManagedExecution execution, MetaDataPatch patch) { diff --git a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java index 7d7d2176ef..8baff15832 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/ManagerNode.java @@ -1,6 +1,5 @@ package com.bakdata.conquery.commands; -import java.net.InetSocketAddress; import java.time.Duration; import java.util.ArrayList; import java.util.Collection; @@ -14,30 +13,17 @@ import javax.ws.rs.client.Client; import com.bakdata.conquery.io.cps.CPSTypeIdResolver; -import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.jackson.PathParamInjector; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.jersey.RESTServer; -import com.bakdata.conquery.io.mina.BinaryJacksonCoder; -import com.bakdata.conquery.io.mina.CQProtocolCodecFilter; -import com.bakdata.conquery.io.mina.ChunkReader; -import com.bakdata.conquery.io.mina.ChunkWriter; -import com.bakdata.conquery.io.mina.MinaAttributes; -import com.bakdata.conquery.io.mina.NetworkSession; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.NamespaceStorage; +import com.bakdata.conquery.mode.Manager; import com.bakdata.conquery.models.auth.AuthorizationController; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.forms.frontendconfiguration.FormScanner; import com.bakdata.conquery.models.i18n.I18n; -import com.bakdata.conquery.models.jobs.Job; -import com.bakdata.conquery.models.jobs.JobManager; -import com.bakdata.conquery.models.jobs.ReactingJob; -import com.bakdata.conquery.models.messages.SlowMessage; -import com.bakdata.conquery.models.messages.namespaces.specific.ShutdownShard; -import com.bakdata.conquery.models.messages.network.MessageToManagerNode; -import com.bakdata.conquery.models.messages.network.NetworkMessageContext; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Worker; import com.bakdata.conquery.resources.ResourcesProvider; @@ -47,8 +33,6 @@ import com.bakdata.conquery.tasks.PermissionCleanupTask; import com.bakdata.conquery.tasks.QueryCleanupTask; import com.bakdata.conquery.tasks.ReloadMetaStorageTask; -import com.bakdata.conquery.tasks.ReportConsistencyTask; -import com.bakdata.conquery.util.io.ConqueryMDC; import com.fasterxml.jackson.databind.DeserializationConfig; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationConfig; @@ -60,11 +44,9 @@ import lombok.Getter; import lombok.NonNull; import lombok.SneakyThrows; +import lombok.experimental.Delegate; import lombok.extern.slf4j.Slf4j; -import org.apache.mina.core.service.IoAcceptor; import org.apache.mina.core.service.IoHandlerAdapter; -import org.apache.mina.core.session.IoSession; -import org.apache.mina.transport.socket.nio.NioSocketAcceptor; import org.glassfish.jersey.internal.inject.AbstractBinder; /** @@ -80,18 +62,14 @@ public class ManagerNode extends IoHandlerAdapter implements Managed { private final String name; - private IoAcceptor acceptor; - private MetaStorage storage; - private JobManager jobManager; private Validator validator; - private ConqueryConfig config; private AdminServlet admin; private AuthorizationController authController; private ScheduledExecutorService maintenanceService; - private DatasetRegistry datasetRegistry; - private Environment environment; private final List providers = new ArrayList<>(); private Client client; + @Delegate(excludes = Managed.class) + private Manager manager; // Resources without authentication private DropwizardResourceConfig unprotectedAuthApi; @@ -108,27 +86,20 @@ public ManagerNode(@NonNull String name) { this.name = name; } - public void run(ConqueryConfig config, Environment environment) throws InterruptedException { - this.environment = environment; - this.config = config; + public void run(Manager manager) throws InterruptedException { + Environment environment = manager.getEnvironment(); + ConqueryConfig config = manager.getConfig(); validator = environment.getValidator(); client = new JerseyClientBuilder(environment).using(config.getJerseyClient()) .build(getName()); - // Instantiate DatasetRegistry and MetaStorage, so they are ready for injection into the object mapper (API + Storage) - // The validator is already injected at this point see Conquery.java - datasetRegistry = new DatasetRegistry(config.getCluster().getEntityBucketSize(), config, this::createInternalObjectMapper); - storage = new MetaStorage(config.getStorage(), datasetRegistry); - datasetRegistry.setMetaStorage(storage); - + this.manager = manager; final ObjectMapper objectMapper = environment.getObjectMapper(); customizeApiObjectMapper(objectMapper); - jobManager = new JobManager("ManagerNode", config.isFailOnError()); - // FormScanner needs to be instantiated before plugins are initialized formScanner = new FormScanner(config); @@ -152,7 +123,7 @@ public void run(ConqueryConfig config, Environment environment) throws Interrupt loadMetaStorage(); - authController = new AuthorizationController(storage, config.getAuthorizationRealms()); + authController = new AuthorizationController(getStorage(), config.getAuthorizationRealms()); environment.lifecycle().manage(authController); unprotectedAuthAdmin = AuthServlet.generalSetup(environment.metrics(), config, environment.admin(), objectMapper); @@ -189,13 +160,13 @@ public void run(ConqueryConfig config, Environment environment) throws Interrupt environment.admin().addTask(formScanner); environment.admin().addTask( - new QueryCleanupTask(storage, Duration.of( + new QueryCleanupTask(getStorage(), Duration.of( config.getQueries().getOldQueriesTime().getQuantity(), config.getQueries().getOldQueriesTime().getUnit().toChronoUnit() ))); - environment.admin().addTask(new PermissionCleanupTask(storage)); - environment.admin().addTask(new ReportConsistencyTask(datasetRegistry)); - environment.admin().addTask(new ReloadMetaStorageTask(storage)); + environment.admin().addTask(new PermissionCleanupTask(getStorage())); + manager.getAdminTasks().forEach(environment.admin()::addTask); + environment.admin().addTask(new ReloadMetaStorageTask(getStorage())); final ShutdownTask shutdown = new ShutdownTask(); environment.admin().addTask(shutdown); @@ -207,8 +178,8 @@ private void configureApiServlet(ConqueryConfig config, DropwizardResourceConfig jerseyConfig.register(new AbstractBinder() { @Override protected void configure() { - bind(storage).to(MetaStorage.class); - bind(datasetRegistry).to(DatasetRegistry.class); + bind(getStorage()).to(MetaStorage.class); + bind(getDatasetRegistry()).to(DatasetRegistry.class); } }); @@ -257,42 +228,15 @@ public void customizeApiObjectMapper(ObjectMapper objectMapper) { * @see ManagerNode#customizeApiObjectMapper(ObjectMapper) */ public ObjectMapper createInternalObjectMapper(Class viewClass) { - final ObjectMapper objectMapper = getConfig().configureObjectMapper(Jackson.BINARY_MAPPER.copy()); - - - final MutableInjectableValues injectableValues = new MutableInjectableValues(); - objectMapper.setInjectableValues(injectableValues); - injectableValues.add(Validator.class, getValidator()); - getDatasetRegistry().injectInto(objectMapper); - getStorage().injectInto(objectMapper); - getConfig().injectInto(objectMapper); - - - if (viewClass != null) { - // Set serialization config - SerializationConfig serializationConfig = objectMapper.getSerializationConfig(); - - serializationConfig = serializationConfig.withView(viewClass); - - objectMapper.setConfig(serializationConfig); - - // Set deserialization config - DeserializationConfig deserializationConfig = objectMapper.getDeserializationConfig(); - - deserializationConfig = deserializationConfig.withView(viewClass); - - objectMapper.setConfig(deserializationConfig); - } - - return objectMapper; + return getInternalObjectMapperCreator().createInternalObjectMapper(viewClass); } private void loadMetaStorage() { log.info("Opening MetaStorage"); - storage.openStores(createInternalObjectMapper(View.Persistence.Manager.class)); + getStorage().openStores(getInternalObjectMapperCreator().createInternalObjectMapper(View.Persistence.Manager.class)); log.info("Loading MetaStorage"); - storage.loadData(); - log.info("MetaStorage loaded {}", storage); + getStorage().loadData(); + log.info("MetaStorage loaded {}", getStorage()); } @SneakyThrows(InterruptedException.class) @@ -302,94 +246,30 @@ public void loadNamespaces() { ExecutorService loaders = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); // Namespaces load their storage themselves, so they can inject Namespace relevant objects into stored objects - final Collection namespaceStorages = config.getStorage().discoverNamespaceStorages(); + final Collection namespaceStorages = getConfig().getStorage().discoverNamespaceStorages(); for (NamespaceStorage namespaceStorage : namespaceStorages) { loaders.submit(() -> { - datasetRegistry.createNamespace(namespaceStorage); + getDatasetRegistry().createNamespace(namespaceStorage); }); } loaders.shutdown(); while (!loaders.awaitTermination(1, TimeUnit.MINUTES)) { - final int coundLoaded = datasetRegistry.getDatasets().size(); + final int coundLoaded = getDatasetRegistry().getDatasets().size(); log.debug("Waiting for Worker namespaces to load. {} are already finished. {} pending.", coundLoaded, namespaceStorages.size() - coundLoaded); } } - @Override - public void sessionOpened(IoSession session) { - ConqueryMDC.setLocation("ManagerNode[" + session.getLocalAddress().toString() + "]"); - log.info("New client {} connected, waiting for identity", session.getRemoteAddress()); - } - - @Override - public void sessionClosed(IoSession session) { - ConqueryMDC.setLocation("ManagerNode[" + session.getLocalAddress().toString() + "]"); - log.info("Client '{}' disconnected ", session.getAttribute(MinaAttributes.IDENTIFIER)); - } - - @Override - public void exceptionCaught(IoSession session, Throwable cause) { - ConqueryMDC.setLocation("ManagerNode[" + session.getLocalAddress().toString() + "]"); - log.error("caught exception", cause); - } - - @Override - public void messageReceived(IoSession session, Object message) { - ConqueryMDC.setLocation("ManagerNode[" + session.getLocalAddress().toString() + "]"); - if (message instanceof MessageToManagerNode toManagerNode) { - - log.trace("ManagerNode received {} from {}", message.getClass().getSimpleName(), session.getRemoteAddress()); - - Job job = new ReactingJob<>(toManagerNode, new NetworkMessageContext.ManagerNodeNetworkContext( - new NetworkSession(session), - datasetRegistry, config.getCluster().getBackpressure() - )); - - if (toManagerNode instanceof SlowMessage slowMessage) { - slowMessage.setProgressReporter(job.getProgressReporter()); - jobManager.addSlowJob(job); - } - else { - jobManager.addFastJob(job); - } - } - else { - log.error("Unknown message type {} in {}", message.getClass(), message); - } - } - @Override public void start() throws Exception { - acceptor = new NioSocketAcceptor(); - - ObjectMapper om = createInternalObjectMapper(View.InternalCommunication.class); - config.configureObjectMapper(om); - BinaryJacksonCoder coder = new BinaryJacksonCoder(datasetRegistry, validator, om); - acceptor.getFilterChain().addLast("codec", new CQProtocolCodecFilter(new ChunkWriter(coder), new ChunkReader(coder, om))); - acceptor.setHandler(this); - acceptor.getSessionConfig().setAll(config.getCluster().getMina()); - acceptor.bind(new InetSocketAddress(config.getCluster().getPort())); - log.info("Started ManagerNode @ {}", acceptor.getLocalAddress()); + manager.start(); } @Override public void stop() throws Exception { - datasetRegistry.getShardNodes().forEach(((socketAddress, shardNodeInformation) -> shardNodeInformation.send(new ShutdownShard()))); - - jobManager.close(); - - datasetRegistry.close(); - - try { - acceptor.dispose(); - } - catch (Exception e) { - log.error(acceptor + " could not be closed", e); - } - + manager.stop(); for (ResourcesProvider provider : providers) { try { provider.close(); @@ -399,11 +279,12 @@ public void stop() throws Exception { } } + try { - storage.close(); + getStorage().close(); } catch (Exception e) { - log.error(storage + " could not be closed", e); + log.error("{} could not be closed", getStorage(), e); } client.close(); diff --git a/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java index 54683dd211..9692a2a79d 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/StandaloneCommand.java @@ -11,6 +11,8 @@ import java.util.concurrent.TimeUnit; import com.bakdata.conquery.Conquery; +import com.bakdata.conquery.mode.cluster.ClusterManager; +import com.bakdata.conquery.mode.cluster.ClusterManagerProvider; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.XodusStoreFactory; import com.bakdata.conquery.util.io.ConqueryMDC; @@ -26,7 +28,8 @@ public class StandaloneCommand extends io.dropwizard.cli.ServerCommand { private final Conquery conquery; - private ManagerNode manager = new ManagerNode(); + private ClusterManager manager; + private ManagerNode managerNode = new ManagerNode(); private final List shardNodes = new Vector<>(); // TODO clean up the command structure, so we can use the Environment from EnvironmentCommand @@ -68,9 +71,10 @@ protected void startStandalone(Environment environment, Namespace namespace, Con managerConfig = config.withStorage(((XodusStoreFactory) config.getStorage()).withDirectory(managerDir)); } + manager = new ClusterManagerProvider().provideManager(managerConfig, environment); - conquery.setManager(manager); - conquery.run(managerConfig, environment); + conquery.setManagerNode(managerNode); + conquery.run(manager); //create thread pool to start multiple ShardNodes at the same time ExecutorService starterPool = Executors.newFixedThreadPool( diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java index ee294b92d8..026a549572 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/MetaStorage.java @@ -18,6 +18,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.RoleId; import com.bakdata.conquery.models.identifiable.ids.specific.UserId; import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.Namespace; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; @@ -35,7 +36,7 @@ public class MetaStorage extends ConqueryStorage implements Injectable { private final StoreFactory storageFactory; @Getter - protected final DatasetRegistry datasetRegistry; + protected final DatasetRegistry datasetRegistry; private IdentifiableStore executions; private IdentifiableStore formConfigs; private IdentifiableStore authUser; diff --git a/backend/src/main/java/com/bakdata/conquery/mode/DelegateManager.java b/backend/src/main/java/com/bakdata/conquery/mode/DelegateManager.java new file mode 100644 index 0000000000..b4bf4ef7cd --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/DelegateManager.java @@ -0,0 +1,48 @@ +package com.bakdata.conquery.mode; + +import java.util.Collection; +import java.util.List; +import java.util.function.Supplier; + +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.jobs.JobManager; +import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.models.worker.ShardNodeInformation; +import io.dropwizard.servlets.tasks.Task; +import io.dropwizard.setup.Environment; +import lombok.Value; + +/** + * Generic manager that contains shared data. + * + * @param type of the namespace + */ +@Value +public class DelegateManager implements Manager { + ConqueryConfig config; + Environment environment; + DatasetRegistry datasetRegistry; + ImportHandler importHandler; + StorageListener storageListener; + Supplier> nodeProvider; + List adminTasks; + InternalObjectMapperCreator internalObjectMapperCreator; + JobManager jobManager; + + @Override + public void start() throws Exception { + } + + @Override + public void stop() throws Exception { + jobManager.close(); + datasetRegistry.close(); + } + + @Override + public MetaStorage getStorage() { + return datasetRegistry.getMetaStorage(); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/ImportHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/ImportHandler.java new file mode 100644 index 0000000000..b91e335596 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/ImportHandler.java @@ -0,0 +1,19 @@ +package com.bakdata.conquery.mode; + +import java.io.InputStream; + +import com.bakdata.conquery.models.datasets.Import; +import com.bakdata.conquery.models.worker.Namespace; + +/** + * Handler of {@link Import} requests. + */ +public interface ImportHandler { + + void updateImport(Namespace namespace, InputStream inputStream); + + void addImport(Namespace namespace, InputStream inputStream); + + void deleteImport(Import imp); + +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java b/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java new file mode 100644 index 0000000000..3706f728a0 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/InternalObjectMapperCreator.java @@ -0,0 +1,68 @@ +package com.bakdata.conquery.mode; + +import javax.annotation.Nullable; +import javax.validation.Validator; + +import com.bakdata.conquery.io.jackson.Jackson; +import com.bakdata.conquery.io.jackson.MutableInjectableValues; +import com.bakdata.conquery.io.jackson.View; +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.Namespace; +import com.fasterxml.jackson.databind.DeserializationConfig; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationConfig; +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +/** + * Creator for internal object mapper in the manager. + */ +@Getter +@RequiredArgsConstructor +public class InternalObjectMapperCreator { + private final ConqueryConfig config; + private final Validator validator; + private DatasetRegistry datasetRegistry = null; + private MetaStorage storage = null; + + public void init(DatasetRegistry datasetRegistry) { + this.datasetRegistry = datasetRegistry; + this.storage = datasetRegistry.getMetaStorage(); + } + + public ObjectMapper createInternalObjectMapper(@Nullable Class viewClass) { + if (datasetRegistry == null || storage == null) { + throw new IllegalStateException("%s must be initialized by calling its init method".formatted(this.getClass().getSimpleName())); + } + + final ObjectMapper objectMapper = getConfig().configureObjectMapper(Jackson.BINARY_MAPPER.copy()); + + final MutableInjectableValues injectableValues = new MutableInjectableValues(); + objectMapper.setInjectableValues(injectableValues); + injectableValues.add(Validator.class, getValidator()); + getDatasetRegistry().injectInto(objectMapper); + getStorage().injectInto(objectMapper); + getConfig().injectInto(objectMapper); + + + if (viewClass != null) { + // Set serialization config + SerializationConfig serializationConfig = objectMapper.getSerializationConfig(); + + serializationConfig = serializationConfig.withView(viewClass); + + objectMapper.setConfig(serializationConfig); + + // Set deserialization config + DeserializationConfig deserializationConfig = objectMapper.getDeserializationConfig(); + + deserializationConfig = deserializationConfig.withView(viewClass); + + objectMapper.setConfig(deserializationConfig); + } + + return objectMapper; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/Manager.java b/backend/src/main/java/com/bakdata/conquery/mode/Manager.java new file mode 100644 index 0000000000..d2cc6fee79 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/Manager.java @@ -0,0 +1,31 @@ +package com.bakdata.conquery.mode; + +import java.util.Collection; +import java.util.List; +import java.util.function.Supplier; + +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.jobs.JobManager; +import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.models.worker.ShardNodeInformation; +import io.dropwizard.lifecycle.Managed; +import io.dropwizard.servlets.tasks.Task; +import io.dropwizard.setup.Environment; + +/** + * A manager provides the implementations that differ by running mode. + */ +public interface Manager extends Managed { + ConqueryConfig getConfig(); + Environment getEnvironment(); + DatasetRegistry getDatasetRegistry(); + ImportHandler getImportHandler(); + StorageListener getStorageListener(); + Supplier> getNodeProvider(); + List getAdminTasks(); + InternalObjectMapperCreator getInternalObjectMapperCreator(); + JobManager getJobManager(); + MetaStorage getStorage(); +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java new file mode 100644 index 0000000000..f0566764c5 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/ManagerProvider.java @@ -0,0 +1,42 @@ +package com.bakdata.conquery.mode; + +import javax.validation.Validator; + +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.jobs.JobManager; +import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.Namespace; +import io.dropwizard.setup.Environment; + +/** + * Provider for {@link Manager}. + */ +public interface ManagerProvider { + + String JOB_MANAGER_NAME = "ManagerNode"; + + Manager provideManager(ConqueryConfig config, Environment environment); + + static JobManager newJobManager(ConqueryConfig config) { + return new JobManager(JOB_MANAGER_NAME, config.isFailOnError()); + } + + static InternalObjectMapperCreator newInternalObjectMapperCreator(ConqueryConfig config, Validator validator) { + return new InternalObjectMapperCreator(config, validator); + } + + static DatasetRegistry createDatasetRegistry(NamespaceHandler namespaceHandler, ConqueryConfig config, + InternalObjectMapperCreator creator) { + DatasetRegistry datasetRegistry = new DatasetRegistry<>( + config.getCluster().getEntityBucketSize(), + config, + creator, + namespaceHandler + ); + MetaStorage storage = new MetaStorage(config.getStorage(), datasetRegistry); + datasetRegistry.setMetaStorage(storage); + return datasetRegistry; + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java new file mode 100644 index 0000000000..b6087f809f --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceHandler.java @@ -0,0 +1,54 @@ +package com.bakdata.conquery.mode; + +import java.util.ArrayList; +import java.util.List; + +import com.bakdata.conquery.io.jackson.Injectable; +import com.bakdata.conquery.io.jackson.View; +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.io.storage.NamespaceStorage; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; +import com.bakdata.conquery.models.index.IndexService; +import com.bakdata.conquery.models.jobs.JobManager; +import com.bakdata.conquery.models.query.FilterSearch; +import com.bakdata.conquery.models.worker.Namespace; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Handler of namespaces in a ConQuery instance. + * + * @param type of the namespace. + */ +public interface NamespaceHandler { + + N createNamespace(NamespaceStorage storage, MetaStorage metaStorage); + + void removeNamespace(DatasetId id, N namespace); + + /** + * Creates the {@link NamespaceSetupData} that is shared by all {@link Namespace} types. + */ + static NamespaceSetupData createNamespaceSetup(NamespaceStorage storage, final ConqueryConfig config, final InternalObjectMapperCreator mapperCreator) { + List injectables = new ArrayList<>(); + final IndexService indexService = new IndexService(config.getCsv().createCsvParserSettings()); + injectables.add(indexService); + ObjectMapper persistenceMapper = mapperCreator.createInternalObjectMapper(View.Persistence.Manager.class); + ObjectMapper communicationMapper = mapperCreator.createInternalObjectMapper(View.InternalCommunication.class); + ObjectMapper preprocessMapper = mapperCreator.createInternalObjectMapper(null); + + injectables.forEach(i -> i.injectInto(persistenceMapper)); + injectables.forEach(i -> i.injectInto(communicationMapper)); + injectables.forEach(i -> i.injectInto(preprocessMapper)); + + // Open and load the stores + storage.openStores(persistenceMapper); + storage.loadData(); + + JobManager jobManager = new JobManager(storage.getDataset().getName(), config.isFailOnError()); + + FilterSearch filterSearch = new FilterSearch(storage, jobManager, config.getCsv(), config.getIndex()); + return new NamespaceSetupData(injectables, indexService, communicationMapper, preprocessMapper, jobManager, filterSearch); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/NamespaceSetupData.java b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceSetupData.java new file mode 100644 index 0000000000..779205b73d --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/NamespaceSetupData.java @@ -0,0 +1,23 @@ +package com.bakdata.conquery.mode; + +import java.util.List; + +import com.bakdata.conquery.io.jackson.Injectable; +import com.bakdata.conquery.models.index.IndexService; +import com.bakdata.conquery.models.jobs.JobManager; +import com.bakdata.conquery.models.query.FilterSearch; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.Value; + +/** + * Data required for the set-up of a namespace. + */ +@Value +public class NamespaceSetupData { + List injectables; + IndexService indexService; + ObjectMapper communicationMapper; + ObjectMapper preprocessMapper; + JobManager jobManager; + FilterSearch filterSearch; +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/StorageListener.java b/backend/src/main/java/com/bakdata/conquery/mode/StorageListener.java new file mode 100644 index 0000000000..ecb7a982c3 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/StorageListener.java @@ -0,0 +1,27 @@ +package com.bakdata.conquery.mode; + +import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.datasets.SecondaryIdDescription; +import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.Concept; + +/** + * Listener for updates of stored entities in ConQuery. + */ +public interface StorageListener { + + void onAddSecondaryId(SecondaryIdDescription secondaryId); + + void onDeleteSecondaryId(SecondaryIdDescription description); + + void onAddTable(Table table); + + void onRemoveTable(Table table); + + void onAddConcept(Concept concept); + + void onDeleteConcept(Concept concept); + + void onUpdateMatchingStats(final Dataset dataset); + +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionManager.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionManager.java new file mode 100644 index 0000000000..b9101d6b62 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterConnectionManager.java @@ -0,0 +1,122 @@ +package com.bakdata.conquery.mode.cluster; + +import java.io.IOException; +import java.net.InetSocketAddress; + +import javax.validation.Validator; + +import com.bakdata.conquery.io.jackson.View; +import com.bakdata.conquery.io.mina.BinaryJacksonCoder; +import com.bakdata.conquery.io.mina.CQProtocolCodecFilter; +import com.bakdata.conquery.io.mina.ChunkReader; +import com.bakdata.conquery.io.mina.ChunkWriter; +import com.bakdata.conquery.io.mina.MinaAttributes; +import com.bakdata.conquery.io.mina.NetworkSession; +import com.bakdata.conquery.mode.InternalObjectMapperCreator; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.jobs.Job; +import com.bakdata.conquery.models.jobs.JobManager; +import com.bakdata.conquery.models.jobs.ReactingJob; +import com.bakdata.conquery.models.messages.SlowMessage; +import com.bakdata.conquery.models.messages.namespaces.specific.ShutdownShard; +import com.bakdata.conquery.models.messages.network.MessageToManagerNode; +import com.bakdata.conquery.models.messages.network.NetworkMessageContext; +import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.DistributedNamespace; +import com.bakdata.conquery.util.io.ConqueryMDC; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.mina.core.service.IoAcceptor; +import org.apache.mina.core.service.IoHandlerAdapter; +import org.apache.mina.core.session.IoSession; +import org.apache.mina.transport.socket.nio.NioSocketAcceptor; + +/** + * Manager of the connection from the manager to the ConQuery shards. + */ +@Slf4j +@RequiredArgsConstructor +public class ClusterConnectionManager extends IoHandlerAdapter { + + private IoAcceptor acceptor; + private final DatasetRegistry datasetRegistry; + private final JobManager jobManager; + private final Validator validator; + private final ConqueryConfig config; + private final InternalObjectMapperCreator internalObjectMapperCreator; + @Getter + private final ClusterState clusterState; + + @Override + public void sessionOpened(IoSession session) { + ConqueryMDC.setLocation("ManagerNode[" + session.getLocalAddress().toString() + "]"); + log.info("New client {} connected, waiting for identity", session.getRemoteAddress()); + } + + @Override + public void sessionClosed(IoSession session) { + ConqueryMDC.setLocation("ManagerNode[" + session.getLocalAddress().toString() + "]"); + log.info("Client '{}' disconnected ", session.getAttribute(MinaAttributes.IDENTIFIER)); + } + + @Override + public void exceptionCaught(IoSession session, Throwable cause) { + ConqueryMDC.setLocation("ManagerNode[" + session.getLocalAddress().toString() + "]"); + log.error("caught exception", cause); + } + + @Override + public void messageReceived(IoSession session, Object message) { + ConqueryMDC.setLocation("ManagerNode[" + session.getLocalAddress().toString() + "]"); + if (message instanceof MessageToManagerNode toManagerNode) { + + log.trace("ManagerNode received {} from {}", message.getClass().getSimpleName(), session.getRemoteAddress()); + + Job job = new ReactingJob<>(toManagerNode, + new NetworkMessageContext.ManagerNodeNetworkContext( + new NetworkSession(session), + datasetRegistry, + clusterState, + config.getCluster().getBackpressure() + )); + + if (toManagerNode instanceof SlowMessage slowMessage) { + slowMessage.setProgressReporter(job.getProgressReporter()); + jobManager.addSlowJob(job); + } + else { + jobManager.addFastJob(job); + } + } + else { + log.error("Unknown message type {} in {}", message.getClass(), message); + } + } + + public void start() throws IOException { + acceptor = new NioSocketAcceptor(); + + ObjectMapper om = internalObjectMapperCreator.createInternalObjectMapper(View.InternalCommunication.class); + config.configureObjectMapper(om); + BinaryJacksonCoder coder = new BinaryJacksonCoder(datasetRegistry, validator, om); + acceptor.getFilterChain().addLast("codec", new CQProtocolCodecFilter(new ChunkWriter(coder), new ChunkReader(coder, om))); + acceptor.setHandler(this); + acceptor.getSessionConfig().setAll(config.getCluster().getMina()); + acceptor.bind(new InetSocketAddress(config.getCluster().getPort())); + log.info("Started ManagerNode @ {}", acceptor.getLocalAddress()); + } + + public void stop() { + clusterState.getShardNodes().forEach(((socketAddress, shardNodeInformation) -> shardNodeInformation.send(new ShutdownShard()))); + + try { + acceptor.dispose(); + } + catch (RuntimeException e) { + log.error("{} could not be closed", acceptor, e); + } + + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterImportHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterImportHandler.java new file mode 100644 index 0000000000..d2e30ab3e4 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterImportHandler.java @@ -0,0 +1,93 @@ +package com.bakdata.conquery.mode.cluster; + +import java.io.InputStream; +import java.util.Collection; + +import com.bakdata.conquery.mode.ImportHandler; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.datasets.Import; +import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.Concept; +import com.bakdata.conquery.models.datasets.concepts.Connector; +import com.bakdata.conquery.models.identifiable.IdMutex; +import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; +import com.bakdata.conquery.models.identifiable.ids.specific.DictionaryId; +import com.bakdata.conquery.models.jobs.ImportJob; +import com.bakdata.conquery.models.messages.namespaces.specific.RemoveImportJob; +import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.DistributedNamespace; +import com.bakdata.conquery.models.worker.Namespace; +import lombok.AllArgsConstructor; +import lombok.SneakyThrows; + +/** + * Handler of {@link Import} requests that realizes them both on the manager and the cluster's shards. + */ +@AllArgsConstructor +public +class ClusterImportHandler implements ImportHandler { + + private final IdMutex sharedDictionaryLocks = new IdMutex<>(); + private final ConqueryConfig config; + private final DatasetRegistry datasetRegistry; + + @SneakyThrows + @Override + public void updateImport(Namespace namespace, InputStream inputStream) { + ImportJob job = ImportJob.createOrUpdate( + datasetRegistry.get(namespace.getDataset().getId()), + inputStream, + config.getCluster().getEntityBucketSize(), + sharedDictionaryLocks, + config, + true + ); + + namespace.getJobManager().addSlowJob(job); + + clearDependentConcepts(namespace.getStorage().getAllConcepts(), job.getTable()); + } + + @SneakyThrows + @Override + public void addImport(Namespace namespace, InputStream inputStream) { + ImportJob job = ImportJob.createOrUpdate( + datasetRegistry.get(namespace.getDataset().getId()), + inputStream, + config.getCluster().getEntityBucketSize(), + sharedDictionaryLocks, + config, + false + ); + namespace.getJobManager().addSlowJob(job); + + clearDependentConcepts(namespace.getStorage().getAllConcepts(), job.getTable()); + } + + @Override + public void deleteImport(Import imp) { + + DatasetId id = imp.getTable().getDataset().getId(); + final DistributedNamespace namespace = datasetRegistry.get(id); + + clearDependentConcepts(namespace.getStorage().getAllConcepts(), imp.getTable()); + + namespace.getStorage().removeImport(imp.getId()); + namespace.getWorkerHandler().sendToAll(new RemoveImportJob(imp)); + + // Remove bucket assignments for consistency report + namespace.getWorkerHandler().removeBucketAssignmentsForImportFormWorkers(imp); + } + + private void clearDependentConcepts(Collection> allConcepts, Table table) { + for (Concept c : allConcepts) { + for (Connector con : c.getConnectors()) { + if (!con.getTable().equals(table)) { + continue; + } + + con.getConcept().clearMatchingStats(); + } + } + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManager.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManager.java new file mode 100644 index 0000000000..809b87a945 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManager.java @@ -0,0 +1,33 @@ +package com.bakdata.conquery.mode.cluster; + +import com.bakdata.conquery.mode.DelegateManager; +import com.bakdata.conquery.mode.Manager; +import com.bakdata.conquery.models.worker.DistributedNamespace; +import io.dropwizard.lifecycle.Managed; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.experimental.Delegate; + +/** + * {@link Manager} for running ConQuery in cluster mode. + */ +@RequiredArgsConstructor +public class ClusterManager implements Manager { + @Delegate(excludes = Managed.class) + private final DelegateManager delegate; + @Getter + private final ClusterConnectionManager connectionManager; + + @Override + public void start() throws Exception { + delegate.start(); + connectionManager.start(); + } + + @Override + public void stop() throws Exception { + delegate.stop(); + connectionManager.stop(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java new file mode 100644 index 0000000000..f8dd53c541 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterManagerProvider.java @@ -0,0 +1,56 @@ +package com.bakdata.conquery.mode.cluster; + +import java.util.Collection; +import java.util.List; +import java.util.function.Supplier; + +import com.bakdata.conquery.mode.DelegateManager; +import com.bakdata.conquery.mode.ImportHandler; +import com.bakdata.conquery.mode.InternalObjectMapperCreator; +import com.bakdata.conquery.mode.ManagerProvider; +import com.bakdata.conquery.mode.NamespaceHandler; +import com.bakdata.conquery.mode.StorageListener; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.jobs.JobManager; +import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.DistributedNamespace; +import com.bakdata.conquery.models.worker.ShardNodeInformation; +import com.bakdata.conquery.tasks.ReportConsistencyTask; +import io.dropwizard.servlets.tasks.Task; +import io.dropwizard.setup.Environment; + +public class ClusterManagerProvider implements ManagerProvider { + + public ClusterManager provideManager(ConqueryConfig config, Environment environment) { + JobManager jobManager = ManagerProvider.newJobManager(config); + InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, environment.getValidator()); + ClusterState clusterState = new ClusterState(); + NamespaceHandler namespaceHandler = new ClusterNamespaceHandler(clusterState, config, creator); + DatasetRegistry datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, creator); + creator.init(datasetRegistry); + + ClusterConnectionManager connectionManager = new ClusterConnectionManager( + datasetRegistry, jobManager, environment.getValidator(), config, creator, clusterState + ); + ImportHandler importHandler = new ClusterImportHandler(config, datasetRegistry); + StorageListener extension = new ClusterStorageListener(jobManager, datasetRegistry); + Supplier> nodeProvider = () -> clusterState.getShardNodes().values(); + List adminTasks = List.of(new ReportConsistencyTask(clusterState)); + + DelegateManager delegate = new DelegateManager<>( + config, + environment, + datasetRegistry, + importHandler, + extension, + nodeProvider, + adminTasks, + creator, + jobManager + ); + + return new ClusterManager(delegate, connectionManager); + } + + +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java new file mode 100644 index 0000000000..30b0237106 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterNamespaceHandler.java @@ -0,0 +1,56 @@ +package com.bakdata.conquery.mode.cluster; + +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.io.storage.NamespaceStorage; +import com.bakdata.conquery.mode.InternalObjectMapperCreator; +import com.bakdata.conquery.mode.NamespaceSetupData; +import com.bakdata.conquery.mode.NamespaceHandler; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; +import com.bakdata.conquery.models.messages.network.specific.AddWorker; +import com.bakdata.conquery.models.messages.network.specific.RemoveWorker; +import com.bakdata.conquery.models.query.DistributedExecutionManager; +import com.bakdata.conquery.models.worker.DistributedNamespace; +import com.bakdata.conquery.models.worker.ShardNodeInformation; +import com.bakdata.conquery.models.worker.WorkerHandler; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class ClusterNamespaceHandler implements NamespaceHandler { + private final ClusterState clusterState; + private final ConqueryConfig config; + private final InternalObjectMapperCreator mapperCreator; + + @Override + public DistributedNamespace createNamespace(NamespaceStorage storage, final MetaStorage metaStorage) { + NamespaceSetupData namespaceData = NamespaceHandler.createNamespaceSetup(storage, config, mapperCreator); + DistributedExecutionManager executionManager = new DistributedExecutionManager(metaStorage, clusterState); + WorkerHandler workerHandler = new WorkerHandler(namespaceData.getCommunicationMapper(), storage); + clusterState.getWorkerHandlers().put(storage.getDataset().getId(), workerHandler); + + DistributedNamespace distributedNamespace = new DistributedNamespace( + namespaceData.getPreprocessMapper(), + namespaceData.getCommunicationMapper(), + storage, + executionManager, + namespaceData.getJobManager(), + namespaceData.getFilterSearch(), + namespaceData.getIndexService(), + namespaceData.getInjectables(), + workerHandler + ); + + for (ShardNodeInformation node : clusterState.getShardNodes().values()) { + node.send(new AddWorker(storage.getDataset())); + } + return distributedNamespace; + } + + + @Override + public void removeNamespace(DatasetId id, DistributedNamespace namespace) { + clusterState.getShardNodes().values().forEach(shardNode -> shardNode.send(new RemoveWorker(namespace.getDataset()))); + clusterState.getWorkerHandlers().keySet().removeIf(worker -> worker.getDataset().getDataset().equals(id)); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterState.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterState.java new file mode 100644 index 0000000000..36356fa9fc --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterState.java @@ -0,0 +1,35 @@ +package com.bakdata.conquery.mode.cluster; + +import java.net.SocketAddress; +import java.util.NoSuchElementException; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; +import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId; +import com.bakdata.conquery.models.worker.ShardNodeInformation; +import com.bakdata.conquery.models.worker.WorkerHandler; +import com.bakdata.conquery.models.worker.WorkerInformation; +import lombok.Value; + +@Value +public class ClusterState { + ConcurrentMap shardNodes = new ConcurrentHashMap<>(); + ConcurrentMap workerHandlers = new ConcurrentHashMap<>(); + + public synchronized void register(ShardNodeInformation node, WorkerInformation info) { + WorkerHandler workerHandler = workerHandlers.get(info.getDataset()); + if (workerHandler == null) { + throw new NoSuchElementException("Trying to register a worker for unknown dataset '%s'. I only know %s".formatted(info.getDataset(), workerHandlers.keySet())); + } + workerHandler.register(node, info); + } + + public WorkerInformation getWorker(final WorkerId workerId, final DatasetId id) { + return Optional.ofNullable(workerHandlers.get(id)) + .flatMap(ns -> ns.getWorkers().getOptional(workerId)) + .orElseThrow(() -> new NoSuchElementException("Unknown worker worker '%s' for dataset '%s'".formatted(workerId, id))); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterStorageListener.java b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterStorageListener.java new file mode 100644 index 0000000000..f16aaaa1b8 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/cluster/ClusterStorageListener.java @@ -0,0 +1,79 @@ +package com.bakdata.conquery.mode.cluster; + +import java.util.Collection; +import java.util.stream.Collectors; + +import com.bakdata.conquery.mode.StorageListener; +import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.datasets.SecondaryIdDescription; +import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.Concept; +import com.bakdata.conquery.models.jobs.JobManager; +import com.bakdata.conquery.models.jobs.SimpleJob; +import com.bakdata.conquery.models.messages.namespaces.specific.RemoveConcept; +import com.bakdata.conquery.models.messages.namespaces.specific.RemoveSecondaryId; +import com.bakdata.conquery.models.messages.namespaces.specific.RemoveTable; +import com.bakdata.conquery.models.messages.namespaces.specific.UpdateConcept; +import com.bakdata.conquery.models.messages.namespaces.specific.UpdateMatchingStatsMessage; +import com.bakdata.conquery.models.messages.namespaces.specific.UpdateSecondaryId; +import com.bakdata.conquery.models.messages.namespaces.specific.UpdateTable; +import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.DistributedNamespace; +import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.models.worker.WorkerHandler; +import lombok.AllArgsConstructor; + +/** + * Propagates changes of stored entities to relevant ConQuery shards in the cluster. + */ +@AllArgsConstructor +public +class ClusterStorageListener implements StorageListener { + + private final JobManager jobManager; + private final DatasetRegistry datasetRegistry; + + @Override + public void onAddSecondaryId(SecondaryIdDescription secondaryId) { + datasetRegistry.get(secondaryId.getDataset().getId()).getWorkerHandler().sendToAll(new UpdateSecondaryId(secondaryId)); + } + + @Override + public void onDeleteSecondaryId(SecondaryIdDescription secondaryId) { + datasetRegistry.get(secondaryId.getDataset().getId()).getWorkerHandler().sendToAll(new RemoveSecondaryId(secondaryId)); + } + + @Override + public void onAddTable(Table table) { + datasetRegistry.get(table.getDataset().getId()).getWorkerHandler().sendToAll(new UpdateTable(table)); + } + + @Override + public void onRemoveTable(Table table) { + datasetRegistry.get(table.getDataset().getId()).getWorkerHandler().sendToAll(new RemoveTable(table)); + } + + @Override + public void onAddConcept(Concept concept) { + WorkerHandler handler = datasetRegistry.get(concept.getDataset().getId()).getWorkerHandler(); + SimpleJob simpleJob = new SimpleJob(String.format("sendToAll : Add %s ", concept.getId()), () -> handler.sendToAll(new UpdateConcept(concept))); + jobManager.addSlowJob(simpleJob); + } + + @Override + public void onDeleteConcept(Concept concept) { + WorkerHandler handler = datasetRegistry.get(concept.getDataset().getId()).getWorkerHandler(); + SimpleJob simpleJob = new SimpleJob("sendToAll: remove " + concept.getId(), () -> handler.sendToAll(new RemoveConcept(concept))); + jobManager.addSlowJob(simpleJob); + } + + @Override + public void onUpdateMatchingStats(final Dataset dataset) { + final Namespace namespace = datasetRegistry.get(dataset.getId()); + final Collection> concepts = namespace.getStorage().getAllConcepts() + .stream() + .filter(concept -> concept.getMatchingStats() == null) + .collect(Collectors.toSet()); + datasetRegistry.get(dataset.getId()).getWorkerHandler().sendToAll(new UpdateMatchingStatsMessage(concepts)); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/FailingImportHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/FailingImportHandler.java new file mode 100644 index 0000000000..e6f0ff65e5 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/FailingImportHandler.java @@ -0,0 +1,29 @@ +package com.bakdata.conquery.mode.local; + +import java.io.InputStream; + +import com.bakdata.conquery.mode.ImportHandler; +import com.bakdata.conquery.models.datasets.Import; +import com.bakdata.conquery.models.worker.Namespace; + +public class FailingImportHandler implements ImportHandler { + + @Override + public void updateImport(Namespace namespace, InputStream inputStream) { + fail(); + } + + @Override + public void addImport(Namespace namespace, InputStream inputStream) { + fail(); + } + + @Override + public void deleteImport(Import imp) { + fail(); + } + + private static void fail() { + throw new UnsupportedOperationException("Imports are not supported when running in SQL mode"); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java new file mode 100644 index 0000000000..e1ee37a178 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java @@ -0,0 +1,40 @@ +package com.bakdata.conquery.mode.local; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.function.Supplier; + +import com.bakdata.conquery.mode.DelegateManager; +import com.bakdata.conquery.mode.InternalObjectMapperCreator; +import com.bakdata.conquery.mode.ManagerProvider; +import com.bakdata.conquery.mode.NamespaceHandler; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.LocalNamespace; +import com.bakdata.conquery.models.worker.ShardNodeInformation; +import io.dropwizard.setup.Environment; + +public class LocalManagerProvider implements ManagerProvider { + + private static final Supplier> EMPTY_NODE_PROVIDER = Collections::emptyList; + + public DelegateManager provideManager(ConqueryConfig config, Environment environment) { + InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, environment.getValidator()); + NamespaceHandler namespaceHandler = new LocalNamespaceHandler(config, creator); + DatasetRegistry datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, creator); + creator.init(datasetRegistry); + + return new DelegateManager<>( + config, + environment, + datasetRegistry, + new FailingImportHandler(), + new LocalStorageListener(), + EMPTY_NODE_PROVIDER, + List.of(), + creator, + ManagerProvider.newJobManager(config) + ); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java new file mode 100644 index 0000000000..3a887ea01a --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java @@ -0,0 +1,42 @@ +package com.bakdata.conquery.mode.local; + +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.io.storage.NamespaceStorage; +import com.bakdata.conquery.mode.InternalObjectMapperCreator; +import com.bakdata.conquery.mode.NamespaceSetupData; +import com.bakdata.conquery.mode.NamespaceHandler; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; +import com.bakdata.conquery.models.query.ExecutionManager; +import com.bakdata.conquery.models.worker.LocalNamespace; +import com.bakdata.conquery.sql.conquery.SqlExecutionManager; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public class LocalNamespaceHandler implements NamespaceHandler { + + private final ConqueryConfig config; + private final InternalObjectMapperCreator mapperCreator; + + @Override + public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaStorage metaStorage) { + NamespaceSetupData namespaceData = NamespaceHandler.createNamespaceSetup(namespaceStorage, config, mapperCreator); + ExecutionManager executionManager = new SqlExecutionManager(); + return new LocalNamespace( + namespaceData.getPreprocessMapper(), + namespaceData.getCommunicationMapper(), + namespaceStorage, + executionManager, + namespaceData.getJobManager(), + namespaceData.getFilterSearch(), + namespaceData.getIndexService(), + namespaceData.getInjectables() + ); + } + + @Override + public void removeNamespace(DatasetId id, LocalNamespace namespace) { + // nothing to do + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalStorageListener.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalStorageListener.java new file mode 100644 index 0000000000..20dcd9e25b --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalStorageListener.java @@ -0,0 +1,40 @@ +package com.bakdata.conquery.mode.local; + +import com.bakdata.conquery.mode.StorageListener; +import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.datasets.SecondaryIdDescription; +import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.Concept; + +public class LocalStorageListener implements StorageListener { + + // When running without shards, no further actions are required + + @Override + public void onAddSecondaryId(SecondaryIdDescription secondaryId) { + } + + @Override + public void onDeleteSecondaryId(SecondaryIdDescription description) { + } + + @Override + public void onAddTable(Table table) { + } + + @Override + public void onRemoveTable(Table table) { + } + + @Override + public void onAddConcept(Concept concept) { + } + + @Override + public void onDeleteConcept(Concept concept) { + } + + @Override + public void onUpdateMatchingStats(Dataset dataset) { + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationHelper.java b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationHelper.java index 43337e3dba..766b3abf54 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationHelper.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/AuthorizationHelper.java @@ -23,6 +23,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.util.QueryUtils.NamespacedIdentifiableCollector; import com.google.common.collect.ArrayListMultimap; import com.google.common.collect.Multimap; @@ -106,7 +107,7 @@ public static void authorizeDownloadDatasets(@NonNull Subject subject, @NonNull /** * Calculates the abilities on all datasets a subject has based on its permissions. */ - public static Map> buildDatasetAbilityMap(Subject subject, DatasetRegistry datasetRegistry) { + public static Map> buildDatasetAbilityMap(Subject subject, DatasetRegistry datasetRegistry) { HashMap> datasetAbilities = new HashMap<>(); for (Dataset dataset : datasetRegistry.getAllDatasets()) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java index 25aa7bc60c..d04ad65584 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ConqueryConfig.java @@ -116,6 +116,11 @@ public class ConqueryConfig extends Configuration implements Injectable { @Valid @NotNull private List plugins = new ArrayList<>(); + + @Valid + @NotNull + private SqlConnectorConfig sqlConnectorConfig = new SqlConnectorConfig(); + /** * null means here that we try to deduce from an attached agent */ diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java new file mode 100644 index 0000000000..747f1840a2 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java @@ -0,0 +1,9 @@ +package com.bakdata.conquery.models.config; + +import lombok.Data; + +@Data +public class SqlConnectorConfig { + + boolean enabled; +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java index 4d84e6150e..12367ec4c7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/PreviewConfig.java @@ -26,6 +26,7 @@ import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.Namespace; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.OptBoolean; @@ -92,7 +93,7 @@ public class PreviewConfig { @JacksonInject(useInput = OptBoolean.FALSE) @NotNull - private DatasetRegistry datasetRegistry; + private DatasetRegistry datasetRegistry; public boolean isGroupingColumn(SecondaryIdDescription desc) { return getGrouping().contains(desc.getId()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/jobs/ImportJob.java b/backend/src/main/java/com/bakdata/conquery/models/jobs/ImportJob.java index 3e9aa63cbe..b14e51b818 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/jobs/ImportJob.java +++ b/backend/src/main/java/com/bakdata/conquery/models/jobs/ImportJob.java @@ -48,7 +48,8 @@ import com.bakdata.conquery.models.preproc.PreprocessedReader; import com.bakdata.conquery.models.preproc.parser.specific.IntegerParser; import com.bakdata.conquery.models.query.entity.Entity; -import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.models.worker.DistributedNamespace; +import com.bakdata.conquery.models.worker.WorkerHandler; import com.bakdata.conquery.models.worker.WorkerInformation; import com.bakdata.conquery.util.ResourceUtil; import com.bakdata.conquery.util.progressreporter.ProgressReporter; @@ -67,7 +68,7 @@ @Slf4j public class ImportJob extends Job { - private final Namespace namespace; + private final DistributedNamespace namespace; @Getter private final Table table; @@ -80,7 +81,7 @@ public class ImportJob extends Job { private static final int NUMBER_OF_STEPS = /* directly in execute = */4; - public static ImportJob createOrUpdate(Namespace namespace, InputStream inputStream, int entityBucketSize, IdMutex sharedDictionaryLocks, ConqueryConfig config, boolean update) + public static ImportJob createOrUpdate(DistributedNamespace namespace, InputStream inputStream, int entityBucketSize, IdMutex sharedDictionaryLocks, ConqueryConfig config, boolean update) throws IOException { try (PreprocessedReader parser = new PreprocessedReader(inputStream, namespace.getPreprocessMapper())) { @@ -113,7 +114,7 @@ public static ImportJob createOrUpdate(Namespace namespace, InputStream inputStr throw new WebApplicationException(String.format("Import[%s] is not present.", importId), Response.Status.NOT_FOUND); } // before updating the import, make sure that all workers removed the last import - namespace.sendToAll(new RemoveImportJob(processedImport)); + namespace.getWorkerHandler().sendToAll(new RemoveImportJob(processedImport)); namespace.getStorage().removeImport(importId); } else if (processedImport != null) { @@ -196,7 +197,7 @@ private static Map createLocalIdReplacements(Map importDictionaries(Namespace namespace, Map dicts, Column[] columns, String importName, Table table) { + private static Map importDictionaries(DistributedNamespace namespace, Map dicts, Column[] columns, String importName, Table table) { // Empty Maps are Coalesced to null by Jackson if (dicts == null) { @@ -256,10 +257,10 @@ private static Map importDictionaries(Namespace names return out; } - private static void distributeDictionary(Namespace namespace, Dictionary dictionary) { + private static void distributeDictionary(DistributedNamespace namespace, Dictionary dictionary) { log.trace("Sending {} to all Workers", dictionary); namespace.getStorage().updateDictionary(dictionary); - namespace.sendToAll(new UpdateDictionary(dictionary)); + namespace.getWorkerHandler().sendToAll(new UpdateDictionary(dictionary)); } @@ -312,7 +313,8 @@ public void execute() throws JSONException, InterruptedException, IOException { final Map> workerAssignments = sendBuckets(container.getStarts(), container.getLengths(), primaryMapping, imp, buckets2LocalEntities, storesSorted); - workerAssignments.forEach(namespace::addBucketsToWorker); + WorkerHandler handler = namespace.getWorkerHandler(); + workerAssignments.forEach(handler::addBucketsToWorker); } @@ -327,9 +329,11 @@ private Map> sendBuckets(Map starts, M for (Map.Entry> bucket2entities : buckets2LocalEntities.entrySet()) { - WorkerInformation responsibleWorker = - Objects.requireNonNull(namespace.getResponsibleWorkerForBucket(bucket2entities.getKey()), () -> "No responsible worker for Bucket#" - + bucket2entities.getKey()); + WorkerInformation responsibleWorker = Objects.requireNonNull( + namespace + .getWorkerHandler() + .getResponsibleWorkerForBucket(bucket2entities.getKey()), + () -> "No responsible worker for Bucket#" + bucket2entities.getKey()); awaitFreeJobQueue(responsibleWorker); @@ -456,11 +460,11 @@ private void distributeWorkerResponsibilities(DictionaryMapping primaryMapping) for (int entity : primaryMapping.target()) { int bucket = Entity.getBucket(entity, bucketSize); - if (namespace.getResponsibleWorkerForBucket(bucket) != null) { + if (namespace.getWorkerHandler().getResponsibleWorkerForBucket(bucket) != null) { continue; } - namespace.addResponsibility(bucket); + namespace.getWorkerHandler().addResponsibility(bucket); } } } @@ -552,7 +556,7 @@ private Import createImport(PreprocessedHeader header, Map } imp.setDictionaries(dictionaries); - namespace.sendToAll(new AddImport(imp)); + namespace.getWorkerHandler().sendToAll(new AddImport(imp)); return imp; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/NamespaceMessage.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/NamespaceMessage.java index 6fd7c1a9ed..b308a38c03 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/NamespaceMessage.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/NamespaceMessage.java @@ -1,14 +1,14 @@ package com.bakdata.conquery.models.messages.namespaces; import com.bakdata.conquery.models.messages.SlowMessage; -import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.models.worker.DistributedNamespace; import com.bakdata.conquery.util.progressreporter.ProgressReporter; import com.fasterxml.jackson.annotation.JsonIgnore; import lombok.Getter; import lombok.Setter; @Getter @Setter -public abstract class NamespaceMessage extends NamespacedMessage implements SlowMessage { +public abstract class NamespaceMessage extends NamespacedMessage implements SlowMessage { @JsonIgnore @Getter @Setter private ProgressReporter progressReporter; diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/CollectQueryResult.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/CollectQueryResult.java index 0054851e13..cf5293e966 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/CollectQueryResult.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/CollectQueryResult.java @@ -4,7 +4,7 @@ import com.bakdata.conquery.models.messages.namespaces.NamespaceMessage; import com.bakdata.conquery.models.messages.namespaces.NamespacedMessage; import com.bakdata.conquery.models.query.results.ShardResult; -import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.models.worker.DistributedNamespace; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; @@ -27,7 +27,7 @@ public class CollectQueryResult extends NamespaceMessage { private ShardResult result; @Override - public void react(Namespace context) throws Exception { + public void react(DistributedNamespace context) throws Exception { log.info("Received {} of size {}", result, result.getResults().size()); context.getExecutionManager().handleQueryResult(result); diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ReportConsistency.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ReportConsistency.java index 2ba3747eb2..0239857c86 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ReportConsistency.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/ReportConsistency.java @@ -11,7 +11,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId; import com.bakdata.conquery.models.messages.namespaces.NamespaceMessage; import com.bakdata.conquery.models.messages.namespaces.NamespacedMessage; -import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.models.worker.DistributedNamespace; import com.google.common.collect.Sets; import lombok.AllArgsConstructor; import lombok.Getter; @@ -40,10 +40,10 @@ public class ReportConsistency extends NamespaceMessage { @Override - public void react(Namespace context) throws Exception { + public void react(DistributedNamespace context) throws Exception { Set managerImports = context.getStorage().getAllImports().stream().map(Import::getId).collect(Collectors.toSet()); - Set assignedWorkerBuckets = context.getBucketsForWorker(workerId); + Set assignedWorkerBuckets = context.getWorkerHandler().getBucketsForWorker(workerId); boolean importsOkay = isConsistent("Imports", managerImports, workerImports, workerId); boolean bucketsOkay = isConsistent("Buckets", assignedWorkerBuckets, workerBuckets, workerId); diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateElementMatchingStats.java b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateElementMatchingStats.java index 4ba0fee85d..5efc018639 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateElementMatchingStats.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/namespaces/specific/UpdateElementMatchingStats.java @@ -10,7 +10,7 @@ import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId; import com.bakdata.conquery.models.messages.namespaces.NamespaceMessage; import com.bakdata.conquery.models.messages.namespaces.NamespacedMessage; -import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.models.worker.DistributedNamespace; import com.fasterxml.jackson.annotation.JsonCreator; import lombok.AllArgsConstructor; import lombok.Getter; @@ -30,7 +30,7 @@ public class UpdateElementMatchingStats extends NamespaceMessage { private final Map, MatchingStats.Entry> values; @Override - public void react(Namespace context) throws Exception { + public void react(DistributedNamespace context) throws Exception { for (Entry, MatchingStats.Entry> entry : values.entrySet()) { try { final ConceptElement target = entry.getKey(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/network/NetworkMessageContext.java b/backend/src/main/java/com/bakdata/conquery/models/messages/network/NetworkMessageContext.java index fd8b905070..dd6d332c7b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/network/NetworkMessageContext.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/network/NetworkMessageContext.java @@ -6,8 +6,10 @@ import com.bakdata.conquery.commands.ShardNode; import com.bakdata.conquery.io.mina.MessageSender; import com.bakdata.conquery.io.mina.NetworkSession; +import com.bakdata.conquery.mode.cluster.ClusterState; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.DistributedNamespace; import com.bakdata.conquery.models.worker.Workers; import lombok.Getter; @@ -52,12 +54,14 @@ public ShardNodeNetworkContext(ShardNode shardNode, NetworkSession session, Work @Getter public static class ManagerNodeNetworkContext extends NetworkMessageContext { - private final DatasetRegistry namespaces; + private final ClusterState clusterState; + private final DatasetRegistry datasetRegistry; - public ManagerNodeNetworkContext(NetworkSession session, DatasetRegistry namespaces, int backpressure) { + public ManagerNodeNetworkContext(NetworkSession session, DatasetRegistry datasetRegistry, ClusterState clusterState, int backpressure) { super(session, backpressure); - this.namespaces = namespaces; + this.datasetRegistry = datasetRegistry; + this.clusterState = clusterState; } } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/AddShardNode.java b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/AddShardNode.java index 2aeaddf4bd..4dcf448d68 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/AddShardNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/AddShardNode.java @@ -26,7 +26,7 @@ public void react(NetworkMessageContext.ManagerNodeNetworkContext context) throw context.getBackpressure() ); - context.getNamespaces().getShardNodes().put(context.getRemoteAddress(), nodeInformation); + context.getClusterState().getShardNodes().put(context.getRemoteAddress(), nodeInformation); log.info("ShardNode `{}` registered.", context.getRemoteAddress()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/ForwardToNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/ForwardToNamespace.java index 15aba46653..fbd9d4d3a3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/ForwardToNamespace.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/ForwardToNamespace.java @@ -9,7 +9,7 @@ import com.bakdata.conquery.models.messages.network.MessageToManagerNode; import com.bakdata.conquery.models.messages.network.NetworkMessage; import com.bakdata.conquery.models.messages.network.NetworkMessageContext.ManagerNodeNetworkContext; -import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.models.worker.DistributedNamespace; import com.bakdata.conquery.util.io.ConqueryMDC; import com.bakdata.conquery.util.progressreporter.ProgressReporter; import lombok.Getter; @@ -25,7 +25,7 @@ public class ForwardToNamespace extends MessageToManagerNode implements SlowMess @Override public void react(ManagerNodeNetworkContext context) throws Exception { - Namespace ns = Objects.requireNonNull(context.getNamespaces().get(datasetId), () -> String.format("Missing dataset `%s`", datasetId)); + DistributedNamespace ns = Objects.requireNonNull(context.getDatasetRegistry().get(datasetId), () -> String.format("Missing dataset `%s`", datasetId)); ConqueryMDC.setLocation(ns.getStorage().getDataset().toString()); message.react(ns); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/RegisterWorker.java b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/RegisterWorker.java index 6b03ec0a9f..ef29e18919 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/RegisterWorker.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/RegisterWorker.java @@ -36,10 +36,10 @@ public void react(ManagerNodeNetworkContext context) throws Exception { } info.setConnectedShardNode(node); - context.getNamespaces().register(node, info); + context.getClusterState().register(node, info); // Request consistency report - context.getNamespaces().getWorkers().get(info.getId()).send(new RequestConsistency()); + context.getClusterState().getWorker(info.getId(), info.getDataset()).send(new RequestConsistency()); } /** @@ -48,7 +48,7 @@ public void react(ManagerNodeNetworkContext context) throws Exception { * @return the found slave or null if none was found */ private ShardNodeInformation getShardNode(ManagerNodeNetworkContext context) { - return context.getNamespaces() + return context.getClusterState() .getShardNodes() .get(context.getRemoteAddress()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/RemoveShardNode.java b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/RemoveShardNode.java index 558eb68214..4f25093694 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/RemoveShardNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/RemoveShardNode.java @@ -24,6 +24,6 @@ public class RemoveShardNode extends MessageToManagerNode { @Override public void react(NetworkMessageContext.ManagerNodeNetworkContext context) throws Exception { log.info("ShardNode {} unregistered.", context.getRemoteAddress()); - context.getNamespaces().getShardNodes().remove(context.getRemoteAddress()); + context.getClusterState().getShardNodes().remove(context.getRemoteAddress()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/UpdateJobManagerStatus.java b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/UpdateJobManagerStatus.java index 3f2c339ecf..a215af08dd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/UpdateJobManagerStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/UpdateJobManagerStatus.java @@ -23,13 +23,14 @@ public class UpdateJobManagerStatus extends MessageToManagerNode { @Override public void react(ManagerNodeNetworkContext context) throws Exception { - final ShardNodeInformation node = context.getNamespaces().getShardNodes().get(context.getRemoteAddress()); + final ShardNodeInformation node = context.getClusterState().getShardNodes().get(context.getRemoteAddress()); if (node == null) { - log.error("Could not find ShardNode `{}`, I only know of {}", context.getRemoteAddress(), context.getNamespaces().getShardNodes().keySet()); + log.error("Could not find ShardNode `{}`, I only know of {}", context.getRemoteAddress(), context.getClusterState().getShardNodes().keySet()); return; } // The shards don't know their own name so we attach it here node.addJobManagerStatus(status.withOrigin(context.getRemoteAddress().toString())); } + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/DistributedExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/models/query/DistributedExecutionManager.java new file mode 100644 index 0000000000..5761dec598 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/query/DistributedExecutionManager.java @@ -0,0 +1,184 @@ +package com.bakdata.conquery.models.query; + +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ExecutionException; +import java.util.stream.Stream; + +import com.bakdata.conquery.apiv1.query.QueryDescription; +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.metrics.ExecutionMetrics; +import com.bakdata.conquery.mode.cluster.ClusterState; +import com.bakdata.conquery.models.auth.AuthorizationHelper; +import com.bakdata.conquery.models.auth.entities.Group; +import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.error.ConqueryError; +import com.bakdata.conquery.models.execution.ExecutionState; +import com.bakdata.conquery.models.execution.InternalExecution; +import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.messages.namespaces.specific.CancelQuery; +import com.bakdata.conquery.models.query.results.EntityResult; +import com.bakdata.conquery.models.query.results.ShardResult; +import com.bakdata.conquery.models.worker.Namespace; +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.RemovalNotification; +import lombok.RequiredArgsConstructor; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; + +@RequiredArgsConstructor +@Slf4j +public class DistributedExecutionManager implements ExecutionManager { + + private final MetaStorage storage; + private final ClusterState clusterState; + + private final Cache>> executionResults = + CacheBuilder.newBuilder() + .softValues() + .removalListener(this::executionRemoved) + .build(); + + /** + * Manage state of evicted Queries, setting them to NEW. + */ + private void executionRemoved(RemovalNotification> removalNotification) { + // If removal was done manually we assume it was also handled properly + if (!removalNotification.wasEvicted()) { + return; + } + + final ManagedExecutionId executionId = removalNotification.getKey(); + + log.warn("Evicted Results for Query[{}] (Reason: {})", executionId, removalNotification.getCause()); + + storage.getExecution(executionId).reset(); + } + + @Override + public ManagedExecution runQuery(Namespace namespace, QueryDescription query, User user, Dataset submittedDataset, ConqueryConfig config, boolean system) { + final ManagedExecution execution = createExecution(query, user, submittedDataset, system); + execute(namespace, execution, config); + + return execution; + + } + + @Override + public void execute(Namespace namespace, ManagedExecution execution, ConqueryConfig config) { + try { + execution.initExecutable(namespace, config); + } + catch (Exception e) { + // ConqueryErrors are usually user input errors so no need to log them at level=ERROR + if (e instanceof ConqueryError) { + log.warn("Failed to initialize Query[{}]", execution.getId(), e); + } + else { + log.error("Failed to initialize Query[{}]", execution.getId(), e); + } + + storage.removeExecution(execution.getId()); + throw e; + } + + log.info("Starting execution[{}]", execution.getQueryId()); + + execution.start(); + + + final String primaryGroupName = AuthorizationHelper.getPrimaryGroup(execution.getOwner(), storage).map(Group::getName).orElse("none"); + ExecutionMetrics.getRunningQueriesCounter(primaryGroupName).inc(); + + if (execution instanceof InternalExecution internalExecution) { + log.info("Executing Query[{}] in Dataset[{}]", execution.getQueryId(), namespace.getDataset().getId()); + clusterState.getWorkerHandlers().get(execution.getDataset().getId()).sendToAll(internalExecution.createExecutionMessage()); + } + } + + @Override + public ManagedExecution createExecution(QueryDescription query, User user, Dataset submittedDataset, boolean system) { + return createQuery(query, UUID.randomUUID(), user, submittedDataset, system); + } + + + // Visible for testing + public ManagedExecution createQuery(QueryDescription query, UUID queryId, User user, Dataset submittedDataset, boolean system) { + // Transform the submitted query into an initialized execution + ManagedExecution managed = query.toManagedExecution(user, submittedDataset, storage); + managed.setSystem(system); + managed.setQueryId(queryId); + + // Store the execution + storage.addExecution(managed); + + return managed; + } + + /** + * Receive part of query result and store into query. + * + * @param result + */ + public > void handleQueryResult(R result) { + + final E query = (E) storage.getExecution(result.getQueryId()); + + if (query.getState() != ExecutionState.RUNNING) { + return; + } + + query.addResult(result); + + // State changed to DONE or FAILED + if (query.getState() != ExecutionState.RUNNING) { + final String primaryGroupName = AuthorizationHelper.getPrimaryGroup(query.getOwner(), storage).map(Group::getName).orElse("none"); + + ExecutionMetrics.getRunningQueriesCounter(primaryGroupName).dec(); + ExecutionMetrics.getQueryStateCounter(query.getState(), primaryGroupName).inc(); + ExecutionMetrics.getQueriesTimeHistogram(primaryGroupName).update(query.getExecutionTime().toMillis()); + } + + } + + + /** + * Register another result for the execution. + */ + + @SneakyThrows(ExecutionException.class) // can only occur if ArrayList::new fails which is unlikely and would have other problems also + public void addQueryResult(ManagedExecution execution, List queryResults) { + // We don't collect all results together into a fat list as that would cause lots of huge re-allocations for little gain. + executionResults.get(execution.getId(), ArrayList::new) + .add(queryResults); + } + + /** + * Discard the query's results. + */ + @Override + public void clearQueryResults(ManagedExecution execution) { + executionResults.invalidate(execution.getId()); + } + + @Override + public Stream streamQueryResults(ManagedExecution execution) { + final List> resultParts = executionResults.getIfPresent(execution.getId()); + + return resultParts == null + ? Stream.empty() + : resultParts.stream().flatMap(List::stream); + + } + + @Override + public void cancelQuery(Dataset dataset, ManagedExecution query) { + clusterState.getWorkerHandlers().get(dataset.getId()).sendToAll(new CancelQuery(query.getId())); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java index 933fe3daab..41e714878b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ExecutionManager.java @@ -1,170 +1,43 @@ package com.bakdata.conquery.models.query; -import java.util.ArrayList; import java.util.List; +import java.util.Set; import java.util.UUID; -import java.util.concurrent.ExecutionException; import java.util.stream.Stream; import com.bakdata.conquery.apiv1.query.QueryDescription; -import com.bakdata.conquery.io.storage.MetaStorage; -import com.bakdata.conquery.metrics.ExecutionMetrics; -import com.bakdata.conquery.models.auth.AuthorizationHelper; -import com.bakdata.conquery.models.auth.entities.Group; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; -import com.bakdata.conquery.models.error.ConqueryError; -import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.execution.InternalExecution; import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; +import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.ShardResult; import com.bakdata.conquery.models.worker.Namespace; -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; import com.google.common.cache.RemovalNotification; -import lombok.RequiredArgsConstructor; -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -@RequiredArgsConstructor -@Slf4j -public class ExecutionManager { +public interface ExecutionManager { - private final MetaStorage storage; - /** - * Manage state of evicted Queries, setting them to NEW. - */ - private void executionRemoved(RemovalNotification> removalNotification) { - - // If removal was done manually we assume it was also handled properly - if (!removalNotification.wasEvicted()) { - return; - } - - final ManagedExecutionId executionId = removalNotification.getKey(); - - log.warn("Evicted Results for Query[{}] (Reason: {})", executionId, removalNotification.getCause()); - - storage.getExecution(executionId).reset(); - } private final Cache>> executionResults = - CacheBuilder.newBuilder() - .softValues() - .removalListener(this::executionRemoved) - .build(); - - public ManagedExecution runQuery(Namespace namespace, QueryDescription query, User user, Dataset submittedDataset, ConqueryConfig config, boolean system) { - final ManagedExecution execution = createExecution(query, user, submittedDataset, system); - execute(namespace, execution, config); - - return execution; - } - - public ManagedExecution createExecution(QueryDescription query, User user, Dataset submittedDataset, boolean system) { - return createQuery(query, UUID.randomUUID(), user, submittedDataset, system); - } - - public void execute(Namespace namespace, ManagedExecution execution, ConqueryConfig config) { - try { - execution.initExecutable(namespace, config); - } - catch (Exception e) { - // ConqueryErrors are usually user input errors so no need to log them at level=ERROR - if (e instanceof ConqueryError) { - log.warn("Failed to initialize Query[{}]", execution.getId(), e); - } - else { - log.error("Failed to initialize Query[{}]", execution.getId(), e); - } - - storage.removeExecution(execution.getId()); - throw e; - } - - log.info("Starting execution[{}]", execution.getQueryId()); - - execution.start(); - - - final String primaryGroupName = AuthorizationHelper.getPrimaryGroup(execution.getOwner(), storage).map(Group::getName).orElse("none"); - ExecutionMetrics.getRunningQueriesCounter(primaryGroupName).inc(); - - if (execution instanceof InternalExecution internalExecution) { - log.info("Executing Query[{}] in Dataset[{}]", execution.getQueryId(), namespace.getDataset().getId()); - namespace.sendToAll(internalExecution.createExecutionMessage()); - } - } + ManagedExecution runQuery(Namespace namespace, QueryDescription query, User user, Dataset submittedDataset, ConqueryConfig config, boolean system); - public ManagedExecution createQuery(QueryDescription query, UUID queryId, User user, Dataset submittedDataset, boolean system) { - // Transform the submitted query into an initialized execution - ManagedExecution managed = query.toManagedExecution(user, submittedDataset, storage); - managed.setSystem(system); - managed.setQueryId(queryId); + void execute(Namespace namespace, ManagedExecution execution, ConqueryConfig config); - // Store the execution - storage.addExecution(managed); + ManagedExecution createExecution(QueryDescription query, User user, Dataset submittedDataset, boolean system); - return managed; - } - - /** - * Receive part of query result and store into query. - * - * @param result - */ - public > void handleQueryResult(R result) { - - - final E query = (E) storage.getExecution(result.getQueryId()); - - if (query.getState() != ExecutionState.RUNNING) { - return; - } - - query.addResult(result); - - // State changed to DONE or FAILED - if (query.getState() != ExecutionState.RUNNING) { - final String primaryGroupName = AuthorizationHelper.getPrimaryGroup(query.getOwner(), storage).map(Group::getName).orElse("none"); - - ExecutionMetrics.getRunningQueriesCounter(primaryGroupName).dec(); - ExecutionMetrics.getQueryStateCounter(query.getState(), primaryGroupName).inc(); - ExecutionMetrics.getQueriesTimeHistogram(primaryGroupName).update(query.getExecutionTime().toMillis()); - } - } - - /** - * Register another result for the execution. - */ - @SneakyThrows(ExecutionException.class) // can only occur if ArrayList::new fails which is unlikely and would have other problems also - public void addQueryResult(ManagedExecution execution, List queryResults) { - // We don't collect all results together into a fat list as that would cause lots of huge re-allocations for little gain. - executionResults.get(execution.getId(), ArrayList::new) - .add(queryResults); - } + void cancelQuery(final Dataset dataset, final ManagedExecution query); /** * Discard the query's results. */ - public void clearQueryResults(ManagedExecution execution) { - executionResults.invalidate(execution.getId()); - } + void clearQueryResults(ManagedExecution execution); /** * Stream the results of the query, if available. */ - public Stream streamQueryResults(ManagedExecution execution) { - final List> resultParts = executionResults.getIfPresent(execution.getId()); - - return resultParts == null - ? Stream.empty() - : resultParts.stream().flatMap(List::stream); - } - - - + Stream streamQueryResults(ManagedExecution execution); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java index 4bd1f5449a..66117dcbc6 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java @@ -4,15 +4,11 @@ import java.util.Collections; import java.util.List; import java.util.Locale; -import java.util.Map; import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; -import java.util.function.Predicate; import java.util.stream.Collectors; import java.util.stream.Stream; -import c10n.C10N; import com.bakdata.conquery.apiv1.execution.ExecutionStatus; import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.apiv1.query.Query; @@ -21,7 +17,6 @@ import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; -import com.bakdata.conquery.internationalization.CQElementC10n; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.entities.Subject; @@ -38,12 +33,13 @@ import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.query.results.ShardResult; +import com.bakdata.conquery.models.worker.DistributedNamespace; import com.bakdata.conquery.models.worker.WorkerInformation; +import com.bakdata.conquery.util.QueryUtils; import com.fasterxml.jackson.annotation.JacksonInject; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.OptBoolean; import com.google.common.base.Preconditions; -import com.google.common.base.Strings; import lombok.Getter; import lombok.NonNull; import lombok.Setter; @@ -57,7 +53,6 @@ @CPSType(base = ManagedExecution.class, id = "MANAGED_QUERY") public class ManagedQuery extends ManagedExecution implements SingleTableResult, InternalExecution { - private static final int MAX_CONCEPT_LABEL_CONCAT_LENGTH = 70; // Needs to be resolved externally before being executed private Query query; /** @@ -129,7 +124,7 @@ public long resultRowCount() { @Override public void start() { super.start(); - involvedWorkers = Collections.synchronizedSet(getNamespace().getWorkers().stream() + involvedWorkers = Collections.synchronizedSet(getNamespace().getWorkerHandler().getWorkers().stream() .map(WorkerInformation::getId) .collect(Collectors.toSet())); } @@ -201,79 +196,7 @@ public QueryDescription getSubmitted() { */ @Override protected String makeDefaultLabel(PrintSettings cfg) { - final StringBuilder sb = new StringBuilder(); - - final Map, List> sortedContents = - Visitable.stream(query) - .collect(Collectors.groupingBy(Visitable::getClass)); - - int sbStartSize = sb.length(); - - // Check for CQExternal - List externals = sortedContents.getOrDefault(CQExternal.class, Collections.emptyList()); - if (!externals.isEmpty()) { - if (sb.length() > 0) { - sb.append(" "); - } - sb.append(C10N.get(CQElementC10n.class, I18n.LOCALE.get()).external()); - } - - // Check for CQReused - if (sortedContents.containsKey(CQReusedQuery.class)) { - if (sb.length() > 0) { - sb.append(" "); - } - sb.append(C10N.get(CQElementC10n.class, I18n.LOCALE.get()).reused()); - } - - - // Check for CQConcept - if (sortedContents.containsKey(CQConcept.class)) { - if (sb.length() > 0) { - sb.append(" "); - } - // Track length of text we are appending for concepts. - final AtomicInteger length = new AtomicInteger(); - - sortedContents.get(CQConcept.class) - .stream() - .map(CQConcept.class::cast) - - .map(c -> makeLabelWithRootAndChild(c, cfg)) - .filter(Predicate.not(Strings::isNullOrEmpty)) - .distinct() - - .takeWhile(elem -> length.addAndGet(elem.length()) < MAX_CONCEPT_LABEL_CONCAT_LENGTH) - .forEach(label -> sb.append(label).append(" ")); - - // Last entry will output one Space that we don't want - if (sb.length() > 0) { - sb.deleteCharAt(sb.length() - 1); - } - - // If not all Concept could be included in the name, point that out - if (length.get() > MAX_CONCEPT_LABEL_CONCAT_LENGTH) { - sb.append(" ").append(C10N.get(CQElementC10n.class, I18n.LOCALE.get()).furtherConcepts()); - } - } - - - // Fallback to id if nothing could be extracted from the query description - if (sbStartSize == sb.length()) { - sb.append(getId().getExecution()); - } - - return sb.toString(); - } - - private static String makeLabelWithRootAndChild(CQConcept cqConcept, PrintSettings cfg) { - String label = cqConcept.getUserOrDefaultLabel(cfg.getLocale()); - if (label == null) { - label = cqConcept.getConcept().getLabel(); - } - - // Concat everything with dashes - return label.replace(" ", "-"); + return QueryUtils.makeQueryLabel(query, cfg, getId()); } @Override @@ -286,4 +209,9 @@ public void visit(Consumer visitor) { visitor.accept(this); query.visit(visitor); } + + public DistributedNamespace getNamespace() { + return (DistributedNamespace) super.getNamespace(); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/entity/Entity.java b/backend/src/main/java/com/bakdata/conquery/models/query/entity/Entity.java index d0470a4a4d..9816d9ee88 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/entity/Entity.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/entity/Entity.java @@ -8,7 +8,7 @@ /** * All associated data to a single entity (usually a person), over all {@link Table}s and {@link com.bakdata.conquery.models.datasets.concepts.Concept}s. * - * @implNote The ManagerNode does not hold any data of Entities, only the ShardNodes do (via Workers). Additionally, all data of a single Entity must be held by a single Worker only (See {@link com.bakdata.conquery.models.worker.Namespace::getResponsibleWorker}). + * @implNote The ManagerNode does not hold any data of Entities, only the ShardNodes do (via Workers). Additionally, all data of a single Entity must be held by a single Worker only (See {@link com.bakdata.conquery.models.worker.WorkerHandler#getResponsibleWorkerForBucket(int)}). */ @RequiredArgsConstructor @ToString(of = "id") diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java index 2ad13d38ab..c690b42a99 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/DatasetRegistry.java @@ -2,34 +2,27 @@ import java.io.Closeable; import java.io.IOException; -import java.net.SocketAddress; import java.util.Collection; import java.util.List; import java.util.NoSuchElementException; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.function.Function; import java.util.stream.Collectors; import javax.validation.Validator; -import javax.validation.constraints.NotNull; -import com.bakdata.conquery.commands.ManagerNode; import com.bakdata.conquery.io.jackson.MutableInjectableValues; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.NamespaceStorage; +import com.bakdata.conquery.mode.InternalObjectMapperCreator; +import com.bakdata.conquery.mode.NamespaceHandler; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.PreviewConfig; import com.bakdata.conquery.models.identifiable.CentralRegistry; -import com.bakdata.conquery.models.identifiable.IdMap; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; -import com.bakdata.conquery.models.messages.network.specific.AddWorker; -import com.bakdata.conquery.models.messages.network.specific.RemoveWorker; -import com.bakdata.conquery.models.query.ExecutionManager; import com.fasterxml.jackson.annotation.JsonIgnoreType; import com.fasterxml.jackson.databind.ObjectMapper; import lombok.Getter; @@ -37,42 +30,30 @@ import lombok.Setter; import lombok.extern.slf4j.Slf4j; -/** - * Holds the necessary information about all datasets on the {@link ManagerNode}. - * This includes meta data of each dataset (not to confuse with {@link MetaStorage}) as well as informations about the - * distributed query engine. - */ @Slf4j @RequiredArgsConstructor @JsonIgnoreType -public class DatasetRegistry extends IdResolveContext implements Closeable { - - private final ConcurrentMap datasets = new ConcurrentHashMap<>(); - @NotNull - @Getter - @Setter - private IdMap workers = new IdMap<>(); // TODO remove this and take it from Namespaces.datasets +public class DatasetRegistry extends IdResolveContext implements Closeable { + private final ConcurrentMap datasets = new ConcurrentHashMap<>(); @Getter private final int entityBucketSize; - @Getter - private final ConcurrentMap shardNodes = new ConcurrentHashMap<>(); - @Getter private final ConqueryConfig config; - private final Function, ObjectMapper> internalObjectMapperCreator; + private final InternalObjectMapperCreator internalObjectMapperCreator; @Getter @Setter private MetaStorage metaStorage; + private final NamespaceHandler namespaceHandler; - public Namespace createNamespace(Dataset dataset, Validator validator) throws IOException { + public N createNamespace(Dataset dataset, Validator validator) throws IOException { // Prepare empty storage NamespaceStorage datasetStorage = new NamespaceStorage(config.getStorage(), "dataset_" + dataset.getName(), validator); - final ObjectMapper persistenceMapper = internalObjectMapperCreator.apply(View.Persistence.Manager.class); + final ObjectMapper persistenceMapper = internalObjectMapperCreator.createInternalObjectMapper(View.Persistence.Manager.class); datasetStorage.openStores(persistenceMapper); datasetStorage.loadData(); @@ -84,42 +65,26 @@ public Namespace createNamespace(Dataset dataset, Validator validator) throws IO return createNamespace(datasetStorage); } - - public Namespace createNamespace(NamespaceStorage datasetStorage) { - final Namespace namespace = Namespace.create( - new ExecutionManager(getMetaStorage()), - datasetStorage, - config, - internalObjectMapperCreator - ); - + public N createNamespace(NamespaceStorage datasetStorage) { + final N namespace = namespaceHandler.createNamespace(datasetStorage, metaStorage); add(namespace); - - // for now we just add one worker to every ShardNode - for (ShardNodeInformation node : getShardNodes().values()) { - node.send(new AddWorker(datasetStorage.getDataset())); - } - return namespace; } - private void add(Namespace ns) { + public void add(N ns) { datasets.put(ns.getStorage().getDataset().getId(), ns); } - public Namespace get(DatasetId dataset) { + public N get(DatasetId dataset) { return datasets.get(dataset); } - + public void removeNamespace(DatasetId id) { - Namespace removed = datasets.remove(id); + N removed = datasets.remove(id); - if(removed != null) { + if (removed != null) { metaStorage.getCentralRegistry().remove(removed.getDataset()); - - getShardNodes().values().forEach(shardNode -> shardNode.send(new RemoveWorker(removed.getDataset()))); - - workers.keySet().removeIf(w->w.getDataset().equals(id)); + namespaceHandler.removeNamespace(id, removed); removed.remove(); } } @@ -132,39 +97,22 @@ public CentralRegistry findRegistry(DatasetId dataset) throws NoSuchElementExcep return datasets.get(dataset).getStorage().getCentralRegistry(); } - + @Override public CentralRegistry getMetaRegistry() { return metaStorage.getCentralRegistry(); } - public synchronized void register(ShardNodeInformation node, WorkerInformation info) { - WorkerInformation old = workers.getOptional(info.getId()).orElse(null); - if (old != null) { - old.setIncludedBuckets(info.getIncludedBuckets()); - old.setConnectedShardNode(node); - } - else { - info.setConnectedShardNode(node); - workers.add(info); - } - - Namespace ns = datasets.get(info.getDataset()); - if (ns == null) { - throw new NoSuchElementException( - "Trying to register a worker for unknown dataset '" + info.getDataset() + "'. I only know " + datasets.keySet()); - } - ns.addWorker(info); - } public List getAllDatasets() { return datasets.values().stream().map(Namespace::getStorage).map(NamespaceStorage::getDataset).collect(Collectors.toList()); } - public Collection getDatasets() { + public Collection getDatasets() { return datasets.values(); } - + + @Override public void close() { for (Namespace namespace : datasets.values()) { try { @@ -178,8 +126,8 @@ public void close() { @Override public MutableInjectableValues inject(MutableInjectableValues values) { - // Make this class also availiable under DatasetRegistry - return super.inject(values) - .add(DatasetRegistry.class, this); + // Make this class also available under DatasetRegistry + return super.inject(values).add(DatasetRegistry.class, this); } + } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java new file mode 100644 index 0000000000..4ef1a5d78f --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/DistributedNamespace.java @@ -0,0 +1,39 @@ +package com.bakdata.conquery.models.worker; + +import java.util.List; + +import com.bakdata.conquery.io.jackson.Injectable; +import com.bakdata.conquery.io.storage.NamespaceStorage; +import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.index.IndexService; +import com.bakdata.conquery.models.jobs.JobManager; +import com.bakdata.conquery.models.query.DistributedExecutionManager; +import com.bakdata.conquery.models.query.FilterSearch; +import com.bakdata.conquery.models.query.entity.Entity; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.Getter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; + + +/** + * Keep track of all data assigned to a single dataset. Each ShardNode has one {@link Worker} per {@link Dataset} / {@link DistributedNamespace}. + * Every Worker is assigned a partition of the loaded {@link Entity}s via {@link Entity::getBucket}. + */ +@Slf4j +@Getter +@ToString(onlyExplicitlyIncluded = true) +public class DistributedNamespace extends LocalNamespace { + + private final WorkerHandler workerHandler; + private final DistributedExecutionManager executionManager; + + public DistributedNamespace(ObjectMapper preprocessMapper, ObjectMapper communicationMapper, NamespaceStorage storage, DistributedExecutionManager executionManager, + JobManager jobManager, FilterSearch filterSearch, IndexService indexService, List injectables, + WorkerHandler workerHandler) { + super(preprocessMapper, communicationMapper, storage, executionManager, jobManager, filterSearch, indexService, injectables); + this.executionManager = executionManager; + this.workerHandler = workerHandler; + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java new file mode 100644 index 0000000000..d77b93adff --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/LocalNamespace.java @@ -0,0 +1,141 @@ +package com.bakdata.conquery.models.worker; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.function.Function; + +import com.bakdata.conquery.io.jackson.Injectable; +import com.bakdata.conquery.io.jackson.View; +import com.bakdata.conquery.io.storage.NamespaceStorage; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.datasets.PreviewConfig; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; +import com.bakdata.conquery.models.identifiable.CentralRegistry; +import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; +import com.bakdata.conquery.models.index.IndexService; +import com.bakdata.conquery.models.jobs.JobManager; +import com.bakdata.conquery.models.jobs.SimpleJob; +import com.bakdata.conquery.models.query.ExecutionManager; +import com.bakdata.conquery.models.query.FilterSearch; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.AccessLevel; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Getter +@ToString(onlyExplicitlyIncluded = true) +@RequiredArgsConstructor +public class LocalNamespace extends IdResolveContext implements Namespace { + + private final ObjectMapper preprocessMapper; + private final ObjectMapper communicationMapper; + @ToString.Include + private final NamespaceStorage storage; + + private final ExecutionManager executionManager; + + // TODO: 01.07.2020 FK: This is not used a lot, as NamespacedMessages are highly convoluted and hard to decouple as is. + private final JobManager jobManager; + + private final FilterSearch filterSearch; + + private final IndexService indexService; + + // Jackson's injectables that are available when deserializing requests (see PathParamInjector) or items from the storage + private final List injectables; + + + @Override + public Dataset getDataset() { + return storage.getDataset(); + } + + @Override + public void close() { + try { + jobManager.close(); + } + catch (Exception e) { + log.error("Unable to close namespace jobmanager of {}", this, e); + } + + try { + log.info("Closing namespace storage of {}", getStorage().getDataset().getId()); + storage.close(); + } + catch (IOException e) { + log.error("Unable to close namespace storage of {}.", this, e); + } + } + + @Override + public void remove() { + try { + jobManager.close(); + } + catch (Exception e) { + log.error("Unable to close namespace jobmanager of {}", this, e); + } + + log.info("Removing namespace storage of {}", getStorage().getDataset().getId()); + storage.removeStorage(); + } + + @Override + public CentralRegistry getCentralRegistry() { + return getStorage().getCentralRegistry(); + } + + @Override + public int getNumberOfEntities() { + return getStorage().getPrimaryDictionary().getSize(); + } + + @Override + public void updateInternToExternMappings() { + storage.getAllConcepts().stream() + .flatMap(c -> c.getConnectors().stream()) + .flatMap(con -> con.getSelects().stream()) + .filter(MappableSingleColumnSelect.class::isInstance) + .map(MappableSingleColumnSelect.class::cast) + .forEach((s) -> jobManager.addSlowJob(new SimpleJob("Update internToExtern Mappings [" + s.getId() + "]", s::loadMapping))); + + storage.getSecondaryIds().stream() + .filter(desc -> desc.getMapping() != null) + .forEach((s) -> jobManager.addSlowJob(new SimpleJob("Update internToExtern Mappings [" + s.getId() + "]", s.getMapping()::init))); + } + + @Override + public void clearIndexCache() { + indexService.evictCache(); + } + + @Override + public PreviewConfig getPreviewConfig() { + return getStorage().getPreviewConfig(); + } + + @Override + public CentralRegistry findRegistry(DatasetId dataset) throws NoSuchElementException { + if (!this.getDataset().getId().equals(dataset)) { + throw new NoSuchElementException("Wrong dataset: '" + dataset + "' (expected: '" + this.getDataset().getId() + "')"); + } + return storage.getCentralRegistry(); + } + + @Override + public CentralRegistry getMetaRegistry() { + throw new UnsupportedOperationException(); + } + + @Override + public ExecutionManager getExecutionManager() { + return executionManager; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java b/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java index 5fc7de4458..e4f16697e0 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/Namespace.java @@ -1,299 +1,59 @@ package com.bakdata.conquery.models.worker; import java.io.Closeable; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; import java.util.List; import java.util.NoSuchElementException; -import java.util.Objects; -import java.util.Set; -import java.util.function.Function; import com.bakdata.conquery.io.jackson.Injectable; -import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.storage.NamespaceStorage; -import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; -import com.bakdata.conquery.models.datasets.Import; import com.bakdata.conquery.models.datasets.PreviewConfig; -import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; import com.bakdata.conquery.models.identifiable.CentralRegistry; -import com.bakdata.conquery.models.identifiable.ids.specific.BucketId; +import com.bakdata.conquery.models.identifiable.Identifiable; +import com.bakdata.conquery.models.identifiable.ids.Id; +import com.bakdata.conquery.models.identifiable.ids.NamespacedId; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId; import com.bakdata.conquery.models.index.IndexService; import com.bakdata.conquery.models.jobs.JobManager; -import com.bakdata.conquery.models.jobs.SimpleJob; -import com.bakdata.conquery.models.messages.namespaces.WorkerMessage; -import com.bakdata.conquery.models.messages.namespaces.specific.UpdateWorkerBucket; import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.models.query.FilterSearch; -import com.bakdata.conquery.models.query.entity.Entity; import com.fasterxml.jackson.databind.ObjectMapper; -import it.unimi.dsi.fastutil.ints.Int2ObjectArrayMap; -import it.unimi.dsi.fastutil.ints.Int2ObjectMap; -import lombok.AccessLevel; -import lombok.Getter; -import lombok.NonNull; -import lombok.RequiredArgsConstructor; -import lombok.ToString; -import lombok.extern.slf4j.Slf4j; +public interface Namespace extends Injectable, Closeable { -/** - * Keep track of all data assigned to a single dataset. Each ShardNode has one {@link Worker} per {@link Dataset} / {@link Namespace}. - * Every Worker is assigned a partition of the loaded {@link Entity}s via {@link Entity::getBucket}. - */ -@Slf4j -@Getter -@ToString(onlyExplicitlyIncluded = true) -@RequiredArgsConstructor(access = AccessLevel.PRIVATE) -public class Namespace extends IdResolveContext implements Closeable { + Dataset getDataset(); - private final ObjectMapper preprocessMapper; - private final ObjectMapper communicationMapper; - @ToString.Include - private final NamespaceStorage storage; + void remove(); - private final ExecutionManager executionManager; + CentralRegistry getCentralRegistry(); - // TODO: 01.07.2020 FK: This is not used a lot, as NamespacedMessages are highly convoluted and hard to decouple as is. - private final JobManager jobManager; + int getNumberOfEntities(); - /** - * All known {@link Worker}s that are part of this Namespace. - */ - private final Set workers = new HashSet<>(); + void updateInternToExternMappings(); - /** - * Map storing the buckets each Worker has been assigned. - */ - private final Int2ObjectMap bucket2WorkerMap = new Int2ObjectArrayMap<>(); + void clearIndexCache(); - private final FilterSearch filterSearch; + PreviewConfig getPreviewConfig(); - private final IndexService indexService; + CentralRegistry findRegistry(DatasetId dataset) throws NoSuchElementException; - // Jackson's injectables that are available when deserializing requests (see PathParamInjector) or items from the storage - private final List injectables; + CentralRegistry getMetaRegistry(); - public static Namespace create(ExecutionManager executionManager, NamespaceStorage storage, ConqueryConfig config, Function, ObjectMapper> mapperCreator) { + ExecutionManager getExecutionManager(); - // Prepare namespace dependent Jackson injectables - List injectables = new ArrayList<>(); - final IndexService indexService = new IndexService(config.getCsv().createCsvParserSettings()); - injectables.add(indexService); - ObjectMapper persistenceMapper = mapperCreator.apply(View.Persistence.Manager.class); - ObjectMapper communicationMapper = mapperCreator.apply(View.InternalCommunication.class); - ObjectMapper preprocessMapper = mapperCreator.apply(null); + ObjectMapper getPreprocessMapper(); - injectables.forEach(i -> i.injectInto(persistenceMapper)); - injectables.forEach(i -> i.injectInto(communicationMapper)); - injectables.forEach(i -> i.injectInto(preprocessMapper)); + ObjectMapper getCommunicationMapper(); - // Open and load the stores - storage.openStores(persistenceMapper); - storage.loadData(); + NamespaceStorage getStorage(); - JobManager jobManager = new JobManager(storage.getDataset().getName(), config.isFailOnError()); + JobManager getJobManager(); - FilterSearch filterSearch = new FilterSearch(storage, jobManager, config.getCsv(), config.getIndex()); + FilterSearch getFilterSearch(); + IndexService getIndexService(); - final Namespace namespace = new Namespace(preprocessMapper, communicationMapper, storage, executionManager, jobManager, filterSearch, indexService, injectables); + List getInjectables(); - - return namespace; - } - - - public void sendToAll(WorkerMessage msg) { - if (workers.isEmpty()) { - throw new IllegalStateException("There are no workers yet"); - } - for (WorkerInformation w : workers) { - w.send(msg); - } - } - - - /** - * Find the assigned worker for the bucket. If there is none return null. - */ - public synchronized WorkerInformation getResponsibleWorkerForBucket(int bucket) { - return bucket2WorkerMap.get(bucket); - } - - /** - * Assign responsibility of a bucket to a Worker. - * - * @implNote Currently the least occupied Worker receives a new Bucket, this can change in later implementations. (For example for dedicated Workers, or entity weightings) - */ - public synchronized void addResponsibility(int bucket) { - WorkerInformation smallest = workers.stream() - .min(Comparator.comparing(si -> si.getIncludedBuckets().size())) - .orElseThrow(() -> new IllegalStateException("Unable to find minimum.")); - - log.debug("Assigning Bucket[{}] to Worker[{}]", bucket, smallest.getId()); - - bucket2WorkerMap.put(bucket, smallest); - - smallest.getIncludedBuckets().add(bucket); - } - - public synchronized void addWorker(WorkerInformation info) { - Objects.requireNonNull(info.getConnectedShardNode(), () -> String.format("No open connections found for Worker[%s]", info.getId())); - - info.setCommunicationWriter(communicationMapper.writer()); - - workers.add(info); - - for (Integer bucket : info.getIncludedBuckets()) { - final WorkerInformation old = bucket2WorkerMap.put(bucket.intValue(), info); - - // This is a completely invalid state from which we should not recover even in production settings. - if (old != null && !old.equals(info)) { - throw new IllegalStateException(String.format("Duplicate claims for Bucket[%d] from %s and %s", bucket, old, info)); - } - } - } - - public Dataset getDataset() { - return storage.getDataset(); - } - - public void close() { - try { - jobManager.close(); - } - catch (Exception e) { - log.error("Unable to close namespace jobmanager of {}", this, e); - } - - try { - log.info("Closing namespace storage of {}", getStorage().getDataset().getId()); - storage.close(); - } - catch (IOException e) { - log.error("Unable to close namespace storage of {}.", this, e); - } - } - - public void remove() { - try { - jobManager.close(); - } - catch (Exception e) { - log.error("Unable to close namespace jobmanager of {}", this, e); - } - - log.info("Removing namespace storage of {}", getStorage().getDataset().getId()); - storage.removeStorage(); - } - - public Set getBucketsForWorker(WorkerId workerId) { - - final WorkerToBucketsMap workerBuckets = storage.getWorkerBuckets(); - if (workerBuckets == null) { - return Collections.emptySet(); - } - return workerBuckets.getBucketsForWorker(workerId); - } - - private synchronized WorkerToBucketsMap createWorkerBucketsMap() { - // Ensure that only one map is created and populated in the storage - WorkerToBucketsMap workerBuckets = storage.getWorkerBuckets(); - if (workerBuckets != null) { - return workerBuckets; - } - storage.setWorkerToBucketsMap(new WorkerToBucketsMap()); - return storage.getWorkerBuckets(); - } - - /** - * Updates the Worker-to-Buckets map, persist it and distributes the update to the shards. - * - * @see Namespace#removeBucketAssignmentsForImportFormWorkers(Import) - */ - public synchronized void addBucketsToWorker(@NonNull WorkerId id, @NonNull Set bucketIds) { - // Ensure that add and remove are not executed at the same time. - // We don't make assumptions about the underlying implementation regarding thread safety - WorkerToBucketsMap workerBuckets = storage.getWorkerBuckets(); - if (workerBuckets == null) { - workerBuckets = createWorkerBucketsMap(); - } - workerBuckets.addBucketForWorker(id, bucketIds); - - storage.setWorkerToBucketsMap(workerBuckets); - - sendUpdatedWorkerInformation(); - } - - public synchronized void removeBucketAssignmentsForImportFormWorkers(@NonNull Import importId) { - - final WorkerToBucketsMap workerBuckets = storage.getWorkerBuckets(); - if (workerBuckets == null) { - return; - } - workerBuckets.removeBucketsOfImport(importId.getId()); - - storage.setWorkerToBucketsMap(workerBuckets); - - sendUpdatedWorkerInformation(); - } - - - private synchronized void sendUpdatedWorkerInformation() { - // While we hold the lock on the namespace distribute the new, consistent state among the workers - for (WorkerInformation w : getWorkers()) { - w.send(new UpdateWorkerBucket(w)); - } - } - - public CentralRegistry getCentralRegistry() { - return getStorage().getCentralRegistry(); - } - - public int getNumberOfEntities() { - return getStorage().getPrimaryDictionary().getSize(); - } - - - public void updateInternToExternMappings() { - storage.getAllConcepts().stream() - .flatMap(c -> c.getConnectors().stream()) - .flatMap(con -> con.getSelects().stream()) - .filter(MappableSingleColumnSelect.class::isInstance) - .map(MappableSingleColumnSelect.class::cast) - .forEach((s) -> jobManager.addSlowJob(new SimpleJob("Update internToExtern Mappings [" + s.getId() + "]", s::loadMapping))); - - storage.getSecondaryIds().stream() - .filter(desc -> desc.getMapping() != null) - .forEach((s) -> jobManager.addSlowJob(new SimpleJob("Update internToExtern Mappings [" + s.getId() + "]", s.getMapping()::init))); - - } - - public void clearIndexCache() { - indexService.evictCache(); - } - - public PreviewConfig getPreviewConfig() { - return getStorage().getPreviewConfig(); - } - - @Override - public CentralRegistry findRegistry(DatasetId dataset) throws NoSuchElementException { - if (!this.getDataset().getId().equals(dataset)) { - throw new NoSuchElementException("Wrong dataset: '" + dataset + "' (expected: '" + this.getDataset().getId() + "')"); - } - return storage.getCentralRegistry(); - } - - @Override - public CentralRegistry getMetaRegistry() { - throw new UnsupportedOperationException(); - } + & NamespacedId, T extends Identifiable> T resolve(ID id); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/WorkerHandler.java b/backend/src/main/java/com/bakdata/conquery/models/worker/WorkerHandler.java new file mode 100644 index 0000000000..a78b17c666 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/WorkerHandler.java @@ -0,0 +1,155 @@ +package com.bakdata.conquery.models.worker; + +import java.util.Collections; +import java.util.Comparator; +import java.util.Objects; +import java.util.Set; + +import com.bakdata.conquery.io.storage.NamespaceStorage; +import com.bakdata.conquery.models.datasets.Import; +import com.bakdata.conquery.models.identifiable.IdMap; +import com.bakdata.conquery.models.identifiable.ids.specific.BucketId; +import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId; +import com.bakdata.conquery.models.messages.namespaces.WorkerMessage; +import com.bakdata.conquery.models.messages.namespaces.specific.UpdateWorkerBucket; +import com.fasterxml.jackson.databind.ObjectMapper; +import it.unimi.dsi.fastutil.ints.Int2ObjectArrayMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectMap; +import lombok.NonNull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +/** + * Handler for worker in a single namespace. + */ +@Slf4j +@RequiredArgsConstructor +public class WorkerHandler { + + private final ObjectMapper communicationMapper; + /** + * All known {@link Worker}s that are part of this Namespace. + */ + private final IdMap workers = new IdMap<>(); + + /** + * Map storing the buckets each Worker has been assigned. + */ + private final Int2ObjectMap bucket2WorkerMap = new Int2ObjectArrayMap<>(); + + private final NamespaceStorage storage; + + public IdMap getWorkers() { + return this.workers; + } + + public void sendToAll(WorkerMessage msg) { + if (workers.isEmpty()) { + throw new IllegalStateException("There are no workers yet"); + } + for (WorkerInformation w : workers.values()) { + w.send(msg); + } + } + + public synchronized void removeBucketAssignmentsForImportFormWorkers(@NonNull Import importId) { + final WorkerToBucketsMap workerBuckets = storage.getWorkerBuckets(); + if (workerBuckets == null) { + return; + } + workerBuckets.removeBucketsOfImport(importId.getId()); + + storage.setWorkerToBucketsMap(workerBuckets); + + sendUpdatedWorkerInformation(); + } + + private synchronized void sendUpdatedWorkerInformation() { + for (WorkerInformation w : this.workers.values()) { + w.send(new UpdateWorkerBucket(w)); + } + } + + private synchronized WorkerToBucketsMap createWorkerBucketsMap() { + // Ensure that only one map is created and populated in the storage + WorkerToBucketsMap workerBuckets = storage.getWorkerBuckets(); + if (workerBuckets != null) { + return workerBuckets; + } + storage.setWorkerToBucketsMap(new WorkerToBucketsMap()); + return storage.getWorkerBuckets(); + } + + public synchronized void addBucketsToWorker(@NonNull WorkerId id, @NonNull Set bucketIds) { + // Ensure that add and remove are not executed at the same time. + // We don't make assumptions about the underlying implementation regarding thread safety + WorkerToBucketsMap workerBuckets = storage.getWorkerBuckets(); + if (workerBuckets == null) { + workerBuckets = createWorkerBucketsMap(); + } + workerBuckets.addBucketForWorker(id, bucketIds); + + storage.setWorkerToBucketsMap(workerBuckets); + + sendUpdatedWorkerInformation(); + } + + public synchronized WorkerInformation getResponsibleWorkerForBucket(int bucket) { + return bucket2WorkerMap.get(bucket); + } + + /** + * @implNote Currently the least occupied Worker receives a new Bucket, this can change in later implementations. (For example for + * dedicated Workers, or entity weightings) + */ + + public synchronized void addResponsibility(int bucket) { + WorkerInformation smallest = workers + .stream() + .min(Comparator.comparing(si -> si.getIncludedBuckets().size())) + .orElseThrow(() -> new IllegalStateException("Unable to find minimum.")); + + log.debug("Assigning Bucket[{}] to Worker[{}]", bucket, smallest.getId()); + + bucket2WorkerMap.put(bucket, smallest); + + smallest.getIncludedBuckets().add(bucket); + } + + public synchronized void addWorker(WorkerInformation info) { + Objects.requireNonNull(info.getConnectedShardNode(), () -> String.format("No open connections found for Worker[%s]", info.getId())); + + info.setCommunicationWriter(communicationMapper.writer()); + + workers.add(info); + + for (Integer bucket : info.getIncludedBuckets()) { + final WorkerInformation old = bucket2WorkerMap.put(bucket.intValue(), info); + + // This is a completely invalid state from which we should not recover even in production settings. + if (old != null && !old.equals(info)) { + throw new IllegalStateException(String.format("Duplicate claims for Bucket[%d] from %s and %s", bucket, old, info)); + } + } + } + + public void register(ShardNodeInformation node, WorkerInformation info) { + WorkerInformation old = this.getWorkers().getOptional(info.getId()).orElse(null); + if (old != null) { + old.setIncludedBuckets(info.getIncludedBuckets()); + old.setConnectedShardNode(node); + } + else { + info.setConnectedShardNode(node); + } + this.addWorker(info); + } + + public Set getBucketsForWorker(WorkerId workerId) { + final WorkerToBucketsMap workerBuckets = storage.getWorkerBuckets(); + if (workerBuckets == null) { + return Collections.emptySet(); + } + return workerBuckets.getBucketsForWorker(workerId); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java index 3272889683..52aa5df7ff 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/AdminServlet.java @@ -83,21 +83,23 @@ public AdminServlet(ManagerNode manager) { jerseyConfig.register(new JacksonMessageBodyProvider(manager.getEnvironment().getObjectMapper())); // freemarker support - adminProcessor = new AdminProcessor( manager.getConfig(), manager.getStorage(), manager.getDatasetRegistry(), manager.getJobManager(), manager.getMaintenanceService(), - manager.getValidator() + manager.getValidator(), + manager.getNodeProvider() ); adminDatasetProcessor = new AdminDatasetProcessor( manager.getConfig(), manager.getValidator(), manager.getDatasetRegistry(), - manager.getJobManager() + manager.getJobManager(), + manager.getImportHandler(), + manager.getStorageListener() ); final AuthCookieFilter authCookieFilter = manager.getConfig().getAuthentication().getAuthCookieFilter(); diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java index 90275f407d..7d65da0ecc 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetProcessor.java @@ -16,6 +16,8 @@ import javax.ws.rs.WebApplicationException; import javax.ws.rs.core.Response; +import com.bakdata.conquery.mode.ImportHandler; +import com.bakdata.conquery.mode.StorageListener; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Dataset; @@ -29,26 +31,15 @@ import com.bakdata.conquery.models.datasets.concepts.filters.specific.SelectFilter; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.MappableSingleColumnSelect; import com.bakdata.conquery.models.exceptions.ValidatorHelper; -import com.bakdata.conquery.models.identifiable.IdMutex; import com.bakdata.conquery.models.identifiable.Identifiable; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import com.bakdata.conquery.models.identifiable.ids.specific.DictionaryId; import com.bakdata.conquery.models.identifiable.ids.specific.TableId; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; import com.bakdata.conquery.models.index.InternToExternMapper; import com.bakdata.conquery.models.index.search.SearchIndex; -import com.bakdata.conquery.models.jobs.ImportJob; import com.bakdata.conquery.models.jobs.JobManager; import com.bakdata.conquery.models.jobs.SimpleJob; -import com.bakdata.conquery.models.messages.namespaces.specific.RemoveConcept; -import com.bakdata.conquery.models.messages.namespaces.specific.RemoveImportJob; -import com.bakdata.conquery.models.messages.namespaces.specific.RemoveSecondaryId; -import com.bakdata.conquery.models.messages.namespaces.specific.RemoveTable; -import com.bakdata.conquery.models.messages.namespaces.specific.UpdateConcept; -import com.bakdata.conquery.models.messages.namespaces.specific.UpdateMatchingStatsMessage; -import com.bakdata.conquery.models.messages.namespaces.specific.UpdateSecondaryId; -import com.bakdata.conquery.models.messages.namespaces.specific.UpdateTable; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.Namespace; import com.univocity.parsers.csv.CsvParser; @@ -64,16 +55,16 @@ @RequiredArgsConstructor(onConstructor_ = {@Inject}) public class AdminDatasetProcessor { - public static final int MAX_IMPORTS_TEXT_LENGTH = 100; private static final String ABBREVIATION_MARKER = "\u2026"; private final ConqueryConfig config; private final Validator validator; - private final DatasetRegistry datasetRegistry; + private final DatasetRegistry datasetRegistry; private final JobManager jobManager; + private final ImportHandler importHandler; + private final StorageListener storageListener; - private final IdMutex sharedDictionaryLocks = new IdMutex<>(); /** * Creates and initializes a new dataset if it does not already exist. @@ -127,8 +118,7 @@ public synchronized void addSecondaryId(Namespace namespace, SecondaryIdDescript log.info("Received new SecondaryId[{}]", secondaryId.getId()); namespace.getStorage().addSecondaryId(secondaryId); - - namespace.sendToAll(new UpdateSecondaryId(secondaryId)); + storageListener.onAddSecondaryId(secondaryId); } /** @@ -141,7 +131,7 @@ public synchronized void deleteSecondaryId(@NonNull SecondaryIdDescription secon final List dependents = namespace.getStorage().getTables().stream() .map(Table::getColumns).flatMap(Arrays::stream) .filter(column -> secondaryId.equals(column.getSecondaryId())) - .collect(Collectors.toList()); + .toList(); if (!dependents.isEmpty()) { final Set tables = dependents.stream().map(Column::getTable).map(Identifiable::getId).collect(Collectors.toSet()); @@ -157,7 +147,7 @@ public synchronized void deleteSecondaryId(@NonNull SecondaryIdDescription secon log.info("Deleting SecondaryId[{}]", secondaryId); namespace.getStorage().removeSecondaryId(secondaryId.getId()); - namespace.sendToAll(new RemoveSecondaryId(secondaryId)); + storageListener.onDeleteSecondaryId(secondaryId); } /** @@ -182,7 +172,7 @@ else if (!table.getDataset().equals(dataset)) { ValidatorHelper.failOnError(log, validator.validate(table)); namespace.getStorage().addTable(table); - namespace.sendToAll(new UpdateTable(table)); + storageListener.onAddTable(table); } @@ -219,7 +209,7 @@ public synchronized void addConcept(@NonNull Dataset dataset, @NonNull Concept namespace.sendToAll(new UpdateConcept(concept)))); + storageListener.onAddConcept(concept); } @@ -268,55 +258,24 @@ public void setStructure(Namespace namespace, StructureNode[] structure) { /** * Reads an Import partially Importing it if not yet present, then submitting it for full import. */ - @SneakyThrows public void addImport(Namespace namespace, InputStream inputStream) throws IOException { - - ImportJob job = ImportJob.createOrUpdate(namespace, inputStream, config.getCluster().getEntityBucketSize(), sharedDictionaryLocks, config, false); - namespace.getJobManager().addSlowJob(job); - - clearDependentConcepts(namespace.getStorage().getAllConcepts(), job.getTable()); + this.importHandler.addImport(namespace, inputStream); } /** * Reads an Import partially Importing it if it is present, then submitting it for full import [Update of an import]. */ - @SneakyThrows public void updateImport(Namespace namespace, InputStream inputStream) throws IOException { - - ImportJob job = ImportJob.createOrUpdate(namespace, inputStream, config.getCluster().getEntityBucketSize(), sharedDictionaryLocks, config, true); - - namespace.getJobManager().addSlowJob(job); - - clearDependentConcepts(namespace.getStorage().getAllConcepts(), job.getTable()); + this.importHandler.updateImport(namespace, inputStream); } /** * Deletes an import. */ public synchronized void deleteImport(Import imp) { - final Namespace namespace = datasetRegistry.get(imp.getTable().getDataset().getId()); - - clearDependentConcepts(namespace.getStorage().getAllConcepts(), imp.getTable()); - - - namespace.getStorage().removeImport(imp.getId()); - namespace.sendToAll(new RemoveImportJob(imp)); - - // Remove bucket assignments for consistency report - namespace.removeBucketAssignmentsForImportFormWorkers(imp); + this.importHandler.deleteImport(imp); } - private void clearDependentConcepts(Collection> allConcepts, Table table) { - for (Concept c : allConcepts) { - for (Connector con : c.getConnectors()) { - if (!con.getTable().equals(table)) { - continue; - } - - con.getConcept().clearMatchingStats(); - } - } - } /** * Deletes a table if it has no dependents or not forced to do so. @@ -339,7 +298,7 @@ public synchronized List deleteTable(Table table, boolean force) { .forEach(this::deleteImport); namespace.getStorage().removeTable(table.getId()); - namespace.sendToAll(new RemoveTable(table)); + storageListener.onRemoveTable(table); } return dependentConcepts.stream().map(Concept::getId).collect(Collectors.toList()); @@ -352,8 +311,7 @@ public synchronized void deleteConcept(Concept concept) { final Namespace namespace = datasetRegistry.get(concept.getDataset().getId()); namespace.getStorage().removeConcept(concept.getId()); - getJobManager() - .addSlowJob(new SimpleJob("sendToAll: remove " + concept.getId(), () -> namespace.sendToAll(new RemoveConcept(concept)))); + storageListener.onDeleteConcept(concept); } /** @@ -368,12 +326,8 @@ public void updateMatchingStats(Dataset dataset) { "Initiate Update Matching Stats and FilterSearch", () -> { - final Collection> concepts = ns.getStorage().getAllConcepts() - .stream() - .filter(concept -> concept.getMatchingStats() == null) - .collect(Collectors.toSet()); - ns.sendToAll(new UpdateMatchingStatsMessage(concepts)); + storageListener.onUpdateMatchingStats(dataset); ns.getFilterSearch().updateSearch(); ns.updateInternToExternMappings(); } @@ -404,13 +358,13 @@ public List deleteInternToExternMapping(InternToExternMapper internTo final Set> dependentConcepts = namespace.getStorage().getAllConcepts().stream() .filter( - c -> c.getSelects().stream() - .filter(MappableSingleColumnSelect.class::isInstance) + c -> c.getSelects().stream() + .filter(MappableSingleColumnSelect.class::isInstance) - .map(MappableSingleColumnSelect.class::cast) - .map(MappableSingleColumnSelect::getMapping) - .anyMatch(internToExternMapper::equals) - ) + .map(MappableSingleColumnSelect.class::cast) + .map(MappableSingleColumnSelect::getMapping) + .anyMatch(internToExternMapper::equals) + ) .collect(Collectors.toSet()); if (force || dependentConcepts.isEmpty()) { diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetsResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetsResource.java index 750f37f058..3ced1d1d56 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetsResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminDatasetsResource.java @@ -15,9 +15,8 @@ import com.bakdata.conquery.io.jersey.ExtraMimeTypes; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; -import lombok.Getter; +import com.bakdata.conquery.models.worker.Namespace; import lombok.RequiredArgsConstructor; -import lombok.Setter; @Produces({ExtraMimeTypes.JSON_STRING, ExtraMimeTypes.SMILE_STRING}) @Consumes({ExtraMimeTypes.JSON_STRING, ExtraMimeTypes.SMILE_STRING}) diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminProcessor.java index 76261723dd..6f3b5a76c5 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminProcessor.java @@ -8,6 +8,7 @@ import java.util.TreeSet; import java.util.concurrent.ScheduledExecutorService; import java.util.function.Predicate; +import java.util.function.Supplier; import java.util.stream.Collectors; import javax.validation.Validator; @@ -51,11 +52,12 @@ public class AdminProcessor { private final ConqueryConfig config; private final MetaStorage storage; - private final DatasetRegistry datasetRegistry; + private final DatasetRegistry datasetRegistry; private final JobManager jobManager; private final ScheduledExecutorService maintenanceService; private final Validator validator; private final ObjectWriter jsonWriter = Jackson.MAPPER.writer(); + private final Supplier> nodeProvider; public void addRoles(List roles) { @@ -277,7 +279,7 @@ public Collection getJobs() { )); } - for (ShardNodeInformation si : getDatasetRegistry().getShardNodes().values()) { + for (ShardNodeInformation si : nodeProvider.get()) { out.addAll(si.getJobManagerStatus()); } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java index 479e0e8751..906fd8e151 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/AdminResource.java @@ -34,6 +34,7 @@ import com.bakdata.conquery.models.jobs.JobManagerStatus; import com.bakdata.conquery.models.messages.network.specific.CancelJobMessage; import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.models.worker.ShardNodeInformation; import com.bakdata.conquery.resources.admin.ui.AdminUIResource; import io.dropwizard.auth.Auth; @@ -78,7 +79,7 @@ public Response cancelJob(@PathParam(JOB_ID) UUID jobId) { processor.getJobManager().cancelJob(jobId); - for (ShardNodeInformation info : processor.getDatasetRegistry().getShardNodes().values()) { + for (ShardNodeInformation info : processor.getNodeProvider().get()) { info.send(new CancelJobMessage(jobId)); } @@ -115,7 +116,7 @@ public FullExecutionStatus[] getQueries(@Auth Subject currentUser, @QueryParam(" final long limit = maybeLimit.orElse(100); final MetaStorage storage = processor.getStorage(); - final DatasetRegistry datasetRegistry = processor.getDatasetRegistry(); + final DatasetRegistry datasetRegistry = processor.getDatasetRegistry(); return storage.getAllExecutions().stream() @@ -136,4 +137,4 @@ public FullExecutionStatus[] getQueries(@Auth Subject currentUser, @QueryParam(" }) .toArray(FullExecutionStatus[]::new); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UIProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UIProcessor.java index 7a4b9f21f6..be2b588645 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UIProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/rest/UIProcessor.java @@ -33,6 +33,7 @@ import com.bakdata.conquery.models.events.CBlock; import com.bakdata.conquery.models.identifiable.ids.specific.UserId; import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.resources.admin.ui.model.FrontendAuthOverview; import com.bakdata.conquery.resources.admin.ui.model.FrontendGroupContent; import com.bakdata.conquery.resources.admin.ui.model.FrontendPermission; @@ -57,7 +58,7 @@ public class UIProcessor { @Getter private final AdminProcessor adminProcessor; - public DatasetRegistry getDatasetRegistry() { + public DatasetRegistry getDatasetRegistry() { return adminProcessor.getDatasetRegistry(); } @@ -66,7 +67,7 @@ public MetaStorage getStorage() { } public UIContext getUIContext() { - return new UIContext(getDatasetRegistry()); + return new UIContext(adminProcessor.getNodeProvider()); } public FrontendAuthOverview getAuthOverview() { diff --git a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/UIContext.java b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/UIContext.java index 940f8bc618..83d48c9271 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/UIContext.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/admin/ui/model/UIContext.java @@ -1,9 +1,13 @@ package com.bakdata.conquery.resources.admin.ui.model; +import java.net.SocketAddress; import java.util.Collection; +import java.util.Map; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; -import com.bakdata.conquery.models.worker.DatasetRegistry; -import com.bakdata.conquery.models.worker.WorkerInformation; +import com.bakdata.conquery.models.worker.ShardNodeInformation; import com.bakdata.conquery.resources.ResourceConstants; import freemarker.template.TemplateModel; import lombok.Getter; @@ -11,25 +15,18 @@ @RequiredArgsConstructor public class UIContext { - + private static final TemplateModel STATIC_URI_ELEMENTS = ResourceConstants.getAsTemplateModel(); - @Getter - private final DatasetRegistry namespaces; + private final Supplier> shardNodeSupplier; @Getter public final TemplateModel staticUriElem = STATIC_URI_ELEMENTS; - public boolean[] getWorkerStatuses() { - boolean[] result = new boolean[namespaces.getShardNodes().values().size()]; - int id = 0; - for(WorkerInformation wi:namespaces.getWorkers().values()) { - result[id++] = wi.isConnected(); - } - return result; - } - - public Collection getWorkers() { - return namespaces.getWorkers().values(); + public Map getShardNodes() { + return shardNodeSupplier.get().stream().collect(Collectors.toMap( + ShardNodeInformation::getRemoteAddress, + Function.identity() + )); } } diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java index a3ff393d83..ab877f9b41 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java @@ -60,7 +60,7 @@ @RequiredArgsConstructor(onConstructor_ = {@Inject}) public class ConceptsProcessor { - private final DatasetRegistry namespaces; + private final DatasetRegistry namespaces; private final Validator validator; private final ConqueryConfig config; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java new file mode 100644 index 0000000000..6778add265 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java @@ -0,0 +1,45 @@ +package com.bakdata.conquery.sql.conquery; + +import java.util.UUID; +import java.util.stream.Stream; + +import com.bakdata.conquery.apiv1.query.QueryDescription; +import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.query.ExecutionManager; +import com.bakdata.conquery.models.query.results.EntityResult; +import com.bakdata.conquery.models.worker.Namespace; + +public class SqlExecutionManager implements ExecutionManager { + @Override + public ManagedExecution runQuery(Namespace namespace, QueryDescription query, User user, Dataset submittedDataset, ConqueryConfig config, boolean system) { + return null; + } + + @Override + public void execute(Namespace namespace, ManagedExecution execution, ConqueryConfig config) { + + } + + @Override + public ManagedExecution createExecution(QueryDescription query, User user, Dataset submittedDataset, boolean system) { + return null; + } + + @Override + public void cancelQuery(Dataset dataset, ManagedExecution query) { + + } + + @Override + public void clearQueryResults(ManagedExecution execution) { + } + + @Override + public Stream streamQueryResults(ManagedExecution execution) { + return null; + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlManagedQuery.java b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlManagedQuery.java new file mode 100644 index 0000000000..13ea8e1212 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlManagedQuery.java @@ -0,0 +1,78 @@ +package com.bakdata.conquery.sql.conquery; + +import java.util.List; +import java.util.function.Consumer; +import java.util.stream.Stream; + +import com.bakdata.conquery.apiv1.query.Query; +import com.bakdata.conquery.apiv1.query.QueryDescription; +import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.query.ColumnDescriptor; +import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.query.SingleTableResult; +import com.bakdata.conquery.models.query.Visitable; +import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.models.query.results.EntityResult; +import com.bakdata.conquery.util.QueryUtils; +import lombok.Getter; +import lombok.Setter; + +@Setter +@Getter +@CPSType(base = ManagedExecution.class, id = "SQL_QUERY") +public class SqlManagedQuery extends ManagedExecution implements SingleTableResult { + private Query query; + + protected SqlManagedQuery(MetaStorage storage) { + super(storage); + } + + public SqlManagedQuery(Query query, User owner, Dataset dataset, MetaStorage storage) { + super(owner, dataset, storage); + this.query = query; + } + + @Override + protected void doInitExecutable() { + + } + + @Override + public QueryDescription getSubmitted() { + return query; + } + + @Override + protected String makeDefaultLabel(PrintSettings cfg) { + return QueryUtils.makeQueryLabel(query, cfg, getId()); + } + + @Override + public void visit(Consumer visitor) { + visitor.accept(this); + } + + @Override + public List generateColumnDescriptions() { + return null; + } + + @Override + public List getResultInfos() { + return null; + } + + @Override + public Stream streamResults() { + return null; + } + + @Override + public long resultRowCount() { + return 0; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conquery/package-info.java b/backend/src/main/java/com/bakdata/conquery/sql/conquery/package-info.java new file mode 100644 index 0000000000..9a7ac7868c --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conquery/package-info.java @@ -0,0 +1,4 @@ +/** + * Integration with the existing ConQuery architecture. + */ +package com.bakdata.conquery.sql.conquery; diff --git a/backend/src/main/java/com/bakdata/conquery/tasks/ReportConsistencyTask.java b/backend/src/main/java/com/bakdata/conquery/tasks/ReportConsistencyTask.java index fe521f008a..533c9fdc8c 100644 --- a/backend/src/main/java/com/bakdata/conquery/tasks/ReportConsistencyTask.java +++ b/backend/src/main/java/com/bakdata/conquery/tasks/ReportConsistencyTask.java @@ -4,21 +4,23 @@ import java.util.List; import java.util.Map; +import com.bakdata.conquery.mode.cluster.ClusterState; import com.bakdata.conquery.models.messages.namespaces.specific.RequestConsistency; -import com.bakdata.conquery.models.worker.DatasetRegistry; import io.dropwizard.servlets.tasks.Task; public class ReportConsistencyTask extends Task { - private final DatasetRegistry datasetRegistry; + private final ClusterState clusterState; - public ReportConsistencyTask(DatasetRegistry datasetRegistry) { + public ReportConsistencyTask(ClusterState clusterState) { super("report-consistency"); - this.datasetRegistry = datasetRegistry; + this.clusterState = clusterState; } @Override public void execute(Map> parameters, PrintWriter output) throws Exception { - datasetRegistry.getWorkers().values().forEach(w -> w.send(new RequestConsistency())); + clusterState.getWorkerHandlers().values().stream() + .flatMap(ns -> ns.getWorkers().stream()) + .forEach(worker -> worker.send(new RequestConsistency())); } } diff --git a/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java b/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java index 914ebf05d2..9778859a0c 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java +++ b/backend/src/main/java/com/bakdata/conquery/util/QueryUtils.java @@ -2,16 +2,21 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Locale; +import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.stream.Collectors; +import c10n.C10N; import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.apiv1.query.QueryDescription; import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; @@ -19,6 +24,7 @@ import com.bakdata.conquery.apiv1.query.concept.specific.CQOr; import com.bakdata.conquery.apiv1.query.concept.specific.CQReusedQuery; import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; +import com.bakdata.conquery.internationalization.CQElementC10n; import com.bakdata.conquery.models.auth.permissions.Ability; import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; import com.bakdata.conquery.models.common.CDateSet; @@ -26,14 +32,17 @@ import com.bakdata.conquery.models.datasets.SecondaryIdDescription; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.models.datasets.concepts.Connector; +import com.bakdata.conquery.models.i18n.I18n; import com.bakdata.conquery.models.identifiable.ids.NamespacedId; import com.bakdata.conquery.models.identifiable.ids.NamespacedIdentifiable; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.NamespacedIdentifiableHolding; +import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.visitor.QueryVisitor; +import com.google.common.base.Strings; import com.google.common.collect.ClassToInstanceMap; import lombok.Getter; import lombok.NonNull; @@ -44,6 +53,7 @@ @UtilityClass public class QueryUtils { + private static final int MAX_CONCEPT_LABEL_CONCAT_LENGTH = 70; /** * Provides a starting operator for consumer chains, that does nothing. */ @@ -213,4 +223,81 @@ public static QueryExecutionContext determineDateAggregatorForContext(QueryExecu } return ctx.withQueryDateAggregator(altValidityDateAggregator.get()); } + + public static String makeQueryLabel(final Visitable query, PrintSettings cfg, ManagedExecutionId id) { + final StringBuilder sb = new StringBuilder(); + + final Map, List> sortedContents = + Visitable.stream(query) + .collect(Collectors.groupingBy(Visitable::getClass)); + + int sbStartSize = sb.length(); + + // Check for CQExternal + List externals = sortedContents.getOrDefault(CQExternal.class, Collections.emptyList()); + if (!externals.isEmpty()) { + if (!sb.isEmpty()) { + sb.append(" "); + } + sb.append(C10N.get(CQElementC10n.class, I18n.LOCALE.get()).external()); + } + + // Check for CQReused + if (sortedContents.containsKey(CQReusedQuery.class)) { + if (!sb.isEmpty()) { + sb.append(" "); + } + sb.append(C10N.get(CQElementC10n.class, I18n.LOCALE.get()).reused()); + } + + + // Check for CQConcept + if (sortedContents.containsKey(CQConcept.class)) { + if (!sb.isEmpty()) { + sb.append(" "); + } + // Track length of text we are appending for concepts. + final AtomicInteger length = new AtomicInteger(); + + sortedContents.get(CQConcept.class) + .stream() + .map(CQConcept.class::cast) + + .map(c -> makeLabelWithRootAndChild(c, cfg)) + .filter(Predicate.not(Strings::isNullOrEmpty)) + .distinct() + + .takeWhile(elem -> length.addAndGet(elem.length()) < MAX_CONCEPT_LABEL_CONCAT_LENGTH) + .forEach(label -> sb.append(label).append(" ")); + + // Last entry will output one Space that we don't want + if (!sb.isEmpty()) { + sb.deleteCharAt(sb.length() - 1); + } + + // If not all Concept could be included in the name, point that out + if (length.get() > MAX_CONCEPT_LABEL_CONCAT_LENGTH) { + sb.append(" ").append(C10N.get(CQElementC10n.class, I18n.LOCALE.get()).furtherConcepts()); + } + } + + + // Fallback to id if nothing could be extracted from the query description + if (sbStartSize == sb.length()) { + sb.append(id.getExecution()); + } + + return sb.toString(); + } + + + private static String makeLabelWithRootAndChild(CQConcept cqConcept, PrintSettings cfg) { + String label = cqConcept.getUserOrDefaultLabel(cfg.getLocale()); + if (label == null) { + label = cqConcept.getConcept().getLabel(); + } + + // Concat everything with dashes + return label.replace(" ", "-"); + } } diff --git a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/index.html.ftl b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/index.html.ftl index e5c39c8f3b..29168ef3d4 100644 --- a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/index.html.ftl +++ b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/index.html.ftl @@ -2,9 +2,9 @@ <@layout.layout>
- <#list ctx.namespaces.shardNodes as key,shardNode> + <#list ctx.shardNodes as key,shardNode> ${key}
- \ No newline at end of file + diff --git a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/template.html.ftl b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/template.html.ftl index 4a9306ec02..3af578bd0e 100644 --- a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/template.html.ftl +++ b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/template.html.ftl @@ -70,7 +70,7 @@
- <#list ctx.namespaces.shardNodes as key,shardNode> + <#list ctx.shardNodes as key,shardNode>
@@ -129,4 +129,4 @@ {"factor": 1000000000, "unit" :"G"}, {"factor": 1000000000000, "unit" : "T" } ] /> <#assign siStr=(num / (siMap[thousands].factor))?string("0.# ") + siMap[thousands].unit /> <#return siStr /> - \ No newline at end of file + diff --git a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java index 15870d993e..3d29b78c03 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/StoredQueriesProcessorTest.java @@ -48,6 +48,7 @@ import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.DistributedNamespace; import com.bakdata.conquery.util.NonPersistentStoreFactory; import com.google.common.collect.ImmutableList; import lombok.SneakyThrows; @@ -59,7 +60,8 @@ public class StoredQueriesProcessorTest { public static final AuthorizationController AUTHORIZATION_CONTROLLER = new AuthorizationController(STORAGE, new DevelopmentAuthorizationConfig()); public static final ConqueryConfig CONFIG = new ConqueryConfig(); - private static final QueryProcessor processor = new QueryProcessor(new DatasetRegistry(0, CONFIG, null), STORAGE, CONFIG); + private static final DatasetRegistry datasetRegistry = new DatasetRegistry<>(0, CONFIG, null, null); + private static final QueryProcessor processor = new QueryProcessor(datasetRegistry, STORAGE, CONFIG); private static final Dataset DATASET_0 = new Dataset() {{ setName("dataset0"); diff --git a/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java b/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java index 6a989bb02d..d74b89ac95 100644 --- a/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java +++ b/backend/src/test/java/com/bakdata/conquery/api/form/config/FormConfigTest.java @@ -46,6 +46,7 @@ import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.IdResolveContext; +import com.bakdata.conquery.models.worker.LocalNamespace; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.util.NonPersistentStoreFactory; import com.fasterxml.jackson.databind.JsonNode; @@ -101,7 +102,7 @@ public void setupTestClass() throws Exception { doAnswer(invocation -> { final DatasetId id = invocation.getArgument(0); - Namespace namespaceMock = Mockito.mock(Namespace.class); + Namespace namespaceMock = Mockito.mock(LocalNamespace.class); if (id.equals(datasetId)) { when(namespaceMock.getDataset()).thenReturn(dataset); } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTest.java b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTest.java index c5c10c8a22..72c052b0a1 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTest.java @@ -44,7 +44,7 @@ abstract class Simple implements IntegrationTest { public void execute(String name, TestConquery testConquery) throws Exception { StandaloneSupport conquery = testConquery.getSupport(name); // Because Shiro works with a static Security manager - testConquery.getStandaloneCommand().getManager().getAuthController().registerStaticSecurityManager(); + testConquery.getStandaloneCommand().getManagerNode().getAuthController().registerStaticSecurityManager(); try { execute(conquery); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/JsonIntegrationTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/JsonIntegrationTest.java index 795a036d49..7dd908dc5e 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/JsonIntegrationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/JsonIntegrationTest.java @@ -48,8 +48,7 @@ public void execute(StandaloneSupport conquery) throws Exception { //ensure the metadata is collected - - conquery.getNamespace().sendToAll(new UpdateMatchingStatsMessage(conquery.getNamespace().getStorage().getAllConcepts())); + conquery.getNamespace().getWorkerHandler().sendToAll(new UpdateMatchingStatsMessage(conquery.getNamespace().getStorage().getAllConcepts())); conquery.waitUntilWorkDone(); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/AdminEndpointTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/AdminEndpointTest.java index 852dc02217..17718ea918 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/AdminEndpointTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/AdminEndpointTest.java @@ -20,7 +20,7 @@ public class AdminEndpointTest implements ProgrammaticIntegrationTest { public void execute(String name, TestConquery testConquery) throws Exception { List expectedEndpoints = READER.readValue(In.resource("/tests/endpoints/adminEndpointInfo.json").asStream()); - DropwizardResourceConfig jerseyConfig = testConquery.getStandaloneCommand().getManager().getAdmin().getJerseyConfig(); + DropwizardResourceConfig jerseyConfig = testConquery.getStandaloneCommand().getManagerNode().getAdmin().getJerseyConfig(); List resources = EndpointTestHelper.collectEndpoints(jerseyConfig); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/AdminUIEndpointTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/AdminUIEndpointTest.java index 2a8ad28d11..0fa0c1d05a 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/AdminUIEndpointTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/AdminUIEndpointTest.java @@ -20,7 +20,7 @@ public class AdminUIEndpointTest implements ProgrammaticIntegrationTest { public void execute(String name, TestConquery testConquery) throws Exception { List expectedEndpoints = READER.readValue(In.resource("/tests/endpoints/adminUIEndpointInfo.json").asStream()); - DropwizardResourceConfig jerseyConfig = testConquery.getStandaloneCommand().getManager().getAdmin().getJerseyConfigUI(); + DropwizardResourceConfig jerseyConfig = testConquery.getStandaloneCommand().getManagerNode().getAdmin().getJerseyConfigUI(); List resources = EndpointTestHelper.collectEndpoints(jerseyConfig); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java index a6e99141d4..603b94445f 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ConceptPermissionTest.java @@ -2,7 +2,6 @@ import static com.bakdata.conquery.integration.common.LoadingUtil.importSecondaryIds; -import com.bakdata.conquery.apiv1.QueryProcessor; import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.integration.IntegrationTest; import com.bakdata.conquery.integration.common.IntegrationUtils; @@ -31,8 +30,7 @@ public void execute(StandaloneSupport conquery) throws Exception { final MetaStorage storage = conquery.getMetaStorage(); final Dataset dataset = conquery.getDataset(); final String testJson = In.resource("/tests/query/SIMPLE_TREECONCEPT_QUERY/SIMPLE_TREECONCEPT_Query.test.json").withUTF8().readAll(); - final QueryTest test = (QueryTest) JsonIntegrationTest.readJson(dataset.getId(), testJson); - final QueryProcessor processor = new QueryProcessor(conquery.getDatasetRegistry(), storage, conquery.getConfig()); + final QueryTest test = JsonIntegrationTest.readJson(dataset.getId(), testJson); final User user = new User("testUser", "testUserLabel", storage); // Manually import data, so we can do our own work. diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java index 3931d12eeb..93617d472f 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/ExternalFormBackendTest.java @@ -52,7 +52,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { log.info("Test health"); assertThat(testConquery.getStandaloneCommand() - .getManager() + .getManagerNode() .getEnvironment() .healthChecks() .runHealthCheck(FORM_BACKEND_ID) @@ -60,7 +60,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { .describedAs("Checking health of form backend").isTrue(); log.info("Get external form configs"); - final FormScanner formScanner = testConquery.getStandaloneCommand().getManager().getFormScanner(); + final FormScanner formScanner = testConquery.getStandaloneCommand().getManagerNode().getFormScanner(); formScanner.execute(Collections.emptyMap(), null); final String externalFormId = FormBackendConfig.createSubTypedId("SOME_EXTERNAL_FORM"); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/MetadataCollectionTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/MetadataCollectionTest.java index 1b81dc4f59..6605a6e6f5 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/MetadataCollectionTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/MetadataCollectionTest.java @@ -23,19 +23,19 @@ public class MetadataCollectionTest extends IntegrationTest.Simple implements Pr public void execute(StandaloneSupport conquery) throws Exception { //read test sepcification String testJson = In.resource("/tests/query/SIMPLE_TREECONCEPT_QUERY/SIMPLE_TREECONCEPT_Query.test.json").withUTF8().readAll(); - + DatasetId dataset = conquery.getDataset().getId(); - + ConqueryTestSpec test = JsonIntegrationTest.readJson(dataset, testJson); ValidatorHelper.failOnError(log, conquery.getValidator().validate(test)); - + test.importRequiredData(conquery); - + //ensure the metadata is collected - conquery.getNamespace().sendToAll(new UpdateMatchingStatsMessage(conquery.getNamespace().getStorage().getAllConcepts())); + conquery.getNamespace().getWorkerHandler().sendToAll(new UpdateMatchingStatsMessage(conquery.getNamespace().getStorage().getAllConcepts())); conquery.waitUntilWorkDone(); - + TreeConcept concept = (TreeConcept) conquery.getNamespace().getStorage().getAllConcepts().iterator().next(); //check the number of matched events diff --git a/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java b/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java index 1cc0b95608..0a0af0aa8e 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/tests/RestartTest.java @@ -46,7 +46,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { Validator validator = Validators.newValidator(); EntityIdMap entityIdMap = IdMapSerialisationTest.createTestPersistentMap(); - ManagerNode manager = testConquery.getStandaloneCommand().getManager(); + ManagerNode manager = testConquery.getStandaloneCommand().getManagerNode(); AdminDatasetProcessor adminDatasetProcessor = manager.getAdmin().getAdminDatasetProcessor(); AdminProcessor adminProcessor = manager.getAdmin().getAdminProcessor(); @@ -179,7 +179,7 @@ public void execute(String name, TestConquery testConquery) throws Exception { assertThat(entityIdMapAfterRestart).isEqualTo(entityIdMap); // We need to reassign the dataset processor because the instance prio to the restart became invalid - adminDatasetProcessor = testConquery.getStandaloneCommand().getManager().getAdmin().getAdminDatasetProcessor(); + adminDatasetProcessor = testConquery.getStandaloneCommand().getManagerNode().getAdmin().getAdminDatasetProcessor(); // Cleanup adminDatasetProcessor.deleteDataset(dataset1); adminDatasetProcessor.deleteDataset(dataset2); @@ -189,4 +189,3 @@ public void execute(String name, TestConquery testConquery) throws Exception { adminDatasetProcessor.deleteDataset(dataset6); } } - diff --git a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java index 379f1f07d6..48cb1582cb 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/AbstractSerializationTest.java @@ -10,8 +10,12 @@ import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.mode.InternalObjectMapperCreator; +import com.bakdata.conquery.mode.cluster.ClusterNamespaceHandler; +import com.bakdata.conquery.mode.cluster.ClusterState; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.DistributedNamespace; import com.bakdata.conquery.util.NonPersistentStoreFactory; import com.fasterxml.jackson.databind.ObjectMapper; import io.dropwizard.jersey.validation.Validators; @@ -23,7 +27,7 @@ public abstract class AbstractSerializationTest { private final Validator validator = Validators.newValidator(); private final ConqueryConfig config = new ConqueryConfig(); - private DatasetRegistry datasetRegistry; + private DatasetRegistry datasetRegistry; private MetaStorage metaStorage; private ObjectMapper managerInternalMapper; @@ -33,15 +37,19 @@ public abstract class AbstractSerializationTest { @BeforeEach public void before() { - datasetRegistry = new DatasetRegistry(0, config, null); + InternalObjectMapperCreator creator = new InternalObjectMapperCreator(config, validator); + datasetRegistry = new DatasetRegistry<>(0, config, null, new ClusterNamespaceHandler(new ClusterState(), config, creator)); metaStorage = new MetaStorage(new NonPersistentStoreFactory(), datasetRegistry); + datasetRegistry.setMetaStorage(metaStorage); + creator.init(datasetRegistry); // Prepare manager node internal mapper final ManagerNode managerNode = mock(ManagerNode.class); when(managerNode.getConfig()).thenReturn(config); when(managerNode.getValidator()).thenReturn(validator); - when(managerNode.getDatasetRegistry()).thenReturn(datasetRegistry); + doReturn(datasetRegistry).when(managerNode).getDatasetRegistry(); when(managerNode.getStorage()).thenReturn(metaStorage); + when(managerNode.getInternalObjectMapperCreator()).thenReturn(creator); when(managerNode.createInternalObjectMapper(any())).thenCallRealMethod(); managerInternalMapper = managerNode.createInternalObjectMapper(View.Persistence.Manager.class); diff --git a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java index 67a53f5b1c..efb6619344 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/jackson/serializer/IdRefrenceTest.java @@ -14,6 +14,7 @@ import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.identifiable.CentralRegistry; import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.DistributedNamespace; import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.bakdata.conquery.util.NonPersistentStoreFactory; import com.fasterxml.jackson.annotation.JsonCreator; @@ -38,7 +39,7 @@ public void testListReferences() throws IOException { registry.register(dataset); registry.register(table); - final DatasetRegistry datasetRegistry = new DatasetRegistry(0, null, null); + final DatasetRegistry datasetRegistry = new DatasetRegistry<>(0, null, null, null); final MetaStorage metaStorage = new MetaStorage(new NonPersistentStoreFactory(),datasetRegistry); diff --git a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java index 74c3145abd..f2b1bece25 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/execution/DefaultLabelTest.java @@ -24,6 +24,7 @@ import com.bakdata.conquery.models.i18n.I18n; import com.bakdata.conquery.models.query.ManagedQuery; import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.worker.LocalNamespace; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.util.NonPersistentStoreFactory; import org.jetbrains.annotations.NotNull; @@ -36,7 +37,7 @@ public class DefaultLabelTest { private final static MetaStorage STORAGE = new NonPersistentStoreFactory().createMetaStorage(); - private static final Namespace NAMESPACE = Mockito.mock(Namespace.class); + private static final Namespace NAMESPACE = Mockito.mock(LocalNamespace.class); private static final Dataset DATASET = new Dataset("dataset"); private static final User user = new User("user","user", STORAGE); diff --git a/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java b/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java index 06056941ab..8043860107 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/query/DefaultColumnNameTest.java @@ -33,6 +33,7 @@ import com.bakdata.conquery.models.query.queryplan.aggregators.Aggregator; import com.bakdata.conquery.models.query.resultinfo.SelectResultInfo; import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; +import com.bakdata.conquery.models.worker.LocalNamespace; import com.bakdata.conquery.models.worker.Namespace; import io.dropwizard.jersey.validation.Validators; import lombok.SneakyThrows; @@ -43,7 +44,7 @@ @Slf4j public class DefaultColumnNameTest { - private static final Namespace NAMESPACE = mock(Namespace.class); + private static final Namespace NAMESPACE = mock(LocalNamespace.class); private static final PrintSettings SETTINGS = new PrintSettings(false, Locale.ENGLISH, NAMESPACE, new ConqueryConfig(), null); private static final Validator VALIDATOR = Validators.newValidator(); diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java b/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java index 160774811d..e31daee764 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java @@ -23,7 +23,7 @@ import com.bakdata.conquery.models.identifiable.ids.Id; import com.bakdata.conquery.models.identifiable.ids.NamespacedId; import com.bakdata.conquery.models.worker.DatasetRegistry; -import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.models.worker.DistributedNamespace; import com.bakdata.conquery.resources.admin.rest.AdminDatasetProcessor; import com.bakdata.conquery.resources.admin.rest.AdminProcessor; import com.google.common.util.concurrent.MoreExecutors; @@ -39,7 +39,7 @@ public class StandaloneSupport { private final TestConquery testConquery; @Getter - private final Namespace namespace; + private final DistributedNamespace namespace; @Getter private final Dataset dataset; @Getter @@ -54,7 +54,7 @@ public class StandaloneSupport { private final User testUser; public AuthorizationController getAuthorizationController() { - return testConquery.getStandaloneCommand().getManager().getAuthController(); + return testConquery.getStandaloneCommand().getManagerNode().getAuthController(); } public void waitUntilWorkDone() { @@ -84,19 +84,19 @@ public void run(Environment environment, net.sourceforge.argparse4j.inf.Namespac public Validator getValidator() { - return testConquery.getStandaloneCommand().getManager().getValidator(); + return testConquery.getStandaloneCommand().getManagerNode().getValidator(); } public MetaStorage getMetaStorage() { - return testConquery.getStandaloneCommand().getManager().getStorage(); + return testConquery.getStandaloneCommand().getManagerNode().getStorage(); } public NamespaceStorage getNamespaceStorage() { - return testConquery.getStandaloneCommand().getManager().getDatasetRegistry().get(dataset.getId()).getStorage(); + return testConquery.getStandaloneCommand().getManagerNode().getDatasetRegistry().get(dataset.getId()).getStorage(); } public DatasetRegistry getDatasetRegistry() { - return testConquery.getStandaloneCommand().getManager().getDatasetRegistry(); + return testConquery.getStandaloneCommand().getManagerNode().getDatasetRegistry(); } public List getShardNodes() { diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java index 1ea114f87f..b67083bd99 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestConquery.java @@ -21,6 +21,7 @@ import com.bakdata.conquery.commands.StandaloneCommand; import com.bakdata.conquery.integration.IntegrationTests; import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.mode.cluster.ClusterState; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.XodusStoreFactory; @@ -29,6 +30,7 @@ import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.DistributedNamespace; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.util.Wait; import com.bakdata.conquery.util.io.Cloner; @@ -123,10 +125,11 @@ public synchronized StandaloneSupport openDataset(DatasetId datasetId) { } private synchronized StandaloneSupport createSupport(DatasetId datasetId, String name) { - DatasetRegistry datasets = standaloneCommand.getManager().getDatasetRegistry(); - Namespace ns = datasets.get(datasetId); + DatasetRegistry datasets = standaloneCommand.getManager().getDatasetRegistry(); + DistributedNamespace ns = datasets.get(datasetId); - assertThat(datasets.getShardNodes()).hasSize(2); + ClusterState clusterState = standaloneCommand.getManager().getConnectionManager().getClusterState(); + assertThat(clusterState.getShardNodes()).hasSize(2); // make tmp subdir and change cfg accordingly File localTmpDir = new File(tmpDir, "tmp_" + name); @@ -139,7 +142,7 @@ private synchronized StandaloneSupport createSupport(DatasetId datasetId, String log.info("Reusing existing folder {} for Support", localTmpDir.getPath()); } - ConqueryConfig localCfg = Cloner.clone(config, Map.of(Validator.class, standaloneCommand.getManager().getEnvironment().getValidator()), IntegrationTests.MAPPER); + ConqueryConfig localCfg = Cloner.clone(config, Map.of(Validator.class, standaloneCommand.getManagerNode().getEnvironment().getValidator()), IntegrationTests.MAPPER); StandaloneSupport support = new StandaloneSupport( @@ -148,8 +151,8 @@ private synchronized StandaloneSupport createSupport(DatasetId datasetId, String ns.getStorage().getDataset(), localTmpDir, localCfg, - standaloneCommand.getManager().getAdmin().getAdminProcessor(), - standaloneCommand.getManager().getAdmin().getAdminDatasetProcessor(), + standaloneCommand.getManagerNode().getAdmin().getAdminProcessor(), + standaloneCommand.getManagerNode().getAdmin().getAdminDatasetProcessor(), // Getting the User from AuthorizationConfig testUser ); @@ -158,7 +161,7 @@ private synchronized StandaloneSupport createSupport(DatasetId datasetId, String .total(Duration.ofSeconds(5)) .stepTime(Duration.ofMillis(5)) .build() - .until(() -> ns.getWorkers().size() == datasets.getShardNodes().size()); + .until(() -> clusterState.getWorkerHandlers().get(datasetId).getWorkers().size() == clusterState.getShardNodes().size()); support.waitUntilWorkDone(); openSupports.add(support); @@ -173,7 +176,7 @@ public synchronized StandaloneSupport getSupport(String name) { name += "[" + count + "]"; } Dataset dataset = new Dataset(name); - standaloneCommand.getManager().getAdmin().getAdminDatasetProcessor().addDataset(dataset); + standaloneCommand.getManagerNode().getAdmin().getAdminDatasetProcessor().addDataset(dataset); return createSupport(dataset.getId(), name); } catch (Exception e) { @@ -223,13 +226,13 @@ public void afterEach() throws Exception { } openSupports.clear(); } - this.getStandaloneCommand().getManager().getStorage().clear(); + this.getStandaloneCommand().getManagerNode().getStorage().clear(); waitUntilWorkDone(); } @SneakyThrows public void removeSupportDataset(StandaloneSupport support) { - standaloneCommand.getManager().getDatasetRegistry().removeNamespace(support.getDataset().getId()); + standaloneCommand.getManagerNode().getDatasetRegistry().removeNamespace(support.getDataset().getId()); } public void removeSupport(StandaloneSupport support) { @@ -265,15 +268,15 @@ public void waitUntilWorkDone() { private boolean isBusy() { boolean busy; - busy = standaloneCommand.getManager().getJobManager().isSlowWorkerBusy(); - busy |= standaloneCommand.getManager() + busy = standaloneCommand.getManagerNode().getJobManager().isSlowWorkerBusy(); + busy |= standaloneCommand.getManagerNode() .getStorage() .getAllExecutions() .stream() .map(ManagedExecution::getState) .anyMatch(ExecutionState.RUNNING::equals); - for (Namespace namespace : standaloneCommand.getManager().getDatasetRegistry().getDatasets()) { + for (Namespace namespace : standaloneCommand.getManagerNode().getDatasetRegistry().getDatasets()) { busy |= namespace.getJobManager().isSlowWorkerBusy(); } @@ -284,8 +287,8 @@ private boolean isBusy() { } public void beforeEach() { - final MetaStorage storage = standaloneCommand.getManager().getStorage(); - testUser = standaloneCommand.getManager().getConfig().getAuthorizationRealms().getInitialUsers().get(0).createOrOverwriteUser(storage); + final MetaStorage storage = standaloneCommand.getManagerNode().getStorage(); + testUser = standaloneCommand.getManagerNode().getConfig().getAuthorizationRealms().getInitialUsers().get(0).createOrOverwriteUser(storage); storage.updateUser(testUser); } } From 1ca91fffdea29922402050b3c7e84c4d53489d8b Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 4 Jul 2023 10:36:07 +0200 Subject: [PATCH 428/679] fix waiting for execturoService --- .../conquery/io/storage/xodus/stores/SerializingStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index d0b18f34ae..4740377dde 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -238,7 +238,7 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { executorService.shutdown(); - while (executorService.awaitTermination(1, TimeUnit.MINUTES)){ + while (!executorService.awaitTermination(1, TimeUnit.MINUTES)){ log.debug("Still waiting for {} to load.", this); } From 99c4ab74c92e3f1e9c0a6d3926c6cc23c0454167 Mon Sep 17 00:00:00 2001 From: Torben Meyer Date: Tue, 4 Jul 2023 11:04:13 +0200 Subject: [PATCH 429/679] Basic support for SQL connector (#3063) * Add skeleton for conversion to SQL queries This PR adds the base classes for converting ConQuery queries to SQL. A couple of notes: * We'll reevaluate the conversion mechanism and might choose to implement it directly in the respective classes * The abstraction of the SQL dialect is not part of this PR, and therefore we have a pretty tight coupling there * The `SqlTestSpecParser` is almost identical to `ConqueryTestSpec`, but I'd like to find the correct abstraction after #3059 --- .github/workflows/test_backend.yml | 7 +- backend/pom.xml | 33 +++ .../mode/local/LocalManagerProvider.java | 7 +- .../mode/local/LocalNamespaceHandler.java | 6 +- .../conquery/models/config/Dialect.java | 17 ++ .../models/config/SqlConnectorConfig.java | 20 ++ .../conquery/models/error/ConqueryError.java | 20 ++ .../conquery/sql/DslContextFactory.java | 29 +++ .../com/bakdata/conquery/sql/SqlContext.java | 11 + .../com/bakdata/conquery/sql/SqlQuery.java | 8 + .../sql/conquery/SqlExecutionManager.java | 48 +++- .../sql/conquery/SqlManagedQuery.java | 55 ++++- .../conquery/sql/conquery/SqlResultInfo.java | 44 ++++ .../conquery/sql/conversion/Converter.java | 26 +++ .../sql/conversion/ConverterService.java | 29 +++ .../sql/conversion/NodeConverter.java | 17 ++ .../sql/conversion/NodeConverterService.java | 32 +++ .../conquery/sql/conversion/SqlConverter.java | 22 ++ .../conversion/context/ConversionContext.java | 43 ++++ .../context/selects/ConceptSelects.java | 69 ++++++ .../context/selects/MergedSelects.java | 94 ++++++++ .../conversion/context/selects/Selects.java | 63 ++++++ .../conversion/context/step/QueryStep.java | 40 ++++ .../context/step/QueryStepTransformer.java | 59 +++++ .../conversion/cqelement/CQAndConverter.java | 81 +++++++ .../cqelement/CQConceptConverter.java | 208 ++++++++++++++++++ .../cqelement/CQDateRestrictionConverter.java | 20 ++ .../conversion/cqelement/CQOrConverter.java | 22 ++ .../ConceptPreprocessingService.java | 138 ++++++++++++ .../conversion/dialect/PostgreSqlDialect.java | 46 ++++ .../dialect/PostgreSqlFunctionProvider.java | 52 +++++ .../sql/conversion/dialect/SqlDialect.java | 84 +++++++ .../dialect/SqlFunctionProvider.java | 57 +++++ .../conversion/filter/FilterConverter.java | 21 ++ .../filter/FilterConverterService.java | 14 ++ .../filter/MultiSelectConverter.java | 19 ++ .../conversion/filter/RealRangeConverter.java | 30 +++ .../conquery/sql/conversion/package-info.java | 4 + .../query/ConceptQueryConverter.java | 44 ++++ .../select/DateDistanceConverter.java | 66 ++++++ .../select/FirstValueConverter.java | 19 ++ .../conversion/select/SelectConverter.java | 14 ++ .../select/SelectConverterService.java | 14 ++ .../conversion/supplier/DateNowSupplier.java | 9 + .../supplier/SystemDateNowSupplier.java | 12 + .../sql/execution/SqlEntityResult.java | 55 +++++ .../sql/execution/SqlExecutionResult.java | 22 ++ .../sql/execution/SqlExecutionService.java | 90 ++++++++ .../conquery/util/io/IdColumnUtil.java | 8 + .../java/com/bakdata/conquery/TestTags.java | 3 + .../integration/ConqueryIntegrationTests.java | 42 ++-- .../integration/IntegrationTests.java | 21 ++ .../json/AbstractQueryEngineTest.java | 2 +- .../integration/json/ConqueryTestSpec.java | 19 +- .../conquery/integration/json/FormTest.java | 2 +- .../integration/sql/CsvTableImporter.java | 170 ++++++++++++++ .../sql/PostgreSqlIntegrationTests.java | 86 ++++++++ .../integration/sql/SqlIntegrationTest.java | 31 +++ .../sql/SqlIntegrationTestSpec.java | 124 +++++++++++ .../integration/sql/SqlStandaloneSupport.java | 93 ++++++++ .../sql/TestPostgreSqlDialect.java | 35 +++ .../util/support/StandaloneSupport.java | 2 +- .../conquery/util/support/TestSupport.java | 28 +++ .../tests/sql/and/different_concept/and.json | 207 +++++++++++++++++ .../sql/and/different_concept/content_1.csv | 13 ++ .../sql/and/different_concept/content_2.csv | 9 + .../sql/and/different_concept/expected.csv | 4 + .../tests/sql/and/same_concept/and.json | 93 ++++++++ .../tests/sql/and/same_concept/content_1.csv | 13 ++ .../tests/sql/and/same_concept/expected.csv | 13 ++ .../date_restriction_date_column/content.csv | 9 + .../date_restriction_date_column.json | 98 +++++++++ .../date_restriction_date_column/expected.csv | 3 + .../content.csv | 9 + .../date_restriction_no_validity_date.json | 82 +++++++ .../expected.csv | 3 + .../date_restriction/daterange/content.csv | 9 + .../date_restriction_date_range.json | 85 +++++++ .../date_restriction/daterange/expected.csv | 3 + .../date_restriction/simple_date/content.csv | 9 + .../date_restriction_simple_date.json | 85 +++++++ .../date_restriction/simple_date/expected.csv | 3 + .../tests/sql/filter/number/content.csv | 13 ++ .../tests/sql/filter/number/expected.csv | 11 + .../tests/sql/filter/number/number.spec.json | 75 +++++++ .../sql/filter/number_only_max/content.csv | 13 ++ .../sql/filter/number_only_max/expected.csv | 5 + .../number_only_max/number_only_max.spec.json | 74 +++++++ .../sql/filter/number_only_min/content.csv | 13 ++ .../sql/filter/number_only_min/expected.csv | 12 + .../number_only_min/number_only_min.spec.json | 74 +++++++ .../tests/sql/filter/select/content.csv | 9 + .../tests/sql/filter/select/expected.csv | 3 + .../tests/sql/filter/select/select.spec.json | 74 +++++++ .../centuries/centuries.spec.json | 87 ++++++++ .../date_distance/centuries/content.csv | 9 + .../date_distance/centuries/expected.csv | 3 + .../days_with_date_restriction/content.csv | 9 + .../days_with_date_restriction.spec.json | 98 +++++++++ .../days_with_date_restriction/expected.csv | 3 + .../days_without_date_restriction/content.csv | 9 + .../days_without_date_restriction.json | 87 ++++++++ .../expected.csv | 3 + .../selects/date_distance/decades/content.csv | 9 + .../date_distance/decades/decades.spec.json | 87 ++++++++ .../date_distance/decades/expected.csv | 3 + .../selects/date_distance/months/content.csv | 9 + .../selects/date_distance/months/expected.csv | 3 + .../date_distance/months/months.spec.json | 87 ++++++++ .../selects/date_distance/years/content.csv | 9 + .../selects/date_distance/years/expected.csv | 3 + .../date_distance/years/years.spec.json | 87 ++++++++ .../selects/validity_date/default/content.csv | 9 + .../validity_date/default/expected.csv | 3 + .../default/validity_date_default.json | 79 +++++++ .../content.csv | 9 + .../expected.csv | 3 + .../validity_date_excluded.json | 80 +++++++ 118 files changed, 4470 insertions(+), 50 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/SqlContext.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/SqlQuery.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlResultInfo.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/Converter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/ConverterService.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverterService.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/context/ConversionContext.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/ConceptSelects.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/MergedSelects.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/Selects.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStep.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStepTransformer.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQConceptConverter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQDateRestrictionConverter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConceptPreprocessingService.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlDialect.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/FilterConverter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/FilterConverterService.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/MultiSelectConverter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/RealRangeConverter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/package-info.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/select/DateDistanceConverter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/select/FirstValueConverter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/select/SelectConverter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/select/SelectConverterService.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/supplier/DateNowSupplier.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/supplier/SystemDateNowSupplier.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/execution/SqlEntityResult.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionResult.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java create mode 100644 backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java create mode 100644 backend/src/test/java/com/bakdata/conquery/integration/sql/PostgreSqlIntegrationTests.java create mode 100644 backend/src/test/java/com/bakdata/conquery/integration/sql/SqlIntegrationTest.java create mode 100644 backend/src/test/java/com/bakdata/conquery/integration/sql/SqlIntegrationTestSpec.java create mode 100644 backend/src/test/java/com/bakdata/conquery/integration/sql/SqlStandaloneSupport.java create mode 100644 backend/src/test/java/com/bakdata/conquery/integration/sql/TestPostgreSqlDialect.java create mode 100644 backend/src/test/java/com/bakdata/conquery/util/support/TestSupport.java create mode 100644 backend/src/test/resources/tests/sql/and/different_concept/and.json create mode 100644 backend/src/test/resources/tests/sql/and/different_concept/content_1.csv create mode 100644 backend/src/test/resources/tests/sql/and/different_concept/content_2.csv create mode 100644 backend/src/test/resources/tests/sql/and/different_concept/expected.csv create mode 100644 backend/src/test/resources/tests/sql/and/same_concept/and.json create mode 100644 backend/src/test/resources/tests/sql/and/same_concept/content_1.csv create mode 100644 backend/src/test/resources/tests/sql/and/same_concept/expected.csv create mode 100644 backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/content.csv create mode 100644 backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/date_restriction_date_column.json create mode 100644 backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/expected.csv create mode 100644 backend/src/test/resources/tests/sql/date_restriction/date_restriction_no_validity_date/content.csv create mode 100644 backend/src/test/resources/tests/sql/date_restriction/date_restriction_no_validity_date/date_restriction_no_validity_date.json create mode 100644 backend/src/test/resources/tests/sql/date_restriction/date_restriction_no_validity_date/expected.csv create mode 100644 backend/src/test/resources/tests/sql/date_restriction/daterange/content.csv create mode 100644 backend/src/test/resources/tests/sql/date_restriction/daterange/date_restriction_date_range.json create mode 100644 backend/src/test/resources/tests/sql/date_restriction/daterange/expected.csv create mode 100644 backend/src/test/resources/tests/sql/date_restriction/simple_date/content.csv create mode 100644 backend/src/test/resources/tests/sql/date_restriction/simple_date/date_restriction_simple_date.json create mode 100644 backend/src/test/resources/tests/sql/date_restriction/simple_date/expected.csv create mode 100644 backend/src/test/resources/tests/sql/filter/number/content.csv create mode 100644 backend/src/test/resources/tests/sql/filter/number/expected.csv create mode 100644 backend/src/test/resources/tests/sql/filter/number/number.spec.json create mode 100644 backend/src/test/resources/tests/sql/filter/number_only_max/content.csv create mode 100644 backend/src/test/resources/tests/sql/filter/number_only_max/expected.csv create mode 100644 backend/src/test/resources/tests/sql/filter/number_only_max/number_only_max.spec.json create mode 100644 backend/src/test/resources/tests/sql/filter/number_only_min/content.csv create mode 100644 backend/src/test/resources/tests/sql/filter/number_only_min/expected.csv create mode 100644 backend/src/test/resources/tests/sql/filter/number_only_min/number_only_min.spec.json create mode 100644 backend/src/test/resources/tests/sql/filter/select/content.csv create mode 100644 backend/src/test/resources/tests/sql/filter/select/expected.csv create mode 100644 backend/src/test/resources/tests/sql/filter/select/select.spec.json create mode 100644 backend/src/test/resources/tests/sql/selects/date_distance/centuries/centuries.spec.json create mode 100644 backend/src/test/resources/tests/sql/selects/date_distance/centuries/content.csv create mode 100644 backend/src/test/resources/tests/sql/selects/date_distance/centuries/expected.csv create mode 100644 backend/src/test/resources/tests/sql/selects/date_distance/days_with_date_restriction/content.csv create mode 100644 backend/src/test/resources/tests/sql/selects/date_distance/days_with_date_restriction/days_with_date_restriction.spec.json create mode 100644 backend/src/test/resources/tests/sql/selects/date_distance/days_with_date_restriction/expected.csv create mode 100644 backend/src/test/resources/tests/sql/selects/date_distance/days_without_date_restriction/content.csv create mode 100644 backend/src/test/resources/tests/sql/selects/date_distance/days_without_date_restriction/days_without_date_restriction.json create mode 100644 backend/src/test/resources/tests/sql/selects/date_distance/days_without_date_restriction/expected.csv create mode 100644 backend/src/test/resources/tests/sql/selects/date_distance/decades/content.csv create mode 100644 backend/src/test/resources/tests/sql/selects/date_distance/decades/decades.spec.json create mode 100644 backend/src/test/resources/tests/sql/selects/date_distance/decades/expected.csv create mode 100644 backend/src/test/resources/tests/sql/selects/date_distance/months/content.csv create mode 100644 backend/src/test/resources/tests/sql/selects/date_distance/months/expected.csv create mode 100644 backend/src/test/resources/tests/sql/selects/date_distance/months/months.spec.json create mode 100644 backend/src/test/resources/tests/sql/selects/date_distance/years/content.csv create mode 100644 backend/src/test/resources/tests/sql/selects/date_distance/years/expected.csv create mode 100644 backend/src/test/resources/tests/sql/selects/date_distance/years/years.spec.json create mode 100644 backend/src/test/resources/tests/sql/selects/validity_date/default/content.csv create mode 100644 backend/src/test/resources/tests/sql/selects/validity_date/default/expected.csv create mode 100644 backend/src/test/resources/tests/sql/selects/validity_date/default/validity_date_default.json create mode 100644 backend/src/test/resources/tests/sql/selects/validity_date/excluded_from_time_aggregation/content.csv create mode 100644 backend/src/test/resources/tests/sql/selects/validity_date/excluded_from_time_aggregation/expected.csv create mode 100644 backend/src/test/resources/tests/sql/selects/validity_date/excluded_from_time_aggregation/validity_date_excluded.json diff --git a/.github/workflows/test_backend.yml b/.github/workflows/test_backend.yml index 00a5fa85d3..e481274689 100644 --- a/.github/workflows/test_backend.yml +++ b/.github/workflows/test_backend.yml @@ -14,7 +14,7 @@ on: jobs: test: runs-on: ubuntu-latest - timeout-minutes: 10 + timeout-minutes: 15 steps: - name: Cache local Maven repository uses: actions/cache@v2 @@ -35,8 +35,11 @@ jobs: - name: Build Backend run: mvn -T 1C install -pl backend -DskipTests -am - name: Unit Test - run: mvn test -T 1C -pl backend -DexcludedGroups="INTEGRATION_PROGRAMMATIC, INTEGRATION_JSON" + run: mvn test -T 1C -pl backend -DexcludedGroups="INTEGRATION_PROGRAMMATIC, INTEGRATION_JSON, INTEGRATION_SQL_BACKEND" - name: Programmatic Integration Tests run: mvn test -T 1C -pl backend -Dgroups="INTEGRATION_PROGRAMMATIC" - name: JSON based Integration Tests run: mvn test -T 1C -pl backend -Dgroups="INTEGRATION_JSON" + - name: SQL based Integration Tests + if: ${{ startsWith(github.head_ref, 'sql/') }} + run: mvn test -T 1C -pl backend -Dgroups="INTEGRATION_SQL_BACKEND" diff --git a/backend/pom.xml b/backend/pom.xml index 033f7ed060..c2cdef06ad 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -346,5 +346,38 @@ auto-service 1.0.1 + + org.jooq + jooq + 3.18.3 + + + org.jooq + jooq-postgres-extensions + 3.18.3 + + + com.zaxxer + HikariCP + 5.0.1 + + + org.testcontainers + testcontainers + 1.17.6 + test + + + org.testcontainers + junit-jupiter + 1.17.6 + test + + + org.testcontainers + postgresql + 1.17.6 + test + diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java index e1ee37a178..794df98a87 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java @@ -13,6 +13,9 @@ import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.LocalNamespace; import com.bakdata.conquery.models.worker.ShardNodeInformation; +import com.bakdata.conquery.sql.DslContextFactory; +import com.bakdata.conquery.sql.SqlContext; +import com.bakdata.conquery.sql.conversion.dialect.PostgreSqlDialect; import io.dropwizard.setup.Environment; public class LocalManagerProvider implements ManagerProvider { @@ -21,7 +24,9 @@ public class LocalManagerProvider implements ManagerProvider { public DelegateManager provideManager(ConqueryConfig config, Environment environment) { InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, environment.getValidator()); - NamespaceHandler namespaceHandler = new LocalNamespaceHandler(config, creator); + // todo(tm): proper injection + SqlContext sqlContext = new SqlContext(config.getSqlConnectorConfig(), new PostgreSqlDialect(DslContextFactory.create(config.getSqlConnectorConfig()))); + NamespaceHandler namespaceHandler = new LocalNamespaceHandler(config, creator, sqlContext); DatasetRegistry datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, creator); creator.init(datasetRegistry); diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java index 3a887ea01a..6bed3c12d8 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalNamespaceHandler.java @@ -3,12 +3,13 @@ import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.mode.InternalObjectMapperCreator; -import com.bakdata.conquery.mode.NamespaceSetupData; import com.bakdata.conquery.mode.NamespaceHandler; +import com.bakdata.conquery.mode.NamespaceSetupData; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.identifiable.ids.specific.DatasetId; import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.models.worker.LocalNamespace; +import com.bakdata.conquery.sql.SqlContext; import com.bakdata.conquery.sql.conquery.SqlExecutionManager; import lombok.RequiredArgsConstructor; @@ -17,11 +18,12 @@ public class LocalNamespaceHandler implements NamespaceHandler { private final ConqueryConfig config; private final InternalObjectMapperCreator mapperCreator; + private final SqlContext sqlContext; @Override public LocalNamespace createNamespace(NamespaceStorage namespaceStorage, MetaStorage metaStorage) { NamespaceSetupData namespaceData = NamespaceHandler.createNamespaceSetup(namespaceStorage, config, mapperCreator); - ExecutionManager executionManager = new SqlExecutionManager(); + ExecutionManager executionManager = new SqlExecutionManager(sqlContext, metaStorage); return new LocalNamespace( namespaceData.getPreprocessMapper(), namespaceData.getCommunicationMapper(), diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java b/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java new file mode 100644 index 0000000000..2ec655aea9 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java @@ -0,0 +1,17 @@ +package com.bakdata.conquery.models.config; + +import lombok.Getter; +import org.jooq.SQLDialect; + +@Getter +public enum Dialect { + + POSTGRESQL(SQLDialect.POSTGRES); + + private final SQLDialect jooqDialect; + + Dialect(SQLDialect jooqDialect) { + this.jooqDialect = jooqDialect; + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java index 747f1840a2..857018cada 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java @@ -1,9 +1,29 @@ package com.bakdata.conquery.models.config; +import lombok.AllArgsConstructor; +import lombok.Builder; import lombok.Data; +import lombok.NoArgsConstructor; @Data +@Builder +@NoArgsConstructor +@AllArgsConstructor public class SqlConnectorConfig { boolean enabled; + + private Dialect dialect; + + /** + * Determines if generated SQL should be formatted. + */ + private boolean withPrettyPrinting; + + private String databaseUsername; + + private String databasePassword; + + private String jdbcConnectionUrl; + private String primaryColumn = "pid"; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/error/ConqueryError.java b/backend/src/main/java/com/bakdata/conquery/models/error/ConqueryError.java index 37a64bddbb..1ee2c907b7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/error/ConqueryError.java +++ b/backend/src/main/java/com/bakdata/conquery/models/error/ConqueryError.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.models.error; +import java.sql.SQLException; import java.util.Collections; import java.util.Map; import java.util.Objects; @@ -377,4 +378,23 @@ public NoSecondaryIdSelectedError() { } } + + @CPSType(base = ConqueryError.class, id = "CQ_SQL_ERROR") + public static class SqlError extends ContextError { + + private static final String SQL_ERROR = "ERROR"; + + private static final String TEMPLATE = "Something went wrong while querying the database: ${" + SQL_ERROR + "}."; + + @JsonCreator + private SqlError() { + super(TEMPLATE); + } + + public SqlError(SQLException sqlException) { + this(); + getContext().put(SQL_ERROR, sqlException.toString()); + } + + } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java new file mode 100644 index 0000000000..16bfe54ecd --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java @@ -0,0 +1,29 @@ +package com.bakdata.conquery.sql; + +import javax.sql.DataSource; + +import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import org.jooq.DSLContext; +import org.jooq.conf.Settings; +import org.jooq.impl.DSL; + +public class DslContextFactory { + + public static DSLContext create(SqlConnectorConfig config) { + HikariConfig hikariConfig = new HikariConfig(); + hikariConfig.setJdbcUrl(config.getJdbcConnectionUrl()); + hikariConfig.setUsername(config.getDatabaseUsername()); + hikariConfig.setPassword(config.getDatabasePassword()); + + DataSource dataSource = new HikariDataSource(hikariConfig); + + return DSL.using( + dataSource, + config.getDialect().getJooqDialect(), + new Settings().withRenderFormatted(config.isWithPrettyPrinting()) + ); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/SqlContext.java b/backend/src/main/java/com/bakdata/conquery/sql/SqlContext.java new file mode 100644 index 0000000000..bca5ca6f2e --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/SqlContext.java @@ -0,0 +1,11 @@ +package com.bakdata.conquery.sql; + +import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; +import lombok.Value; + +@Value +public class SqlContext { + SqlConnectorConfig config; + SqlDialect sqlDialect; +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/SqlQuery.java b/backend/src/main/java/com/bakdata/conquery/sql/SqlQuery.java new file mode 100644 index 0000000000..1d93f04749 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/SqlQuery.java @@ -0,0 +1,8 @@ +package com.bakdata.conquery.sql; + +import lombok.Value; + +@Value +public class SqlQuery { + String sqlString; +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java index 6778add265..9ab60cacde 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlExecutionManager.java @@ -1,9 +1,11 @@ package com.bakdata.conquery.sql.conquery; -import java.util.UUID; + import java.util.stream.Stream; +import com.bakdata.conquery.apiv1.query.Query; import com.bakdata.conquery.apiv1.query.QueryDescription; +import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Dataset; @@ -11,35 +13,67 @@ import com.bakdata.conquery.models.query.ExecutionManager; import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.sql.SqlContext; +import com.bakdata.conquery.sql.SqlQuery; +import com.bakdata.conquery.sql.conversion.SqlConverter; +import com.bakdata.conquery.sql.execution.SqlExecutionResult; +import com.bakdata.conquery.sql.execution.SqlExecutionService; +import lombok.extern.slf4j.Slf4j; +@Slf4j public class SqlExecutionManager implements ExecutionManager { + private final MetaStorage metaStorage; + private final SqlExecutionService executionService; + private final SqlConverter converter; + + public SqlExecutionManager(final SqlContext context, MetaStorage metaStorage) { + this.metaStorage = metaStorage; + this.executionService = new SqlExecutionService(context.getSqlDialect().getDSLContext()); + this.converter = new SqlConverter(context.getSqlDialect(), context.getConfig()); + } + @Override - public ManagedExecution runQuery(Namespace namespace, QueryDescription query, User user, Dataset submittedDataset, ConqueryConfig config, boolean system) { - return null; + public SqlManagedQuery runQuery(Namespace namespace, QueryDescription query, User user, Dataset submittedDataset, ConqueryConfig config, boolean system) { + SqlManagedQuery execution = createExecution(query, user, submittedDataset, system); + execution.initExecutable(namespace, config); + execution.start(); + // todo(tm): Non-blocking execution + SqlExecutionResult result = this.executionService.execute(execution); + execution.finish(result); + return execution; } @Override public void execute(Namespace namespace, ManagedExecution execution, ConqueryConfig config) { + if (!(execution instanceof SqlManagedQuery)) { + throw new UnsupportedOperationException("The SQL execution manager can only execute SQL queries, but got a %s".formatted(execution.getClass())); + } + this.executionService.execute(((SqlManagedQuery) execution)); } @Override - public ManagedExecution createExecution(QueryDescription query, User user, Dataset submittedDataset, boolean system) { - return null; + public SqlManagedQuery createExecution(QueryDescription query, User user, Dataset submittedDataset, boolean system) { + Query castQuery = (Query) query; + SqlQuery converted = this.converter.convert(castQuery); + SqlManagedQuery sqlManagedQuery = new SqlManagedQuery(castQuery, user, submittedDataset, metaStorage, converted); + metaStorage.addExecution(sqlManagedQuery); + return sqlManagedQuery; } @Override public void cancelQuery(Dataset dataset, ManagedExecution query) { - + // unsupported for now } @Override public void clearQueryResults(ManagedExecution execution) { + // unsupported for now } @Override public Stream streamQueryResults(ManagedExecution execution) { - return null; + throw new UnsupportedOperationException("Streaming for now not supported"); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlManagedQuery.java b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlManagedQuery.java index 13ea8e1212..a4eb2bfe78 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlManagedQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlManagedQuery.java @@ -1,7 +1,11 @@ package com.bakdata.conquery.sql.conquery; +import java.util.ArrayList; import java.util.List; +import java.util.Locale; import java.util.function.Consumer; +import java.util.function.Function; +import java.util.stream.Collectors; import java.util.stream.Stream; import com.bakdata.conquery.apiv1.query.Query; @@ -10,14 +14,22 @@ import com.bakdata.conquery.io.storage.MetaStorage; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.i18n.I18n; import com.bakdata.conquery.models.query.ColumnDescriptor; import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.SingleTableResult; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.models.query.resultinfo.UniqueNamer; import com.bakdata.conquery.models.query.results.EntityResult; +import com.bakdata.conquery.models.types.ResultType; +import com.bakdata.conquery.sql.SqlQuery; +import com.bakdata.conquery.sql.execution.SqlExecutionResult; import com.bakdata.conquery.util.QueryUtils; +import com.google.common.base.Preconditions; import lombok.Getter; import lombok.Setter; @@ -26,19 +38,22 @@ @CPSType(base = ManagedExecution.class, id = "SQL_QUERY") public class SqlManagedQuery extends ManagedExecution implements SingleTableResult { private Query query; + private SqlQuery sqlQuery; + private SqlExecutionResult result; protected SqlManagedQuery(MetaStorage storage) { super(storage); } - public SqlManagedQuery(Query query, User owner, Dataset dataset, MetaStorage storage) { + public SqlManagedQuery(Query query, User owner, Dataset dataset, MetaStorage storage, SqlQuery sqlQuery) { super(owner, dataset, storage); this.query = query; + this.sqlQuery = sqlQuery; } @Override protected void doInitExecutable() { - + query.resolve(new QueryResolveContext(getNamespace(), getConfig(), getStorage(), null)); } @Override @@ -58,21 +73,49 @@ public void visit(Consumer visitor) { @Override public List generateColumnDescriptions() { - return null; + // todo(tm): This is basically a duplicate from ManagedQuery, but sets the ResultType to String because the SQL connector doesn't convert types for now. + // As soon as the connector properly handles types, we can extract this into a helper and use it for both this and ManagedQuery. + Preconditions.checkArgument(isInitialized(), "The execution must have been initialized first"); + List columnDescriptions = new ArrayList<>(); + + final Locale locale = I18n.LOCALE.get(); + + PrintSettings settings = new PrintSettings(true, locale, getNamespace(), getConfig(), null); + + UniqueNamer uniqNamer = new UniqueNamer(settings); + + // First add the id columns to the descriptor list. The are the first columns + for (ResultInfo header : getConfig().getIdColumns().getIdResultInfos()) { + columnDescriptions.add(ColumnDescriptor.builder() + .label(uniqNamer.getUniqueName(header)) + .type(ResultType.StringT.getINSTANCE().typeInfo()) + .semantics(header.getSemantics()) + .build()); + } + + final UniqueNamer collector = new UniqueNamer(settings); + getResultInfos().forEach(info -> columnDescriptions.add(info.asColumnDescriptor(settings, collector))); + return columnDescriptions; } @Override public List getResultInfos() { - return null; + // See above: For now, the SQL connector doesn't handle types + return query.getResultInfos().stream().map(SqlResultInfo::new).collect(Collectors.toList()); } @Override public Stream streamResults() { - return null; + return result.getTable().stream(); } @Override public long resultRowCount() { - return 0; + return result.getRowCount(); + } + + public void finish(final SqlExecutionResult result) { + this.result = result; + super.finish(ExecutionState.DONE); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlResultInfo.java b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlResultInfo.java new file mode 100644 index 0000000000..5ab7aa2102 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conquery/SqlResultInfo.java @@ -0,0 +1,44 @@ +package com.bakdata.conquery.sql.conquery; + +import java.util.Set; + +import com.bakdata.conquery.models.query.PrintSettings; +import com.bakdata.conquery.models.query.resultinfo.ResultInfo; +import com.bakdata.conquery.models.types.ResultType; +import com.bakdata.conquery.models.types.SemanticType; + +/** + * Temporary result info that sets all {@link ResultType} to {@link com.bakdata.conquery.models.types.ResultType.StringT}. + */ +public class SqlResultInfo extends ResultInfo { + private final ResultInfo delegate; + + public SqlResultInfo(ResultInfo delegate) { + this.delegate = delegate; + } + + @Override + public String userColumnName(PrintSettings printSettings) { + return delegate.userColumnName(printSettings); + } + + @Override + public String defaultColumnName(PrintSettings printSettings) { + return delegate.defaultColumnName(printSettings); + } + + @Override + public ResultType getType() { + return ResultType.StringT.getINSTANCE(); + } + + @Override + public Set getSemantics() { + return delegate.getSemantics(); + } + + @Override + public String getDescription() { + return delegate.getDescription(); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/Converter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/Converter.java new file mode 100644 index 0000000000..2557139ab9 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/Converter.java @@ -0,0 +1,26 @@ +package com.bakdata.conquery.sql.conversion; + +import java.util.Optional; + +import com.bakdata.conquery.sql.conversion.context.ConversionContext; + +/** + * A converter converts an input into a result object if the input matches the conversion class. + * + * @param type that can be converted + * @param type of the result + */ +public interface Converter { + + default Optional tryConvert(I input, ConversionContext context) { + if (getConversionClass().isInstance(input)) { + return Optional.ofNullable(convert(getConversionClass().cast(input), context)); + } + return Optional.empty(); + } + + Class getConversionClass(); + + R convert(final C convert, final ConversionContext context); + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/ConverterService.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ConverterService.java new file mode 100644 index 0000000000..cddd02e0d1 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/ConverterService.java @@ -0,0 +1,29 @@ +package com.bakdata.conquery.sql.conversion; + +import java.util.List; + +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.google.common.collect.MoreCollectors; + +/** + * Converts an input to a result with an applicable converter. + * + * @param type that can be converted + * @param type of the result + * @see Converter + */ +public abstract class ConverterService { + + private final List> converters; + + protected ConverterService(List> converters) { + this.converters = converters; + } + + public R convert(C selectNode, ConversionContext context) { + return converters.stream() + .flatMap(converter -> converter.tryConvert(selectNode, context).stream()) + .collect(MoreCollectors.onlyElement()); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverter.java new file mode 100644 index 0000000000..7eb6cb0c0c --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverter.java @@ -0,0 +1,17 @@ +package com.bakdata.conquery.sql.conversion; + +import com.bakdata.conquery.models.query.Visitable; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; + +/** + * Interface for converters that implement the translation of a ConQuery query to an SQL query. + * + *

+ * A ConQuery is a graph that has a {@link com.bakdata.conquery.apiv1.query.QueryDescription} as its root. + * The children of the root are of type {@link com.bakdata.conquery.apiv1.query.CQElement}. + * + * @param type of the node to convert + */ +public interface NodeConverter extends Converter { + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverterService.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverterService.java new file mode 100644 index 0000000000..f174a48265 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/NodeConverterService.java @@ -0,0 +1,32 @@ +package com.bakdata.conquery.sql.conversion; + +import com.bakdata.conquery.apiv1.query.QueryDescription; +import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.models.query.Visitable; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; + +/** + * Entry point for converting {@link QueryDescription} to an SQL query. + */ +public class NodeConverterService extends ConverterService { + + private final SqlDialect dialect; + private final SqlConnectorConfig config; + + public NodeConverterService(SqlDialect dialect, SqlConnectorConfig config) { + super(dialect.getNodeConverters()); + this.dialect = dialect; + this.config = config; + } + + public ConversionContext convert(QueryDescription queryDescription) { + ConversionContext initialCtx = ConversionContext.builder() + .config(config) + .nodeConverterService(this) + .sqlDialect(this.dialect) + .build(); + return convert(queryDescription, initialCtx); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java new file mode 100644 index 0000000000..8ee38b2f56 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/SqlConverter.java @@ -0,0 +1,22 @@ +package com.bakdata.conquery.sql.conversion; + +import com.bakdata.conquery.apiv1.query.QueryDescription; +import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.sql.SqlQuery; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; +import org.jooq.conf.ParamType; + +public class SqlConverter { + + private final NodeConverterService nodeConverterService; + + public SqlConverter(SqlDialect dialect, SqlConnectorConfig config) { + this.nodeConverterService = new NodeConverterService(dialect, config); + } + + public SqlQuery convert(QueryDescription queryDescription) { + ConversionContext converted = nodeConverterService.convert(queryDescription); + return new SqlQuery(converted.getFinalQuery().getSQL(ParamType.INLINED)); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/ConversionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/ConversionContext.java new file mode 100644 index 0000000000..dc838ecab5 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/ConversionContext.java @@ -0,0 +1,43 @@ +package com.bakdata.conquery.sql.conversion.context; + +import java.util.List; + +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.sql.conversion.NodeConverterService; +import com.bakdata.conquery.sql.conversion.context.step.QueryStep; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; +import lombok.Builder; +import lombok.Singular; +import lombok.Value; +import lombok.With; +import org.jooq.Record; +import org.jooq.Select; + +@Value +@With +@Builder(toBuilder = true) +public class ConversionContext { + + SqlConnectorConfig config; + NodeConverterService nodeConverterService; + SqlDialect sqlDialect; + @Singular + List querySteps; + Select finalQuery; + CDateRange dateRestrictionRange; + int queryStepCounter; + + + public boolean dateRestrictionActive() { + return this.dateRestrictionRange != null; + } + + public ConversionContext withQueryStep(QueryStep queryStep) { + return this.toBuilder() + .queryStep(queryStep) + .queryStepCounter(queryStepCounter + 1) + .build(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/ConceptSelects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/ConceptSelects.java new file mode 100644 index 0000000000..8e42dde561 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/ConceptSelects.java @@ -0,0 +1,69 @@ +package com.bakdata.conquery.sql.conversion.context.selects; + +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Stream; + +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import lombok.Builder; +import lombok.Value; +import lombok.With; +import org.jooq.Field; + +/** + * {@link ConceptSelects} represent all select fields of a {@link CQConcept}. + */ +@Value +@With +@Builder(toBuilder = true) +public class ConceptSelects implements Selects { + + Field primaryColumn; + Optional> dateRestriction; + Optional> validityDate; + List> eventSelect; + List> eventFilter; + List> groupSelect; + List> groupFilter; + + @Override + public ConceptSelects byName(String qualifier) { + return builder() + .primaryColumn(this.mapFieldToQualifier(qualifier, this.primaryColumn)) + .dateRestriction(this.mapFieldStreamToQualifier(qualifier, this.dateRestriction.stream()).findFirst()) + .validityDate(this.mapFieldStreamToQualifier(qualifier, this.validityDate.stream()).findFirst()) + .eventSelect(this.mapFieldStreamToQualifier(qualifier, this.eventSelect.stream()).toList()) + .eventFilter(this.mapFieldStreamToQualifier(qualifier, this.eventFilter.stream()).toList()) + .groupSelect(this.mapFieldStreamToQualifier(qualifier, this.groupSelect.stream()).toList()) + .groupFilter(this.mapFieldStreamToQualifier(qualifier, this.groupFilter.stream()).toList()) + .build(); + } + + @Override + public List> all() { + return Stream.concat( + this.primaryColumnAndValidityDate(), + this.explicitSelects().stream() + ).toList(); + } + + private Stream> primaryColumnAndValidityDate() { + return Stream.concat( + Stream.of(this.primaryColumn), + this.validityDate.stream() + ); + } + + @Override + public List> explicitSelects() { + return Stream.of( + this.dateRestriction.stream(), + this.eventSelect.stream(), + this.eventFilter.stream(), + this.groupSelect.stream(), + this.groupFilter.stream() + ).flatMap(Function.identity()).toList(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/MergedSelects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/MergedSelects.java new file mode 100644 index 0000000000..f164b4c292 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/MergedSelects.java @@ -0,0 +1,94 @@ +package com.bakdata.conquery.sql.conversion.context.selects; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import com.bakdata.conquery.sql.conversion.context.step.QueryStep; +import lombok.AccessLevel; +import lombok.AllArgsConstructor; +import lombok.Value; +import org.jooq.Field; + +/** + * {@link MergedSelects} represent the combination of multiple {@link Selects}. + * Default selects fields of multiple {@link Selects} will be merged and special select fields like the primary column + * or validity dates will be unified or aggregated due to defined policies. + */ +@Value +@AllArgsConstructor(access = AccessLevel.PRIVATE) +public class MergedSelects implements Selects { + + Field primaryColumn; + + Optional> validityDate; + + /** + * A merged list of all select fields, except the primary column and validity date, + * of the {@link QueryStep}'s passed to the {@link MergedSelects} constructor. + * Each field name is qualified with its associated CTE name. + */ + List> mergedSelects; + + public MergedSelects(List querySteps) { + this.primaryColumn = this.extractPrimaryColumnSelect(querySteps); + this.validityDate = this.extractValidityDate(querySteps); + this.mergedSelects = this.mergeSelects(querySteps); + } + + private Field extractPrimaryColumnSelect(List querySteps) { + // as we join all QuerySteps / CTEs onto the same primary column, + // it's sufficient to obtain it from the first one in the list + QueryStep firstQueryStep = querySteps.iterator().next(); + return this.mapFieldToQualifier(firstQueryStep.getCteName(), firstQueryStep.getSelects().getPrimaryColumn()); + } + + private Optional> extractValidityDate(List querySteps) { + // TODO: date aggregation... + if (querySteps.isEmpty()) { + return Optional.empty(); + } + QueryStep firstQueryStep = querySteps.get(0); + return this.mapFieldStreamToQualifier(firstQueryStep.getCteName(), firstQueryStep.getSelects().getValidityDate().stream()) + .findFirst(); + } + + private List> mergeSelects(List queriesToJoin) { + return queriesToJoin.stream() + .flatMap(queryStep -> queryStep.getSelects().explicitSelects().stream() + .map(field -> this.mapFieldToQualifier(queryStep.getCteName(), field))) + .toList(); + } + + @Override + public MergedSelects byName(String qualifier) { + return new MergedSelects( + this.mapFieldToQualifier(qualifier, this.primaryColumn), + this.mapFieldStreamToQualifier(qualifier, this.validityDate.stream()).findFirst(), + this.mergedSelects.stream() + .map(field -> this.mapFieldToQualifier(qualifier, field)) + .toList() + ); + } + + @Override + public List> all() { + return Stream.concat( + this.primaryColumnAndValidityDate(), + this.mergedSelects.stream() + ).toList(); + } + + private Stream> primaryColumnAndValidityDate() { + return Stream.concat( + Stream.of(this.primaryColumn), + this.validityDate.stream() + ); + } + + @Override + public List> explicitSelects() { + return this.mergedSelects; + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/Selects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/Selects.java new file mode 100644 index 0000000000..6229c247a9 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/Selects.java @@ -0,0 +1,63 @@ +package com.bakdata.conquery.sql.conversion.context.selects; + +import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; + +import org.jooq.Field; +import org.jooq.impl.DSL; + +public interface Selects { + + Field getPrimaryColumn(); + + Optional> getValidityDate(); + + /** + * Returns the selected columns as fully qualified reference. + * + * @param qualifier the table name that creates these selects + * @return selects as fully qualified reference + * @see Selects#mapFieldToQualifier(String, Field) + */ + Selects byName(String qualifier); + + /** + * @return A list of all select fields including the primary column and validity date. + */ + List> all(); + + /** + * List of columns that the user explicitly referenced, either via a filter or a select. + * + * @return A list of all select fields WITHOUT implicitly selected columns like the primary column and validity date. + */ + List> explicitSelects(); + + default Stream> mapFieldStreamToQualifier(String qualifier, Stream> objectField) { + return objectField.map(column -> this.mapFieldToQualifier(qualifier, column)); + } + + /** + * Converts a select to its fully qualified reference. + * + *

+ *

Example:

+ * 
{@code
+	 * with a as (select c1 - c2 as c
+	 * from t1)
+	 * select t1.c
+	 * from a
+	 * }
+ *

+ * This function maps the select {@code c1 - c2 as c} to {@code t1.c}. + * + * @param qualifier + * @param field + * @return + */ + default Field mapFieldToQualifier(String qualifier, Field field) { + return DSL.field(DSL.name(qualifier, field.getName())); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStep.java new file mode 100644 index 0000000000..9d72ec56bd --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStep.java @@ -0,0 +1,40 @@ +package com.bakdata.conquery.sql.conversion.context.step; + +import java.util.List; + +import com.bakdata.conquery.sql.conversion.context.selects.Selects; +import lombok.Builder; +import lombok.Value; +import org.jooq.Condition; +import org.jooq.Record; +import org.jooq.TableLike; +import org.jooq.impl.DSL; + +/** + * Intermediate representation of an SQL query. + */ +@Value +@Builder +public class QueryStep { + + String cteName; + Selects selects; + TableLike fromTable; + List conditions; + /** + * The CTEs referenced by this QueryStep + */ + List predecessors; + + public static TableLike toTableLike(String fromTableName) { + return DSL.table(DSL.name(fromTableName)); + } + + /** + * @return All selects re-mapped to a qualifier, which is the cteName of this QueryStep. + */ + public Selects getQualifiedSelects() { + return this.selects.byName(this.cteName); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStepTransformer.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStepTransformer.java new file mode 100644 index 0000000000..e41c457235 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStepTransformer.java @@ -0,0 +1,59 @@ +package com.bakdata.conquery.sql.conversion.context.step; + +import java.util.List; +import java.util.stream.Stream; + +import org.jooq.CommonTableExpression; +import org.jooq.DSLContext; +import org.jooq.Record; +import org.jooq.Select; +import org.jooq.impl.DSL; + +/** + * Transformer for translating the intermediate representation of {@link QueryStep} into the final SQL query. + */ +public class QueryStepTransformer { + + private final DSLContext dslContext; + + public QueryStepTransformer(DSLContext dslContext) { + this.dslContext = dslContext; + } + + /** + * Converts a given {@link QueryStep} into an executable SELECT statement. + */ + public Select toSelectQuery(QueryStep queryStep) { + return this.dslContext.with(this.constructPredecessorCteList(queryStep)) + .select(queryStep.getSelects().all()) + .from(queryStep.getFromTable()) + .where(queryStep.getConditions()); + } + + private List> constructPredecessorCteList(QueryStep queryStep) { + return queryStep.getPredecessors().stream() + .flatMap(predecessor -> this.toCteList(predecessor).stream()) + .toList(); + } + + private List> toCteList(QueryStep queryStep) { + return Stream.concat( + this.predecessorCtes(queryStep), + Stream.of(this.toCte(queryStep)) + ).toList(); + } + + private Stream> predecessorCtes(QueryStep queryStep) { + return queryStep.getPredecessors().stream() + .flatMap(predecessor -> this.toCteList(predecessor).stream()); + } + + private CommonTableExpression toCte(QueryStep queryStep) { + return DSL.name(queryStep.getCteName()).as( + this.dslContext.select(queryStep.getSelects().all()) + .from(queryStep.getFromTable()) + .where(queryStep.getConditions()) + ); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java new file mode 100644 index 0000000000..98228286e4 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java @@ -0,0 +1,81 @@ +package com.bakdata.conquery.sql.conversion.cqelement; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import com.bakdata.conquery.apiv1.query.CQElement; +import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.context.selects.MergedSelects; +import com.bakdata.conquery.sql.conversion.context.step.QueryStep; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.TableLike; +import org.jooq.impl.DSL; + +public class CQAndConverter implements NodeConverter { + + @Override + public ConversionContext convert(CQAnd node, ConversionContext context) { + // if the AND node has a single child, the AND node is a noop + // otherwise, the converted children need to be logically combined before we obtain the final query + if (node.getChildren().size() == 1) { + return context.getNodeConverterService().convert(node.getChildren().get(0), context); + } + + ConversionContext childrenContext = context; + for (CQElement child : node.getChildren()) { + childrenContext = context.getNodeConverterService().convert(child, childrenContext); + } + + List queriesToJoin = childrenContext.getQuerySteps(); + QueryStep andQueryStep = QueryStep.builder() + .cteName(this.constructAndQueryStepLabel(queriesToJoin)) + .selects(new MergedSelects(queriesToJoin)) + .fromTable(this.constructJoinedTable(queriesToJoin)) + .conditions(Collections.emptyList()) + .predecessors(queriesToJoin) + .build(); + + return context.withQueryStep(andQueryStep); + } + + private String constructAndQueryStepLabel(List queriesToJoin) { + return queriesToJoin.stream() + .map(QueryStep::getCteName) + .collect(Collectors.joining("_AND_")); + } + + private TableLike constructJoinedTable(List queriesToJoin) { + + Table joinedQuery = this.getIntitialJoinTable(queriesToJoin); + + for (int i = 0; i < queriesToJoin.size() - 1; i++) { + + QueryStep leftPartQS = queriesToJoin.get(i); + QueryStep rightPartQS = queriesToJoin.get(i + 1); + + Field leftPartPrimaryColumn = leftPartQS.getQualifiedSelects().getPrimaryColumn(); + Field rightPartPrimaryColumn = rightPartQS.getQualifiedSelects().getPrimaryColumn(); + + joinedQuery = joinedQuery + .innerJoin(rightPartQS.getCteName()) + .on(leftPartPrimaryColumn.eq(rightPartPrimaryColumn)); + } + + return joinedQuery; + } + + private Table getIntitialJoinTable(List queriesToJoin) { + return DSL.table(DSL.name(queriesToJoin.get(0).getCteName())); + } + + + @Override + public Class getConversionClass() { + return CQAnd.class; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQConceptConverter.java new file mode 100644 index 0000000000..db4e9b43b2 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQConceptConverter.java @@ -0,0 +1,208 @@ +package com.bakdata.conquery.sql.conversion.cqelement; + +import java.util.Collections; +import java.util.List; +import java.util.Locale; +import java.util.Optional; + +import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.context.selects.ConceptSelects; +import com.bakdata.conquery.sql.conversion.context.step.QueryStep; +import com.bakdata.conquery.sql.conversion.filter.FilterConverterService; +import com.bakdata.conquery.sql.conversion.select.SelectConverterService; +import org.jooq.Condition; +import org.jooq.Field; + +public class CQConceptConverter implements NodeConverter { + + private final FilterConverterService filterConverterService; + private final SelectConverterService selectConverterService; + + public CQConceptConverter(FilterConverterService filterConverterService, SelectConverterService selectConverterService) { + this.filterConverterService = filterConverterService; + this.selectConverterService = selectConverterService; + } + + @Override + public Class getConversionClass() { + return CQConcept.class; + } + + @Override + public ConversionContext convert(CQConcept node, ConversionContext context) { + + if (node.getTables().size() > 1) { + throw new UnsupportedOperationException("Can't handle concepts with multiple tables for now."); + } + + ConceptPreprocessingService preprocessingService = new ConceptPreprocessingService(node, context); + CQTable table = node.getTables().get(0); + String conceptLabel = this.getConceptLabel(node, context); + + QueryStep preprocessingStep = preprocessingService.buildPreprocessingQueryStepForTable(conceptLabel, table); + QueryStep dateRestriction = this.buildDateRestrictionQueryStep(context, node, conceptLabel, preprocessingStep); + QueryStep eventSelect = this.buildEventSelectQueryStep(context, table, conceptLabel, dateRestriction); + QueryStep eventFilter = this.buildEventFilterQueryStep(context, table, conceptLabel, eventSelect); + QueryStep finalStep = this.buildFinalQueryStep(conceptLabel, eventFilter); + + return context.withQueryStep(finalStep); + } + + private String getConceptLabel(CQConcept node, ConversionContext context) { + // only relevant for debugging purposes as it will be part of the generated SQL query + // we prefix each cte name of a concept with an incrementing counter to prevent naming collisions if the same concept is selected multiple times + return "%s_%s".formatted( + context.getQueryStepCounter(), + node.getUserOrDefaultLabel(Locale.ENGLISH) + .toLowerCase() + .replace(' ', '_') + .replaceAll("\\s", "_") + ); + } + + /** + * selects: + * - all of previous step + */ + private QueryStep buildDateRestrictionQueryStep( + ConversionContext context, + CQConcept node, + String conceptLabel, + QueryStep previous + ) { + if (((ConceptSelects) previous.getSelects()).getDateRestriction().isEmpty()) { + return previous; + } + + ConceptSelects dateRestrictionSelects = this.prepareDateRestrictionSelects(node, previous); + List dateRestriction = this.buildDateRestriction(context, previous); + + return QueryStep.builder() + .cteName(createCteName(conceptLabel, "_date_restriction")) + .fromTable(QueryStep.toTableLike(previous.getCteName())) + .selects(dateRestrictionSelects) + .conditions(dateRestriction) + .predecessors(List.of(previous)) + .build(); + } + + /** + * selects: + * - all of previous steps + * - transformed columns with selects + */ + private QueryStep buildEventSelectQueryStep( + ConversionContext context, + CQTable table, + String conceptLabel, QueryStep previous + ) { + if (table.getSelects().isEmpty()) { + return previous; + } + + ConceptSelects eventSelectSelects = this.prepareEventSelectSelects(context, table, previous); + + return QueryStep.builder() + .cteName(createCteName(conceptLabel, "_event_select")) + .fromTable(QueryStep.toTableLike(previous.getCteName())) + .selects(eventSelectSelects) + .conditions(Collections.emptyList()) + .predecessors(List.of(previous)) + .build(); + } + + /** + * selects: + * - all of previous step + * - remove filter + */ + private QueryStep buildEventFilterQueryStep( + ConversionContext context, + CQTable table, + String conceptLabel, + QueryStep previous + ) { + if (table.getFilters().isEmpty()) { + return previous; + } + + ConceptSelects eventFilterSelects = this.prepareEventFilterSelects(previous); + List eventFilterConditions = this.buildEventFilterConditions(context, table); + + return QueryStep.builder() + .cteName(createCteName(conceptLabel, "_event_filter")) + .fromTable(QueryStep.toTableLike(previous.getCteName())) + .selects(eventFilterSelects) + .conditions(eventFilterConditions) + .predecessors(List.of(previous)) + .build(); + } + + private ConceptSelects prepareDateRestrictionSelects(CQConcept node, QueryStep previous) { + ConceptSelects.ConceptSelectsBuilder selectsBuilder = ((ConceptSelects) previous.getQualifiedSelects()).toBuilder(); + selectsBuilder.dateRestriction(Optional.empty()); + if (node.isExcludeFromTimeAggregation()) { + selectsBuilder.validityDate(Optional.empty()); + } + return selectsBuilder.build(); + } + + private List buildDateRestriction(ConversionContext context, QueryStep previous) { + return ((ConceptSelects) previous.getSelects()).getDateRestriction() + .map(dateRestrictionColumn -> getDateRestrictionAsCondition(context, previous, dateRestrictionColumn)) + .orElseGet(Collections::emptyList); + } + + private static List getDateRestrictionAsCondition(ConversionContext context, QueryStep previous, Field dateRestrictionColumn) { + return previous.getSelects().getValidityDate().stream() + .map(validityDateColumn -> context.getSqlDialect().getFunction().dateRestriction(dateRestrictionColumn, validityDateColumn)) + .toList(); + } + + private ConceptSelects prepareEventSelectSelects( + ConversionContext context, + CQTable table, + QueryStep previous + ) { + return ((ConceptSelects) previous.getQualifiedSelects()).withEventSelect(this.getEventSelects(context, table)); + } + + private ConceptSelects prepareEventFilterSelects(QueryStep previous) { + return ((ConceptSelects) previous.getQualifiedSelects()).withEventFilter(Collections.emptyList()); + } + + private List buildEventFilterConditions(ConversionContext context, CQTable table) { + return table.getFilters().stream() + .map(filterValue -> this.filterConverterService.convert(filterValue, context)) + .toList(); + } + + private List> getEventSelects(ConversionContext context, CQTable table) { + return table.getSelects().stream() + .map(select -> (Field) this.selectConverterService.convert(select, context)) + .toList(); + } + + /** + * selects: + * - all of previous step + */ + private QueryStep buildFinalQueryStep(String conceptLabel, QueryStep previous) { + ConceptSelects finalSelects = ((ConceptSelects) previous.getQualifiedSelects()); + return QueryStep.builder() + .cteName(createCteName(conceptLabel, "")) + .fromTable(QueryStep.toTableLike(previous.getCteName())) + .selects(finalSelects) + .conditions(Collections.emptyList()) + .predecessors(List.of(previous)) + .build(); + } + + private static String createCteName(String conceptLabel, String suffix) { + return "concept_%s%s".formatted(conceptLabel, suffix); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQDateRestrictionConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQDateRestrictionConverter.java new file mode 100644 index 0000000000..37e3c1d56f --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQDateRestrictionConverter.java @@ -0,0 +1,20 @@ +package com.bakdata.conquery.sql.conversion.cqelement; + +import com.bakdata.conquery.apiv1.query.concept.specific.CQDateRestriction; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; + +public class CQDateRestrictionConverter implements NodeConverter { + + @Override + public ConversionContext convert(CQDateRestriction node, ConversionContext context) { + ConversionContext childContext = context.withDateRestrictionRange(CDateRange.of(node.getDateRange())); + return context.getNodeConverterService().convert(node.getChild(), childContext).withDateRestrictionRange(null); + } + + @Override + public Class getConversionClass() { + return CQDateRestriction.class; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java new file mode 100644 index 0000000000..b0fd9f9aad --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java @@ -0,0 +1,22 @@ +package com.bakdata.conquery.sql.conversion.cqelement; + +import com.bakdata.conquery.apiv1.query.concept.specific.CQOr; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; + +public class CQOrConverter implements NodeConverter { + + @Override + public ConversionContext convert(CQOr node, ConversionContext context) { + if (node.getChildren().size() > 1) { + throw new IllegalArgumentException("Multiple children are not yet supported"); + } + + return context.getNodeConverterService().convert(node.getChildren().get(0), context); + } + + @Override + public Class getConversionClass() { + return CQOr.class; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConceptPreprocessingService.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConceptPreprocessingService.java new file mode 100644 index 0000000000..cefe56a169 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConceptPreprocessingService.java @@ -0,0 +1,138 @@ + +package com.bakdata.conquery.sql.conversion.cqelement; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; +import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.context.selects.ConceptSelects; +import com.bakdata.conquery.sql.conversion.context.step.QueryStep; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import org.checkerframework.checker.units.qual.C; +import org.jooq.Field; +import org.jooq.impl.DSL; + +public class ConceptPreprocessingService { + + private static final String DATE_RESTRICTION_COLUMN_NAME = "date_restriction"; + private static final String VALIDITY_DATE_COLUMN_NAME = "validity_date"; + private final CQConcept concept; + private final ConversionContext context; + private final SqlFunctionProvider sqlFunctionProvider; + + public ConceptPreprocessingService(CQConcept concept, ConversionContext context) { + this.concept = concept; + this.context = context; + this.sqlFunctionProvider = this.context.getSqlDialect().getFunction(); + } + + /** + * selects: + * - (primary column) + * - date restriction + * - validity date + * - any filter (group/event) + * - any select (group/event) + */ + public QueryStep buildPreprocessingQueryStepForTable(String conceptLabel, CQTable table) { + + ConceptSelects.ConceptSelectsBuilder selectsBuilder = ConceptSelects.builder(); + + selectsBuilder.primaryColumn(DSL.field(context.getConfig().getPrimaryColumn())); + selectsBuilder.dateRestriction(this.getDateRestrictionSelect(table)); + selectsBuilder.validityDate(this.getValidityDateSelect(table)); + + List> conceptSelectFields = this.getColumnSelectReferences(table); + List> conceptFilterFields = this.getColumnFilterReferences(table); + + // deduplicate because a concepts selects and filters can require the same columns + // and selecting the same columns several times will cause SQL errors + List> deduplicatedFilterFields = conceptFilterFields.stream() + .filter(field -> !conceptSelectFields.contains(field)) + .toList(); + + selectsBuilder.eventSelect(conceptSelectFields); + selectsBuilder.eventFilter(deduplicatedFilterFields); + + // not part of preprocessing yet + selectsBuilder.groupSelect(Collections.emptyList()) + .groupFilter(Collections.emptyList()); + + return QueryStep.builder() + .cteName(this.getPreprocessingStepLabel(conceptLabel)) + .fromTable(QueryStep.toTableLike(this.getFromTableName(table))) + .selects(selectsBuilder.build()) + .conditions(Collections.emptyList()) + .predecessors(Collections.emptyList()) + .build(); + } + + private Optional> getDateRestrictionSelect(CQTable table) { + if (!this.context.dateRestrictionActive() || !this.tableHasValidityDates(table)) { + return Optional.empty(); + } + CDateRange dateRestrictionRange = this.context.getDateRestrictionRange(); + Field dateRestriction = this.sqlFunctionProvider.daterange(dateRestrictionRange) + .as(DATE_RESTRICTION_COLUMN_NAME); + return Optional.of(dateRestriction); + } + + private Optional> getValidityDateSelect(CQTable table) { + if (!this.validityDateIsRequired(table)) { + return Optional.empty(); + } + Field validityDateRange = this.sqlFunctionProvider.daterange(table.findValidityDateColumn()) + .as(VALIDITY_DATE_COLUMN_NAME); + return Optional.of(validityDateRange); + } + + /** + * @return True, if a date restriction is active and the node is not excluded from time aggregation + * OR there is no date restriction, but still existing validity dates which are included in time aggregation. + */ + private boolean validityDateIsRequired(CQTable table) { + return this.tableHasValidityDates(table) + && !this.concept.isExcludeFromTimeAggregation(); + } + + private boolean tableHasValidityDates(CQTable table) { + return !table.getConnector() + .getValidityDates() + .isEmpty(); + } + + private List> getColumnSelectReferences(CQTable table) { + return table.getSelects().stream() + .flatMap(select -> select.getRequiredColumns().stream().map(column -> this.mapColumnOntoTable(column, table))) + .toList(); + } + + private List> getColumnFilterReferences(CQTable table) { + return table.getFilters().stream() + .map(FilterValue::getFilter) + .flatMap(filter -> filter.getRequiredColumns().stream().map(column -> this.mapColumnOntoTable(column, table))) + .toList(); + } + + private String getFromTableName(CQTable table) { + return table.getConnector() + .getTable() + .getName(); + } + + private Field mapColumnOntoTable(Column column, CQTable table) { + return DSL.field(DSL.name(this.getFromTableName(table), column.getName())); + } + + private String getPreprocessingStepLabel(String conceptLabel) { + return "concept_%s_preprocessing".formatted(conceptLabel); + } + + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlDialect.java new file mode 100644 index 0000000000..406a412792 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlDialect.java @@ -0,0 +1,46 @@ +package com.bakdata.conquery.sql.conversion.dialect; + +import java.util.List; + +import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; +import com.bakdata.conquery.models.datasets.concepts.select.Select; +import com.bakdata.conquery.models.query.Visitable; +import com.bakdata.conquery.sql.conversion.filter.FilterConverter; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.select.SelectConverter; +import org.jooq.DSLContext; + +public class PostgreSqlDialect implements SqlDialect { + + private final DSLContext dslContext; + + public PostgreSqlDialect(DSLContext dslContext) { + this.dslContext = dslContext; + } + + @Override + public DSLContext getDSLContext() { + return this.dslContext; + } + + @Override + public List> getNodeConverters() { + return getDefaultNodeConverters(); + } + + @Override + public List>> getFilterConverters() { + return getDefaultFilterConverters(); + } + + @Override + public List> getSelectConverters() { + return getDefaultSelectConverters(); + } + + @Override + public SqlFunctionProvider getFunction() { + return new PostgreSqlFunctionProvider(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java new file mode 100644 index 0000000000..5ee2069393 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java @@ -0,0 +1,52 @@ +package com.bakdata.conquery.sql.conversion.dialect; + +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.datasets.Column; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.impl.DSL; + +/** + * Provider of SQL functions for PostgresSQL. + * + * @see PostgreSQL Documentation + */ +public class PostgreSqlFunctionProvider implements SqlFunctionProvider { + + @Override + public Condition dateRestriction(Field dateRestrictionColumn, Field validityDateColumn) { + // the && operator checks if two ranges overlap (see https://www.postgresql.org/docs/15/functions-range.html) + return DSL.condition( + "{0} && {1}", + dateRestrictionColumn, + validityDateColumn + ); + } + + @Override + public Field daterange(CDateRange dateRestriction) { + return DSL.field( + "daterange({0}::date, {1}::date, '[]')", + DSL.val(dateRestriction.getMin().toString()), + DSL.val(dateRestriction.getMax().toString()) + ); + } + + @Override + public Field daterange(Column column) { + return switch (column.getType()) { + // if validityDateColumn is a DATE_RANGE we can make use of Postgres' integrated daterange type. + case DATE_RANGE -> DSL.field(column.getName()); + // if the validity date column is not of daterange type, we construct it manually + case DATE -> DSL.field( + "daterange({0}, {0}, '[]')", + DSL.field(column.getName()) + ); + default -> throw new IllegalArgumentException( + "Given column type '%s' can't be converted to a proper date restriction." + .formatted(column.getType()) + ); + }; + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java new file mode 100644 index 0000000000..ff703e2d54 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java @@ -0,0 +1,84 @@ +package com.bakdata.conquery.sql.conversion.dialect; + +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; +import com.bakdata.conquery.models.datasets.concepts.select.Select; +import com.bakdata.conquery.models.query.Visitable; +import com.bakdata.conquery.sql.conversion.Converter; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.context.step.QueryStepTransformer; +import com.bakdata.conquery.sql.conversion.cqelement.CQAndConverter; +import com.bakdata.conquery.sql.conversion.cqelement.CQConceptConverter; +import com.bakdata.conquery.sql.conversion.cqelement.CQDateRestrictionConverter; +import com.bakdata.conquery.sql.conversion.cqelement.CQOrConverter; +import com.bakdata.conquery.sql.conversion.filter.FilterConverter; +import com.bakdata.conquery.sql.conversion.filter.FilterConverterService; +import com.bakdata.conquery.sql.conversion.filter.MultiSelectConverter; +import com.bakdata.conquery.sql.conversion.filter.RealRangeConverter; +import com.bakdata.conquery.sql.conversion.query.ConceptQueryConverter; +import com.bakdata.conquery.sql.conversion.select.DateDistanceConverter; +import com.bakdata.conquery.sql.conversion.select.FirstValueConverter; +import com.bakdata.conquery.sql.conversion.select.SelectConverter; +import com.bakdata.conquery.sql.conversion.select.SelectConverterService; +import com.bakdata.conquery.sql.conversion.supplier.SystemDateNowSupplier; +import org.jooq.DSLContext; + +public interface SqlDialect { + + SqlFunctionProvider getFunction(); + + List> getNodeConverters(); + + List>> getFilterConverters(); + + List> getSelectConverters(); + + DSLContext getDSLContext(); + + default List> getDefaultNodeConverters() { + return List.of( + new CQDateRestrictionConverter(), + new CQAndConverter(), + new CQOrConverter(), + new CQConceptConverter(new FilterConverterService(getFilterConverters()), new SelectConverterService(getSelectConverters())), + new ConceptQueryConverter(new QueryStepTransformer(getDSLContext())) + ); + } + + default List>> getDefaultFilterConverters() { + return List.of( + new MultiSelectConverter(), + new RealRangeConverter() + ); + } + + default List> customizeSelectConverters(List> substitutes) { + return customize(getDefaultSelectConverters(), substitutes); + } + + default List> getDefaultSelectConverters() { + return List.of( + new FirstValueConverter(), + new DateDistanceConverter(new SystemDateNowSupplier()) + ); + } + + private static > List customize(List defaults, List substitutes) { + Map, C> substituteMap = getSubstituteMap(substitutes); + return defaults.stream() + .map(converter -> substituteMap.getOrDefault(converter.getConversionClass(), converter)) + .collect(Collectors.toList()); + } + + private static > Map, C> getSubstituteMap(List substitutes) { + return substitutes.stream() + .collect(Collectors.toMap( + Converter::getConversionClass, + Function.identity() + )); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java new file mode 100644 index 0000000000..67d41cefcd --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -0,0 +1,57 @@ +package com.bakdata.conquery.sql.conversion.dialect; + +import java.sql.Date; + +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.events.MajorTypeId; +import org.jooq.Condition; +import org.jooq.DatePart; +import org.jooq.Field; +import org.jooq.impl.DSL; + +/** + * Provider of SQL functions. + */ +public interface SqlFunctionProvider { + + String DEFAULT_DATE_FORMAT = "yyyy-mm-dd"; + + + Condition dateRestriction(Field dateRestrictionColumn, Field validityDateColumn); + + /** + * @return A daterange for a date restriction. + */ + Field daterange(CDateRange dateRestriction); + + /** + * @return A daterange for an existing column. + */ + Field daterange(Column column); + + default Field toDate(String dateColumn) { + return DSL.toDate(dateColumn, DEFAULT_DATE_FORMAT); + } + + default Field dateDistance(DatePart timeUnit, Date endDate, Column startDateColumn) { + if (startDateColumn.getType() != MajorTypeId.DATE) { + throw new UnsupportedOperationException("Can't calculate date distance to column of type " + + startDateColumn.getType()); + } + // we can now safely cast to Field of type Date + Field startDate = DSL.field(startDateColumn.getName(), Date.class); + return DSL.dateDiff(timeUnit, startDate, endDate); + } + + default Condition in(String columnName, String[] values) { + return DSL.field(columnName) + .in(values); + } + + default Field first(String columnName) { + // TODO: this is just a temporary placeholder + return DSL.field(columnName); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/FilterConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/FilterConverter.java new file mode 100644 index 0000000000..1b645ba0fa --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/FilterConverter.java @@ -0,0 +1,21 @@ +package com.bakdata.conquery.sql.conversion.filter; + +import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; +import com.bakdata.conquery.models.datasets.concepts.filters.SingleColumnFilter; +import com.bakdata.conquery.sql.conversion.Converter; +import org.jooq.Condition; + +/** + * Converts a {@link com.bakdata.conquery.apiv1.query.concept.filter.FilterValue} + * to a condition for a SQL WHERE clause. + * + * @param The type of Filter this converter is responsible for. + */ +public interface FilterConverter> extends Converter { + + static String getColumnName(FilterValue filter) { + // works for now but we might have to distinguish later if we encounter non-SingleColumnFilters + return ((SingleColumnFilter) filter.getFilter()).getColumn().getName(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/FilterConverterService.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/FilterConverterService.java new file mode 100644 index 0000000000..3c835fca5e --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/FilterConverterService.java @@ -0,0 +1,14 @@ +package com.bakdata.conquery.sql.conversion.filter; + +import java.util.List; + +import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; +import com.bakdata.conquery.sql.conversion.ConverterService; +import org.jooq.Condition; + +public class FilterConverterService extends ConverterService, Condition> { + + public FilterConverterService(List> converters) { + super(converters); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/MultiSelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/MultiSelectConverter.java new file mode 100644 index 0000000000..14239afd0a --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/MultiSelectConverter.java @@ -0,0 +1,19 @@ +package com.bakdata.conquery.sql.conversion.filter; + +import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import org.jooq.Condition; + +public class MultiSelectConverter implements FilterConverter { + + @Override + public Condition convert(FilterValue.CQBigMultiSelectFilter filter, ConversionContext context) { + return context.getSqlDialect().getFunction() + .in(FilterConverter.getColumnName(filter), filter.getValue()); + } + + @Override + public Class getConversionClass() { + return FilterValue.CQBigMultiSelectFilter.class; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/RealRangeConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/RealRangeConverter.java new file mode 100644 index 0000000000..ec77749315 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/RealRangeConverter.java @@ -0,0 +1,30 @@ +package com.bakdata.conquery.sql.conversion.filter; + +import java.util.Optional; +import java.util.stream.Stream; + +import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.impl.DSL; + +public class RealRangeConverter implements FilterConverter { + + @Override + public Condition convert(FilterValue.CQRealRangeFilter filter, ConversionContext context) { + Field field = DSL.field(FilterConverter.getColumnName(filter)); + + Optional greaterOrEqualCondition = Optional.ofNullable(filter.getValue().getMin()).map(field::greaterOrEqual); + Optional lessOrEqualCondition = Optional.ofNullable(filter.getValue().getMax()).map(field::lessOrEqual); + return Stream.concat(greaterOrEqualCondition.stream(), lessOrEqualCondition.stream()) + .reduce(Condition::and) + .orElseThrow(() -> new IllegalArgumentException("Missing min or max value for real range filter.")); + } + + @Override + public Class getConversionClass() { + return FilterValue.CQRealRangeFilter.class; + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/package-info.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/package-info.java new file mode 100644 index 0000000000..21a2e2c710 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/package-info.java @@ -0,0 +1,4 @@ +/** + * Module implements the conversion of {@link com.bakdata.conquery.apiv1.query.QueryDescription} to a SQL query. + */ +package com.bakdata.conquery.sql.conversion; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java new file mode 100644 index 0000000000..9dd3367ed8 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java @@ -0,0 +1,44 @@ +package com.bakdata.conquery.sql.conversion.query; + +import java.util.List; + +import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.context.step.QueryStep; +import com.bakdata.conquery.sql.conversion.context.step.QueryStepTransformer; +import org.jooq.Record; +import org.jooq.Select; + +public class ConceptQueryConverter implements NodeConverter { + + private final QueryStepTransformer queryStepTransformer; + + public ConceptQueryConverter(QueryStepTransformer queryStepTransformer) { + this.queryStepTransformer = queryStepTransformer; + } + + @Override + public ConversionContext convert(ConceptQuery node, ConversionContext context) { + + ConversionContext contextAfterConversion = context.getNodeConverterService() + .convert(node.getRoot(), context); + + QueryStep preFinalStep = contextAfterConversion.getQuerySteps().iterator().next(); + QueryStep finalStep = QueryStep.builder() + .cteName(null) // the final QueryStep won't be converted to a CTE + .selects(preFinalStep.getQualifiedSelects()) + .fromTable(QueryStep.toTableLike(preFinalStep.getCteName())) + .conditions(preFinalStep.getConditions()) + .predecessors(List.of(preFinalStep)) + .build(); + + Select finalQuery = this.queryStepTransformer.toSelectQuery(finalStep); + return context.withFinalQuery(finalQuery); + } + + @Override + public Class getConversionClass() { + return ConceptQuery.class; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/DateDistanceConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/DateDistanceConverter.java new file mode 100644 index 0000000000..d70800c39f --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/DateDistanceConverter.java @@ -0,0 +1,66 @@ +package com.bakdata.conquery.sql.conversion.select; + +import java.sql.Date; +import java.time.LocalDate; +import java.time.temporal.ChronoUnit; +import java.util.Map; +import java.util.Objects; + +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DateDistanceSelect; +import com.bakdata.conquery.models.events.MajorTypeId; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.supplier.DateNowSupplier; +import org.jooq.DatePart; +import org.jooq.Field; + +public class DateDistanceConverter implements SelectConverter { + + private static final Map DATE_CONVERSION = Map.of( + ChronoUnit.DECADES, DatePart.DECADE, + ChronoUnit.YEARS, DatePart.YEAR, + ChronoUnit.DAYS, DatePart.DAY, + ChronoUnit.MONTHS, DatePart.MONTH, + ChronoUnit.CENTURIES, DatePart.CENTURY + ); + private final DateNowSupplier dateNowSupplier; + + public DateDistanceConverter(DateNowSupplier dateNowSupplier) { + this.dateNowSupplier = dateNowSupplier; + } + + @Override + public Field convert(DateDistanceSelect select, ConversionContext context) { + DatePart timeUnit = DATE_CONVERSION.get(select.getTimeUnit()); + if (timeUnit == null) { + throw new UnsupportedOperationException("Chrono unit %s is not supported".formatted(select.getTimeUnit())); + } + Column startDateColumn = select.getColumn(); + Date endDate = getEndDate(context); + + if (startDateColumn.getType() != MajorTypeId.DATE) { + throw new UnsupportedOperationException("Can't calculate date distance to column of type " + + startDateColumn.getType()); + } + return context.getSqlDialect().getFunction().dateDistance(timeUnit, endDate, startDateColumn) + .as(select.getLabel()); + } + + private Date getEndDate(ConversionContext context) { + LocalDate endDate; + // if a date restriction is set, the max of the date restriction equals the end date of the date distance + if (Objects.nonNull(context.getDateRestrictionRange())) { + endDate = context.getDateRestrictionRange().getMax(); + } + else { + // otherwise the current date is the upper bound + endDate = dateNowSupplier.getLocalDateNow(); + } + return Date.valueOf(endDate); + } + + @Override + public Class getConversionClass() { + return DateDistanceSelect.class; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/FirstValueConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/FirstValueConverter.java new file mode 100644 index 0000000000..50a66ddb03 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/FirstValueConverter.java @@ -0,0 +1,19 @@ +package com.bakdata.conquery.sql.conversion.select; + +import com.bakdata.conquery.models.datasets.concepts.select.connector.FirstValueSelect; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import org.jooq.Field; + +public class FirstValueConverter implements SelectConverter { + + public Field convert(FirstValueSelect select, ConversionContext context) { + SqlFunctionProvider fn = context.getSqlDialect().getFunction(); + return fn.first(select.getColumn().getName()); + } + + @Override + public Class getConversionClass() { + return FirstValueSelect.class; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/SelectConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/SelectConverter.java new file mode 100644 index 0000000000..1ec492c90b --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/SelectConverter.java @@ -0,0 +1,14 @@ +package com.bakdata.conquery.sql.conversion.select; + +import com.bakdata.conquery.models.datasets.concepts.select.Select; +import com.bakdata.conquery.sql.conversion.Converter; +import org.jooq.Field; + +/** + * Converts a {@link com.bakdata.conquery.models.datasets.concepts.select.Select} to a field for a SQL SELECT statement. + * + * @param The type of Select this converter is responsible for. + */ +public interface SelectConverter extends Converter> { + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/SelectConverterService.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/SelectConverterService.java new file mode 100644 index 0000000000..88527eede2 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/SelectConverterService.java @@ -0,0 +1,14 @@ +package com.bakdata.conquery.sql.conversion.select; + +import java.util.List; + +import com.bakdata.conquery.models.datasets.concepts.select.Select; +import com.bakdata.conquery.sql.conversion.ConverterService; +import org.jooq.Field; + +public class SelectConverterService extends ConverterService> { + + public SelectConverterService(List> converters) { + super(converters); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/supplier/DateNowSupplier.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/supplier/DateNowSupplier.java new file mode 100644 index 0000000000..3798ed9d76 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/supplier/DateNowSupplier.java @@ -0,0 +1,9 @@ +package com.bakdata.conquery.sql.conversion.supplier; + +import java.time.LocalDate; + +public interface DateNowSupplier { + + LocalDate getLocalDateNow(); + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/supplier/SystemDateNowSupplier.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/supplier/SystemDateNowSupplier.java new file mode 100644 index 0000000000..5b24826dd0 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/supplier/SystemDateNowSupplier.java @@ -0,0 +1,12 @@ +package com.bakdata.conquery.sql.conversion.supplier; + +import java.time.LocalDate; + +public class SystemDateNowSupplier implements DateNowSupplier { + + @Override + public LocalDate getLocalDateNow() { + return LocalDate.now(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlEntityResult.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlEntityResult.java new file mode 100644 index 0000000000..9fbe011230 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlEntityResult.java @@ -0,0 +1,55 @@ +package com.bakdata.conquery.sql.execution; + +import java.util.Collections; +import java.util.List; +import java.util.function.UnaryOperator; +import java.util.stream.Stream; + +import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.query.results.EntityResult; +import com.fasterxml.jackson.annotation.JsonCreator; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +@ToString +@Getter +@Setter +@AllArgsConstructor(onConstructor_=@JsonCreator) +@CPSType(id="SQL_RESULT", base= EntityResult.class) +public class SqlEntityResult implements EntityResult { + + private final int entityId; + private final String id; + private Object[] values; + + public String getId() { + return id; + } + + @Override + public int getEntityId() { + return entityId; + } + + @Override + public Stream streamValues() { + return Stream.ofNullable(values); + } + + @Override + public int columnCount() { + return values.length; + } + + @Override + public void modifyResultLinesInplace(UnaryOperator lineModifier) { + values = lineModifier.apply(values); + } + + @Override + public List listResultLines() { + return Collections.singletonList(values); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionResult.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionResult.java new file mode 100644 index 0000000000..d4b828437a --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionResult.java @@ -0,0 +1,22 @@ +package com.bakdata.conquery.sql.execution; + +import java.util.List; + +import com.bakdata.conquery.models.query.results.EntityResult; +import com.bakdata.conquery.models.query.results.SinglelineEntityResult; +import lombok.Value; + +@Value +public class SqlExecutionResult { + + List columnNames; + List table; + int rowCount; + + public SqlExecutionResult(List columnNames, List table) { + this.columnNames = columnNames; + this.table = table; + this.rowCount = table.size(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java new file mode 100644 index 0000000000..5bea87bd0d --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java @@ -0,0 +1,90 @@ +package com.bakdata.conquery.sql.execution; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.IntStream; + +import com.bakdata.conquery.models.error.ConqueryError; +import com.bakdata.conquery.models.query.results.EntityResult; +import com.bakdata.conquery.models.query.results.SinglelineEntityResult; +import com.bakdata.conquery.sql.conquery.SqlManagedQuery; +import com.google.common.base.Stopwatch; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.jooq.DSLContext; + +@RequiredArgsConstructor +@Slf4j +public class SqlExecutionService { + + private final DSLContext dslContext; + + public SqlExecutionResult execute(SqlManagedQuery sqlQuery) { + log.info("Starting SQL execution[{}]", sqlQuery.getQueryId()); + Stopwatch stopwatch = Stopwatch.createStarted(); + SqlExecutionResult result = dslContext.connectionResult(connection -> this.createStatementAndExecute(sqlQuery, connection)); + log.info("Finished SQL execution[{}] with {} results within {}", sqlQuery.getQueryId(), result.getRowCount(), stopwatch.elapsed()); + return result; + } + + private SqlExecutionResult createStatementAndExecute(SqlManagedQuery sqlQuery, Connection connection) { + + try (Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery(sqlQuery.getSqlQuery().getSqlString())) { + int columnCount = resultSet.getMetaData().getColumnCount(); + List columnNames = this.getColumnNames(resultSet, columnCount); + List resultTable = this.createResultTable(resultSet, columnCount); + + return new SqlExecutionResult(columnNames, resultTable); + } + catch (SQLException e) { + throw new ConqueryError.SqlError(e); + } + } + + private List createResultTable(ResultSet resultSet, int columnCount) throws SQLException { + List resultTable = new ArrayList<>(resultSet.getFetchSize()); + while (resultSet.next()) { + Object[] resultRow = this.getResultRow(resultSet, columnCount); + resultTable.add(new SqlEntityResult(resultSet.getRow(), resultSet.getObject(1).toString(), resultRow)); + } + return resultTable; + } + + private List getColumnNames(ResultSet resultSet, int columnCount) { + // JDBC ResultSet indices start with 1 + return IntStream.rangeClosed(2, columnCount) + .mapToObj(columnIndex -> this.getColumnName(resultSet, columnIndex)) + .toList(); + } + + private String getColumnName(ResultSet resultSet, int columnIndex) { + try { + return resultSet.getMetaData().getColumnName(columnIndex); + } + catch (SQLException e) { + throw new ConqueryError.SqlError(e); + } + } + + private Object[] getResultRow(ResultSet resultSet, int columnCount) { + // JDBC ResultSet indices start with 1 and we skip the first column because it contains the id + return IntStream.rangeClosed(2, columnCount) + .mapToObj(columnIndex -> this.getValueOfColumn(resultSet, columnIndex)) + .toArray(); + } + + private String getValueOfColumn(ResultSet resultSet, int columnIndex) { + try { + return resultSet.getString(columnIndex); + } + catch (SQLException e) { + throw new ConqueryError.SqlError(e); + } + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java b/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java index a3c0ec6654..3463601bff 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java +++ b/backend/src/main/java/com/bakdata/conquery/util/io/IdColumnUtil.java @@ -12,9 +12,13 @@ import com.bakdata.conquery.models.execution.ManagedExecution; import com.bakdata.conquery.models.identifiable.mapping.AutoIncrementingPseudomizer; import com.bakdata.conquery.models.identifiable.mapping.EntityIdMap; +import com.bakdata.conquery.models.identifiable.mapping.EntityPrintId; import com.bakdata.conquery.models.identifiable.mapping.FullIdPrinter; import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; +import com.bakdata.conquery.models.query.results.EntityResult; import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.sql.conquery.SqlManagedQuery; +import com.bakdata.conquery.sql.execution.SqlEntityResult; import lombok.experimental.UtilityClass; @UtilityClass @@ -53,6 +57,10 @@ public static IdPrinter getIdPrinter(Subject owner, ManagedExecution execution, .orElseThrow(); if (owner.isPermitted(execution.getDataset(), Ability.PRESERVE_ID)) { + // todo(tm): The integration of ids in the sql connector needs to be properly managed + if (execution instanceof SqlManagedQuery) { + return entityResult -> EntityPrintId.from(((SqlEntityResult) entityResult).getId()); + } return new FullIdPrinter(namespace.getStorage().getPrimaryDictionary(), namespace.getStorage().getIdMapping(), size, pos); } diff --git a/backend/src/test/java/com/bakdata/conquery/TestTags.java b/backend/src/test/java/com/bakdata/conquery/TestTags.java index 96ddc92423..2884330831 100644 --- a/backend/src/test/java/com/bakdata/conquery/TestTags.java +++ b/backend/src/test/java/com/bakdata/conquery/TestTags.java @@ -5,7 +5,10 @@ public class TestTags { public static final String INTEGRATION_PROGRAMMATIC = "INTEGRATION_PROGRAMMATIC"; public static final String INTEGRATION_JSON = "INTEGRATION_JSON"; + public static final String INTEGRATION_SQL_BACKEND = "INTEGRATION_SQL_BACKEND"; + public static final String TEST_DIRECTORY_ENVIRONMENT_VARIABLE = "CONQUERY_TEST_DIRECTORY"; + public static final String SQL_BACKEND_TEST_DIRECTORY_ENVIRONMENT_VARIABLE = "SQL_TEST_DIRECTORY"; public static final String TEST_PROGRAMMATIC_REGEX_FILTER = "CONQUERY_TEST_PROGRAMMATIC_REGEX_FILTER"; diff --git a/backend/src/test/java/com/bakdata/conquery/integration/ConqueryIntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/ConqueryIntegrationTests.java index 7b9b7ebb4a..7899e06272 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/ConqueryIntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/ConqueryIntegrationTests.java @@ -1,30 +1,32 @@ package com.bakdata.conquery.integration; +import java.util.List; +import java.util.stream.Stream; + import com.bakdata.conquery.TestTags; import org.junit.jupiter.api.DynamicNode; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.TestFactory; -import java.util.List; -import java.util.stream.Stream; - public class ConqueryIntegrationTests extends IntegrationTests { - public ConqueryIntegrationTests() { - super("tests/", "com.bakdata.conquery.integration"); - } - - @Override - @TestFactory - @Tag(TestTags.INTEGRATION_JSON) - public List jsonTests() { - return super.jsonTests(); - } - - @Override - @TestFactory - @Tag(TestTags.INTEGRATION_PROGRAMMATIC) - public Stream programmaticTests() { - return super.programmaticTests(); - } + + public ConqueryIntegrationTests() { + super("tests/", "com.bakdata.conquery.integration"); + } + + @Override + @TestFactory + @Tag(TestTags.INTEGRATION_JSON) + public List jsonTests() { + return super.jsonTests(); + } + + @Override + @TestFactory + @Tag(TestTags.INTEGRATION_PROGRAMMATIC) + public Stream programmaticTests() { + return super.programmaticTests(); + } + } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java index 17e9b2f1e8..bd2fd0f5bc 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java @@ -7,6 +7,7 @@ import java.io.InputStream; import java.net.URI; import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; @@ -21,13 +22,17 @@ import com.bakdata.conquery.TestTags; import com.bakdata.conquery.integration.json.JsonIntegrationTest; +import com.bakdata.conquery.integration.sql.SqlIntegrationTest; import com.bakdata.conquery.integration.tests.ProgrammaticIntegrationTest; import com.bakdata.conquery.io.cps.CPSTypeIdResolver; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; import com.bakdata.conquery.util.support.ConfigOverride; import com.bakdata.conquery.util.support.TestConquery; +import com.codahale.metrics.SharedMetricRegistries; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.base.Strings; @@ -127,6 +132,22 @@ public Stream programmaticTests() { .map(this::createDynamicProgrammaticTestNode); } + + @SneakyThrows + public Stream sqlTests(SqlDialect sqlDialect, SqlConnectorConfig sqlConfig) { + SharedMetricRegistries.setDefault("test"); + final Path testRootDir = Path.of(Objects.requireNonNullElse( + System.getenv(TestTags.SQL_BACKEND_TEST_DIRECTORY_ENVIRONMENT_VARIABLE), + SqlIntegrationTest.SQL_TEST_DIR + )); + + Stream paths = Files.walk(testRootDir); + List dynamicTestStream = paths.filter(path -> !Files.isDirectory(path) && path.toString().endsWith(".json")) + .map(path -> SqlIntegrationTest.fromPath(path, sqlDialect, sqlConfig)) + .map(test -> DynamicTest.dynamicTest(test.getTestSpec().getLabel(), test)).toList(); + return dynamicTestStream.stream(); + } + private DynamicTest createDynamicProgrammaticTestNode(ProgrammaticIntegrationTest test) { return DynamicTest.dynamicTest( test.getClass().getSimpleName(), diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java index ed1f11342e..a64c65993c 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/AbstractQueryEngineTest.java @@ -31,7 +31,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j -public abstract class AbstractQueryEngineTest extends ConqueryTestSpec { +public abstract class AbstractQueryEngineTest extends ConqueryTestSpec { @Override diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java index 41aef7a599..69fbcb62a9 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/ConqueryTestSpec.java @@ -22,6 +22,7 @@ import com.bakdata.conquery.models.worker.SingletonNamespaceCollection; import com.bakdata.conquery.util.NonPersistentStoreFactory; import com.bakdata.conquery.util.support.StandaloneSupport; +import com.bakdata.conquery.util.support.TestSupport; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.JavaType; @@ -36,7 +37,7 @@ @JsonTypeInfo(use = JsonTypeInfo.Id.CUSTOM, include = JsonTypeInfo.As.PROPERTY, property = "type") @Slf4j @CPSBase -public abstract class ConqueryTestSpec { +public abstract class ConqueryTestSpec { @Getter @Setter @@ -58,9 +59,9 @@ public ConqueryConfig overrideConfig(ConqueryConfig config) { return config.withStorage(new NonPersistentStoreFactory()); } - public abstract void executeTest(StandaloneSupport support) throws Exception; + public abstract void executeTest(S support) throws Exception; - public abstract void importRequiredData(StandaloneSupport support) throws Exception; + public abstract void importRequiredData(S support) throws Exception; @Override @@ -68,19 +69,19 @@ public String toString() { return label; } - public static T parseSubTree(StandaloneSupport support, JsonNode node, Class expectedClass) throws IOException, JSONException { + public static T parseSubTree(TestSupport support, JsonNode node, Class expectedClass) throws IOException, JSONException { return parseSubTree(support, node, expectedClass, null); } - public static T parseSubTree(StandaloneSupport support, JsonNode node, Class expectedClass, Consumer modifierBeforeValidation) throws IOException, JSONException { + public static T parseSubTree(TestSupport support, JsonNode node, Class expectedClass, Consumer modifierBeforeValidation) throws IOException, JSONException { return parseSubTree(support, node, Jackson.MAPPER.getTypeFactory().constructParametricType(expectedClass, new JavaType[0]), modifierBeforeValidation); } - public static T parseSubTree(StandaloneSupport support, JsonNode node, JavaType expectedType) throws IOException, JSONException { + public static T parseSubTree(TestSupport support, JsonNode node, JavaType expectedType) throws IOException, JSONException { return parseSubTree(support, node, expectedType, null); } - public static T parseSubTree(StandaloneSupport support, JsonNode node, JavaType expectedType, Consumer modifierBeforeValidation) throws IOException, JSONException { + public static T parseSubTree(TestSupport support, JsonNode node, JavaType expectedType, Consumer modifierBeforeValidation) throws IOException, JSONException { final ObjectMapper om = Jackson.MAPPER.copy(); ObjectMapper mapper = support.getDataset().injectIntoNew( new SingletonNamespaceCollection(support.getNamespace().getStorage().getCentralRegistry(), support.getMetaStorage().getCentralRegistry()) @@ -102,7 +103,7 @@ public static T parseSubTree(StandaloneSupport support, JsonNode node, Java return result; } - public static List parseSubTreeList(StandaloneSupport support, ArrayNode node, Class expectedType, Consumer modifierBeforeValidation) throws IOException, JSONException { + public static List parseSubTreeList(TestSupport support, ArrayNode node, Class expectedType, Consumer modifierBeforeValidation) throws IOException, JSONException { final ObjectMapper om = Jackson.MAPPER.copy(); ObjectMapper mapper = support.getDataset().injectInto( new SingletonNamespaceCollection(support.getNamespace().getStorage().getCentralRegistry()).injectIntoNew( @@ -150,7 +151,7 @@ public static List parseSubTreeList(StandaloneSupport support, ArrayNode @RequiredArgsConstructor private static class DatasetPlaceHolderFiller extends DeserializationProblemHandler { - private final StandaloneSupport support; + private final TestSupport support; @Override public Object handleWeirdStringValue(DeserializationContext ctxt, Class targetType, String valueToConvert, String failureMsg) throws IOException { diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java index 12fd1c29ea..01a69b3148 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/FormTest.java @@ -54,7 +54,7 @@ @Getter @Setter @CPSType(id = "FORM_TEST", base = ConqueryTestSpec.class) -public class FormTest extends ConqueryTestSpec { +public class FormTest extends ConqueryTestSpec { /* * parse form as json first, because it may contain namespaced ids, that can only be resolved after diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java new file mode 100644 index 0000000000..78a3d72366 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java @@ -0,0 +1,170 @@ +package com.bakdata.conquery.integration.sql; + + +import java.io.IOException; +import java.math.BigDecimal; +import java.nio.file.Files; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.Date; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +import com.bakdata.conquery.integration.common.RequiredColumn; +import com.bakdata.conquery.integration.common.RequiredTable; +import com.bakdata.conquery.integration.common.ResourceFile; +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.config.CSVConfig; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.events.MajorTypeId; +import com.bakdata.conquery.models.preproc.parser.specific.DateRangeParser; +import com.bakdata.conquery.models.query.results.EntityResult; +import com.bakdata.conquery.sql.execution.SqlEntityResult; +import com.google.common.base.Strings; +import com.univocity.parsers.csv.CsvParser; +import lombok.SneakyThrows; +import org.jooq.DSLContext; +import org.jooq.DataType; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.RowN; +import org.jooq.Table; +import org.jooq.conf.ParamType; +import org.jooq.impl.BuiltInDataType; +import org.jooq.impl.DSL; +import org.jooq.impl.SQLDataType; +import org.jooq.postgres.extensions.types.DateRange; + +public class CsvTableImporter { + + private final DSLContext dslContext; + private final DateRangeParser dateRangeParser; + private final CsvParser csvReader; + + public CsvTableImporter(DSLContext dslContext) { + this.dslContext = dslContext; + this.dateRangeParser = new DateRangeParser(new ConqueryConfig()); + this.csvReader = new CSVConfig().withSkipHeader(true).createParser(); + } + + /** + * Imports the table into the database that is connected to the {@link org.jooq.DSLContext DSLContext} + * of this {@link com.bakdata.conquery.integration.sql.CsvTableImporter CSVTableImporter}. + */ + public void importTableIntoDatabase(RequiredTable requiredTable) { + + Table table = DSL.table(requiredTable.getName()); + List allRequiredColumns = this.getAllRequiredColumns(requiredTable); + List> columns = this.createFieldsForColumns(allRequiredColumns); + List content = this.getTablesContentFromCSV(requiredTable.getCsv(), allRequiredColumns); + + // because we currently won't shut down the container between the testcases, we drop tables upfront if they + // exist to ensure consistency if table names of different testcases are the same + String dropTableStatement = dslContext.dropTableIfExists(table) + .getSQL(ParamType.INLINED); + + String createTableStatement = dslContext.createTable(table) + .columns(columns) + .getSQL(ParamType.INLINED); + + String insertIntoTableStatement = dslContext.insertInto(table, columns) + .valuesOfRows(content) + .getSQL(ParamType.INLINED); + + // we directly use JDBC because JOOQ can't cope with PostgreSQL custom types + dslContext.connection((Connection connection) -> { + try (Statement statement = connection.createStatement()) { + statement.execute(dropTableStatement); + statement.execute(createTableStatement); + statement.execute(insertIntoTableStatement); + } + }); + } + + public List readExpectedEntities(Path csv) throws IOException { + List rawEntities = this.csvReader.parseAll(Files.newInputStream(csv)); + List results = new ArrayList<>(rawEntities.size()); + for (int i = 0; i < rawEntities.size(); i++) { + String[] row = rawEntities.get(i); + results.add(new SqlEntityResult(i + 1, row[0], Arrays.copyOfRange(row, 1, row.length))); + } + return results; + } + + + private List> createFieldsForColumns(List requiredColumns) { + return requiredColumns.stream() + .map(this::createField) + .collect(Collectors.toList()); + } + + private List getAllRequiredColumns(RequiredTable table) { + ArrayList requiredColumns = new ArrayList<>(); + requiredColumns.add(table.getPrimaryColumn()); + requiredColumns.addAll(Arrays.stream(table.getColumns()).toList()); + return requiredColumns; + } + + private Field createField(RequiredColumn requiredColumn) { + DataType dataType = switch (requiredColumn.getType()) { + case STRING -> SQLDataType.VARCHAR; + case INTEGER -> SQLDataType.INTEGER; + case BOOLEAN -> SQLDataType.BOOLEAN; + case REAL -> SQLDataType.REAL; + case DECIMAL, MONEY -> SQLDataType.DECIMAL; + case DATE -> SQLDataType.DATE; + case DATE_RANGE -> new BuiltInDataType<>(DateRange.class, "daterange"); + }; + return DSL.field(requiredColumn.getName(), dataType); + } + + @SneakyThrows + private List getTablesContentFromCSV(ResourceFile csvFile, List requiredColumns) { + List rawContent = this.csvReader.parseAll(csvFile.stream()); + List> castedContent = this.castContent(rawContent, requiredColumns); + return castedContent.stream() + .map(DSL::row) + .toList(); + } + + /** + * Casts all values of each row to the corresponding type of the column the value refers to. + */ + private List> castContent(List rawContent, List requiredColumns) { + List> castedContent = new ArrayList<>(rawContent.size()); + for (String[] row : rawContent) { + List castEntriesOfRow = new ArrayList<>(row.length); + for (int i = 0; i < row.length; i++) { + MajorTypeId type = requiredColumns.get(i).getType(); + castEntriesOfRow.add(this.castEntryAccordingToColumnType(row[i], type)); + } + castedContent.add(castEntriesOfRow); + } + return castedContent; + } + + private Object castEntryAccordingToColumnType(String entry, MajorTypeId type) { + + // if the entry from the CSV is empty, the value in the database should be null + if (Strings.isNullOrEmpty(entry)) { + return null; + } + + return switch (type) { + case STRING -> entry; + case BOOLEAN -> Boolean.valueOf(entry); + case INTEGER -> Integer.valueOf(entry); + case REAL -> Float.valueOf(entry); + case DECIMAL, MONEY -> new BigDecimal(entry); + case DATE -> Date.valueOf(entry); + case DATE_RANGE -> { + CDateRange dateRange = this.dateRangeParser.parse(entry); + yield DateRange.dateRange(Date.valueOf(dateRange.getMin()), Date.valueOf(dateRange.getMax())); + } + }; + } + +} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/PostgreSqlIntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/PostgreSqlIntegrationTests.java new file mode 100644 index 0000000000..9ee02002ba --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/PostgreSqlIntegrationTests.java @@ -0,0 +1,86 @@ +package com.bakdata.conquery.integration.sql; + +import java.util.stream.Stream; + +import com.bakdata.conquery.TestTags; +import com.bakdata.conquery.apiv1.query.ConceptQuery; +import com.bakdata.conquery.integration.IntegrationTests; +import com.bakdata.conquery.models.config.Dialect; +import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.models.error.ConqueryError; +import com.bakdata.conquery.sql.DslContextFactory; +import com.bakdata.conquery.sql.SqlQuery; +import com.bakdata.conquery.sql.conquery.SqlManagedQuery; +import com.bakdata.conquery.sql.execution.SqlExecutionService; +import lombok.extern.slf4j.Slf4j; +import org.assertj.core.api.Assertions; +import org.jooq.DSLContext; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestFactory; +import org.testcontainers.containers.PostgreSQLContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; + +@Testcontainers +@Slf4j +public class PostgreSqlIntegrationTests extends IntegrationTests { + + private static final DockerImageName postgreSqlImageName = DockerImageName.parse("postgres:alpine3.17"); + private static final String databaseName = "test"; + private static final String username = "user"; + private static final String password = "pass"; + private static DSLContext dslContext; + private static SqlConnectorConfig sqlConfig; + + public PostgreSqlIntegrationTests() { + super("tests/", "com.bakdata.conquery.integration"); + } + + @Container + private static final PostgreSQLContainer postgresqlContainer = new PostgreSQLContainer<>(postgreSqlImageName) + .withDatabaseName(databaseName) + .withUsername(username) + .withPassword(password); + + + @BeforeAll + static void before() { + postgresqlContainer.start(); + sqlConfig = SqlConnectorConfig.builder() + .dialect(Dialect.POSTGRESQL) + .jdbcConnectionUrl(postgresqlContainer.getJdbcUrl()) + .databaseUsername(username) + .databasePassword(password) + .withPrettyPrinting(true) + .primaryColumn("pid") + .build(); + dslContext = DslContextFactory.create(sqlConfig); + } + + @Test + @Tag(TestTags.INTEGRATION_SQL_BACKEND) + public void shouldThrowException() { + SqlExecutionService executionService = new SqlExecutionService(dslContext); + SqlManagedQuery validQuery = new SqlManagedQuery(new ConceptQuery(), null, null, null, new SqlQuery("SELECT 1")); + Assertions.assertThatNoException().isThrownBy(() -> executionService.execute(validQuery)); + + // executing an empty query should throw an SQL error + SqlManagedQuery emptyQuery = new SqlManagedQuery(new ConceptQuery(), null, null, null, new SqlQuery("")); + Assertions.assertThatThrownBy(() -> executionService.execute(emptyQuery)) + .isInstanceOf(ConqueryError.SqlError.class) + .hasMessageContaining("Something went wrong while querying the database: org.postgresql.util.PSQLException"); + } + + + @TestFactory + @Tag(TestTags.INTEGRATION_SQL_BACKEND) + public Stream sqlBackendTests() { + return super.sqlTests(new TestPostgreSqlDialect(dslContext), sqlConfig); + } + + +} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlIntegrationTest.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlIntegrationTest.java new file mode 100644 index 0000000000..c64c775951 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlIntegrationTest.java @@ -0,0 +1,31 @@ +package com.bakdata.conquery.integration.sql; + +import java.io.IOException; +import java.nio.file.Path; + +import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.models.exceptions.JSONException; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; +import lombok.AllArgsConstructor; +import lombok.Getter; +import org.junit.jupiter.api.function.Executable; + +@AllArgsConstructor +@Getter +public class SqlIntegrationTest implements Executable { + + public static final String SQL_TEST_DIR = "src/test/resources/tests/sql"; + + private final SqlStandaloneSupport support; + private final SqlIntegrationTestSpec testSpec; + + public void execute() throws IOException, JSONException { + testSpec.importRequiredData(support); + testSpec.executeTest(support); + } + + public static SqlIntegrationTest fromPath(final Path path, final SqlDialect sqlDialect, final SqlConnectorConfig sqlConfig) { + return new SqlIntegrationTest(new SqlStandaloneSupport(sqlDialect, sqlConfig), SqlIntegrationTestSpec.fromJsonSpec(path)); + } + +} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlIntegrationTestSpec.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlIntegrationTestSpec.java new file mode 100644 index 0000000000..83bc210090 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlIntegrationTestSpec.java @@ -0,0 +1,124 @@ +package com.bakdata.conquery.integration.sql; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import javax.validation.Valid; +import javax.validation.constraints.NotNull; + +import com.bakdata.conquery.apiv1.query.Query; +import com.bakdata.conquery.integration.common.RequiredData; +import com.bakdata.conquery.integration.common.RequiredTable; +import com.bakdata.conquery.integration.json.ConqueryTestSpec; +import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.io.jackson.Jackson; +import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.Concept; +import com.bakdata.conquery.models.exceptions.JSONException; +import com.bakdata.conquery.models.query.results.EntityResult; +import com.bakdata.conquery.models.query.results.SinglelineEntityResult; +import com.bakdata.conquery.sql.conquery.SqlManagedQuery; +import com.bakdata.conquery.sql.execution.SqlEntityResult; +import com.bakdata.conquery.sql.execution.SqlExecutionResult; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectReader; +import com.fasterxml.jackson.databind.node.ArrayNode; +import lombok.Getter; +import lombok.Setter; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.assertj.core.api.Assertions; + +@Getter +@Setter +@CPSType(id = "SQL_TEST", base = ConqueryTestSpec.class) +@Slf4j +public class SqlIntegrationTestSpec extends ConqueryTestSpec { + + private static final String EXPECTED_SQL_FILENAME = "expected.sql"; + + @NotNull + @JsonProperty("query") + private JsonNode rawQuery; + + @JsonIgnore + private String description; + + @NotNull + private String expectedCsv; + + @Valid + @NotNull + private RequiredData content; + + @NotNull + @JsonProperty("concepts") + private ArrayNode rawConcepts; + + @JsonIgnore + private Query query; + + @JsonIgnore + private Path specDir; + + + @SneakyThrows + public static SqlIntegrationTestSpec fromJsonSpec(Path path) { + SqlIntegrationTestSpec test = readSpecFromJson(path); + test.setSpecDir(path.getParent()); + return test; + } + + private static SqlIntegrationTestSpec readSpecFromJson(Path path) throws IOException { + final ObjectReader objectReader = Jackson.MAPPER.readerFor(SqlIntegrationTestSpec.class); + return objectReader.readValue(Files.readString(path)); + } + + @Override + public void executeTest(SqlStandaloneSupport support) throws IOException { + for (RequiredTable table : content.getTables()) { + support.getTableImporter().importTableIntoDatabase(table); + } + + SqlManagedQuery managedQuery = support.getExecutionManager() + .runQuery(support.getNamespace(), getQuery(), support.getTestUser(), support.getDataset(), support.getConfig(), false); + log.info("Execute query: \n{}", managedQuery.getSqlQuery().getSqlString()); + + SqlExecutionResult result = managedQuery.getResult(); + List resultCsv = result.getTable(); + Path expectedCsvFile = this.specDir.resolve(this.expectedCsv); + List expectedCsv = support.getTableImporter().readExpectedEntities(expectedCsvFile); + Assertions.assertThat(resultCsv).usingRecursiveFieldByFieldElementComparator().containsExactlyElementsOf(expectedCsv); + } + + @Override + public void importRequiredData(SqlStandaloneSupport support) throws IOException, JSONException { + importTables(support); + importConcepts(support); + Query parsedQuery = ConqueryTestSpec.parseSubTree(support, getRawQuery(), Query.class); + setQuery(parsedQuery); + } + + private void importTables(SqlStandaloneSupport support) { + for (RequiredTable rTable : getContent().getTables()) { + final Table table = rTable.toTable(support.getDataset(), support.getNamespaceStorage().getCentralRegistry()); + support.getNamespaceStorage().addTable(table); + } + } + + private void importConcepts(SqlStandaloneSupport support) throws IOException, JSONException { + List> + concepts = + ConqueryTestSpec.parseSubTreeList(support, getRawConcepts(), Concept.class, concept -> concept.setDataset(support.getDataset())); + + for (Concept concept : concepts) { + support.getNamespaceStorage().updateConcept(concept); + } + } + + +} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlStandaloneSupport.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlStandaloneSupport.java new file mode 100644 index 0000000000..32fb9525d9 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlStandaloneSupport.java @@ -0,0 +1,93 @@ +package com.bakdata.conquery.integration.sql; + +import javax.validation.Validator; + +import com.bakdata.conquery.integration.IntegrationTests; +import com.bakdata.conquery.io.jackson.Jackson; +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.io.storage.NamespaceStorage; +import com.bakdata.conquery.mode.InternalObjectMapperCreator; +import com.bakdata.conquery.mode.local.LocalNamespaceHandler; +import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.worker.DatasetRegistry; +import com.bakdata.conquery.models.worker.LocalNamespace; +import com.bakdata.conquery.models.worker.Namespace; +import com.bakdata.conquery.sql.SqlContext; +import com.bakdata.conquery.sql.conquery.SqlExecutionManager; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; +import com.bakdata.conquery.util.NonPersistentStoreFactory; +import com.bakdata.conquery.util.support.TestSupport; +import io.dropwizard.jersey.validation.Validators; +import lombok.Value; + +@Value +public class SqlStandaloneSupport implements TestSupport { + + private static final Validator VALIDATOR = Validators.newValidator(); + Dataset dataset; + Namespace namespace; + ConqueryConfig config; + MetaStorage metaStorage; + User testUser; + + CsvTableImporter tableImporter; + SqlExecutionManager executionManager; + + public SqlStandaloneSupport(final SqlDialect sqlDialect, final SqlConnectorConfig sqlConfig) { + this.dataset = new Dataset("test"); + NamespaceStorage storage = new NamespaceStorage(new NonPersistentStoreFactory(), "", VALIDATOR) { + }; + storage.openStores(Jackson.MAPPER.copy()); + storage.updateDataset(dataset); + config = IntegrationTests.DEFAULT_CONFIG; + config.setSqlConnectorConfig(sqlConfig); + InternalObjectMapperCreator creator = new InternalObjectMapperCreator(config, getValidator()); + SqlContext context = new SqlContext(sqlConfig, sqlDialect); + LocalNamespaceHandler localNamespaceHandler = new LocalNamespaceHandler(config, creator, context); + DatasetRegistry registry = new DatasetRegistry<>(0, config, creator, localNamespaceHandler); + + metaStorage = new MetaStorage(new NonPersistentStoreFactory(), registry); + metaStorage.openStores(Jackson.MAPPER.copy()); + registry.setMetaStorage(metaStorage); + creator.init(registry); + + testUser = getConfig().getAuthorizationRealms().getInitialUsers().get(0).createOrOverwriteUser(metaStorage); + metaStorage.updateUser(testUser); + namespace = registry.createNamespace(storage); + tableImporter = new CsvTableImporter(sqlDialect.getDSLContext()); + executionManager = (SqlExecutionManager) namespace.getExecutionManager(); + } + + @Override + public Namespace getNamespace() { + return namespace; + } + + @Override + public Validator getValidator() { + return VALIDATOR; + } + + @Override + public MetaStorage getMetaStorage() { + return metaStorage; + } + + @Override + public NamespaceStorage getNamespaceStorage() { + return namespace.getStorage(); + } + + @Override + public ConqueryConfig getConfig() { + return config; + } + + @Override + public User getTestUser() { + return testUser; + } +} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/TestPostgreSqlDialect.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/TestPostgreSqlDialect.java new file mode 100644 index 0000000000..09ee227eb3 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/TestPostgreSqlDialect.java @@ -0,0 +1,35 @@ +package com.bakdata.conquery.integration.sql; + +import com.bakdata.conquery.models.datasets.concepts.select.Select; +import com.bakdata.conquery.sql.conversion.select.SelectConverter; +import com.bakdata.conquery.sql.conversion.select.DateDistanceConverter; +import com.bakdata.conquery.sql.conversion.dialect.PostgreSqlDialect; +import com.bakdata.conquery.sql.conversion.supplier.DateNowSupplier; +import org.jooq.DSLContext; + +import java.time.LocalDate; +import java.util.List; + +public class TestPostgreSqlDialect extends PostgreSqlDialect { + + public TestPostgreSqlDialect(DSLContext dslContext) { + super(dslContext); + } + + @Override + public List> getSelectConverters() { + return this.customizeSelectConverters(List.of( + new DateDistanceConverter(new MockDateNowSupplier()) + )); + } + + private class MockDateNowSupplier implements DateNowSupplier { + + @Override + public LocalDate getLocalDateNow() { + return LocalDate.parse("2023-03-28"); + } + + } + +} diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java b/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java index e31daee764..89f5a7a925 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java +++ b/backend/src/test/java/com/bakdata/conquery/util/support/StandaloneSupport.java @@ -35,7 +35,7 @@ @Slf4j @RequiredArgsConstructor -public class StandaloneSupport { +public class StandaloneSupport implements TestSupport { private final TestConquery testConquery; @Getter diff --git a/backend/src/test/java/com/bakdata/conquery/util/support/TestSupport.java b/backend/src/test/java/com/bakdata/conquery/util/support/TestSupport.java new file mode 100644 index 0000000000..e7f9d6a7f6 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/util/support/TestSupport.java @@ -0,0 +1,28 @@ +package com.bakdata.conquery.util.support; + +import javax.validation.Validator; + +import com.bakdata.conquery.io.storage.MetaStorage; +import com.bakdata.conquery.io.storage.NamespaceStorage; +import com.bakdata.conquery.models.auth.entities.User; +import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.worker.Namespace; + +public interface TestSupport { + + Dataset getDataset(); + + Namespace getNamespace(); + + Validator getValidator(); + + MetaStorage getMetaStorage(); + + NamespaceStorage getNamespaceStorage(); + + ConqueryConfig getConfig(); + + User getTestUser(); + +} diff --git a/backend/src/test/resources/tests/sql/and/different_concept/and.json b/backend/src/test/resources/tests/sql/and/different_concept/and.json new file mode 100644 index 0000000000..9e855976df --- /dev/null +++ b/backend/src/test/resources/tests/sql/and/different_concept/and.json @@ -0,0 +1,207 @@ +{ + "label": "Simple AND query for 3 different concepts", + "expectedCsv": "expected.csv", + "type": "SQL_TEST", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "type": "CONCEPT", + "label": "vs", + "ids": [ + "number" + ], + "tables": [ + { + "id": "number.number_connector", + "filters": [ + { + "filter": "number.number_connector.value", + "type": "REAL_RANGE", + "value": { + "min": 0, + "max": 1 + } + } + ], + "selects": [ + "number.number_connector.value" + ] + } + ] + }, + { + "ids": [ + "geschlecht_select" + ], + "type": "CONCEPT", + "label": "Geschlecht SELECT", + "tables": [ + { + "id": "geschlecht_select.geschlecht_connector", + "filters": [ + { + "filter": "geschlecht_select.geschlecht_connector.geschlecht", + "type": "BIG_MULTI_SELECT", + "value": [ + "f" + ] + } + ], + "selects": [ + "geschlecht_select.geschlecht_connector.geschlecht" + ] + } + ] + }, + { + "ids": [ + "language_select" + ], + "type": "CONCEPT", + "label": "Language SELECT", + "tables": [ + { + "id": "language_select.language_connector", + "filters": [ + { + "filter": "language_select.language_connector.language", + "type": "BIG_MULTI_SELECT", + "value": [ + "de" + ] + } + ], + "selects": [ + "language_select.language_connector.language" + ] + } + ] + } + ] + } + }, + "concepts": [ + { + "label": "number", + "type": "TREE", + "connectors": [ + { + "label": "number_connector", + "table": "table1", + "validityDates": { + "label": "datum", + "column": "table1.datum" + }, + "filters": { + "label": "value", + "description": "xy", + "column": "table1.value", + "type": "NUMBER" + }, + "selects": { + "name": "value", + "column": "table1.value", + "type": "FIRST" + } + } + ] + }, + { + "label": "geschlecht_select", + "type": "TREE", + "connectors": [ + { + "label": "geschlecht_connector", + "table": "table2", + "validityDates": { + "label": "datum", + "column": "table2.datum" + }, + "filters": { + "label": "geschlecht", + "description": "Geschlecht zur gegebenen Datumseinschränkung", + "column": "table2.geschlecht", + "type": "SELECT" + }, + "selects": { + "name": "geschlecht", + "column": "table2.geschlecht", + "type": "FIRST" + } + } + ] + }, + { + "label": "language_select", + "type": "TREE", + "connectors": [ + { + "label": "language_connector", + "table": "table2", + "validityDates": { + "label": "datum", + "column": "table2.datum" + }, + "filters": { + "label": "language", + "description": "Sprache", + "column": "table2.language", + "type": "SELECT" + }, + "selects": { + "name": "language", + "column": "table2.language", + "type": "FIRST" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/and/different_concept/content_1.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "value", + "type": "REAL" + }, + { + "name": "datum", + "type": "DATE_RANGE" + } + ] + }, + { + "csv": "tests/sql/and/different_concept/content_2.csv", + "name": "table2", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + }, + { + "name": "geschlecht", + "type": "STRING" + }, + { + "name": "language", + "type": "STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/and/different_concept/content_1.csv b/backend/src/test/resources/tests/sql/and/different_concept/content_1.csv new file mode 100644 index 0000000000..1851eed139 --- /dev/null +++ b/backend/src/test/resources/tests/sql/and/different_concept/content_1.csv @@ -0,0 +1,13 @@ +pid,value,datum +1,1,"2014-06-30/2015-06-30" +2,1.01,"2014-06-30/2015-06-30" +1,1,"2015-02-03/2015-06-30" +1,0.5,"2014-06-30/2015-06-30" +3,0.5,"2014-04-30/2014-06-30" +4,1,"2014-06-30/2015-06-30" +5,0.5,"2014-04-30/2014-06-30" +5,1,"2014-06-30/2015-06-30" +6,1,"2014-04-30/2014-06-30" +7,1,"2014-02-05/2014-02-20" +8,1,"2014-04-30/2014-06-30" +7,-1,"2014-06-30/2015-06-30" diff --git a/backend/src/test/resources/tests/sql/and/different_concept/content_2.csv b/backend/src/test/resources/tests/sql/and/different_concept/content_2.csv new file mode 100644 index 0000000000..dc012de238 --- /dev/null +++ b/backend/src/test/resources/tests/sql/and/different_concept/content_2.csv @@ -0,0 +1,9 @@ +pid,datum,geschlecht,language +1,2012-01-01,"f","de" +2,2010-07-15,"m","fr" +3,2013-11-10,"f","en" +4,2012-11-11,"m","" +5,2007-11-11,"","" +6,2012-11-11,"","de" +7,2012-11-11,"mf","de" +8,2012-11-11,"fm","fr" diff --git a/backend/src/test/resources/tests/sql/and/different_concept/expected.csv b/backend/src/test/resources/tests/sql/and/different_concept/expected.csv new file mode 100644 index 0000000000..beeae56d79 --- /dev/null +++ b/backend/src/test/resources/tests/sql/and/different_concept/expected.csv @@ -0,0 +1,4 @@ +pid,datum,value,geschlecht,language +1,"[2014-06-30,2015-06-30)",1,f,de +1,"[2015-02-03,2015-06-30)",1,f,de +1,"[2014-06-30,2015-06-30)",0.5,f,de diff --git a/backend/src/test/resources/tests/sql/and/same_concept/and.json b/backend/src/test/resources/tests/sql/and/same_concept/and.json new file mode 100644 index 0000000000..81226f3d8f --- /dev/null +++ b/backend/src/test/resources/tests/sql/and/same_concept/and.json @@ -0,0 +1,93 @@ +{ + "label": "Simple AND query for same concept", + "expectedCsv": "expected.csv", + "type": "SQL_TEST", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "type": "CONCEPT", + "label": "vs", + "ids": [ + "number" + ], + "tables": [ + { + "id": "number.number_connector", + "filters": [ + { + "filter": "number.number_connector.value", + "type": "REAL_RANGE", + "value": { + "min": 0.5, + "max": 1 + } + } + ] + } + ] + }, + { + "type": "CONCEPT", + "label": "vs", + "ids": [ + "number" + ], + "tables": [ + { + "id": "number.number_connector", + "filters": [ + { + "filter": "number.number_connector.value", + "type": "REAL_RANGE", + "value": { + "min": 1, + "max": 2 + } + } + ] + } + ] + } + ] + } + }, + "concepts": [ + { + "label": "number", + "type": "TREE", + "connectors": [ + { + "label": "number_connector", + "table": "table1", + "filters": { + "label": "value", + "description": "xy", + "column": "table1.value", + "type": "NUMBER" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/and/same_concept/content_1.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "value", + "type": "REAL" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/and/same_concept/content_1.csv b/backend/src/test/resources/tests/sql/and/same_concept/content_1.csv new file mode 100644 index 0000000000..5ceffe16ca --- /dev/null +++ b/backend/src/test/resources/tests/sql/and/same_concept/content_1.csv @@ -0,0 +1,13 @@ +pid,value +1,1 +2,1.01 +1,1 +1,0.5 +3,0.5 +4,1 +5,0.5 +5,1 +6,1 +7,1 +8,1 +7,-1 diff --git a/backend/src/test/resources/tests/sql/and/same_concept/expected.csv b/backend/src/test/resources/tests/sql/and/same_concept/expected.csv new file mode 100644 index 0000000000..4d99aa75d0 --- /dev/null +++ b/backend/src/test/resources/tests/sql/and/same_concept/expected.csv @@ -0,0 +1,13 @@ +pid +1 +1 +1 +1 +1 +1 +4 +5 +5 +6 +7 +8 diff --git a/backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/content.csv b/backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/content.csv new file mode 100644 index 0000000000..6f280c47ef --- /dev/null +++ b/backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/content.csv @@ -0,0 +1,9 @@ +pid,datum,datum_alt,geschlecht +1,"2012-06-30/2015-06-30",2012-01-01,"f" +2,"2012-06-30/2015-06-30",2010-07-15,"m" +3,"2012-02-03/2012-06-30",2012-11-10,"f" +4,"2010-06-30/2015-06-30",2012-11-11,"m" +5,"2011-04-30/2014-06-30",2007-11-11,"" +6,"2015-06-30/2016-06-30",2012-11-11,"" +7,"2014-04-30/2015-06-30",2012-11-11,"mf" +8,"2012-04-30/2014-06-30",2012-11-11,"fm" diff --git a/backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/date_restriction_date_column.json b/backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/date_restriction_date_column.json new file mode 100644 index 0000000000..cecd7086e0 --- /dev/null +++ b/backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/date_restriction_date_column.json @@ -0,0 +1,98 @@ +{ + "label": "Date restriction with multiple validity dates and dateColumn", + "type": "SQL_TEST", + "expectedCsv": "expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "type": "DATE_RESTRICTION", + "dateRange": { + "min": "2012-01-01", + "max": "2012-12-31" + }, + "child": { + "ids": [ + "geschlecht_select" + ], + "type": "CONCEPT", + "label": "Geschlecht SELECT", + "tables": [ + { + "id": "geschlecht_select.geschlecht_connector", + "filters": [ + { + "filter": "geschlecht_select.geschlecht_connector.geschlecht", + "type": "BIG_MULTI_SELECT", + "value": [ + "f" + ] + } + ], + "dateColumn" : { + "value" : "geschlecht_select.geschlecht_connector.datum_alt" + } + } + ] + } + } + ] + } + }, + "concepts": [ + { + "label": "geschlecht_select", + "type": "TREE", + "connectors": [ + { + "label": "geschlecht_connector", + "table": "table1", + "validityDates": [ + { + "label": "datum", + "column": "table1.datum" + }, + { + "label": "datum_alt", + "column": "table1.datum_alt" + } + ], + "filters": { + "label": "geschlecht", + "description": "Geschlecht zur gegebenen Datumseinschränkung", + "column": "table1.geschlecht", + "type": "SELECT" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/date_restriction/date_restriction_date_column/content.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE_RANGE" + }, + { + "name": "datum_alt", + "type": "DATE" + }, + { + "name": "geschlecht", + "type": "STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/expected.csv b/backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/expected.csv new file mode 100644 index 0000000000..2b81453f9f --- /dev/null +++ b/backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/expected.csv @@ -0,0 +1,3 @@ +pid,datum_alt +1,"[2012-01-01,2012-01-02)" +3,"[2012-11-10,2012-11-11)" diff --git a/backend/src/test/resources/tests/sql/date_restriction/date_restriction_no_validity_date/content.csv b/backend/src/test/resources/tests/sql/date_restriction/date_restriction_no_validity_date/content.csv new file mode 100644 index 0000000000..212025dec4 --- /dev/null +++ b/backend/src/test/resources/tests/sql/date_restriction/date_restriction_no_validity_date/content.csv @@ -0,0 +1,9 @@ +pid,datum,geschlecht +1,2012-01-01,"f" +2,2010-07-15,"m" +3,2012-11-10,"f" +4,2012-11-11,"m" +5,2007-11-11,"" +6,2012-11-11,"" +7,2012-11-11,"mf" +8,2012-11-11,"fm" diff --git a/backend/src/test/resources/tests/sql/date_restriction/date_restriction_no_validity_date/date_restriction_no_validity_date.json b/backend/src/test/resources/tests/sql/date_restriction/date_restriction_no_validity_date/date_restriction_no_validity_date.json new file mode 100644 index 0000000000..bd3ee6c472 --- /dev/null +++ b/backend/src/test/resources/tests/sql/date_restriction/date_restriction_no_validity_date/date_restriction_no_validity_date.json @@ -0,0 +1,82 @@ +{ + "label": "Date restriction query without validity date", + "description": "If a date restriction is active, but there is no validity date defined to apply the date restriction on, the date restriction filter should not be applied.", + "type": "SQL_TEST", + "expectedCsv": "expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "type": "DATE_RESTRICTION", + "dateRange": { + "min": "2022-01-01", + "max": "2022-12-31" + }, + "child": { + "ids": [ + "geschlecht_select" + ], + "type": "CONCEPT", + "label": "Geschlecht SELECT", + "tables": [ + { + "id": "geschlecht_select.geschlecht_connector", + "filters": [ + { + "filter": "geschlecht_select.geschlecht_connector.geschlecht", + "type": "BIG_MULTI_SELECT", + "value": [ + "f" + ] + } + ] + } + ] + } + } + ] + } + }, + "concepts": [ + { + "label": "geschlecht_select", + "type": "TREE", + "connectors": [ + { + "label": "geschlecht_connector", + "table": "table1", + "filters": { + "label": "geschlecht", + "description": "Geschlecht zur gegebenen Datumseinschränkung", + "column": "table1.geschlecht", + "type": "SELECT" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/date_restriction/date_restriction_no_validity_date/content.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + }, + { + "name": "geschlecht", + "type": "STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/date_restriction/date_restriction_no_validity_date/expected.csv b/backend/src/test/resources/tests/sql/date_restriction/date_restriction_no_validity_date/expected.csv new file mode 100644 index 0000000000..b6a85aedc6 --- /dev/null +++ b/backend/src/test/resources/tests/sql/date_restriction/date_restriction_no_validity_date/expected.csv @@ -0,0 +1,3 @@ +pid +1 +3 diff --git a/backend/src/test/resources/tests/sql/date_restriction/daterange/content.csv b/backend/src/test/resources/tests/sql/date_restriction/daterange/content.csv new file mode 100644 index 0000000000..f3da646264 --- /dev/null +++ b/backend/src/test/resources/tests/sql/date_restriction/daterange/content.csv @@ -0,0 +1,9 @@ +pid,datum,geschlecht +1,"2012-06-30/2015-06-30","f" +2,"2012-06-30/2015-06-30","m" +3,"2012-02-03/2012-06-30","f" +4,"2010-06-30/2015-06-30","m" +5,"2011-04-30/2014-06-30","" +6,"2015-06-30/2016-06-30","" +7,"2014-04-30/2015-06-30","mf" +8,"2012-04-30/2014-06-30","fm" diff --git a/backend/src/test/resources/tests/sql/date_restriction/daterange/date_restriction_date_range.json b/backend/src/test/resources/tests/sql/date_restriction/daterange/date_restriction_date_range.json new file mode 100644 index 0000000000..4942aacbfa --- /dev/null +++ b/backend/src/test/resources/tests/sql/date_restriction/daterange/date_restriction_date_range.json @@ -0,0 +1,85 @@ +{ + "label": "Date restriction query with daterange validity date", + "type": "SQL_TEST", + "expectedCsv": "expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "type": "DATE_RESTRICTION", + "dateRange": { + "min": "2012-01-01", + "max": "2012-12-31" + }, + "child": { + "ids": [ + "geschlecht_select" + ], + "type": "CONCEPT", + "label": "Geschlecht SELECT", + "tables": [ + { + "id": "geschlecht_select.geschlecht_connector", + "filters": [ + { + "filter": "geschlecht_select.geschlecht_connector.geschlecht", + "type": "BIG_MULTI_SELECT", + "value": [ + "f" + ] + } + ] + } + ] + } + } + ] + } + }, + "concepts": [ + { + "label": "geschlecht_select", + "type": "TREE", + "connectors": [ + { + "label": "geschlecht_connector", + "table": "table1", + "validityDates": { + "label": "datum", + "column": "table1.datum" + }, + "filters": { + "label": "geschlecht", + "description": "Geschlecht zur gegebenen Datumseinschränkung", + "column": "table1.geschlecht", + "type": "SELECT" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/date_restriction/daterange/content.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE_RANGE" + }, + { + "name": "geschlecht", + "type": "STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/date_restriction/daterange/expected.csv b/backend/src/test/resources/tests/sql/date_restriction/daterange/expected.csv new file mode 100644 index 0000000000..a8bff966b3 --- /dev/null +++ b/backend/src/test/resources/tests/sql/date_restriction/daterange/expected.csv @@ -0,0 +1,3 @@ +pid,datum +1,"[2012-06-30,2015-06-30)" +3,"[2012-02-03,2012-06-30)" diff --git a/backend/src/test/resources/tests/sql/date_restriction/simple_date/content.csv b/backend/src/test/resources/tests/sql/date_restriction/simple_date/content.csv new file mode 100644 index 0000000000..212025dec4 --- /dev/null +++ b/backend/src/test/resources/tests/sql/date_restriction/simple_date/content.csv @@ -0,0 +1,9 @@ +pid,datum,geschlecht +1,2012-01-01,"f" +2,2010-07-15,"m" +3,2012-11-10,"f" +4,2012-11-11,"m" +5,2007-11-11,"" +6,2012-11-11,"" +7,2012-11-11,"mf" +8,2012-11-11,"fm" diff --git a/backend/src/test/resources/tests/sql/date_restriction/simple_date/date_restriction_simple_date.json b/backend/src/test/resources/tests/sql/date_restriction/simple_date/date_restriction_simple_date.json new file mode 100644 index 0000000000..4749faac8d --- /dev/null +++ b/backend/src/test/resources/tests/sql/date_restriction/simple_date/date_restriction_simple_date.json @@ -0,0 +1,85 @@ +{ + "label": "Date restriction query with simple date validity date", + "type": "SQL_TEST", + "expectedCsv": "expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "type": "DATE_RESTRICTION", + "dateRange": { + "min": "2012-01-01", + "max": "2012-12-31" + }, + "child": { + "ids": [ + "geschlecht_select" + ], + "type": "CONCEPT", + "label": "Geschlecht SELECT", + "tables": [ + { + "id": "geschlecht_select.geschlecht_connector", + "filters": [ + { + "filter": "geschlecht_select.geschlecht_connector.geschlecht", + "type": "BIG_MULTI_SELECT", + "value": [ + "f" + ] + } + ] + } + ] + } + } + ] + } + }, + "concepts": [ + { + "label": "geschlecht_select", + "type": "TREE", + "connectors": [ + { + "label": "geschlecht_connector", + "table": "table1", + "validityDates": { + "label": "datum", + "column": "table1.datum" + }, + "filters": { + "label": "geschlecht", + "description": "Geschlecht zur gegebenen Datumseinschränkung", + "column": "table1.geschlecht", + "type": "SELECT" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/date_restriction/simple_date/content.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + }, + { + "name": "geschlecht", + "type": "STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/date_restriction/simple_date/expected.csv b/backend/src/test/resources/tests/sql/date_restriction/simple_date/expected.csv new file mode 100644 index 0000000000..c34539ab81 --- /dev/null +++ b/backend/src/test/resources/tests/sql/date_restriction/simple_date/expected.csv @@ -0,0 +1,3 @@ +pid,datum +1,"[2012-01-01,2012-01-02)" +3,"[2012-11-10,2012-11-11)" diff --git a/backend/src/test/resources/tests/sql/filter/number/content.csv b/backend/src/test/resources/tests/sql/filter/number/content.csv new file mode 100644 index 0000000000..1851eed139 --- /dev/null +++ b/backend/src/test/resources/tests/sql/filter/number/content.csv @@ -0,0 +1,13 @@ +pid,value,datum +1,1,"2014-06-30/2015-06-30" +2,1.01,"2014-06-30/2015-06-30" +1,1,"2015-02-03/2015-06-30" +1,0.5,"2014-06-30/2015-06-30" +3,0.5,"2014-04-30/2014-06-30" +4,1,"2014-06-30/2015-06-30" +5,0.5,"2014-04-30/2014-06-30" +5,1,"2014-06-30/2015-06-30" +6,1,"2014-04-30/2014-06-30" +7,1,"2014-02-05/2014-02-20" +8,1,"2014-04-30/2014-06-30" +7,-1,"2014-06-30/2015-06-30" diff --git a/backend/src/test/resources/tests/sql/filter/number/expected.csv b/backend/src/test/resources/tests/sql/filter/number/expected.csv new file mode 100644 index 0000000000..ce715976a6 --- /dev/null +++ b/backend/src/test/resources/tests/sql/filter/number/expected.csv @@ -0,0 +1,11 @@ +pid +1 +1 +1 +3 +4 +5 +5 +6 +7 +8 diff --git a/backend/src/test/resources/tests/sql/filter/number/number.spec.json b/backend/src/test/resources/tests/sql/filter/number/number.spec.json new file mode 100644 index 0000000000..46dd0f5425 --- /dev/null +++ b/backend/src/test/resources/tests/sql/filter/number/number.spec.json @@ -0,0 +1,75 @@ +{ + "label": "Single Number-Real-Range Filter Query", + "type": "SQL_TEST", + "expectedCsv": "expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "ids": [ + "number" + ], + "type": "CONCEPT", + "label": "vs", + "tables": [ + { + "id": "number.number_connector", + "filters": [ + { + "filter": "number.number_connector.value", + "type": "REAL_RANGE", + "value": { + "min": 0.5, + "max": 1 + } + } + ] + } + ] + } + ] + } + }, + "concepts": [ + { + "label": "number", + "type": "TREE", + "connectors": [ + { + "label": "number_connector", + "table": "table1", + "filters": { + "label": "value", + "description": "xy", + "column": "table1.value", + "type": "NUMBER" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/filter/number/content.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "value", + "type": "REAL" + }, + { + "name": "datum", + "type": "DATE_RANGE" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/filter/number_only_max/content.csv b/backend/src/test/resources/tests/sql/filter/number_only_max/content.csv new file mode 100644 index 0000000000..1851eed139 --- /dev/null +++ b/backend/src/test/resources/tests/sql/filter/number_only_max/content.csv @@ -0,0 +1,13 @@ +pid,value,datum +1,1,"2014-06-30/2015-06-30" +2,1.01,"2014-06-30/2015-06-30" +1,1,"2015-02-03/2015-06-30" +1,0.5,"2014-06-30/2015-06-30" +3,0.5,"2014-04-30/2014-06-30" +4,1,"2014-06-30/2015-06-30" +5,0.5,"2014-04-30/2014-06-30" +5,1,"2014-06-30/2015-06-30" +6,1,"2014-04-30/2014-06-30" +7,1,"2014-02-05/2014-02-20" +8,1,"2014-04-30/2014-06-30" +7,-1,"2014-06-30/2015-06-30" diff --git a/backend/src/test/resources/tests/sql/filter/number_only_max/expected.csv b/backend/src/test/resources/tests/sql/filter/number_only_max/expected.csv new file mode 100644 index 0000000000..ce668fc0b4 --- /dev/null +++ b/backend/src/test/resources/tests/sql/filter/number_only_max/expected.csv @@ -0,0 +1,5 @@ +pid +1 +3 +5 +7 diff --git a/backend/src/test/resources/tests/sql/filter/number_only_max/number_only_max.spec.json b/backend/src/test/resources/tests/sql/filter/number_only_max/number_only_max.spec.json new file mode 100644 index 0000000000..1b49d2b5aa --- /dev/null +++ b/backend/src/test/resources/tests/sql/filter/number_only_max/number_only_max.spec.json @@ -0,0 +1,74 @@ +{ + "label": "Single Number-Real-Range Filter Query (only max val)", + "type": "SQL_TEST", + "expectedCsv": "expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "ids": [ + "number" + ], + "type": "CONCEPT", + "label": "vs", + "tables": [ + { + "id": "number.number_connector", + "filters": [ + { + "filter": "number.number_connector.value", + "type": "REAL_RANGE", + "value": { + "max": 0.5 + } + } + ] + } + ] + } + ] + } + }, + "concepts": [ + { + "label": "number", + "type": "TREE", + "connectors": [ + { + "label": "number_connector", + "table": "table1", + "filters": { + "label": "value", + "description": "xy", + "column": "table1.value", + "type": "NUMBER" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/filter/number_only_max/content.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "value", + "type": "REAL" + }, + { + "name": "datum", + "type": "DATE_RANGE" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/filter/number_only_min/content.csv b/backend/src/test/resources/tests/sql/filter/number_only_min/content.csv new file mode 100644 index 0000000000..1851eed139 --- /dev/null +++ b/backend/src/test/resources/tests/sql/filter/number_only_min/content.csv @@ -0,0 +1,13 @@ +pid,value,datum +1,1,"2014-06-30/2015-06-30" +2,1.01,"2014-06-30/2015-06-30" +1,1,"2015-02-03/2015-06-30" +1,0.5,"2014-06-30/2015-06-30" +3,0.5,"2014-04-30/2014-06-30" +4,1,"2014-06-30/2015-06-30" +5,0.5,"2014-04-30/2014-06-30" +5,1,"2014-06-30/2015-06-30" +6,1,"2014-04-30/2014-06-30" +7,1,"2014-02-05/2014-02-20" +8,1,"2014-04-30/2014-06-30" +7,-1,"2014-06-30/2015-06-30" diff --git a/backend/src/test/resources/tests/sql/filter/number_only_min/expected.csv b/backend/src/test/resources/tests/sql/filter/number_only_min/expected.csv new file mode 100644 index 0000000000..9b1503f7a8 --- /dev/null +++ b/backend/src/test/resources/tests/sql/filter/number_only_min/expected.csv @@ -0,0 +1,12 @@ +pid +1 +2 +1 +1 +3 +4 +5 +5 +6 +7 +8 diff --git a/backend/src/test/resources/tests/sql/filter/number_only_min/number_only_min.spec.json b/backend/src/test/resources/tests/sql/filter/number_only_min/number_only_min.spec.json new file mode 100644 index 0000000000..918c2d521d --- /dev/null +++ b/backend/src/test/resources/tests/sql/filter/number_only_min/number_only_min.spec.json @@ -0,0 +1,74 @@ +{ + "label": "Single Number-Real-Range Filter Query (only min val)", + "type": "SQL_TEST", + "expectedCsv": "expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "ids": [ + "number" + ], + "type": "CONCEPT", + "label": "vs", + "tables": [ + { + "id": "number.number_connector", + "filters": [ + { + "filter": "number.number_connector.value", + "type": "REAL_RANGE", + "value": { + "min": 0.5 + } + } + ] + } + ] + } + ] + } + }, + "concepts": [ + { + "label": "number", + "type": "TREE", + "connectors": [ + { + "label": "number_connector", + "table": "table1", + "filters": { + "label": "value", + "description": "xy", + "column": "table1.value", + "type": "NUMBER" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/filter/number_only_min/content.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "value", + "type": "REAL" + }, + { + "name": "datum", + "type": "DATE_RANGE" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/filter/select/content.csv b/backend/src/test/resources/tests/sql/filter/select/content.csv new file mode 100644 index 0000000000..db93b08bd4 --- /dev/null +++ b/backend/src/test/resources/tests/sql/filter/select/content.csv @@ -0,0 +1,9 @@ +pid,datum,geschlecht +1,2012-01-01,"f" +2,2010-07-15,"m" +3,2013-11-10,"f" +4,2012-11-11,"m" +5,2007-11-11,"" +6,2012-11-11,"" +7,2012-11-11,"mf" +8,2012-11-11,"fm" diff --git a/backend/src/test/resources/tests/sql/filter/select/expected.csv b/backend/src/test/resources/tests/sql/filter/select/expected.csv new file mode 100644 index 0000000000..b5b5670cd1 --- /dev/null +++ b/backend/src/test/resources/tests/sql/filter/select/expected.csv @@ -0,0 +1,3 @@ +pid +1 +3 \ No newline at end of file diff --git a/backend/src/test/resources/tests/sql/filter/select/select.spec.json b/backend/src/test/resources/tests/sql/filter/select/select.spec.json new file mode 100644 index 0000000000..11ae8586f4 --- /dev/null +++ b/backend/src/test/resources/tests/sql/filter/select/select.spec.json @@ -0,0 +1,74 @@ +{ + "label": "Single Big-Multi-Select Filter Query", + "type": "SQL_TEST", + "expectedCsv": "expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "ids":[ + "geschlecht_select" + ], + "type":"CONCEPT", + "label":"Geschlecht SELECT", + "tables":[ + { + "id":"geschlecht_select.geschlecht_connector", + "filters":[ + { + "filter":"geschlecht_select.geschlecht_connector.geschlecht", + "type":"BIG_MULTI_SELECT", + "value":[ + "f" + ] + } + ] + } + ] + } + ] + } + }, + "concepts":[ + { + "label":"geschlecht_select", + "type":"TREE", + "connectors":[ + { + "label":"geschlecht_connector", + "table":"table1", + "filters":{ + "label":"geschlecht", + "description":"Geschlecht zur gegebenen Datumseinschränkung", + "column":"table1.geschlecht", + "type":"SELECT" + } + } + ] + } + ], + "content":{ + "tables":[ + { + "csv":"tests/sql/filter/select/content.csv", + "name":"table1", + "primaryColumn":{ + "name":"pid", + "type":"STRING" + }, + "columns":[ + { + "name":"datum", + "type":"DATE" + }, + { + "name":"geschlecht", + "type":"STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/centuries/centuries.spec.json b/backend/src/test/resources/tests/sql/selects/date_distance/centuries/centuries.spec.json new file mode 100644 index 0000000000..97e6b8f6e1 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/date_distance/centuries/centuries.spec.json @@ -0,0 +1,87 @@ +{ + "label": "DATE_DISTANCE select query with timeUnit CENTURIES", + "type": "SQL_TEST", + "expectedCsv": "expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "ids":[ + "geschlecht_select" + ], + "type":"CONCEPT", + "label":"Geschlecht SELECT", + "tables":[ + { + "id":"geschlecht_select.geschlecht_connector", + "filters":[ + { + "filter":"geschlecht_select.geschlecht_connector.geschlecht", + "type":"BIG_MULTI_SELECT", + "value":[ + "f" + ] + } + ], + "selects": [ + "geschlecht_select.geschlecht_connector.date_distance_centuries" + ] + } + ] + } + ] + } + }, + "concepts":[ + { + "label":"geschlecht_select", + "type":"TREE", + "connectors":[ + { + "label":"geschlecht_connector", + "table":"table1", + "filters":{ + "label":"geschlecht", + "description":"Geschlecht zur gegebenen Datumseinschränkung", + "column":"table1.geschlecht", + "type":"SELECT" + }, + "selects": [ + { + "column": "table1.datum", + "default": true, + "label": "date_distance_centuries", + "name": "date_distance_centuries", + "timeUnit": "CENTURIES", + "type": "DATE_DISTANCE" + } + ] + } + ] + } + ], + "content":{ + "tables":[ + { + "csv":"tests/sql/selects/date_distance/centuries/content.csv", + "name":"table1", + "primaryColumn":{ + "name":"pid", + "type":"STRING" + }, + "columns":[ + { + "name":"datum", + "type":"DATE" + }, + { + "name":"geschlecht", + "type":"STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/centuries/content.csv b/backend/src/test/resources/tests/sql/selects/date_distance/centuries/content.csv new file mode 100644 index 0000000000..5058be62da --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/date_distance/centuries/content.csv @@ -0,0 +1,9 @@ +pid,datum,geschlecht +1,1920-01-01,"f" +2,2010-07-15,"m" +3,2010-11-10,"f" +4,2013-11-11,"m" +5,2007-11-11,"" +6,2014-11-11,"" +7,2015-11-11,"mf" +8,2011-11-11,"fm" diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/centuries/expected.csv b/backend/src/test/resources/tests/sql/selects/date_distance/centuries/expected.csv new file mode 100644 index 0000000000..c0043272fe --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/date_distance/centuries/expected.csv @@ -0,0 +1,3 @@ +pid,date_distance_centuries +1,1 +3,0 diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/days_with_date_restriction/content.csv b/backend/src/test/resources/tests/sql/selects/date_distance/days_with_date_restriction/content.csv new file mode 100644 index 0000000000..77a2fec9a1 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/date_distance/days_with_date_restriction/content.csv @@ -0,0 +1,9 @@ +pid,datum,geschlecht +1,2012-01-01,"f" +2,2010-07-15,"m" +3,2012-11-10,"f" +4,2013-11-11,"m" +5,2007-11-11,"" +6,2014-11-11,"" +7,2015-11-11,"mf" +8,2011-11-11,"fm" diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/days_with_date_restriction/days_with_date_restriction.spec.json b/backend/src/test/resources/tests/sql/selects/date_distance/days_with_date_restriction/days_with_date_restriction.spec.json new file mode 100644 index 0000000000..f18f1934b2 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/date_distance/days_with_date_restriction/days_with_date_restriction.spec.json @@ -0,0 +1,98 @@ +{ + "type": "SQL_TEST", + "label": "DATE_DISTANCE select query with timeUnit DAYS and date restriction set", + "expectedCsv": "expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "type": "DATE_RESTRICTION", + "dateRange": { + "min": "2012-01-01", + "max": "2012-12-31" + }, + "child": { + "ids":[ + "geschlecht_select" + ], + "type":"CONCEPT", + "label":"Geschlecht SELECT", + "tables":[ + { + "id":"geschlecht_select.geschlecht_connector", + "filters":[ + { + "filter":"geschlecht_select.geschlecht_connector.geschlecht", + "type":"BIG_MULTI_SELECT", + "value":[ + "f" + ] + } + ], + "selects": [ + "geschlecht_select.geschlecht_connector.date_distance_days" + ] + } + ] + } + } + ] + } + }, + "concepts":[ + { + "label":"geschlecht_select", + "type":"TREE", + "connectors":[ + { + "label":"geschlecht_connector", + "table":"table1", + "validityDates": { + "label": "datum", + "column": "table1.datum" + }, + "filters":{ + "label":"geschlecht", + "description":"Geschlecht zur gegebenen Datumseinschränkung", + "column":"table1.geschlecht", + "type":"SELECT" + }, + "selects": [ + { + "column": "table1.datum", + "default": true, + "label": "date_distance_days", + "name": "date_distance_days", + "timeUnit": "DAYS", + "type": "DATE_DISTANCE" + } + ] + } + ] + } + ], + "content":{ + "tables":[ + { + "csv":"tests/sql/selects/date_distance/days_with_date_restriction/content.csv", + "name":"table1", + "primaryColumn":{ + "name":"pid", + "type":"STRING" + }, + "columns":[ + { + "name":"datum", + "type":"DATE" + }, + { + "name":"geschlecht", + "type":"STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/days_with_date_restriction/expected.csv b/backend/src/test/resources/tests/sql/selects/date_distance/days_with_date_restriction/expected.csv new file mode 100644 index 0000000000..f434fa729b --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/date_distance/days_with_date_restriction/expected.csv @@ -0,0 +1,3 @@ +pid,datum,date_distance_days +1,"[2012-01-01,2012-01-02)",365 +3,"[2012-11-10,2012-11-11)",51 diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/days_without_date_restriction/content.csv b/backend/src/test/resources/tests/sql/selects/date_distance/days_without_date_restriction/content.csv new file mode 100644 index 0000000000..5195678965 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/date_distance/days_without_date_restriction/content.csv @@ -0,0 +1,9 @@ +pid,datum,geschlecht +1,2012-01-01,"f" +2,2010-07-15,"m" +3,2013-11-10,"f" +4,2013-11-11,"m" +5,2007-11-11,"" +6,2014-11-11,"" +7,2015-11-11,"mf" +8,2011-11-11,"fm" diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/days_without_date_restriction/days_without_date_restriction.json b/backend/src/test/resources/tests/sql/selects/date_distance/days_without_date_restriction/days_without_date_restriction.json new file mode 100644 index 0000000000..4c54123139 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/date_distance/days_without_date_restriction/days_without_date_restriction.json @@ -0,0 +1,87 @@ +{ + "type": "SQL_TEST", + "label": "DATE_DISTANCE select query with timeUnit DAYS and without date restriction set", + "expectedCsv": "expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "ids":[ + "geschlecht_select" + ], + "type":"CONCEPT", + "label":"Geschlecht SELECT", + "tables":[ + { + "id":"geschlecht_select.geschlecht_connector", + "filters":[ + { + "filter":"geschlecht_select.geschlecht_connector.geschlecht", + "type":"BIG_MULTI_SELECT", + "value":[ + "f" + ] + } + ], + "selects": [ + "geschlecht_select.geschlecht_connector.date_distance_days" + ] + } + ] + } + ] + } + }, + "concepts":[ + { + "label":"geschlecht_select", + "type":"TREE", + "connectors":[ + { + "label":"geschlecht_connector", + "table":"table1", + "filters":{ + "label":"geschlecht", + "description":"Geschlecht zur gegebenen Datumseinschränkung", + "column":"table1.geschlecht", + "type":"SELECT" + }, + "selects": [ + { + "column": "table1.datum", + "default": true, + "label": "date_distance_days", + "name": "date_distance_days", + "timeUnit": "DAYS", + "type": "DATE_DISTANCE" + } + ] + } + ] + } + ], + "content":{ + "tables":[ + { + "csv":"tests/sql/selects/date_distance/days_without_date_restriction/content.csv", + "name":"table1", + "primaryColumn":{ + "name":"pid", + "type":"STRING" + }, + "columns":[ + { + "name":"datum", + "type":"DATE" + }, + { + "name":"geschlecht", + "type":"STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/days_without_date_restriction/expected.csv b/backend/src/test/resources/tests/sql/selects/date_distance/days_without_date_restriction/expected.csv new file mode 100644 index 0000000000..f6fe682a1e --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/date_distance/days_without_date_restriction/expected.csv @@ -0,0 +1,3 @@ +pid,date_distance_days +1,4104 +3,3425 diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/decades/content.csv b/backend/src/test/resources/tests/sql/selects/date_distance/decades/content.csv new file mode 100644 index 0000000000..a27a9a8243 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/date_distance/decades/content.csv @@ -0,0 +1,9 @@ +pid,datum,geschlecht +1,2012-01-01,"f" +2,2010-07-15,"m" +3,2020-11-10,"f" +4,2013-11-11,"m" +5,2007-11-11,"" +6,2014-11-11,"" +7,2015-11-11,"mf" +8,2011-11-11,"fm" diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/decades/decades.spec.json b/backend/src/test/resources/tests/sql/selects/date_distance/decades/decades.spec.json new file mode 100644 index 0000000000..586cc74a15 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/date_distance/decades/decades.spec.json @@ -0,0 +1,87 @@ +{ + "type": "SQL_TEST", + "label": "DATE_DISTANCE select query with timeUnit DECADES", + "expectedCsv": "expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "ids":[ + "geschlecht_select" + ], + "type":"CONCEPT", + "label":"Geschlecht SELECT", + "tables":[ + { + "id":"geschlecht_select.geschlecht_connector", + "filters":[ + { + "filter":"geschlecht_select.geschlecht_connector.geschlecht", + "type":"BIG_MULTI_SELECT", + "value":[ + "f" + ] + } + ], + "selects": [ + "geschlecht_select.geschlecht_connector.date_distance_decades" + ] + } + ] + } + ] + } + }, + "concepts":[ + { + "label":"geschlecht_select", + "type":"TREE", + "connectors":[ + { + "label":"geschlecht_connector", + "table":"table1", + "filters":{ + "label":"geschlecht", + "description":"Geschlecht zur gegebenen Datumseinschränkung", + "column":"table1.geschlecht", + "type":"SELECT" + }, + "selects": [ + { + "column": "table1.datum", + "default": true, + "label": "date_distance_decades", + "name": "date_distance_decades", + "timeUnit": "DECADES", + "type": "DATE_DISTANCE" + } + ] + } + ] + } + ], + "content":{ + "tables":[ + { + "csv":"tests/sql/selects/date_distance/decades/content.csv", + "name":"table1", + "primaryColumn":{ + "name":"pid", + "type":"STRING" + }, + "columns":[ + { + "name":"datum", + "type":"DATE" + }, + { + "name":"geschlecht", + "type":"STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/decades/expected.csv b/backend/src/test/resources/tests/sql/selects/date_distance/decades/expected.csv new file mode 100644 index 0000000000..171337e385 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/date_distance/decades/expected.csv @@ -0,0 +1,3 @@ +pid,date_distance_decades +1,1 +3,0 diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/months/content.csv b/backend/src/test/resources/tests/sql/selects/date_distance/months/content.csv new file mode 100644 index 0000000000..c2d4f04aef --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/date_distance/months/content.csv @@ -0,0 +1,9 @@ +pid,datum,geschlecht +1,2012-01-01,"f" +2,2010-07-15,"m" +3,2010-11-10,"f" +4,2013-11-11,"m" +5,2007-11-11,"" +6,2014-11-11,"" +7,2015-11-11,"mf" +8,2011-11-11,"fm" diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/months/expected.csv b/backend/src/test/resources/tests/sql/selects/date_distance/months/expected.csv new file mode 100644 index 0000000000..d0212b39b1 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/date_distance/months/expected.csv @@ -0,0 +1,3 @@ +pid,date_distance_months +1,134 +3,148 diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/months/months.spec.json b/backend/src/test/resources/tests/sql/selects/date_distance/months/months.spec.json new file mode 100644 index 0000000000..467c8aa561 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/date_distance/months/months.spec.json @@ -0,0 +1,87 @@ +{ + "type": "SQL_TEST", + "label": "DATE_DISTANCE select query with timeUnit MONTHS", + "expectedCsv": "expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "ids":[ + "geschlecht_select" + ], + "type":"CONCEPT", + "label":"Geschlecht SELECT", + "tables":[ + { + "id":"geschlecht_select.geschlecht_connector", + "filters":[ + { + "filter":"geschlecht_select.geschlecht_connector.geschlecht", + "type":"BIG_MULTI_SELECT", + "value":[ + "f" + ] + } + ], + "selects": [ + "geschlecht_select.geschlecht_connector.date_distance_months" + ] + } + ] + } + ] + } + }, + "concepts":[ + { + "label":"geschlecht_select", + "type":"TREE", + "connectors":[ + { + "label":"geschlecht_connector", + "table":"table1", + "filters":{ + "label":"geschlecht", + "description":"Geschlecht zur gegebenen Datumseinschränkung", + "column":"table1.geschlecht", + "type":"SELECT" + }, + "selects": [ + { + "column": "table1.datum", + "default": true, + "label": "date_distance_months", + "name": "date_distance_months", + "timeUnit": "MONTHS", + "type": "DATE_DISTANCE" + } + ] + } + ] + } + ], + "content":{ + "tables":[ + { + "csv":"tests/sql/selects/date_distance/months/content.csv", + "name":"table1", + "primaryColumn":{ + "name":"pid", + "type":"STRING" + }, + "columns":[ + { + "name":"datum", + "type":"DATE" + }, + { + "name":"geschlecht", + "type":"STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/years/content.csv b/backend/src/test/resources/tests/sql/selects/date_distance/years/content.csv new file mode 100644 index 0000000000..c2d4f04aef --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/date_distance/years/content.csv @@ -0,0 +1,9 @@ +pid,datum,geschlecht +1,2012-01-01,"f" +2,2010-07-15,"m" +3,2010-11-10,"f" +4,2013-11-11,"m" +5,2007-11-11,"" +6,2014-11-11,"" +7,2015-11-11,"mf" +8,2011-11-11,"fm" diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/years/expected.csv b/backend/src/test/resources/tests/sql/selects/date_distance/years/expected.csv new file mode 100644 index 0000000000..c0ec9df19d --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/date_distance/years/expected.csv @@ -0,0 +1,3 @@ +pid,date_distance_years +1,11 +3,13 diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/years/years.spec.json b/backend/src/test/resources/tests/sql/selects/date_distance/years/years.spec.json new file mode 100644 index 0000000000..fba8c65151 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/date_distance/years/years.spec.json @@ -0,0 +1,87 @@ +{ + "type": "SQL_TEST", + "label": "DATE_DISTANCE select query with timeUnit YEARS", + "expectedCsv": "expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "ids":[ + "geschlecht_select" + ], + "type":"CONCEPT", + "label":"Geschlecht SELECT", + "tables":[ + { + "id":"geschlecht_select.geschlecht_connector", + "filters":[ + { + "filter":"geschlecht_select.geschlecht_connector.geschlecht", + "type":"BIG_MULTI_SELECT", + "value":[ + "f" + ] + } + ], + "selects": [ + "geschlecht_select.geschlecht_connector.date_distance_years" + ] + } + ] + } + ] + } + }, + "concepts":[ + { + "label":"geschlecht_select", + "type":"TREE", + "connectors":[ + { + "label":"geschlecht_connector", + "table":"table1", + "filters":{ + "label":"geschlecht", + "description":"Geschlecht zur gegebenen Datumseinschränkung", + "column":"table1.geschlecht", + "type":"SELECT" + }, + "selects": [ + { + "column": "table1.datum", + "default": true, + "label": "date_distance_years", + "name": "date_distance_years", + "timeUnit": "YEARS", + "type": "DATE_DISTANCE" + } + ] + } + ] + } + ], + "content":{ + "tables":[ + { + "csv":"tests/sql/selects/date_distance/years/content.csv", + "name":"table1", + "primaryColumn":{ + "name":"pid", + "type":"STRING" + }, + "columns":[ + { + "name":"datum", + "type":"DATE" + }, + { + "name":"geschlecht", + "type":"STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/selects/validity_date/default/content.csv b/backend/src/test/resources/tests/sql/selects/validity_date/default/content.csv new file mode 100644 index 0000000000..db93b08bd4 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/validity_date/default/content.csv @@ -0,0 +1,9 @@ +pid,datum,geschlecht +1,2012-01-01,"f" +2,2010-07-15,"m" +3,2013-11-10,"f" +4,2012-11-11,"m" +5,2007-11-11,"" +6,2012-11-11,"" +7,2012-11-11,"mf" +8,2012-11-11,"fm" diff --git a/backend/src/test/resources/tests/sql/selects/validity_date/default/expected.csv b/backend/src/test/resources/tests/sql/selects/validity_date/default/expected.csv new file mode 100644 index 0000000000..a09d9ed8b4 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/validity_date/default/expected.csv @@ -0,0 +1,3 @@ +pid,datum +1,"[2012-01-01,2012-01-02)" +3,"[2013-11-10,2013-11-11)" diff --git a/backend/src/test/resources/tests/sql/selects/validity_date/default/validity_date_default.json b/backend/src/test/resources/tests/sql/selects/validity_date/default/validity_date_default.json new file mode 100644 index 0000000000..705aeb28f5 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/validity_date/default/validity_date_default.json @@ -0,0 +1,79 @@ +{ + "type": "SQL_TEST", + "label": "Validity date as default select without date restriction set", + "description": "If validity dates exist and the concept is not excluded from time aggregation, validity dates should be part of the final selects - regardless if a date restriction is set.", + "expectedCsv": "expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "ids":[ + "geschlecht_select" + ], + "type":"CONCEPT", + "label":"Geschlecht SELECT", + "tables":[ + { + "id":"geschlecht_select.geschlecht_connector", + "filters":[ + { + "filter":"geschlecht_select.geschlecht_connector.geschlecht", + "type":"BIG_MULTI_SELECT", + "value":[ + "f" + ] + } + ] + } + ] + } + ] + } + }, + "concepts":[ + { + "label":"geschlecht_select", + "type":"TREE", + "connectors":[ + { + "label":"geschlecht_connector", + "table":"table1", + "validityDates":{ + "label":"datum", + "column":"table1.datum" + }, + "filters":{ + "label":"geschlecht", + "description":"Geschlecht zur gegebenen Datumseinschränkung", + "column":"table1.geschlecht", + "type":"SELECT" + } + } + ] + } + ], + "content":{ + "tables":[ + { + "csv":"tests/sql/selects/validity_date/excluded_from_time_aggregation/content.csv", + "name":"table1", + "primaryColumn":{ + "name":"pid", + "type":"STRING" + }, + "columns":[ + { + "name":"datum", + "type":"DATE" + }, + { + "name":"geschlecht", + "type":"STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/selects/validity_date/excluded_from_time_aggregation/content.csv b/backend/src/test/resources/tests/sql/selects/validity_date/excluded_from_time_aggregation/content.csv new file mode 100644 index 0000000000..db93b08bd4 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/validity_date/excluded_from_time_aggregation/content.csv @@ -0,0 +1,9 @@ +pid,datum,geschlecht +1,2012-01-01,"f" +2,2010-07-15,"m" +3,2013-11-10,"f" +4,2012-11-11,"m" +5,2007-11-11,"" +6,2012-11-11,"" +7,2012-11-11,"mf" +8,2012-11-11,"fm" diff --git a/backend/src/test/resources/tests/sql/selects/validity_date/excluded_from_time_aggregation/expected.csv b/backend/src/test/resources/tests/sql/selects/validity_date/excluded_from_time_aggregation/expected.csv new file mode 100644 index 0000000000..b6a85aedc6 --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/validity_date/excluded_from_time_aggregation/expected.csv @@ -0,0 +1,3 @@ +pid +1 +3 diff --git a/backend/src/test/resources/tests/sql/selects/validity_date/excluded_from_time_aggregation/validity_date_excluded.json b/backend/src/test/resources/tests/sql/selects/validity_date/excluded_from_time_aggregation/validity_date_excluded.json new file mode 100644 index 0000000000..152f6f6c7d --- /dev/null +++ b/backend/src/test/resources/tests/sql/selects/validity_date/excluded_from_time_aggregation/validity_date_excluded.json @@ -0,0 +1,80 @@ +{ + "type": "SQL_TEST", + "label": "Validity date excluded from time aggregation", + "description": "If a concept is excluded from time aggregation, validity dates should not be part of the final select.", + "expectedCsv": "expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "ids": [ + "geschlecht_select" + ], + "type": "CONCEPT", + "label": "Geschlecht SELECT", + "excludeFromTimeAggregation": true, + "tables": [ + { + "id": "geschlecht_select.geschlecht_connector", + "filters": [ + { + "filter": "geschlecht_select.geschlecht_connector.geschlecht", + "type": "BIG_MULTI_SELECT", + "value": [ + "f" + ] + } + ] + } + ] + } + ] + } + }, + "concepts": [ + { + "label": "geschlecht_select", + "type": "TREE", + "connectors": [ + { + "label": "geschlecht_connector", + "table": "table1", + "validityDates": { + "label": "datum", + "column": "table1.datum" + }, + "filters": { + "label": "geschlecht", + "description": "Geschlecht zur gegebenen Datumseinschränkung", + "column": "table1.geschlecht", + "type": "SELECT" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/selects/validity_date/default/content.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + }, + { + "name": "geschlecht", + "type": "STRING" + } + ] + } + ] + } +} From 5c4ef0b61c5b91f2aef660e3f4e04a0b71fac628 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 4 Jul 2023 11:28:30 +0200 Subject: [PATCH 430/679] Disable date picker temporarily (#3120) --- frontend/src/js/ui-components/InputDate/InputDate.tsx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/frontend/src/js/ui-components/InputDate/InputDate.tsx b/frontend/src/js/ui-components/InputDate/InputDate.tsx index d99ec77f55..00e57bc56b 100644 --- a/frontend/src/js/ui-components/InputDate/InputDate.tsx +++ b/frontend/src/js/ui-components/InputDate/InputDate.tsx @@ -45,6 +45,10 @@ type Props = Omit & { onCalendarSelect?: (val: string) => void; }; +// TODO: Remove this once we have solved +// - that the date picker overlays other fields in forms +const TEMPORARILY_DISABLED_DATE_PICKER = true; + const InputDate = forwardRef( ( { @@ -126,6 +130,7 @@ const InputDate = forwardRef( customInput={createElement(HiddenInput)} calendarContainer={StyledCalendar} calendarStartDay={1} + disabled={TEMPORARILY_DISABLED_DATE_PICKER} /> ); From a0f8f9f96a167d14389cdded1698f6ce6a52f7f7 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 4 Jul 2023 12:35:48 +0200 Subject: [PATCH 431/679] use ThreadLocal keyReader --- .../xodus/stores/SerializingStore.java | 334 +++++++++--------- 1 file changed, 168 insertions(+), 166 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index 4740377dde..4a6b3425f2 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -59,7 +59,7 @@ public class SerializingStore implements Store { /** * Deserializer for keys */ - private final ObjectReader keyReader; + private final ThreadLocal keyReader; /** * Serializer for values @@ -123,14 +123,14 @@ public , CLASS_V extends Class> SerializingSto keyWriter = objectMapper.writerFor(keyType); - keyReader = objectMapper.readerFor(keyType); + keyReader = ThreadLocal.withInitial(() -> objectMapper.readerFor(keyType)); removeUnreadablesFromUnderlyingStore = removeUnreadableFromStore; unreadableValuesDumpDir = unreadableDataDumpDirectory; if (shouldDumpUnreadables()) { - if(!unreadableValuesDumpDir.exists() && !unreadableValuesDumpDir.mkdirs()) { + if (!unreadableValuesDumpDir.exists() && !unreadableValuesDumpDir.mkdirs()) { throw new IllegalStateException("Could not create dump directory: " + unreadableValuesDumpDir); } else if (!unreadableValuesDumpDir.isDirectory()) { @@ -155,19 +155,51 @@ public void add(KEY key, VALUE value) { store.add(writeKey(key), writeValue(value)); } + /** + * Serialize key with {@code keyWriter}. + */ + private ByteIterable writeKey(KEY key) { + return write(key, keyWriter); + } + + /** + * Serialize value with {@code valueWriter}. + */ + private ByteIterable writeValue(VALUE value) { + return write(value, valueWriter); + } + + /** + * Try writing object with writer. + */ + private ByteIterable write(Object obj, ObjectWriter writer) { + try { + final byte[] bytes = writer.writeValueAsBytes(obj); + if (log.isTraceEnabled()) { + final String json = JacksonUtil.toJsonDebug(bytes); + log.trace("Written ({}): {}", valueType.getName(), json); + } + return new ArrayByteIterable(bytes); + } + catch (JsonProcessingException e) { + throw new RuntimeException("Failed to write " + obj, e); + } + } + @Override public VALUE get(KEY key) { final ByteIterable binValue = store.get(writeKey(key)); try { - return readValue(binValue); - } catch (Exception e) { + return readValue(binValue); + } + catch (Exception e) { - if(unreadableValuesDumpDir != null) { + if (unreadableValuesDumpDir != null) { dumpToFile(binValue, key.toString(), e, unreadableValuesDumpDir, store.getName(), objectMapper); } - if(removeUnreadablesFromUnderlyingStore) { + if (removeUnreadablesFromUnderlyingStore) { remove(key); // Null seems to be an acceptable return value in this case return null; @@ -179,6 +211,115 @@ public VALUE get(KEY key) { } } + /** + * Deserialize value with {@code valueReader}. + */ + private VALUE readValue(ByteIterable value) { + return read(valueReader, value); + } + + /** + * Dumps the content of an unreadable value to a file as a json (it tries to parse it as an object and than tries to dump it as a json). + * + * @param obj The object to dump. + * @param keyOfDump The key under which the unreadable value is accessible. It is used for the file name. + * @param reason The exception causing us to dump the file + * @param unreadableDumpDir The director to dump to. The method assumes that the directory exists and is okay to write to. + * @param storeName The name of the store which is also used in the dump file name. + */ + private static void dumpToFile(@NonNull ByteIterable obj, @NonNull String keyOfDump, Exception reason, @NonNull File unreadableDumpDir, String storeName, ObjectMapper objectMapper) { + // Create dump filehandle + final File dumpfile = makeDumpFileName(keyOfDump, unreadableDumpDir, storeName); + final File exceptionFileName = makeExceptionFileName(keyOfDump, unreadableDumpDir, storeName); + + if (dumpfile.exists() || exceptionFileName.exists()) { + log.trace("Abort dumping of file {} because it already exists.", dumpfile); + return; + } + + if (!dumpfile.getParentFile().exists() && !dumpfile.getParentFile().mkdirs()) { + throw new IllegalStateException("Could not create `%s`.".formatted(dumpfile.getParentFile())); + } + + //TODO FK: dump in a separate thread so we are not blocking the reader thread. + + // Write json + try { + log.info("Dumping value of key {} to {} (because it cannot be deserialized anymore).", keyOfDump, dumpfile.getCanonicalPath()); + + final JsonNode dump = objectMapper.readerFor(JsonNode.class).readValue(obj.getBytesUnsafe(), 0, obj.getLength()); + Jackson.MAPPER.writer().writeValue(dumpfile, dump); + } + catch (IOException e) { + log.error("Failed to dump unreadable value of key `{}` to file `{}`", keyOfDump, dumpfile, e); + } + + try (PrintStream out = new PrintStream(exceptionFileName)) { + reason.printStackTrace(out); + } + catch (IOException e) { + log.error("Failed to dump exception for `{}` to file `{}`.", keyOfDump, exceptionFileName, e); + } + + } + + @Override + public void remove(KEY key) { + log.trace("Removing value to key {} from Store[{}]", key, store.getName()); + store.remove(writeKey(key)); + } + + /** + * Try read value with reader. + */ + private T read(ObjectReader reader, ByteIterable obj) { + if (obj == null) { + return null; + } + try { + return reader.readValue(obj.getBytesUnsafe(), 0, obj.getLength()); + } + catch (IOException e) { + throw new RuntimeException("Failed to read " + JacksonUtil.toJsonDebug(obj.getBytesUnsafe()), e); + } + } + + /** + * Generates a valid file name from the key of the dump object, the store and the current time. + * However, it does not ensure that there is no file with such a name. + *

+ * Current implementation is `$unreadableDumpDir/$today/$store/$key.json` + */ + @NotNull + public static File makeDumpFileName(@NotNull String keyOfDump, @NotNull File unreadableDumpDir, @NotNull String storeName) { + return unreadableDumpDir.toPath() + .resolve(DateTimeFormatter.BASIC_ISO_DATE.format(LocalDateTime.now())) + .resolve(storeName) + .resolve(sanitiseFileName(keyOfDump) + "." + DUMP_FILE_EXTENSION) + .toFile(); + + } + + /** + * Generates a valid file name from the key of the dump object, the store and the current time. + * However, it does not ensure that there is no file with such a name. + *

+ * Current implementation is `$unreadableDumpDir/$today/$store/$key.exception` + */ + @NotNull + public static File makeExceptionFileName(@NotNull String keyOfDump, @NotNull File unreadableDumpDir, @NotNull String storeName) { + return unreadableDumpDir.toPath() + .resolve(DateTimeFormatter.BASIC_ISO_DATE.format(LocalDateTime.now())) + .resolve(storeName) + .resolve(sanitiseFileName(keyOfDump) + "." + EXCEPTION_FILE_EXTENSION) + .toFile(); + + } + + private static String sanitiseFileName(@NotNull String name) { + return FileUtil.SAVE_FILENAME_REPLACEMENT_MATCHER.matcher(name).replaceAll("_"); + } + /** * Iterates a given consumer over the entries of this store. * Depending on the {@link XodusStoreFactory} corrupt entries may be dump to a file and/or removed from the store. @@ -238,21 +379,22 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { executorService.shutdown(); - while (!executorService.awaitTermination(1, TimeUnit.MINUTES)){ + while (!executorService.awaitTermination(1, TimeUnit.MINUTES)) { log.debug("Still waiting for {} to load.", this); } // Print some statistics final int total = result.getTotalProcessed(); log.debug( - String.format( - "While processing store %s:\n\tEntries processed:\t%d\n\tKey read failure:\t%d (%.2f%%)\n\tValue read failure:\t%d (%.2f%%)", - store.getName(), - total, - result.getFailedKeys(), - total > 0 ? (float) result.getFailedKeys() / total * 100 : 0, - result.getFailedValues(), - total > 0 ? (float) result.getFailedValues() / total * 100 : 0)); + String.format( + "While processing store %s:\n\tEntries processed:\t%d\n\tKey read failure:\t%d (%.2f%%)\n\tValue read failure:\t%d (%.2f%%)", + store.getName(), + total, + result.getFailedKeys(), + total > 0 ? (float) result.getFailedKeys() / total * 100 : 0, + result.getFailedValues(), + total > 0 ? (float) result.getFailedValues() / total * 100 : 0 + )); // Remove corrupted entries from the store if configured so if (removeUnreadablesFromUnderlyingStore) { @@ -288,6 +430,13 @@ private TYPE getDeserializedAndDumpFailed(ByteIterable serial, Function T read(ObjectReader reader, ByteIterable obj) { - if (obj == null) { - return null; - } - try { - return reader.readValue(obj.getBytesUnsafe(), 0, obj.getLength()); - } - catch (IOException e) { - throw new RuntimeException("Failed to read " + JacksonUtil.toJsonDebug(obj.getBytesUnsafe()), e); - } - } - - /** - * Dumps the content of an unreadable value to a file as a json (it tries to parse it as an object and than tries to dump it as a json). - * - * @param obj The object to dump. - * @param keyOfDump The key under which the unreadable value is accessible. It is used for the file name. - * @param reason The exception causing us to dump the file - * @param unreadableDumpDir The director to dump to. The method assumes that the directory exists and is okay to write to. - * @param storeName The name of the store which is also used in the dump file name. - */ - private static void dumpToFile(@NonNull ByteIterable obj, @NonNull String keyOfDump, Exception reason, @NonNull File unreadableDumpDir, String storeName, ObjectMapper objectMapper) { - // Create dump filehandle - final File dumpfile = makeDumpFileName(keyOfDump, unreadableDumpDir, storeName); - final File exceptionFileName = makeExceptionFileName(keyOfDump, unreadableDumpDir, storeName); - - if (dumpfile.exists() || exceptionFileName.exists()) { - log.trace("Abort dumping of file {} because it already exists.", dumpfile); - return; - } - - if(!dumpfile.getParentFile().exists() && !dumpfile.getParentFile().mkdirs()){ - throw new IllegalStateException("Could not create `%s`.".formatted(dumpfile.getParentFile())); - } - - //TODO FK: dump in a separate thread so we are not blocking the reader thread. - - // Write json - try { - log.info("Dumping value of key {} to {} (because it cannot be deserialized anymore).", keyOfDump, dumpfile.getCanonicalPath()); - - final JsonNode dump = objectMapper.readerFor(JsonNode.class).readValue(obj.getBytesUnsafe(), 0, obj.getLength()); - Jackson.MAPPER.writer().writeValue(dumpfile, dump); - } - catch (IOException e) { - log.error("Failed to dump unreadable value of key `{}` to file `{}`", keyOfDump, dumpfile, e); - } - - try(PrintStream out = new PrintStream(exceptionFileName)) { - reason.printStackTrace(out); - } - catch (IOException e) { - log.error("Failed to dump exception for `{}` to file `{}`.", keyOfDump, exceptionFileName, e); - } - - } - - /** - * Generates a valid file name from the key of the dump object, the store and the current time. - * However, it does not ensure that there is no file with such a name. - * - * Current implementation is `$unreadableDumpDir/$today/$store/$key.json` - */ - @NotNull - public static File makeDumpFileName(@NotNull String keyOfDump, @NotNull File unreadableDumpDir, @NotNull String storeName) { - return unreadableDumpDir.toPath() - .resolve(DateTimeFormatter.BASIC_ISO_DATE.format(LocalDateTime.now())) - .resolve(storeName) - .resolve(sanitiseFileName(keyOfDump) + "." + DUMP_FILE_EXTENSION) - .toFile(); - - } - - private static String sanitiseFileName(@NotNull String name) { - return FileUtil.SAVE_FILENAME_REPLACEMENT_MATCHER.matcher(name).replaceAll("_"); - } - - /** - * Generates a valid file name from the key of the dump object, the store and the current time. - * However, it does not ensure that there is no file with such a name. - * - * Current implementation is `$unreadableDumpDir/$today/$store/$key.exception` - */ - @NotNull - public static File makeExceptionFileName(@NotNull String keyOfDump, @NotNull File unreadableDumpDir, @NotNull String storeName) { - return unreadableDumpDir.toPath() - .resolve(DateTimeFormatter.BASIC_ISO_DATE.format(LocalDateTime.now())) - .resolve(storeName) - .resolve(sanitiseFileName(keyOfDump) + "." + EXCEPTION_FILE_EXTENSION) - .toFile(); - - } - @Override public void fillCache() { } @@ -487,15 +489,15 @@ public static class IterationStatistic { private int totalProcessed; private int failedKeys; private int failedValues; - + public void incrTotalProcessed() { totalProcessed++; } - + public void incrFailedKeys() { failedKeys++; } - + public void incrFailedValues() { failedValues++; } From 52fb71103fb814d23d2b96be36d3b8547f6f303b Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 4 Jul 2023 12:42:54 +0200 Subject: [PATCH 432/679] use ThreadLocal valueReader --- .../conquery/io/storage/xodus/stores/SerializingStore.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index 4a6b3425f2..0eaf8a7063 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -69,7 +69,7 @@ public class SerializingStore implements Store { /** * Deserializer for values */ - private final ObjectReader valueReader; + private final ThreadLocal valueReader; /** * Optional validator used for serialization. @@ -119,7 +119,7 @@ public , CLASS_V extends Class> SerializingSto valueWriter = objectMapper.writerFor(this.valueType); - valueReader = objectMapper.readerFor(this.valueType); + valueReader = ThreadLocal.withInitial(() -> objectMapper.readerFor(this.valueType)); keyWriter = objectMapper.writerFor(keyType); @@ -215,7 +215,7 @@ public VALUE get(KEY key) { * Deserialize value with {@code valueReader}. */ private VALUE readValue(ByteIterable value) { - return read(valueReader, value); + return read(valueReader.get(), value); } /** From 6860f8dac519291eedb00bb1202cd7303de40749 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 4 Jul 2023 14:39:21 +0200 Subject: [PATCH 433/679] use ArrayBlockingQueue to limit buffered values --- .../io/storage/xodus/stores/SerializingStore.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index 0eaf8a7063..ac169c1b5f 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -7,8 +7,9 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collection; +import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.function.Supplier; @@ -331,7 +332,10 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { final IterationStatistic result = new IterationStatistic(); final ArrayList unreadables = new ArrayList<>(); - final ExecutorService executorService = Executors.newWorkStealingPool(10); + final ExecutorService executorService = new ThreadPoolExecutor(0, Integer.MAX_VALUE, + 60L, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(100) + ); store.forEach((k, v) -> { executorService.submit(() -> { From 89f56bedf76e752099ca2042fa809dbc7cd19c97 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 4 Jul 2023 14:49:12 +0200 Subject: [PATCH 434/679] Iterate editor v2 minor issues --- frontend/src/js/editor-v2/EditorV2.tsx | 8 ++++++-- .../src/js/editor-v2/time-connection/TimeConnection.tsx | 2 +- frontend/src/localization/de.json | 8 ++++---- frontend/src/localization/en.json | 4 ++-- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/frontend/src/js/editor-v2/EditorV2.tsx b/frontend/src/js/editor-v2/EditorV2.tsx index 4eff775dd8..e817400983 100644 --- a/frontend/src/js/editor-v2/EditorV2.tsx +++ b/frontend/src/js/editor-v2/EditorV2.tsx @@ -15,7 +15,7 @@ import { useHotkeys } from "react-hotkeys-hook"; import { useTranslation } from "react-i18next"; import IconButton from "../button/IconButton"; -import { nodeIsConceptQueryNode } from "../model/node"; +import { nodeIsConceptQueryNode, useActiveState } from "../model/node"; import { EmptyQueryEditorDropzone } from "../standard-query-editor/EmptyQueryEditorDropzone"; import { DragItemConceptTreeNode, @@ -92,6 +92,8 @@ const useEditorState = () => { return findNodeById(tree, selectedNodeId); }, [tree, selectedNodeId]); + const { active: selectedNodeActive } = useActiveState(selectedNode?.data); + const onReset = useCallback(() => { setTree(undefined); }, []); @@ -114,6 +116,7 @@ const useEditorState = () => { updateTreeNode, onReset, selectedNode, + selectedNodeActive, setSelectedNodeId, }; }; @@ -142,6 +145,7 @@ export function EditorV2({ updateTreeNode, onReset, selectedNode, + selectedNodeActive, setSelectedNodeId, } = useEditorState(); @@ -311,7 +315,7 @@ export function EditorV2({ { e.stopPropagation(); onOpenQueryNodeEditor(); diff --git a/frontend/src/js/editor-v2/time-connection/TimeConnection.tsx b/frontend/src/js/editor-v2/time-connection/TimeConnection.tsx index c57748319c..4519726816 100644 --- a/frontend/src/js/editor-v2/time-connection/TimeConnection.tsx +++ b/frontend/src/js/editor-v2/time-connection/TimeConnection.tsx @@ -38,7 +38,7 @@ const Interval = styled("span")` `; const Operator = styled("span")` font-weight: bold; - color: ${({ theme }) => theme.col.palette[2]}; + color: ${({ theme }) => theme.col.palette.at(-2)}; `; export const TimeConnection = memo( diff --git a/frontend/src/localization/de.json b/frontend/src/localization/de.json index 4d429dc251..f8c7415572 100644 --- a/frontend/src/localization/de.json +++ b/frontend/src/localization/de.json @@ -67,7 +67,7 @@ "hasSecondaryId": "Analyse-Ebene aktiv", "removeNode": "Knoten entfernen", "removeColumn": "Spalte entfernen", - "clear": "Editor-Oberfläche zurücksetzen", + "clear": "Editor zurücksetzen", "clearConfirm": "Jetzt zurücksetzen", "hasDefaultSettings": "Standardeinstellungen", "hasNonDefaultSettings": "Eigene Einstellungen" @@ -318,8 +318,8 @@ "copyFrom": "Kopieren von ...", "copying": "Kopieren" }, - "clear": "Formular leeren", - "clearConfirm": "Formular jetzt leeren" + "clear": "Formular zurücksetzen", + "clearConfirm": "Jetzt zurücksetzen" } }, "uploadQueryResultsModal": { @@ -524,7 +524,7 @@ "time": "ZEIT", "and": "UND", "or": "ODER", - "clear": "Editor vollständig zurücksetzen", + "clear": "Editor zurücksetzen", "clearConfirm": "Jetzt zurücksetzen", "flip": "Drehen", "dates": "Datum", diff --git a/frontend/src/localization/en.json b/frontend/src/localization/en.json index 69aed6ea75..99e69b3a9c 100644 --- a/frontend/src/localization/en.json +++ b/frontend/src/localization/en.json @@ -66,7 +66,7 @@ "hasSecondaryId": "Analysis layer active", "removeNode": "Remove node", "removeColumn": "Remove column", - "clear": "Clear the Editor", + "clear": "Clear settings", "clearConfirm": "Clear now", "hasDefaultSettings": "Uses default settings", "hasNonDefaultSettings": "Has changed settings" @@ -524,7 +524,7 @@ "time": "TIME", "and": "AND", "or": "OR", - "clear": "Reset editor completely", + "clear": "Reset editor", "clearConfirm": "Reset now", "flip": "Flip", "dates": "Dates", From 4ef65234d613168eb99a692fcf35e047e9978ab1 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 4 Jul 2023 14:51:37 +0200 Subject: [PATCH 435/679] set an upper bound to threadcount --- .../conquery/io/storage/xodus/stores/SerializingStore.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index ac169c1b5f..4032c78c06 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -332,8 +332,8 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { final IterationStatistic result = new IterationStatistic(); final ArrayList unreadables = new ArrayList<>(); - final ExecutorService executorService = new ThreadPoolExecutor(0, Integer.MAX_VALUE, - 60L, TimeUnit.SECONDS, + final ExecutorService executorService = new ThreadPoolExecutor(5, 20, + 10L, TimeUnit.SECONDS, new ArrayBlockingQueue<>(100) ); From 300c585897fafb8b5f9bb5af90231f8681d3b974 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 4 Jul 2023 15:21:37 +0200 Subject: [PATCH 436/679] use SynchronousQueue --- .../io/storage/xodus/stores/SerializingStore.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index 4032c78c06..472a3fab88 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -7,8 +7,7 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collection; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.function.Function; @@ -332,9 +331,9 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { final IterationStatistic result = new IterationStatistic(); final ArrayList unreadables = new ArrayList<>(); - final ExecutorService executorService = new ThreadPoolExecutor(5, 20, - 10L, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(100) + final ThreadPoolExecutor executorService = new ThreadPoolExecutor(5, 20, + 10L, TimeUnit.SECONDS, + new SynchronousQueue<>() ); store.forEach((k, v) -> { From ca921dd298277a25da429c722a11fc0e26d9b466 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 4 Jul 2023 15:23:16 +0200 Subject: [PATCH 437/679] Reset editor v2 on dataset switch --- frontend/src/js/editor-v2/EditorV2.tsx | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/frontend/src/js/editor-v2/EditorV2.tsx b/frontend/src/js/editor-v2/EditorV2.tsx index e817400983..bf1f574a13 100644 --- a/frontend/src/js/editor-v2/EditorV2.tsx +++ b/frontend/src/js/editor-v2/EditorV2.tsx @@ -10,11 +10,12 @@ import { faTrash, } from "@fortawesome/free-solid-svg-icons"; import { createId } from "@paralleldrive/cuid2"; -import { useCallback, useMemo, useState } from "react"; +import { useCallback, useEffect, useMemo, useState } from "react"; import { useHotkeys } from "react-hotkeys-hook"; import { useTranslation } from "react-i18next"; import IconButton from "../button/IconButton"; +import { useDatasetId } from "../dataset/selectors"; import { nodeIsConceptQueryNode, useActiveState } from "../model/node"; import { EmptyQueryEditorDropzone } from "../standard-query-editor/EmptyQueryEditorDropzone"; import { @@ -121,6 +122,13 @@ const useEditorState = () => { }; }; +const useResetOnDatasetChange = (onReset: () => void) => { + const datasetId = useDatasetId(); + useEffect(() => { + onReset(); + }, [datasetId, onReset]); +}; + export function EditorV2({ featureDates, featureNegate, @@ -149,6 +157,8 @@ export function EditorV2({ setSelectedNodeId, } = useEditorState(); + useResetOnDatasetChange(onReset); + const onFlip = useCallback(() => { if (!selectedNode || !selectedNode.children) return; From 2fa9761d9d3249329f683963cf1646ea9b8872a8 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 4 Jul 2023 15:45:28 +0200 Subject: [PATCH 438/679] block provider when at capacity --- .../conquery/io/storage/xodus/stores/SerializingStore.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index 472a3fab88..5140084be4 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -7,7 +7,8 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collection; -import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.Executors; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.function.Function; @@ -333,7 +334,9 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { final ThreadPoolExecutor executorService = new ThreadPoolExecutor(5, 20, 10L, TimeUnit.SECONDS, - new SynchronousQueue<>() + new ArrayBlockingQueue<>(100), + Executors.defaultThreadFactory(), + new ThreadPoolExecutor.CallerRunsPolicy() ); store.forEach((k, v) -> { From 4cd202a2e71fd857690d96c0b62a07ce461db930 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Tue, 4 Jul 2023 16:03:37 +0200 Subject: [PATCH 439/679] Change dragging text to moving from copying. This is the new behaviour --- .../js/external-forms/form-concept-group/FormConceptGroup.tsx | 2 +- frontend/src/localization/de.json | 3 ++- frontend/src/localization/en.json | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 885801c081..832c8a26f9 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -202,7 +202,7 @@ const FormConceptGroup = (props: Props) => { } dropzoneChildren={({ isOver, item }) => isOver && isMovedObject(item) - ? t("externalForms.common.concept.copying") + ? t("externalForms.common.concept.moving") : props.attributeDropzoneText } dropBetween={(i: number) => { diff --git a/frontend/src/localization/de.json b/frontend/src/localization/de.json index 1b5843d915..3a62bb3054 100644 --- a/frontend/src/localization/de.json +++ b/frontend/src/localization/de.json @@ -311,7 +311,8 @@ "concept": { "expand": "Untergeordnete Attribute einbeziehen / wieder ausschließen.", "copyFrom": "Kopieren von ...", - "copying": "Kopieren" + "copying": "Kopieren", + "moving": "Verschieben" }, "clear": "Formular leeren", "clearConfirm": "Formular jetzt leeren" diff --git a/frontend/src/localization/en.json b/frontend/src/localization/en.json index 52d4c1f0ef..f158fb3b02 100644 --- a/frontend/src/localization/en.json +++ b/frontend/src/localization/en.json @@ -312,7 +312,7 @@ "concept": { "expand": "Include / exclude sub-attributes", "copyFrom": "Copy from ...", - "copying": "Copy" + "moving": "Move" }, "clear": "Clear form", "clearConfirm": "Clear form now" From 64e0aa6ce4af07e5318f87c6f9888dbea891041b Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 4 Jul 2023 16:09:26 +0200 Subject: [PATCH 440/679] tune down size of queue --- .../conquery/io/storage/xodus/stores/SerializingStore.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index 5140084be4..9f647215f0 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -332,9 +332,9 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { final IterationStatistic result = new IterationStatistic(); final ArrayList unreadables = new ArrayList<>(); - final ThreadPoolExecutor executorService = new ThreadPoolExecutor(5, 20, + final ThreadPoolExecutor executorService = new ThreadPoolExecutor(5, 10, 10L, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(100), + new ArrayBlockingQueue<>(10), Executors.defaultThreadFactory(), new ThreadPoolExecutor.CallerRunsPolicy() ); From 54fcc2ff6c8e934af3af4bc26f33488c42fc265c Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 4 Jul 2023 16:18:20 +0200 Subject: [PATCH 441/679] Remove query dates validation --- frontend/src/js/model/query.ts | 13 ------------ .../StandardQueryRunner.tsx | 21 ++----------------- 2 files changed, 2 insertions(+), 32 deletions(-) diff --git a/frontend/src/js/model/query.ts b/frontend/src/js/model/query.ts index 2370abf0e4..a94bdefc18 100644 --- a/frontend/src/js/model/query.ts +++ b/frontend/src/js/model/query.ts @@ -2,7 +2,6 @@ import { exists } from "../common/helpers/exists"; import type { StandardQueryStateT } from "../standard-query-editor/queryReducer"; import type { PreviousQueryQueryNodeType, - QueryGroupType, StandardQueryNodeT, } from "../standard-query-editor/types"; import { TIMEBASED_OPERATOR_TYPES } from "../timebased-query-editor/reducer"; @@ -43,15 +42,3 @@ export function isQueryExpandable(node: StandardQueryNodeT) { export function validateQueryLength(query: StandardQueryStateT) { return query.length > 0; } - -function elementHasValidDates(element: StandardQueryNodeT) { - return !element.excludeTimestamps; -} - -function groupHasValidDates(group: QueryGroupType) { - return !group.exclude && group.elements.some(elementHasValidDates); -} - -export function validateQueryDates(query: StandardQueryStateT) { - return !query || query.length === 0 || query.some(groupHasValidDates); -} diff --git a/frontend/src/js/standard-query-editor/StandardQueryRunner.tsx b/frontend/src/js/standard-query-editor/StandardQueryRunner.tsx index 8ebc3a42bb..9a2c0ea5e3 100644 --- a/frontend/src/js/standard-query-editor/StandardQueryRunner.tsx +++ b/frontend/src/js/standard-query-editor/StandardQueryRunner.tsx @@ -1,9 +1,8 @@ -import { useTranslation } from "react-i18next"; import { useSelector } from "react-redux"; import type { DatasetT } from "../api/types"; import type { StateT } from "../app/reducers"; -import { validateQueryLength, validateQueryDates } from "../model/query"; +import { validateQueryLength } from "../model/query"; import QueryRunner from "../query-runner/QueryRunner"; import { useStartQuery, useStopQuery } from "../query-runner/actions"; import type { QueryRunnerStateT } from "../query-runner/reducer"; @@ -18,18 +17,6 @@ function validateDataset(datasetId: DatasetT["id"] | null) { return datasetId !== null; } -function useButtonTooltip(hasQueryValidDates: boolean) { - const { t } = useTranslation(); - - if (!hasQueryValidDates) { - return t("queryRunner.errorDates"); - } - - // Potentially add further validation and more detailed messages - - return undefined; -} - const StandardQueryRunner = () => { const datasetId = useSelector( (state) => state.datasets.selectedDatasetId, @@ -47,12 +34,9 @@ const StandardQueryRunner = () => { const queryId = queryRunner.runningQuery; const isDatasetValid = validateDataset(datasetId); - const hasQueryValidDates = validateQueryDates(query); - const isQueryValid = validateQueryLength(query) && hasQueryValidDates; + const isQueryValid = validateQueryLength(query); const queryStartStopReady = validateQueryStartStop(queryRunner); - const buttonTooltip = useButtonTooltip(hasQueryValidDates); - const startStandardQuery = useStartQuery("standard"); const stopStandardQuery = useStopQuery("standard"); @@ -72,7 +56,6 @@ const StandardQueryRunner = () => { return ( Date: Tue, 4 Jul 2023 16:21:47 +0200 Subject: [PATCH 442/679] Remove translations --- frontend/src/localization/de.json | 1 - frontend/src/localization/en.json | 1 - 2 files changed, 2 deletions(-) diff --git a/frontend/src/localization/de.json b/frontend/src/localization/de.json index f8c7415572..1d7e781dcf 100644 --- a/frontend/src/localization/de.json +++ b/frontend/src/localization/de.json @@ -38,7 +38,6 @@ "showingMismatches": "Zeige vollständig" }, "queryRunner": { - "errorDates": "Ungültig: Alle Datumsbereiche sind von Zeitberechnung ausgeschlossen", "start": "Anfrage starten", "stop": "Anfrage stoppen", "stopSuccess": "Anfrage gestoppt.", diff --git a/frontend/src/localization/en.json b/frontend/src/localization/en.json index 99e69b3a9c..a8830b0292 100644 --- a/frontend/src/localization/en.json +++ b/frontend/src/localization/en.json @@ -37,7 +37,6 @@ "showingMismatches": "Showing full trees" }, "queryRunner": { - "errorDates": "Invalid: all date ranges excluded from time calculation", "start": "Start Query", "stop": "Stop Query", "stopSuccess": "Query stopped", From 87aff4a1f83af14eebceb671aeccaa87da1f1b92 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Tue, 4 Jul 2023 16:25:18 +0200 Subject: [PATCH 443/679] Minor changes in readability --- .../form-components/DropzoneBetweenElements.tsx | 4 ++-- .../js/external-forms/form-concept-group/FormConceptGroup.tsx | 1 + frontend/src/localization/de.json | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index 8a6ff2945c..eb80639a8a 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -21,7 +21,7 @@ const Root = styled("div")<{ border-radius: ${({ theme }) => theme.borderRadius}; `; -const DropzoneContainer = styled("div")<{ +const Expander = styled("div")<{ height: number; }>` overflow: hidden; @@ -55,7 +55,7 @@ const BetweenElements = ({ return ( <> - {isOver && } + {isOver && } ); }; diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 832c8a26f9..ad8ba7277e 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -91,6 +91,7 @@ interface Props { const DropzoneListItem = styled("div")` margin-top: -30px; `; + const Row = styled("div")` display: flex; align-items: center; diff --git a/frontend/src/localization/de.json b/frontend/src/localization/de.json index 3a62bb3054..82e8c5d994 100644 --- a/frontend/src/localization/de.json +++ b/frontend/src/localization/de.json @@ -311,7 +311,6 @@ "concept": { "expand": "Untergeordnete Attribute einbeziehen / wieder ausschließen.", "copyFrom": "Kopieren von ...", - "copying": "Kopieren", "moving": "Verschieben" }, "clear": "Formular leeren", From 5bb55fca935e1d656d9a8578b6e63d8eaf0baf57 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 4 Jul 2023 16:27:37 +0200 Subject: [PATCH 444/679] use very minimal queue and threadPool specs --- .../conquery/io/storage/xodus/stores/SerializingStore.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index 9f647215f0..46c85356c8 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -332,9 +332,9 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { final IterationStatistic result = new IterationStatistic(); final ArrayList unreadables = new ArrayList<>(); - final ThreadPoolExecutor executorService = new ThreadPoolExecutor(5, 10, - 10L, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(10), + final ThreadPoolExecutor executorService = new ThreadPoolExecutor(5, 5, + 60L, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(5), Executors.defaultThreadFactory(), new ThreadPoolExecutor.CallerRunsPolicy() ); From 2cecaad93b595afb39352f2dce75af2c469e0a0c Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 4 Jul 2023 17:52:24 +0200 Subject: [PATCH 445/679] fixes synchronization of jobManagerStatus --- .../models/worker/ShardNodeInformation.java | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java b/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java index e369c9819b..279ddfce02 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java @@ -57,8 +57,10 @@ private String getLatenessMetricName() { * Calculate the time in Milliseconds since we last received a {@link JobManagerStatus} from the corresponding shard. */ private long getMillisSinceLastStatus() { - if(getJobManagerStatus().isEmpty()){ - return -1; + synchronized (jobManagerStatus) { + if (getJobManagerStatus().isEmpty()) { + return -1; + } } return lastStatusTime.until(LocalDateTime.now(), ChronoUnit.MILLIS); @@ -71,10 +73,6 @@ public void awaitClose() { SharedMetricRegistries.getDefault().remove(getLatenessMetricName()); } - public long calculatePressure() { - return jobManagerStatus.stream().mapToLong(status -> status.getJobs().size()).sum(); - } - public void addJobManagerStatus(JobManagerStatus incoming) { lastStatusTime = LocalDateTime.now(); @@ -82,23 +80,24 @@ public void addJobManagerStatus(JobManagerStatus incoming) { // replace with new status jobManagerStatus.remove(incoming); jobManagerStatus.add(incoming); - } - if (calculatePressure() < backpressure) { - synchronized (jobManagerSync) { + if (calculatePressure() < backpressure) { jobManagerSync.notifyAll(); } } + } - public void waitForFreeJobQueue() throws InterruptedException { - if (jobManagerStatus.isEmpty()) { - return; + public long calculatePressure() { + synchronized (jobManagerStatus) { + return jobManagerStatus.stream().mapToLong(status -> status.getJobs().size()).sum(); } + } + public void waitForFreeJobQueue() throws InterruptedException { if (calculatePressure() >= backpressure) { - log.trace("Have to wait for free JobQueue (size = {})", jobManagerStatus.size()); synchronized (jobManagerSync) { + log.trace("Have to wait for free JobQueue (size = {})", jobManagerStatus.size()); jobManagerSync.wait(); } } From 8ba544a4a7c61ffc7b75923f7180bf52055dc18a Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 5 Jul 2023 09:33:11 +0200 Subject: [PATCH 446/679] removes Entities from toString --- .../conquery/models/query/queryplan/specific/ExternalNode.java | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java index 9780a429fd..292e11d736 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ExternalNode.java @@ -38,7 +38,6 @@ public class ExternalNode extends QPNode { private final Map> extraAggregators; private CDateSet contained; - @ToString.Include public Set getEntities() { return includedEntities.keySet(); } From bc741a8390bec1b7bd6628fbfecf3ebe221026c7 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 6 Jul 2023 11:14:22 +0200 Subject: [PATCH 447/679] restructure backpressure calculation to avoid calculation in consumers --- .../models/worker/ShardNodeInformation.java | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java b/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java index 279ddfce02..fe15ed6524 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java +++ b/backend/src/main/java/com/bakdata/conquery/models/worker/ShardNodeInformation.java @@ -4,6 +4,7 @@ import java.time.temporal.ChronoUnit; import java.util.HashSet; import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; import com.bakdata.conquery.io.mina.MessageSender; import com.bakdata.conquery.io.mina.NetworkSession; @@ -35,7 +36,7 @@ public class ShardNodeInformation extends MessageSender.Simple jobManagerStatus = new HashSet<>(); - + private final AtomicBoolean full = new AtomicBoolean(false); private LocalDateTime lastStatusTime = LocalDateTime.now(); public ShardNodeInformation(NetworkSession session, int backpressure) { @@ -57,12 +58,6 @@ private String getLatenessMetricName() { * Calculate the time in Milliseconds since we last received a {@link JobManagerStatus} from the corresponding shard. */ private long getMillisSinceLastStatus() { - synchronized (jobManagerStatus) { - if (getJobManagerStatus().isEmpty()) { - return -1; - } - } - return lastStatusTime.until(LocalDateTime.now(), ChronoUnit.MILLIS); } @@ -81,25 +76,34 @@ public void addJobManagerStatus(JobManagerStatus incoming) { jobManagerStatus.remove(incoming); jobManagerStatus.add(incoming); - if (calculatePressure() < backpressure) { - jobManagerSync.notifyAll(); + + final long pressure = calculatePressure(); + final boolean isFull = pressure > backpressure; + + full.set(isFull); + + if (!isFull) { + synchronized (jobManagerSync) { + jobManagerSync.notifyAll(); + } } } + } - public long calculatePressure() { - synchronized (jobManagerStatus) { - return jobManagerStatus.stream().mapToLong(status -> status.getJobs().size()).sum(); - } + private long calculatePressure() { + return jobManagerStatus.stream().mapToLong(status -> status.getJobs().size()).sum(); } public void waitForFreeJobQueue() throws InterruptedException { - if (calculatePressure() >= backpressure) { - synchronized (jobManagerSync) { - log.trace("Have to wait for free JobQueue (size = {})", jobManagerStatus.size()); - jobManagerSync.wait(); - } + if (!full.get()) { + return; + } + + synchronized (jobManagerSync) { + log.trace("Have to wait for free JobQueue"); + jobManagerSync.wait(); } } } From 71dc9ea1e40e069dd5b59c4498626f69292690b3 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Fri, 7 Jul 2023 13:27:20 +0200 Subject: [PATCH 448/679] minor changes in codestyle and using less recomputation for css classes --- .../DropzoneBetweenElements.tsx | 20 +++++++------------ .../form-components/DropzoneList.tsx | 2 +- .../form-concept-group/FormConceptGroup.tsx | 2 -- 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index eb80639a8a..aed94e7734 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -9,32 +9,27 @@ interface Props { acceptedDropTypes: string[]; } -const Root = styled("div")<{ - height: number; -}>` +const RootHeightBase = 40; + +const Root = styled("div")` width: 100%; left: 0; top: -15px; - height: ${({ height }) => height + 40}px; right: 0; position: relative; border-radius: ${({ theme }) => theme.borderRadius}; `; -const Expander = styled("div")<{ - height: number; -}>` +const Expander = styled("div")` overflow: hidden; - margin-top: ${({ height }) => -height}px; display: block; - height: ${({ height }) => height}px; `; const BetweenElements = ({ acceptedDropTypes, onDrop, }: Props) => { - const [height, setHeight] = useState(40); + const [height, setHeight] = useState(0); const [{ isOver }, addZoneRef] = useDrop({ accept: acceptedDropTypes, @@ -43,7 +38,6 @@ const BetweenElements = ({ if (item.type === "CONCEPT_TREE_NODE") { return setHeight(item.dragContext.height); } - return setHeight(0); }, collect: (monitor) => ({ @@ -54,8 +48,8 @@ const BetweenElements = ({ return ( <> - - {isOver && } + + {isOver && } ); }; diff --git a/frontend/src/js/external-forms/form-components/DropzoneList.tsx b/frontend/src/js/external-forms/form-components/DropzoneList.tsx index cd1c91e2af..af55b0566d 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneList.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneList.tsx @@ -19,7 +19,7 @@ import DropzoneBetweenElements from "./DropzoneBetweenElements"; const ListItem = styled("div")` position: relative; - padding: 0px 5px 0px 5px; + padding: 0 5px; box-shadow: 0 0 3px 0 rgba(0, 0, 0, 0.1); background-color: white; border-radius: ${({ theme }) => theme.borderRadius}; diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index ad8ba7277e..60e49afd0b 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -282,7 +282,6 @@ const FormConceptGroup = (props: Props) => { ); }} items={props.value.map((row, i) => ( - <> {props.renderRowPrefix ? props.renderRowPrefix({ @@ -421,7 +420,6 @@ const FormConceptGroup = (props: Props) => { )} /> - ))} /> {isCopyModalOpen && ( From aa0b743c72e568cdde0a8c8ddd2b3b466ed02558 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Fri, 7 Jul 2023 13:40:18 +0200 Subject: [PATCH 449/679] add height Factor, format --- .../DropzoneBetweenElements.tsx | 10 +- .../form-concept-group/FormConceptGroup.tsx | 246 +++++++++--------- 2 files changed, 129 insertions(+), 127 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index aed94e7734..466d164e54 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -10,6 +10,7 @@ interface Props { } const RootHeightBase = 40; +const HeightFactor = 0.5; const Root = styled("div")` width: 100%; @@ -36,7 +37,7 @@ const BetweenElements = ({ drop: onDrop, hover(item) { if (item.type === "CONCEPT_TREE_NODE") { - return setHeight(item.dragContext.height); + return setHeight(item.dragContext.height * HeightFactor); } }, @@ -48,8 +49,11 @@ const BetweenElements = ({ return ( <> - - {isOver && } + + {isOver && } ); }; diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 60e49afd0b..a14c8ea8da 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -282,144 +282,142 @@ const FormConceptGroup = (props: Props) => { ); }} items={props.value.map((row, i) => ( - - {props.renderRowPrefix - ? props.renderRowPrefix({ - value: props.value, - onChange: props.onChange, - row, - i, - }) - : null} - {row.concepts.length > 1 && ( - - - {t("externalForms.common.connectedWith")}: - - { - props.onChange( - setValueProperties(props.value, i, { - connector: val, - }), + + {props.renderRowPrefix + ? props.renderRowPrefix({ + value: props.value, + onChange: props.onChange, + row, + i, + }) + : null} + {row.concepts.length > 1 && ( + + + {t("externalForms.common.connectedWith")}: + + { + props.onChange( + setValueProperties(props.value, i, { + connector: val, + }), + ); + }} + options={[ + { value: "OR", label: t("common.or") }, + { value: "AND", label: t("common.and") }, + ]} + /> + + )} + + props.onChange(addConcept(props.value, i, null)) + } + onRemoveClick={(j) => + props.onChange( + props.value && props.value[i].concepts.length === 1 + ? removeValue(props.value, i) + : removeConcept(props.value, i, j), + ) + } + items={row.concepts.map((concept, j) => + concept ? ( + + setEditedFormQueryNodePosition({ + valueIdx: i, + conceptIdx: j, + }) + } + deleteInForm={() => { + return props.onChange( + props.value[i].concepts.length === 1 + ? removeValue(props.value, i) + : removeConcept(props.value, i, j), ); }} - options={[ - { value: "OR", label: t("common.or") }, - { value: "AND", label: t("common.and") }, - ]} + expand={{ + onClick: () => + props.onChange( + onToggleIncludeSubnodes( + props.value, + i, + j, + !concept.includeSubnodes, + newValue, + ), + ), + expandable: + !props.disallowMultipleColumns && + hasConceptChildren(concept), + active: !!concept.includeSubnodes, + }} /> - - )} - - props.onChange(addConcept(props.value, i, null)) - } - onRemoveClick={(j) => - props.onChange( - props.value && props.value[i].concepts.length === 1 - ? removeValue(props.value, i) - : removeConcept(props.value, i, j), - ) - } - items={row.concepts.map((concept, j) => - concept ? ( - - setEditedFormQueryNodePosition({ + ) : ( + */ + acceptedDropTypes={DROP_TYPES} + onImportLines={(lines) => + onImportLines(lines, { valueIdx: i, conceptIdx: j }) + } + onDrop={(item: DragItemConceptTreeNode | DragItemFile) => { + if (item.type === "__NATIVE_FILE__") { + onDropFile(item.files[0], { valueIdx: i, conceptIdx: j, - }) - } - deleteInForm={() => { - return props.onChange( - props.value[i].concepts.length === 1 - ? removeValue(props.value, i) - : removeConcept(props.value, i, j), - ); - }} - expand={{ - onClick: () => - props.onChange( - onToggleIncludeSubnodes( - props.value, - i, - j, - !concept.includeSubnodes, - newValue, - ), - ), - expandable: - !props.disallowMultipleColumns && - hasConceptChildren(concept), - active: !!concept.includeSubnodes, - }} - /> - ) : ( - */ - acceptedDropTypes={DROP_TYPES} - onImportLines={(lines) => - onImportLines(lines, { valueIdx: i, conceptIdx: j }) + }); + + return; } - onDrop={( - item: DragItemConceptTreeNode | DragItemFile, - ) => { - if (item.type === "__NATIVE_FILE__") { - onDropFile(item.files[0], { - valueIdx: i, - conceptIdx: j, - }); - - return; - } - - if (props.isValidConcept && !props.isValidConcept(item)) - return null; - - if (isMovedObject(item)) { - return props.onChange( - setConcept( - getValues(props.fieldName), - i, - j, - copyConcept(item), - ), - ); - } + if (props.isValidConcept && !props.isValidConcept(item)) + return null; + + if (isMovedObject(item)) { return props.onChange( setConcept( - props.value, + getValues(props.fieldName), i, j, - initializeConcept(item, defaults, tableConfig), + copyConcept(item), ), ); - }} - > - {({ isOver, item }) => - isOver && isMovedObject(item) - ? t("externalForms.common.concept.copying") - : props.conceptDropzoneText } - - ), - )} - /> - + + return props.onChange( + setConcept( + props.value, + i, + j, + initializeConcept(item, defaults, tableConfig), + ), + ); + }} + > + {({ isOver, item }) => + isOver && isMovedObject(item) + ? t("externalForms.common.concept.copying") + : props.conceptDropzoneText + } + + ), + )} + /> + ))} /> {isCopyModalOpen && ( From 54ccc7efc63d5e85855b020ca3aa22c43391b9f2 Mon Sep 17 00:00:00 2001 From: Jonas Arnhold Date: Tue, 11 Jul 2023 11:33:28 +0200 Subject: [PATCH 450/679] Add SQL OR conversion (#3126) --- .../context/selects/MergedSelects.java | 16 +- .../context/step/LogicalOperation.java | 6 + .../conversion/context/step/StepJoiner.java | 89 ++++++++ .../conversion/cqelement/CQAndConverter.java | 75 +------ .../conversion/cqelement/CQOrConverter.java | 18 +- .../ConceptPreprocessingService.java | 10 +- .../dialect/SqlFunctionProvider.java | 26 +++ .../sql/or/different_concept/content_1.csv | 13 ++ .../sql/or/different_concept/content_2.csv | 9 + .../sql/or/different_concept/expected.csv | 4 + .../sql/or/different_concept/or.spec.json | 207 ++++++++++++++++++ .../tests/sql/or/same_concept/content_1.csv | 13 ++ .../tests/sql/or/same_concept/expected.csv | 3 + .../tests/sql/or/same_concept/or.spec.json | 93 ++++++++ 14 files changed, 496 insertions(+), 86 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/LogicalOperation.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/StepJoiner.java create mode 100644 backend/src/test/resources/tests/sql/or/different_concept/content_1.csv create mode 100644 backend/src/test/resources/tests/sql/or/different_concept/content_2.csv create mode 100644 backend/src/test/resources/tests/sql/or/different_concept/expected.csv create mode 100644 backend/src/test/resources/tests/sql/or/different_concept/or.spec.json create mode 100644 backend/src/test/resources/tests/sql/or/same_concept/content_1.csv create mode 100644 backend/src/test/resources/tests/sql/or/same_concept/expected.csv create mode 100644 backend/src/test/resources/tests/sql/or/same_concept/or.spec.json diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/MergedSelects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/MergedSelects.java index f164b4c292..1ce36e8ae8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/MergedSelects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/MergedSelects.java @@ -9,6 +9,7 @@ import lombok.AllArgsConstructor; import lombok.Value; import org.jooq.Field; +import org.jooq.impl.DSL; /** * {@link MergedSelects} represent the combination of multiple {@link Selects}. @@ -19,6 +20,8 @@ @AllArgsConstructor(access = AccessLevel.PRIVATE) public class MergedSelects implements Selects { + String PRIMARY_COLUMN_ALIAS = "primary_column"; + Field primaryColumn; Optional> validityDate; @@ -31,16 +34,17 @@ public class MergedSelects implements Selects { List> mergedSelects; public MergedSelects(List querySteps) { - this.primaryColumn = this.extractPrimaryColumnSelect(querySteps); + this.primaryColumn = this.coalescePrimaryColumns(querySteps); this.validityDate = this.extractValidityDate(querySteps); this.mergedSelects = this.mergeSelects(querySteps); } - private Field extractPrimaryColumnSelect(List querySteps) { - // as we join all QuerySteps / CTEs onto the same primary column, - // it's sufficient to obtain it from the first one in the list - QueryStep firstQueryStep = querySteps.iterator().next(); - return this.mapFieldToQualifier(firstQueryStep.getCteName(), firstQueryStep.getSelects().getPrimaryColumn()); + private Field coalescePrimaryColumns(List querySteps) { + List> primaryColumns = querySteps.stream() + .map(queryStep -> this.mapFieldToQualifier(queryStep.getCteName(), queryStep.getSelects().getPrimaryColumn())) + .toList(); + return DSL.coalesce((Object) primaryColumns.get(0), primaryColumns.subList(1, primaryColumns.size()).toArray()) + .as(PRIMARY_COLUMN_ALIAS); } private Optional> extractValidityDate(List querySteps) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/LogicalOperation.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/LogicalOperation.java new file mode 100644 index 0000000000..9f0351c854 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/LogicalOperation.java @@ -0,0 +1,6 @@ +package com.bakdata.conquery.sql.conversion.context.step; + +public enum LogicalOperation { + AND, + OR +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/StepJoiner.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/StepJoiner.java new file mode 100644 index 0000000000..84b103f955 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/StepJoiner.java @@ -0,0 +1,89 @@ +package com.bakdata.conquery.sql.conversion.context.step; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import com.bakdata.conquery.apiv1.query.CQElement; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.context.selects.MergedSelects; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.TableLike; +import org.jooq.TableOnConditionStep; +import org.jooq.impl.DSL; + +public class StepJoiner { + + public static ConversionContext joinChildren(Iterable children, ConversionContext context, LogicalOperation logicalOperation) { + + ConversionContext childrenContext = context; + for (CQElement childNode : children) { + childrenContext = context.getNodeConverterService().convert(childNode, childrenContext); + } + + List queriesToJoin = childrenContext.getQuerySteps(); + QueryStep andQueryStep = QueryStep.builder() + .cteName(constructJoinedQueryStepLabel(queriesToJoin, logicalOperation)) + .selects(new MergedSelects(queriesToJoin)) + .fromTable(constructJoinedTable(queriesToJoin, logicalOperation, context)) + .conditions(Collections.emptyList()) + .predecessors(queriesToJoin) + .build(); + + return context.withQuerySteps(List.of(andQueryStep)); + } + + private static String constructJoinedQueryStepLabel(List queriesToJoin, LogicalOperation logicalOperation) { + + String labelConnector = switch (logicalOperation) { + case AND -> "_AND_"; + case OR -> "_OR_"; + }; + + return queriesToJoin.stream() + .map(QueryStep::getCteName) + .collect(Collectors.joining(labelConnector)); + } + + private static TableLike constructJoinedTable(List queriesToJoin, LogicalOperation logicalOperation, ConversionContext context) { + + Table joinedQuery = getIntitialJoinTable(queriesToJoin); + + SqlFunctionProvider functionProvider = context.getSqlDialect().getFunction(); + JoinType joinType = switch (logicalOperation) { + case AND -> functionProvider::innerJoin; + case OR -> functionProvider::fullOuterJoin; + }; + + for (int i = 0; i < queriesToJoin.size() - 1; i++) { + + QueryStep leftPartQS = queriesToJoin.get(i); + QueryStep rightPartQS = queriesToJoin.get(i + 1); + + Field leftPartPrimaryColumn = leftPartQS.getQualifiedSelects().getPrimaryColumn(); + Field rightPartPrimaryColumn = rightPartQS.getQualifiedSelects().getPrimaryColumn(); + + joinedQuery = joinType.join(joinedQuery, rightPartQS, leftPartPrimaryColumn, rightPartPrimaryColumn); + } + + return joinedQuery; + } + + private static Table getIntitialJoinTable(List queriesToJoin) { + return DSL.table(DSL.name(queriesToJoin.get(0).getCteName())); + } + + @FunctionalInterface + private interface JoinType { + TableOnConditionStep join( + Table leftPartQueryBase, + QueryStep rightPartQS, + Field leftPartPrimaryColumn, + Field rightPartPrimaryColumn + ); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java index 98228286e4..31404da21d 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQAndConverter.java @@ -1,81 +1,24 @@ package com.bakdata.conquery.sql.conversion.cqelement; -import java.util.Collections; -import java.util.List; -import java.util.stream.Collectors; - -import com.bakdata.conquery.apiv1.query.CQElement; import com.bakdata.conquery.apiv1.query.concept.specific.CQAnd; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.context.ConversionContext; -import com.bakdata.conquery.sql.conversion.context.selects.MergedSelects; -import com.bakdata.conquery.sql.conversion.context.step.QueryStep; -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Table; -import org.jooq.TableLike; -import org.jooq.impl.DSL; +import com.bakdata.conquery.sql.conversion.context.step.LogicalOperation; +import com.bakdata.conquery.sql.conversion.context.step.StepJoiner; public class CQAndConverter implements NodeConverter { @Override - public ConversionContext convert(CQAnd node, ConversionContext context) { - // if the AND node has a single child, the AND node is a noop - // otherwise, the converted children need to be logically combined before we obtain the final query - if (node.getChildren().size() == 1) { - return context.getNodeConverterService().convert(node.getChildren().get(0), context); - } - - ConversionContext childrenContext = context; - for (CQElement child : node.getChildren()) { - childrenContext = context.getNodeConverterService().convert(child, childrenContext); - } - - List queriesToJoin = childrenContext.getQuerySteps(); - QueryStep andQueryStep = QueryStep.builder() - .cteName(this.constructAndQueryStepLabel(queriesToJoin)) - .selects(new MergedSelects(queriesToJoin)) - .fromTable(this.constructJoinedTable(queriesToJoin)) - .conditions(Collections.emptyList()) - .predecessors(queriesToJoin) - .build(); - - return context.withQueryStep(andQueryStep); - } - - private String constructAndQueryStepLabel(List queriesToJoin) { - return queriesToJoin.stream() - .map(QueryStep::getCteName) - .collect(Collectors.joining("_AND_")); + public Class getConversionClass() { + return CQAnd.class; } - private TableLike constructJoinedTable(List queriesToJoin) { - - Table joinedQuery = this.getIntitialJoinTable(queriesToJoin); - - for (int i = 0; i < queriesToJoin.size() - 1; i++) { - - QueryStep leftPartQS = queriesToJoin.get(i); - QueryStep rightPartQS = queriesToJoin.get(i + 1); - - Field leftPartPrimaryColumn = leftPartQS.getQualifiedSelects().getPrimaryColumn(); - Field rightPartPrimaryColumn = rightPartQS.getQualifiedSelects().getPrimaryColumn(); - - joinedQuery = joinedQuery - .innerJoin(rightPartQS.getCteName()) - .on(leftPartPrimaryColumn.eq(rightPartPrimaryColumn)); + @Override + public ConversionContext convert(CQAnd andNode, ConversionContext context) { + if (andNode.getChildren().size() == 1) { + return context.getNodeConverterService().convert(andNode.getChildren().get(0), context); } - - return joinedQuery; - } - - private Table getIntitialJoinTable(List queriesToJoin) { - return DSL.table(DSL.name(queriesToJoin.get(0).getCteName())); + return StepJoiner.joinChildren(andNode.getChildren(), context, LogicalOperation.AND); } - - @Override - public Class getConversionClass() { - return CQAnd.class; - } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java index b0fd9f9aad..7df18dedaf 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQOrConverter.java @@ -3,20 +3,22 @@ import com.bakdata.conquery.apiv1.query.concept.specific.CQOr; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.context.step.LogicalOperation; +import com.bakdata.conquery.sql.conversion.context.step.StepJoiner; public class CQOrConverter implements NodeConverter { @Override - public ConversionContext convert(CQOr node, ConversionContext context) { - if (node.getChildren().size() > 1) { - throw new IllegalArgumentException("Multiple children are not yet supported"); - } - - return context.getNodeConverterService().convert(node.getChildren().get(0), context); + public Class getConversionClass() { + return CQOr.class; } @Override - public Class getConversionClass() { - return CQOr.class; + public ConversionContext convert(CQOr orNode, ConversionContext context) { + if (orNode.getChildren().size() == 1) { + return context.getNodeConverterService().convert(orNode.getChildren().get(0), context); + } + return StepJoiner.joinChildren(orNode.getChildren(), context, LogicalOperation.OR); } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConceptPreprocessingService.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConceptPreprocessingService.java index cefe56a169..2800d8b30a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConceptPreprocessingService.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConceptPreprocessingService.java @@ -1,4 +1,3 @@ - package com.bakdata.conquery.sql.conversion.cqelement; import java.util.Collections; @@ -14,14 +13,13 @@ import com.bakdata.conquery.sql.conversion.context.selects.ConceptSelects; import com.bakdata.conquery.sql.conversion.context.step.QueryStep; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; -import org.checkerframework.checker.units.qual.C; import org.jooq.Field; import org.jooq.impl.DSL; public class ConceptPreprocessingService { private static final String DATE_RESTRICTION_COLUMN_NAME = "date_restriction"; - private static final String VALIDITY_DATE_COLUMN_NAME = "validity_date"; + private static final String VALIDITY_DATE_COLUMN_NAME_SUFFIX = "_validity_date"; private final CQConcept concept; private final ConversionContext context; private final SqlFunctionProvider sqlFunctionProvider; @@ -46,7 +44,7 @@ public QueryStep buildPreprocessingQueryStepForTable(String conceptLabel, CQTabl selectsBuilder.primaryColumn(DSL.field(context.getConfig().getPrimaryColumn())); selectsBuilder.dateRestriction(this.getDateRestrictionSelect(table)); - selectsBuilder.validityDate(this.getValidityDateSelect(table)); + selectsBuilder.validityDate(this.getValidityDateSelect(table, conceptLabel)); List> conceptSelectFields = this.getColumnSelectReferences(table); List> conceptFilterFields = this.getColumnFilterReferences(table); @@ -83,12 +81,12 @@ private Optional> getDateRestrictionSelect(CQTable table) { return Optional.of(dateRestriction); } - private Optional> getValidityDateSelect(CQTable table) { + private Optional> getValidityDateSelect(CQTable table, String conceptLabel) { if (!this.validityDateIsRequired(table)) { return Optional.empty(); } Field validityDateRange = this.sqlFunctionProvider.daterange(table.findValidityDateColumn()) - .as(VALIDITY_DATE_COLUMN_NAME); + .as(conceptLabel + VALIDITY_DATE_COLUMN_NAME_SUFFIX); return Optional.of(validityDateRange); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index 67d41cefcd..cdfde5aa06 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -5,9 +5,13 @@ import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.events.MajorTypeId; +import com.bakdata.conquery.sql.conversion.context.step.QueryStep; import org.jooq.Condition; import org.jooq.DatePart; import org.jooq.Field; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.TableOnConditionStep; import org.jooq.impl.DSL; /** @@ -54,4 +58,26 @@ default Field first(String columnName) { return DSL.field(columnName); } + default TableOnConditionStep innerJoin( + Table leftPartQueryBase, + QueryStep rightPartQS, + Field leftPartPrimaryColumn, + Field rightPartPrimaryColumn + ) { + return leftPartQueryBase + .innerJoin(DSL.name(rightPartQS.getCteName())) + .on(leftPartPrimaryColumn.eq(rightPartPrimaryColumn)); + } + + default TableOnConditionStep fullOuterJoin( + Table leftPartQueryBase, + QueryStep rightPartQS, + Field leftPartPrimaryColumn, + Field rightPartPrimaryColumn + ) { + return leftPartQueryBase + .fullOuterJoin(DSL.name(rightPartQS.getCteName())) + .on(leftPartPrimaryColumn.eq(rightPartPrimaryColumn)); + } + } diff --git a/backend/src/test/resources/tests/sql/or/different_concept/content_1.csv b/backend/src/test/resources/tests/sql/or/different_concept/content_1.csv new file mode 100644 index 0000000000..1851eed139 --- /dev/null +++ b/backend/src/test/resources/tests/sql/or/different_concept/content_1.csv @@ -0,0 +1,13 @@ +pid,value,datum +1,1,"2014-06-30/2015-06-30" +2,1.01,"2014-06-30/2015-06-30" +1,1,"2015-02-03/2015-06-30" +1,0.5,"2014-06-30/2015-06-30" +3,0.5,"2014-04-30/2014-06-30" +4,1,"2014-06-30/2015-06-30" +5,0.5,"2014-04-30/2014-06-30" +5,1,"2014-06-30/2015-06-30" +6,1,"2014-04-30/2014-06-30" +7,1,"2014-02-05/2014-02-20" +8,1,"2014-04-30/2014-06-30" +7,-1,"2014-06-30/2015-06-30" diff --git a/backend/src/test/resources/tests/sql/or/different_concept/content_2.csv b/backend/src/test/resources/tests/sql/or/different_concept/content_2.csv new file mode 100644 index 0000000000..dc012de238 --- /dev/null +++ b/backend/src/test/resources/tests/sql/or/different_concept/content_2.csv @@ -0,0 +1,9 @@ +pid,datum,geschlecht,language +1,2012-01-01,"f","de" +2,2010-07-15,"m","fr" +3,2013-11-10,"f","en" +4,2012-11-11,"m","" +5,2007-11-11,"","" +6,2012-11-11,"","de" +7,2012-11-11,"mf","de" +8,2012-11-11,"fm","fr" diff --git a/backend/src/test/resources/tests/sql/or/different_concept/expected.csv b/backend/src/test/resources/tests/sql/or/different_concept/expected.csv new file mode 100644 index 0000000000..dea40ae400 --- /dev/null +++ b/backend/src/test/resources/tests/sql/or/different_concept/expected.csv @@ -0,0 +1,4 @@ +pid,validity_date_1,,value,geschlecht,language +7,"[2014-06-30,2015-06-30)",-1,mf, +8,,,,fr +2,,,,fr diff --git a/backend/src/test/resources/tests/sql/or/different_concept/or.spec.json b/backend/src/test/resources/tests/sql/or/different_concept/or.spec.json new file mode 100644 index 0000000000..4fd7f736e4 --- /dev/null +++ b/backend/src/test/resources/tests/sql/or/different_concept/or.spec.json @@ -0,0 +1,207 @@ +{ + "label": "Simple OR query for 3 different concepts", + "expectedCsv": "expected.csv", + "type": "SQL_TEST", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "OR", + "children": [ + { + "type": "CONCEPT", + "label": "vs", + "ids": [ + "number" + ], + "tables": [ + { + "id": "number.number_connector", + "filters": [ + { + "filter": "number.number_connector.value", + "type": "REAL_RANGE", + "value": { + "min": -1, + "max": 0 + } + } + ], + "selects": [ + "number.number_connector.value" + ] + } + ] + }, + { + "ids": [ + "geschlecht_select" + ], + "type": "CONCEPT", + "label": "Geschlecht SELECT", + "tables": [ + { + "id": "geschlecht_select.geschlecht_connector", + "filters": [ + { + "filter": "geschlecht_select.geschlecht_connector.geschlecht", + "type": "BIG_MULTI_SELECT", + "value": [ + "mf" + ] + } + ], + "selects": [ + "geschlecht_select.geschlecht_connector.geschlecht" + ] + } + ] + }, + { + "ids": [ + "language_select" + ], + "type": "CONCEPT", + "label": "Language SELECT", + "tables": [ + { + "id": "language_select.language_connector", + "filters": [ + { + "filter": "language_select.language_connector.language", + "type": "BIG_MULTI_SELECT", + "value": [ + "fr" + ] + } + ], + "selects": [ + "language_select.language_connector.language" + ] + } + ] + } + ] + } + }, + "concepts": [ + { + "label": "number", + "type": "TREE", + "connectors": [ + { + "label": "number_connector", + "table": "table1", + "validityDates": { + "label": "datum", + "column": "table1.datum" + }, + "filters": { + "label": "value", + "description": "xy", + "column": "table1.value", + "type": "NUMBER" + }, + "selects": { + "name": "value", + "column": "table1.value", + "type": "FIRST" + } + } + ] + }, + { + "label": "geschlecht_select", + "type": "TREE", + "connectors": [ + { + "label": "geschlecht_connector", + "table": "table2", + "validityDates": { + "label": "datum", + "column": "table2.datum" + }, + "filters": { + "label": "geschlecht", + "description": "Geschlecht zur gegebenen Datumseinschränkung", + "column": "table2.geschlecht", + "type": "SELECT" + }, + "selects": { + "name": "geschlecht", + "column": "table2.geschlecht", + "type": "FIRST" + } + } + ] + }, + { + "label": "language_select", + "type": "TREE", + "connectors": [ + { + "label": "language_connector", + "table": "table2", + "validityDates": { + "label": "datum", + "column": "table2.datum" + }, + "filters": { + "label": "language", + "description": "Sprache", + "column": "table2.language", + "type": "SELECT" + }, + "selects": { + "name": "language", + "column": "table2.language", + "type": "FIRST" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/or/different_concept/content_1.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "value", + "type": "REAL" + }, + { + "name": "datum", + "type": "DATE_RANGE" + } + ] + }, + { + "csv": "tests/sql/or/different_concept/content_2.csv", + "name": "table2", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + }, + { + "name": "geschlecht", + "type": "STRING" + }, + { + "name": "language", + "type": "STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/or/same_concept/content_1.csv b/backend/src/test/resources/tests/sql/or/same_concept/content_1.csv new file mode 100644 index 0000000000..5ceffe16ca --- /dev/null +++ b/backend/src/test/resources/tests/sql/or/same_concept/content_1.csv @@ -0,0 +1,13 @@ +pid,value +1,1 +2,1.01 +1,1 +1,0.5 +3,0.5 +4,1 +5,0.5 +5,1 +6,1 +7,1 +8,1 +7,-1 diff --git a/backend/src/test/resources/tests/sql/or/same_concept/expected.csv b/backend/src/test/resources/tests/sql/or/same_concept/expected.csv new file mode 100644 index 0000000000..4869420d12 --- /dev/null +++ b/backend/src/test/resources/tests/sql/or/same_concept/expected.csv @@ -0,0 +1,3 @@ +pid +7 +2 diff --git a/backend/src/test/resources/tests/sql/or/same_concept/or.spec.json b/backend/src/test/resources/tests/sql/or/same_concept/or.spec.json new file mode 100644 index 0000000000..8a8dec1b9e --- /dev/null +++ b/backend/src/test/resources/tests/sql/or/same_concept/or.spec.json @@ -0,0 +1,93 @@ +{ + "label": "Simple OR query for same concept", + "expectedCsv": "expected.csv", + "type": "SQL_TEST", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "OR", + "children": [ + { + "type": "CONCEPT", + "label": "vs", + "ids": [ + "number" + ], + "tables": [ + { + "id": "number.number_connector", + "filters": [ + { + "filter": "number.number_connector.value", + "type": "REAL_RANGE", + "value": { + "min": -1, + "max": 0 + } + } + ] + } + ] + }, + { + "type": "CONCEPT", + "label": "vs", + "ids": [ + "number" + ], + "tables": [ + { + "id": "number.number_connector", + "filters": [ + { + "filter": "number.number_connector.value", + "type": "REAL_RANGE", + "value": { + "min": 1.0001, + "max": 1.5 + } + } + ] + } + ] + } + ] + } + }, + "concepts": [ + { + "label": "number", + "type": "TREE", + "connectors": [ + { + "label": "number_connector", + "table": "table1", + "filters": { + "label": "value", + "description": "xy", + "column": "table1.value", + "type": "NUMBER" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/or/same_concept/content_1.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "value", + "type": "REAL" + } + ] + } + ] + } +} From 09f2b862e45ced8524592136f19f9e15bad2ba05 Mon Sep 17 00:00:00 2001 From: Jonas Arnhold Date: Tue, 11 Jul 2023 17:53:28 +0200 Subject: [PATCH 451/679] Add SQL NOT conversion (#3128) --- .../conversion/context/ConversionContext.java | 5 +- .../cqelement/CQNegationConverter.java | 29 ++++++++ .../sql/conversion/dialect/SqlDialect.java | 16 ++--- .../filter/FilterConverterService.java | 16 ++++- .../test/resources/tests/sql/not/content.csv | 9 +++ .../test/resources/tests/sql/not/expected.csv | 5 ++ .../resources/tests/sql/not/not.spec.json | 72 +++++++++++++++++++ 7 files changed, 139 insertions(+), 13 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java create mode 100644 backend/src/test/resources/tests/sql/not/content.csv create mode 100644 backend/src/test/resources/tests/sql/not/expected.csv create mode 100644 backend/src/test/resources/tests/sql/not/not.spec.json diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/ConversionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/ConversionContext.java index dc838ecab5..b07383dacf 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/ConversionContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/ConversionContext.java @@ -1,7 +1,5 @@ package com.bakdata.conquery.sql.conversion.context; -import java.util.List; - import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.sql.conversion.NodeConverterService; @@ -14,6 +12,8 @@ import org.jooq.Record; import org.jooq.Select; +import java.util.List; + @Value @With @Builder(toBuilder = true) @@ -25,6 +25,7 @@ public class ConversionContext { @Singular List querySteps; Select finalQuery; + boolean negation; CDateRange dateRestrictionRange; int queryStepCounter; diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java new file mode 100644 index 0000000000..ce1db05bd7 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQNegationConverter.java @@ -0,0 +1,29 @@ +package com.bakdata.conquery.sql.conversion.cqelement; + +import com.bakdata.conquery.apiv1.query.CQElement; +import com.bakdata.conquery.apiv1.query.concept.specific.CQNegation; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; + +public class CQNegationConverter implements NodeConverter { + + @Override + public Class getConversionClass() { + return CQNegation.class; + } + + @Override + public ConversionContext convert(CQNegation negationNode, ConversionContext context) { + return this.convertChildWithNegationActive(negationNode.getChild(), context); + } + + private ConversionContext convertChildWithNegationActive(CQElement child, ConversionContext context) { + // TODO: handle negation properly after GroupSelect/GroupFilter has been implemented + // - anti-join vs. negating conditions + // - handle double negation + return context.getNodeConverterService() + .convert(child, context.withNegation(true)) + .withNegation(false); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java index ff703e2d54..8ee2a08256 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java @@ -1,20 +1,12 @@ package com.bakdata.conquery.sql.conversion.dialect; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; - import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.sql.conversion.Converter; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.context.step.QueryStepTransformer; -import com.bakdata.conquery.sql.conversion.cqelement.CQAndConverter; -import com.bakdata.conquery.sql.conversion.cqelement.CQConceptConverter; -import com.bakdata.conquery.sql.conversion.cqelement.CQDateRestrictionConverter; -import com.bakdata.conquery.sql.conversion.cqelement.CQOrConverter; +import com.bakdata.conquery.sql.conversion.cqelement.*; import com.bakdata.conquery.sql.conversion.filter.FilterConverter; import com.bakdata.conquery.sql.conversion.filter.FilterConverterService; import com.bakdata.conquery.sql.conversion.filter.MultiSelectConverter; @@ -27,6 +19,11 @@ import com.bakdata.conquery.sql.conversion.supplier.SystemDateNowSupplier; import org.jooq.DSLContext; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + public interface SqlDialect { SqlFunctionProvider getFunction(); @@ -44,6 +41,7 @@ default List> getDefaultNodeConverters() { new CQDateRestrictionConverter(), new CQAndConverter(), new CQOrConverter(), + new CQNegationConverter(), new CQConceptConverter(new FilterConverterService(getFilterConverters()), new SelectConverterService(getSelectConverters())), new ConceptQueryConverter(new QueryStepTransformer(getDSLContext())) ); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/FilterConverterService.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/FilterConverterService.java index 3c835fca5e..1b5a4253af 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/FilterConverterService.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/FilterConverterService.java @@ -1,14 +1,26 @@ package com.bakdata.conquery.sql.conversion.filter; -import java.util.List; - import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; import com.bakdata.conquery.sql.conversion.ConverterService; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; import org.jooq.Condition; +import org.jooq.impl.DSL; + +import java.util.List; public class FilterConverterService extends ConverterService, Condition> { public FilterConverterService(List> converters) { super(converters); } + + @Override + public Condition convert(FilterValue filterValue, ConversionContext context) { + Condition condition = super.convert(filterValue, context); + if (!context.isNegation()) { + return condition; + } + return DSL.not(condition); + } + } diff --git a/backend/src/test/resources/tests/sql/not/content.csv b/backend/src/test/resources/tests/sql/not/content.csv new file mode 100644 index 0000000000..8dab40f969 --- /dev/null +++ b/backend/src/test/resources/tests/sql/not/content.csv @@ -0,0 +1,9 @@ +pid,datum,geschlecht +1,2012-01-01,"f" +2,2010-07-15,"m" +3,2013-11-10,"f" +4,2012-11-11,"m" +5,2007-11-11, +6,2012-11-11, +7,2012-11-11,"mf" +8,2012-11-11,"fm" diff --git a/backend/src/test/resources/tests/sql/not/expected.csv b/backend/src/test/resources/tests/sql/not/expected.csv new file mode 100644 index 0000000000..416640214f --- /dev/null +++ b/backend/src/test/resources/tests/sql/not/expected.csv @@ -0,0 +1,5 @@ +pid +2 +4 +7 +8 diff --git a/backend/src/test/resources/tests/sql/not/not.spec.json b/backend/src/test/resources/tests/sql/not/not.spec.json new file mode 100644 index 0000000000..bf1611ece6 --- /dev/null +++ b/backend/src/test/resources/tests/sql/not/not.spec.json @@ -0,0 +1,72 @@ +{ + "label": "Simple Negation Query", + "type": "SQL_TEST", + "expectedCsv": "expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "NEGATION", + "child": { + "ids": [ + "geschlecht_select" + ], + "type": "CONCEPT", + "label": "Geschlecht SELECT", + "tables": [ + { + "id": "geschlecht_select.geschlecht_connector", + "filters": [ + { + "filter": "geschlecht_select.geschlecht_connector.geschlecht", + "type": "BIG_MULTI_SELECT", + "value": [ + "f" + ] + } + ] + } + ] + } + } + }, + "concepts": [ + { + "label": "geschlecht_select", + "type": "TREE", + "connectors": [ + { + "label": "geschlecht_connector", + "table": "table1", + "filters": { + "label": "geschlecht", + "description": "Geschlecht zur gegebenen Datumseinschränkung", + "column": "table1.geschlecht", + "type": "SELECT" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/not/content.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + }, + { + "name": "geschlecht", + "type": "STRING" + } + ] + } + ] + } +} From 448700c3ae22f9d85a8509f18265bc245a926872 Mon Sep 17 00:00:00 2001 From: Marco Korinth Date: Wed, 12 Jul 2023 15:56:17 +0200 Subject: [PATCH 452/679] feat: admin-ui/jobs redesign and dynamic data loading --- .../conquery/resources/admin/ui/jobs.html.ftl | 223 ++++++++++++------ .../admin/ui/templates/accordion.html.ftl | 14 +- .../admin/ui/templates/breadcrumbs.html.ftl | 2 +- 3 files changed, 162 insertions(+), 77 deletions(-) diff --git a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl index 26e0847442..a31b245c3d 100644 --- a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl +++ b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/jobs.html.ftl @@ -1,71 +1,156 @@ <#import "templates/template.html.ftl" as layout> +<#import "templates/breadcrumbs.html.ftl" as breadcrumbs> +<#import "templates/accordion.html.ftl" as accordion> + <@layout.layout> - <#list c as node, status> -
-
-
-
-
- ${node} - - updated ${status.ageString} ago - ${status.jobs?size} - -
-
-

- <#list status.jobs as job> - - - - - - -
- ${job.label} - -
-
-
- 		- <#if !job.cancelled> - - <#else> -
Cancelled
- -
-
-
-
- - - - -
-
- -
- -
-
- \ No newline at end of file + + + <@breadcrumbs.breadcrumbs + labels=["Jobs"] + /> + +
+
+ + +
+
+ + <@accordion.accordionGroup id="nodesAccordionGroup" class="mt-3"> + + + +
+ <@accordion.accordion summary="" id="categoryTemplate"> + +
+
+ +
+ updated ago + +
+
+
+
+ +
No jobs in this node
+ +
+
+
+
+
+
+
+
+
diff --git a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/accordion.html.ftl b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/accordion.html.ftl index 21b0a695be..c6ba5b66ad 100644 --- a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/accordion.html.ftl +++ b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/accordion.html.ftl @@ -1,10 +1,10 @@ -<#macro accordionGroup class="" style=""> -
+<#macro accordionGroup id="" class="" style=""> +
<#nested />
-<#macro accordion summary infoText="" class="" style=""> -
+<#macro accordion summary infoText="" id="" class="" style=""> +
${summary}
-
${infoText}
+
${infoText}
-
+
<#nested />
@@ -28,4 +28,4 @@ }
- \ No newline at end of file + diff --git a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/breadcrumbs.html.ftl b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/breadcrumbs.html.ftl index 9e8afc8118..5086348b3d 100644 --- a/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/breadcrumbs.html.ftl +++ b/backend/src/main/resources/com/bakdata/conquery/resources/admin/ui/templates/breadcrumbs.html.ftl @@ -1,4 +1,4 @@ -<#macro breadcrumbs labels links class=""> +<#macro breadcrumbs labels links=[] class="">
)}
From 15fa5cd7dc9e8d1fc54dc7381bb8b49d5d91244d Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 14 Aug 2023 14:38:21 +0200 Subject: [PATCH 482/679] simplify code and prevent bleeding --- .../DropzoneBetweenElements.tsx | 18 ++-- .../form-concept-group/FormConceptNode.tsx | 83 ++++++++++--------- 2 files changed, 52 insertions(+), 49 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index 398efd8fe7..f6319279d2 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -11,7 +11,7 @@ interface Props { const RootHeightBase = 30; const LineHeight = 3; - +const MarginTopOffsetOver = 5; const Root = styled("div")` width: 100%; left: 0; @@ -23,13 +23,13 @@ const Root = styled("div")` const Line = styled("div")` overflow: hidden; display: block; - background-color: ${({ theme }) => theme.col.blueGrayLight}; + background-color: ${({ theme }) => theme.col.blueGrayDark}; margin: 1px 0; height: ${LineHeight}px; border-radius: 2px; `; -const BetweenElements = ({ +const DropzoneBetweenElements = ({ acceptedDropTypes, onDrop, lastElement, @@ -44,10 +44,9 @@ const BetweenElements = ({ }); const rootHeightMultiplier = lastElement ? 0.5 : 1; - const rootDefaultMarginTop = (lastElement ? -15 : -5) - LineHeight; - const rootOverMarginTop = lastElement ? -23 : -10; + const rootMarginTop = (lastElement ? -15 : -5) - LineHeight; const rootDefaultTop = lastElement ? -5 : -10; - const rootOverTop = (lastElement ? -2 : -15) - LineHeight; + const rootOverTop = (lastElement ? -5 : -15) - LineHeight; return ( <> @@ -57,9 +56,8 @@ const BetweenElements = ({ style={{ height: RootHeightBase * rootHeightMultiplier + - (isOver && !lastElement ? 0 : LineHeight) + - (lastElement ? LineHeight + 4 : 0), - marginTop: isOver ? rootOverMarginTop : rootDefaultMarginTop, + (isOver ? 0 : LineHeight), + marginTop: (isOver ? MarginTopOffsetOver : 0) + rootMarginTop, top: isOver ? rootOverTop : rootDefaultTop, }} > @@ -67,4 +65,4 @@ const BetweenElements = ({ ); }; -export default BetweenElements; +export default DropzoneBetweenElements; diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx index 55298d813a..087f49e7c6 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptNode.tsx @@ -33,7 +33,6 @@ const Root = styled("div")<{ display: grid; grid-template-columns: 1fr auto; font-size: ${({ theme }) => theme.font.sm}; - margin-top: 5px; `; const Label = styled("p")` @@ -69,6 +68,10 @@ const RootNode = styled("p")` word-break: break-word; `; +const TopMargin = styled("div")` + margin-top: 5px; +`; + interface PropsT { valueIdx: number; conceptIdx: number; @@ -131,46 +134,48 @@ const FormConceptNode: FC = ({ : undefined; return ( - canNodeBeDropped(conceptNode, item)} - highlightDroppable - > - { - ref.current = instance; - drag(instance); - }} - active={hasNonDefaultSettings || hasFilterValues} - onClick={onClick} + + canNodeBeDropped(conceptNode, item)} + highlightDroppable > -
- - <> - {rootNodeLabel && {rootNodeLabel}} - - {conceptNode && !!conceptNode.description && ( - {conceptNode.description} - )} - - -
- - {expand && expand.expandable && ( - - { - e.stopPropagation(); - expand.onClick(); - }} - /> + { + ref.current = instance; + drag(instance); + }} + active={hasNonDefaultSettings || hasFilterValues} + onClick={onClick} + > +
+ + <> + {rootNodeLabel && {rootNodeLabel}} + + {conceptNode && !!conceptNode.description && ( + {conceptNode.description} + )} + - )} - - - +
+ + {expand && expand.expandable && ( + + { + e.stopPropagation(); + expand.onClick(); + }} + /> + + )} + + + + ); }; From 0ac13f2ffe1c25b9f93ae5eadbc35e6c591ed63e Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Mon, 14 Aug 2023 15:27:37 +0200 Subject: [PATCH 483/679] Send WorkerMessage in ForwardToWorker gzipped --- .../network/specific/ForwardToWorker.java | 52 ++++++++++++------- 1 file changed, 33 insertions(+), 19 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/ForwardToWorker.java b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/ForwardToWorker.java index 3703763c2c..daba9b5fbc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/ForwardToWorker.java +++ b/backend/src/main/java/com/bakdata/conquery/models/messages/network/specific/ForwardToWorker.java @@ -1,6 +1,12 @@ package com.bakdata.conquery.models.messages.network.specific; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.OutputStream; import java.util.Objects; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.identifiable.ids.specific.WorkerId; @@ -14,7 +20,6 @@ import com.bakdata.conquery.util.io.ConqueryMDC; import com.bakdata.conquery.util.progressreporter.ProgressReporter; import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import lombok.AccessLevel; @@ -25,7 +30,8 @@ import lombok.ToString; /** - * Messages are sent serialized and only deserialized when they are being processed. This ensures that messages that were sent just shortly before to setup state later messages depend upon is correct. + * @implNote Messages are sent serialized and only deserialized when they are being processed. This ensures that messages that were sent just shortly before to setup state later messages depend upon is correct. + * @implNote Messages are additionally sent gzipped, to avoid hogging memory with long queues. */ @CPSType(id = "FORWARD_TO_WORKER", base = NetworkMessage.class) @RequiredArgsConstructor(access = AccessLevel.PROTECTED) @@ -33,45 +39,53 @@ @ToString(of = {"workerId", "text"}) public class ForwardToWorker extends MessageToShardNode implements SlowMessage { - @SneakyThrows(JsonProcessingException.class) + private final WorkerId workerId; + private final byte[] messageRaw; + // We cache these on the sender side. + @Getter(onMethod_ = @JsonIgnore(false)) + private final boolean slowMessage; + private final String text; + @JsonIgnore + @Setter + private ProgressReporter progressReporter; + public static ForwardToWorker create(WorkerId worker, WorkerMessage message, ObjectWriter writer) { return new ForwardToWorker( worker, - writer.writeValueAsBytes(message), + serializeMessage(message, writer), true, message.toString() ); } - private final WorkerId workerId; - private final byte[] messageRaw; + @SneakyThrows(IOException.class) + private static byte[] serializeMessage(WorkerMessage message, ObjectWriter writer) { + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (OutputStream outputStream = new GZIPOutputStream(baos)) { + writer.writeValue(outputStream, message); + } - // We cache these on the sender side. - @Getter(onMethod_ = @JsonIgnore(false)) - private final boolean slowMessage; - private final String text; + return baos.toByteArray(); + } - @JsonIgnore - @Setter - private ProgressReporter progressReporter; + private static WorkerMessage deserializeMessage(byte[] messageRaw, ObjectMapper mapper) throws java.io.IOException { + return mapper.readerFor(WorkerMessage.class).readValue(new GZIPInputStream(new ByteArrayInputStream(messageRaw))); + } @Override public void react(ShardNodeNetworkContext context) throws Exception { - Worker worker = Objects.requireNonNull(context.getWorkers().getWorker(workerId)); + final Worker worker = Objects.requireNonNull(context.getWorkers().getWorker(workerId)); ConqueryMDC.setLocation(worker.toString()); // Jobception: this is to ensure that no subsequent message is deserialized before one message is processed - worker.getJobManager().addSlowJob(new SimpleJob("Deserialize and process WorkerMessage", () -> { + worker.getJobManager().addSlowJob(new SimpleJob("Process %s".formatted(getText()), () -> { - WorkerMessage message = deserializeMessage(messageRaw, worker.getCommunicationMapper()); + final WorkerMessage message = deserializeMessage(messageRaw, worker.getCommunicationMapper()); message.setProgressReporter(progressReporter); message.react(worker); })); } - private static WorkerMessage deserializeMessage(byte[] messageRaw, ObjectMapper binaryMapper) throws java.io.IOException { - return binaryMapper.readerFor(WorkerMessage.class).readValue(messageRaw); - } } From 9ef3448394f4248a04cc42fb118a65f198f03b96 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Mon, 14 Aug 2023 16:14:31 +0200 Subject: [PATCH 484/679] gzip compresses SerializingStore --- .../xodus/stores/SerializingStore.java | 343 +++++++++--------- 1 file changed, 181 insertions(+), 162 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index 6cf3fcddbd..589120ba35 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -1,7 +1,11 @@ package com.bakdata.conquery.io.storage.xodus.stores; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.io.PrintStream; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; @@ -9,6 +13,8 @@ import java.util.Collection; import java.util.function.Function; import java.util.function.Supplier; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; import javax.validation.Validator; @@ -18,7 +24,6 @@ import com.bakdata.conquery.models.config.XodusStoreFactory; import com.bakdata.conquery.models.exceptions.ValidatorHelper; import com.bakdata.conquery.util.io.FileUtil; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; @@ -126,7 +131,7 @@ public , CLASS_V extends Class> SerializingSto unreadableValuesDumpDir = unreadableDataDumpDirectory; if (shouldDumpUnreadables()) { - if(!unreadableValuesDumpDir.exists() && !unreadableValuesDumpDir.mkdirs()) { + if (!unreadableValuesDumpDir.exists() && !unreadableValuesDumpDir.mkdirs()) { throw new IllegalStateException("Could not create dump directory: " + unreadableValuesDumpDir); } else if (!unreadableValuesDumpDir.isDirectory()) { @@ -151,19 +156,56 @@ public void add(KEY key, VALUE value) { store.add(writeKey(key), writeValue(value)); } + /** + * Serialize key with {@code keyWriter}. + */ + private ByteIterable writeKey(KEY key) { + return write(key, keyWriter); + } + + /** + * Serialize value with {@code valueWriter}. + */ + private ByteIterable writeValue(VALUE value) { + return write(value, valueWriter); + } + + /** + * Try writing object with writer. + */ + private ByteIterable write(Object obj, ObjectWriter writer) { + try { + final ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + try (final OutputStream outputStream = new GZIPOutputStream(baos)) { + writer.writeValue(outputStream, obj); + } + + baos.close(); + + final byte[] bytes = baos.toByteArray(); + + return new ArrayByteIterable(bytes); + } + catch (IOException e) { + throw new RuntimeException("Failed to write " + obj, e); + } + } + @Override public VALUE get(KEY key) { final ByteIterable binValue = store.get(writeKey(key)); try { - return readValue(binValue); - } catch (Exception e) { + return readValue(binValue); + } + catch (Exception e) { - if(unreadableValuesDumpDir != null) { - dumpToFile(binValue, key.toString(), e, unreadableValuesDumpDir, store.getName(), objectMapper); + if (unreadableValuesDumpDir != null) { + dumpToFile(binValue.getBytesUnsafe(), key.toString(), e, unreadableValuesDumpDir, store.getName(), objectMapper); } - if(removeUnreadablesFromUnderlyingStore) { + if (removeUnreadablesFromUnderlyingStore) { remove(key); // Null seems to be an acceptable return value in this case return null; @@ -175,6 +217,124 @@ public VALUE get(KEY key) { } } + /** + * Deserialize value with {@code valueReader}. + */ + private VALUE readValue(ByteIterable value) { + return read(valueReader, value); + } + + /** + * Dumps the content of an unreadable value to a file as a json (it tries to parse it as an object and than tries to dump it as a json). + * + * @param gzippedObj The object to dump. + * @param keyOfDump The key under which the unreadable value is accessible. It is used for the file name. + * @param reason The exception causing us to dump the file + * @param unreadableDumpDir The director to dump to. The method assumes that the directory exists and is okay to write to. + * @param storeName The name of the store which is also used in the dump file name. + */ + private static void dumpToFile(@NonNull byte[] gzippedObj, @NonNull String keyOfDump, Exception reason, @NonNull File unreadableDumpDir, String storeName, ObjectMapper objectMapper) { + // Create dump filehandle + final File dumpfile = makeDumpFileName(keyOfDump, unreadableDumpDir, storeName); + final File exceptionFileName = makeExceptionFileName(keyOfDump, unreadableDumpDir, storeName); + + if (dumpfile.exists() || exceptionFileName.exists()) { + log.trace("Abort dumping of file {} because it already exists.", dumpfile); + return; + } + + if (!dumpfile.getParentFile().exists() && !dumpfile.getParentFile().mkdirs()) { + throw new IllegalStateException("Could not create `%s`.".formatted(dumpfile.getParentFile())); + } + + //TODO FK: dump in a separate thread so we are not blocking the reader thread. + + // Write json + try { + log.info("Dumping value of key {} to {} (because it cannot be deserialized anymore).", keyOfDump, dumpfile.getCanonicalPath()); + + final JsonNode dump = objectMapper.readerFor(JsonNode.class).readValue(debugUnGzip(gzippedObj)); + Jackson.MAPPER.writer().writeValue(dumpfile, dump); + } + catch (IOException e) { + log.error("Failed to dump unreadable value of key `{}` to file `{}`", keyOfDump, dumpfile, e); + } + + try (PrintStream out = new PrintStream(exceptionFileName)) { + reason.printStackTrace(out); + } + catch (IOException e) { + log.error("Failed to dump exception for `{}` to file `{}`.", keyOfDump, exceptionFileName, e); + } + + } + + private static byte[] debugUnGzip(byte[] bytes) throws IOException { + return new GZIPInputStream(new ByteArrayInputStream(bytes)).readAllBytes(); + } + + @Override + public void remove(KEY key) { + log.trace("Removing value to key {} from Store[{}]", key, store.getName()); + store.remove(writeKey(key)); + } + + /** + * Try read value with reader. + */ + private T read(ObjectReader reader, ByteIterable obj) { + if (obj == null) { + return null; + } + try (final InputStream inputStream = new GZIPInputStream(new ByteArrayInputStream(obj.getBytesUnsafe(), 0, obj.getLength()))) { + return reader.readValue(inputStream); + } + catch (IOException e) { + try { + throw new RuntimeException("Failed to read " + JacksonUtil.toJsonDebug(debugUnGzip(obj.getBytesUnsafe())), e); + } + catch (IOException ex) { + throw new RuntimeException(ex); + } + } + } + + /** + * Generates a valid file name from the key of the dump object, the store and the current time. + * However, it does not ensure that there is no file with such a name. + *

+ * Current implementation is `$unreadableDumpDir/$today/$store/$key.json` + */ + @NotNull + public static File makeDumpFileName(@NotNull String keyOfDump, @NotNull File unreadableDumpDir, @NotNull String storeName) { + return unreadableDumpDir.toPath() + .resolve(DateTimeFormatter.BASIC_ISO_DATE.format(LocalDateTime.now())) + .resolve(storeName) + .resolve(sanitiseFileName(keyOfDump) + "." + DUMP_FILE_EXTENSION) + .toFile(); + + } + + /** + * Generates a valid file name from the key of the dump object, the store and the current time. + * However, it does not ensure that there is no file with such a name. + *

+ * Current implementation is `$unreadableDumpDir/$today/$store/$key.exception` + */ + @NotNull + public static File makeExceptionFileName(@NotNull String keyOfDump, @NotNull File unreadableDumpDir, @NotNull String storeName) { + return unreadableDumpDir.toPath() + .resolve(DateTimeFormatter.BASIC_ISO_DATE.format(LocalDateTime.now())) + .resolve(storeName) + .resolve(sanitiseFileName(keyOfDump) + "." + EXCEPTION_FILE_EXTENSION) + .toFile(); + + } + + private static String sanitiseFileName(@NotNull String name) { + return FileUtil.SAVE_FILENAME_REPLACEMENT_MATCHER.matcher(name).replaceAll("_"); + } + /** * Iterates a given consumer over the entries of this store. * Depending on the {@link XodusStoreFactory} corrupt entries may be dump to a file and/or removed from the store. @@ -228,15 +388,14 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { }); // Print some statistics final int total = result.getTotalProcessed(); - log.debug( - String.format( - "While processing store %s:\n\tEntries processed:\t%d\n\tKey read failure:\t%d (%.2f%%)\n\tValue read failure:\t%d (%.2f%%)", + log.debug("While processing store %s:\n\tEntries processed:\t%d\n\tKey read failure:\t%d (%.2f%%)\n\tValue read failure:\t%d (%.2f%%)".formatted( store.getName(), total, result.getFailedKeys(), total > 0 ? (float) result.getFailedKeys() / total * 100 : 0, result.getFailedValues(), - total > 0 ? (float) result.getFailedValues() / total * 100 : 0)); + total > 0 ? (float) result.getFailedValues() / total * 100 : 0 + )); // Remove corrupted entries from the store if configured so if (removeUnreadablesFromUnderlyingStore) { @@ -266,52 +425,12 @@ private TYPE getDeserializedAndDumpFailed(ByteIterable serial, Function T read(ObjectReader reader, ByteIterable obj) { - if (obj == null) { - return null; - } - try { - return reader.readValue(obj.getBytesUnsafe(), 0, obj.getLength()); - } - catch (IOException e) { - throw new RuntimeException("Failed to read " + JacksonUtil.toJsonDebug(obj.getBytesUnsafe()), e); - } - } - - /** - * Dumps the content of an unreadable value to a file as a json (it tries to parse it as an object and than tries to dump it as a json). - * - * @param obj The object to dump. - * @param keyOfDump The key under which the unreadable value is accessible. It is used for the file name. - * @param reason The exception causing us to dump the file - * @param unreadableDumpDir The director to dump to. The method assumes that the directory exists and is okay to write to. - * @param storeName The name of the store which is also used in the dump file name. - */ - private static void dumpToFile(@NonNull ByteIterable obj, @NonNull String keyOfDump, Exception reason, @NonNull File unreadableDumpDir, String storeName, ObjectMapper objectMapper) { - // Create dump filehandle - final File dumpfile = makeDumpFileName(keyOfDump, unreadableDumpDir, storeName); - final File exceptionFileName = makeExceptionFileName(keyOfDump, unreadableDumpDir, storeName); - - if (dumpfile.exists() || exceptionFileName.exists()) { - log.trace("Abort dumping of file {} because it already exists.", dumpfile); - return; - } - - if(!dumpfile.getParentFile().exists() && !dumpfile.getParentFile().mkdirs()){ - throw new IllegalStateException("Could not create `%s`.".formatted(dumpfile.getParentFile())); - } - - //TODO FK: dump in a separate thread so we are not blocking the reader thread. - - // Write json - try { - log.info("Dumping value of key {} to {} (because it cannot be deserialized anymore).", keyOfDump, dumpfile.getCanonicalPath()); - - final JsonNode dump = objectMapper.readerFor(JsonNode.class).readValue(obj.getBytesUnsafe(), 0, obj.getLength()); - Jackson.MAPPER.writer().writeValue(dumpfile, dump); - } - catch (IOException e) { - log.error("Failed to dump unreadable value of key `{}` to file `{}`", keyOfDump, dumpfile, e); + @Override + public void update(KEY key, VALUE value) { + if (!valueType.isInstance(value)) { + throw new IllegalStateException("The element %s is not of the required type %s".formatted(value, valueType)); } - try(PrintStream out = new PrintStream(exceptionFileName)) { - reason.printStackTrace(out); - } - catch (IOException e) { - log.error("Failed to dump exception for `{}` to file `{}`.", keyOfDump, exceptionFileName, e); + if (validateOnWrite) { + ValidatorHelper.failOnError(log, validator.validate(value)); } - } - - /** - * Generates a valid file name from the key of the dump object, the store and the current time. - * However, it does not ensure that there is no file with such a name. - * - * Current implementation is `$unreadableDumpDir/$today/$store/$key.json` - */ - @NotNull - public static File makeDumpFileName(@NotNull String keyOfDump, @NotNull File unreadableDumpDir, @NotNull String storeName) { - return unreadableDumpDir.toPath() - .resolve(DateTimeFormatter.BASIC_ISO_DATE.format(LocalDateTime.now())) - .resolve(storeName) - .resolve(sanitiseFileName(keyOfDump) + "." + DUMP_FILE_EXTENSION) - .toFile(); - - } - - private static String sanitiseFileName(@NotNull String name) { - return FileUtil.SAVE_FILENAME_REPLACEMENT_MATCHER.matcher(name).replaceAll("_"); - } - - /** - * Generates a valid file name from the key of the dump object, the store and the current time. - * However, it does not ensure that there is no file with such a name. - * - * Current implementation is `$unreadableDumpDir/$today/$store/$key.exception` - */ - @NotNull - public static File makeExceptionFileName(@NotNull String keyOfDump, @NotNull File unreadableDumpDir, @NotNull String storeName) { - return unreadableDumpDir.toPath() - .resolve(DateTimeFormatter.BASIC_ISO_DATE.format(LocalDateTime.now())) - .resolve(storeName) - .resolve(sanitiseFileName(keyOfDump) + "." + EXCEPTION_FILE_EXTENSION) - .toFile(); - + store.update(writeKey(key), writeValue(value)); } @Override @@ -471,15 +490,15 @@ public static class IterationStatistic { private int totalProcessed; private int failedKeys; private int failedValues; - + public void incrTotalProcessed() { totalProcessed++; } - + public void incrFailedKeys() { failedKeys++; } - + public void incrFailedValues() { failedValues++; } From 8d812845a8ee30cbf270b0af5bc5579159080bbd Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 14 Aug 2023 16:33:18 +0200 Subject: [PATCH 485/679] Simplify code - fix jumping --- .../DropzoneBetweenElements.tsx | 15 ++-- .../form-concept-group/FormConceptGroup.tsx | 71 ++++++---------- .../form-concept-group/FormConceptNode.tsx | 82 +++++++++---------- 3 files changed, 74 insertions(+), 94 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index f6319279d2..d6d9a0511a 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -11,7 +11,6 @@ interface Props { const RootHeightBase = 30; const LineHeight = 3; -const MarginTopOffsetOver = 5; const Root = styled("div")` width: 100%; left: 0; @@ -29,7 +28,9 @@ const Line = styled("div")` border-radius: 2px; `; -const DropzoneBetweenElements = ({ +const DropzoneBetweenElements = < + DroppableObject extends PossibleDroppableObject, +>({ acceptedDropTypes, onDrop, lastElement, @@ -44,9 +45,10 @@ const DropzoneBetweenElements = @@ -56,8 +58,9 @@ const DropzoneBetweenElements = diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index e540ec3a1a..2cce604693 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -107,6 +107,10 @@ const SxDescription = styled(Description)` font-size: ${({ theme }) => theme.font.xs}; `; +const SxFormConceptNode = styled(FormConceptNode)` + margin-top: 5px; +`; + export interface EditedFormQueryNodePosition { valueIdx: number; conceptIdx: number; @@ -208,50 +212,36 @@ const FormConceptGroup = (props: Props) => { if (props.isValidConcept && !props.isValidConcept(item)) return null; + const concept = isMovedObject(item) + ? copyConcept(item) + : initializeConcept(item, defaults, tableConfig); + let newPropsValue = props.value; + let insertIndex = i; if (isMovedObject(item)) { - let insertIndex = - i > item.dragContext.movedFromAndIdx && - item.dragContext.movedFromOrIdx === 0 - ? i - 1 - : i; - if (item.dragContext.movedFromFieldName === props.fieldName) { - const updatedValue = - props.value[item.dragContext.movedFromAndIdx].concepts - .length === 1 - ? removeValue(props.value, item.dragContext.movedFromAndIdx) + const { movedFromFieldName, movedFromAndIdx, movedFromOrIdx } = + item.dragContext; + + if (movedFromFieldName === props.fieldName) { + if (i > movedFromAndIdx && movedFromOrIdx === 0) { + insertIndex = i - 1; + } + newPropsValue = + props.value[movedFromAndIdx].concepts.length === 1 + ? removeValue(props.value, movedFromAndIdx) : removeConcept( props.value, - item.dragContext.movedFromAndIdx, - item.dragContext.movedFromOrIdx, + movedFromAndIdx, + movedFromOrIdx, ); - return props.onChange( - addConcept( - insertValue(updatedValue, insertIndex, newValue), - insertIndex, - copyConcept(item), - ), - ); } else { if (exists(item.dragContext.deleteFromOtherField)) { item.dragContext.deleteFromOtherField(); } - - return props.onChange( - addConcept( - insertValue(props.value, insertIndex, newValue), - insertIndex, - copyConcept(item), - ), - ); } } return props.onChange( - addConcept( - insertValue(props.value, i, newValue), - i, - initializeConcept(item, defaults, tableConfig), - ), + addConcept(insertValue(newPropsValue, insertIndex, newValue), insertIndex, concept), ); }; }} @@ -274,21 +264,14 @@ const FormConceptGroup = (props: Props) => { if (props.isValidConcept && !props.isValidConcept(item)) return; - if (isMovedObject(item)) { - return props.onChange( - addConcept( - addValue(props.value, newValue), - props.value.length, - copyConcept(item), - ), - ); - } - + const concept = isMovedObject(item) + ? copyConcept(item) + : initializeConcept(item, defaults, tableConfig); return props.onChange( addConcept( addValue(props.value, newValue), props.value.length, // Assuming the last index has increased after addValue - initializeConcept(item, defaults, tableConfig), + concept, ), ); }} @@ -338,7 +321,7 @@ const FormConceptGroup = (props: Props) => { } items={row.concepts.map((concept, j) => concept ? ( - = ({ : undefined; return ( - - canNodeBeDropped(conceptNode, item)} - highlightDroppable + canNodeBeDropped(conceptNode, item)} + highlightDroppable + > + { + ref.current = instance; + drag(instance); + }} + active={hasNonDefaultSettings || hasFilterValues} + onClick={onClick} > - { - ref.current = instance; - drag(instance); - }} - active={hasNonDefaultSettings || hasFilterValues} - onClick={onClick} - > -

- - <> - {rootNodeLabel && {rootNodeLabel}} - - {conceptNode && !!conceptNode.description && ( - {conceptNode.description} - )} - +
+ + <> + {rootNodeLabel && {rootNodeLabel}} + + {conceptNode && !!conceptNode.description && ( + {conceptNode.description} + )} + + +
+ + {expand && expand.expandable && ( + + { + e.stopPropagation(); + expand.onClick(); + }} + /> -
- - {expand && expand.expandable && ( - - { - e.stopPropagation(); - expand.onClick(); - }} - /> - - )} - - - - + )} + + + ); }; From a55e791a01567781b7b0387425baeb736d314eac Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 14 Aug 2023 16:35:43 +0200 Subject: [PATCH 486/679] format --- .../external-forms/form-concept-group/FormConceptGroup.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 2cce604693..bf381b6db3 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -241,7 +241,11 @@ const FormConceptGroup = (props: Props) => { } return props.onChange( - addConcept(insertValue(newPropsValue, insertIndex, newValue), insertIndex, concept), + addConcept( + insertValue(newPropsValue, insertIndex, newValue), + insertIndex, + concept, + ), ); }; }} From 8ebadddeba41a13b9115c70070301df93adab1ec Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Mon, 14 Aug 2023 17:52:12 +0200 Subject: [PATCH 487/679] introduce a set of clunky parameters to allow MigrateCommand from non-gzipped stores to gzipped stores. --- .../conquery/commands/MigrateCommand.java | 105 ++++++++++++------ 1 file changed, 74 insertions(+), 31 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/commands/MigrateCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/MigrateCommand.java index 416ea09024..f63c78daf9 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/MigrateCommand.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/MigrateCommand.java @@ -1,14 +1,21 @@ package com.bakdata.conquery.commands; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.util.Arrays; import java.util.List; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.XodusStoreFactory; import com.bakdata.conquery.util.io.ConqueryMDC; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -27,11 +34,13 @@ import jetbrains.exodus.env.StoreConfig; import jetbrains.exodus.env.Transaction; import kotlin.jvm.functions.Function4; +import lombok.SneakyThrows; import lombok.extern.slf4j.Slf4j; import net.sourceforge.argparse4j.impl.Arguments; import net.sourceforge.argparse4j.inf.Namespace; import net.sourceforge.argparse4j.inf.Subparser; import org.codehaus.groovy.control.CompilerConfiguration; +import org.jetbrains.annotations.NotNull; /** * Command allowing script based migration of databases. Especially useful for data that cannot be easily recreated after reimports, such as {@link com.bakdata.conquery.models.auth.entities.User}s and {@link com.bakdata.conquery.models.execution.ManagedExecution}s. @@ -56,7 +65,6 @@ @Slf4j public class MigrateCommand extends ConqueryCommand { - public MigrateCommand() { super("migrate", "Run a migration script on a store."); } @@ -75,6 +83,18 @@ public void configure(Subparser subparser) { .required(true) .type(Arguments.fileType()); + subparser + .addArgument("--in-gzip") + .help("If true, values are ungzipped before deserialization.") + .setDefault(true) + .type(Arguments.booleanType()); + + subparser + .addArgument("--out-gzip") + .help("If true, values are gzipped before writing.") + .setDefault(true) + .type(Arguments.booleanType()); + subparser .addArgument("--script") .help("Migration Script returning a closure implementing MigrationScriptFactory. See supplementary example.groovy for details.\nSignature: String env, String store, String key, ObjectNode value -> return new Tuple(key,value)") @@ -88,6 +108,10 @@ protected void run(io.dropwizard.setup.Environment environment, Namespace namesp final File inStoreDirectory = namespace.get("in"); final File outStoreDirectory = namespace.get("out"); + final boolean inGzip = namespace.getBoolean("in-gzip"); + final boolean outGzip = namespace.getBoolean("out-gzip"); + + final long logsize = ((XodusStoreFactory) configuration.getStorage()).getXodus().getLogFileSize().toKilobytes(); @@ -99,11 +123,11 @@ protected void run(io.dropwizard.setup.Environment environment, Namespace namesp } // Create Groovy Shell and parse script - CompilerConfiguration config = new CompilerConfiguration(); + final CompilerConfiguration config = new CompilerConfiguration(); config.setScriptBaseClass(MigrationScriptFactory.class.getName()); - GroovyShell groovy = new GroovyShell(config); + final GroovyShell groovy = new GroovyShell(config); - MigrationScriptFactory factory = (MigrationScriptFactory) groovy.parse(In.file((File) namespace.get("script")).readAll()); + final MigrationScriptFactory factory = (MigrationScriptFactory) groovy.parse(In.file((File) namespace.get("script")).readAll()); final Function4 migrator = factory.run(); @@ -116,25 +140,12 @@ protected void run(io.dropwizard.setup.Environment environment, Namespace namesp final File environmentDirectory = new File(outStoreDirectory, xenv.getName()); environmentDirectory.mkdirs(); - processEnvironment(xenv, logsize, environmentDirectory, migrator, mapper); + processEnvironment(xenv, logsize, environmentDirectory, migrator, mapper, inGzip, outGzip); }); } - - /** - * Class defining the interface for the Groovy-Script. - */ - public abstract static class MigrationScriptFactory extends Script { - - /** - * Environment -> Store -> Key -> Value -> (Key, Value) - */ - @Override - public abstract Function4 run(); - } - - private void processEnvironment(File inStoreDirectory, long logSize, File outStoreDirectory, Function4 migrator, ObjectMapper mapper) { + private void processEnvironment(File inStoreDirectory, long logSize, File outStoreDirectory, Function4 migrator, ObjectMapper mapper, boolean inGzip, boolean outGzip) { final jetbrains.exodus.env.Environment inEnvironment = Environments.newInstance( inStoreDirectory, new EnvironmentConfig().setLogFileSize(logSize) @@ -175,7 +186,7 @@ private void processEnvironment(File inStoreDirectory, long logSize, File outSto continue; } - doMigrate(inStore, outStore, migrator, mapper); + migrateStore(inStore, outStore, migrator, mapper, inGzip, outGzip); log.info("Done writing {}.", store); } @@ -191,7 +202,7 @@ private void processEnvironment(File inStoreDirectory, long logSize, File outSto inEnvironment.close(); } - private void doMigrate(Store inStore, Store outStore, Function4 migrator, ObjectMapper mapper) { + private void migrateStore(Store inStore, Store outStore, Function4 migrator, ObjectMapper mapper, boolean inGzip, boolean outGzip) { final Environment inEnvironment = inStore.getEnvironment(); final Environment outEnvironment = outStore.getEnvironment(); @@ -211,13 +222,12 @@ private void doMigrate(Store inStore, Store outStore, Function4 migrated = - migrator.invoke(inEnvironment.getLocation(), inStore.getName(), key, node); + final Tuple migrated = migrator.invoke(inEnvironment.getLocation(), inStore.getName(), key, value); // => Effectively delete the object if (migrated == null) { @@ -226,18 +236,18 @@ private void doMigrate(Store inStore, Store outStore, Function4 Store -> Key -> Value -> (Key, Value) + */ + @Override + public abstract Function4 run(); + } } From 5b355607f50c758c16e90ffa2bf3bd3a40c03e55 Mon Sep 17 00:00:00 2001 From: Jonas Arnhold Date: Tue, 15 Aug 2023 12:52:16 +0200 Subject: [PATCH 488/679] Add support for multi column validity dates (#3129) Add support for multi column validity dates in SQL and Legacy queryengine * Replace findValidityDateColumn() with findValidityDate() Co-authored-by: Torben Meyer Co-authored-by: awildturtok <1553491+awildturtok@users.noreply.github.com> --- .../apiv1/query/TableExportQuery.java | 25 ++-- .../apiv1/query/concept/filter/CQTable.java | 13 +- .../query/concept/specific/CQConcept.java | 9 +- .../models/datasets/concepts/Concept.java | 5 +- .../datasets/concepts/ValidityDate.java | 103 ++++++++++++-- .../conquery/models/events/Bucket.java | 15 +- .../conquery/models/events/EmptyBucket.java | 3 +- .../models/query/QueryExecutionContext.java | 4 +- .../query/queryplan/TableExportQueryPlan.java | 33 +++-- .../specific/EventDateUnionAggregator.java | 17 +-- .../specific/EventDurationSumAggregator.java | 10 +- .../specific/QuarterAggregator.java | 14 +- .../specific/SpecialDateUnion.java | 18 ++- .../specific/value/FirstValueAggregator.java | 19 +-- .../specific/value/LastValueAggregator.java | 13 +- .../queryplan/specific/ValidityDateNode.java | 24 ++-- .../models/query/resultinfo/UniqueNamer.java | 7 +- .../context/selects/ConceptSelects.java | 38 +++-- .../context/selects/MergedSelects.java | 87 +++++++----- .../conversion/context/selects/Selects.java | 5 +- .../cqelement/CQConceptConverter.java | 45 +++--- .../ConceptPreprocessingService.java | 19 +-- .../dialect/PostgreSqlFunctionProvider.java | 132 ++++++++++++++---- .../dialect/SqlFunctionProvider.java | 43 +++--- .../conversion/filter/FilterConverter.java | 6 +- .../query/ConceptQueryConverter.java | 32 ++++- .../select/DateDistanceConverter.java | 26 ++-- .../select/FirstValueConverter.java | 5 +- .../conquery/sql/models/ColumnDateRange.java | 84 +++++++++++ .../integration/IntegrationTests.java | 6 +- .../sql/SqlIntegrationTestSpec.java | 12 +- .../frontend/FilterSearchItemTest.java | 27 +++- .../types/SerialisationObjectsUtil.java | 3 +- .../util/SerialisationObjectsUtil.java | 3 +- .../sql/date_restriction/only_max/content.csv | 9 ++ .../date_restriction/only_max/expected.csv | 3 + .../only_max/only_max.spec.json | 84 +++++++++++ .../sql/date_restriction/only_min/content.csv | 9 ++ .../date_restriction/only_min/expected.csv | 3 + .../only_min/only_min.spec.json | 84 +++++++++++ tutorial/mimic_iii_demo/age_gender.ipynb | 2 +- tutorial/mimic_iii_demo/icd9.ipynb | 2 +- 42 files changed, 813 insertions(+), 288 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/models/ColumnDateRange.java create mode 100644 backend/src/test/resources/tests/sql/date_restriction/only_max/content.csv create mode 100644 backend/src/test/resources/tests/sql/date_restriction/only_max/expected.csv create mode 100644 backend/src/test/resources/tests/sql/date_restriction/only_max/only_max.spec.json create mode 100644 backend/src/test/resources/tests/sql/date_restriction/only_min/content.csv create mode 100644 backend/src/test/resources/tests/sql/date_restriction/only_min/expected.csv create mode 100644 backend/src/test/resources/tests/sql/date_restriction/only_min/only_min.spec.json diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java index 993cee86fc..bd5116638f 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/TableExportQuery.java @@ -34,6 +34,7 @@ import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.models.datasets.concepts.Connector; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.models.datasets.concepts.tree.ConceptTreeNode; import com.bakdata.conquery.models.datasets.concepts.tree.TreeConcept; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; @@ -148,6 +149,13 @@ public void resolve(QueryResolveContext context) { final Map secondaryIdPositions = calculateSecondaryIdPositions(currentPosition); + final Set validityDates = tables.stream() + .map(CQConcept::getTables) + .flatMap(Collection::stream) + .map(CQTable::findValidityDate) + .filter(Objects::nonNull) + .collect(Collectors.toSet()); + // We need to know if a column is a concept column so we can prioritize it if it is also a SecondaryId final Set conceptColumns = tables.stream() .map(CQConcept::getTables) @@ -157,7 +165,7 @@ public void resolve(QueryResolveContext context) { .filter(Objects::nonNull) .collect(Collectors.toSet()); - positions = calculateColumnPositions(currentPosition, tables, secondaryIdPositions, conceptColumns); + positions = calculateColumnPositions(currentPosition, tables, secondaryIdPositions, conceptColumns, validityDates); resultInfos = createResultInfos(secondaryIdPositions, conceptColumns); } @@ -179,22 +187,21 @@ private Map calculateSecondaryIdPositions(Atomi return secondaryIdPositions; } - private static Map calculateColumnPositions(AtomicInteger currentPosition, List tables, Map secondaryIdPositions, Set conceptColumns) { + private static Map calculateColumnPositions(AtomicInteger currentPosition, List tables, Map secondaryIdPositions, Set conceptColumns, Set validityDates) { final Map positions = new HashMap<>(); for (CQConcept concept : tables) { for (CQTable table : concept.getTables()) { - final Column validityDateColumn = table.findValidityDateColumn(); - - if (validityDateColumn != null) { - positions.putIfAbsent(validityDateColumn, 0); - } - // Set column positions, set SecondaryId positions to precomputed ones. for (Column column : table.getConnector().getTable().getColumns()) { + // ValidityDates are handled separately in column=0 + if (validityDates.stream().anyMatch(vd -> vd.containsColumn(column))) { + continue; + } + if (positions.containsKey(column)) { continue; } @@ -341,4 +348,4 @@ public void visit(Consumer visitor) { public RequiredEntities collectRequiredEntities(QueryExecutionContext context) { return query.collectRequiredEntities(context); } -} \ No newline at end of file +} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/CQTable.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/CQTable.java index bea2268426..b858c3741d 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/CQTable.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/CQTable.java @@ -10,8 +10,8 @@ import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.io.jackson.serializer.NsIdRef; import com.bakdata.conquery.io.jackson.serializer.NsIdRefCollection; -import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.Connector; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.query.QueryResolveContext; import com.fasterxml.jackson.annotation.JsonBackReference; @@ -75,18 +75,17 @@ public void resolve(QueryResolveContext context) { } @CheckForNull - public Column findValidityDateColumn() { + public ValidityDate findValidityDate() { - // if no dateColumn is provided, we use the default instead which is always the first one. - // Set to null if none-available in the connector. if (dateColumn != null) { - return dateColumn.getValue().getColumn(); + return dateColumn.getValue(); } if (!connector.getValidityDates().isEmpty()) { - return connector.getValidityDates().get(0).getColumn(); + return connector.getValidityDates().get(0); } return null; } -} \ No newline at end of file + +} diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java index f9ea072431..e425c8f106 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/specific/CQConcept.java @@ -25,6 +25,7 @@ import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.datasets.concepts.ConceptElement; import com.bakdata.conquery.models.datasets.concepts.Connector; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.identifiable.ids.NamespacedIdentifiable; import com.bakdata.conquery.models.query.DateAggregationMode; @@ -226,7 +227,7 @@ public QPNode createQueryPlan(QueryPlanContext context, ConceptQueryPlan plan) { final QPNode conceptSpecificNode = - getConcept().createConceptQuery(context, filters, aggregators, eventDateUnionAggregators, selectValidityDateColumn(table)); + getConcept().createConceptQuery(context, filters, aggregators, eventDateUnionAggregators, selectValidityDate(table)); // Link up the ExistsAggregators to the node existsAggregators.forEach(agg -> agg.setReference(conceptSpecificNode)); @@ -272,14 +273,14 @@ private static List> createAggregators(ConceptQueryPlan plan, List .collect(Collectors.toList()); } - private Column selectValidityDateColumn(CQTable table) { + private ValidityDate selectValidityDate(CQTable table) { if (table.getDateColumn() != null) { - return table.getDateColumn().getValue().getColumn(); + return table.getDateColumn().getValue(); } //else use this first defined validity date column if (!table.getConnector().getValidityDates().isEmpty()) { - return table.getConnector().getValidityDates().get(0).getColumn(); + return table.getConnector().getValidityDates().get(0); } return null; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java index d4c51d7c3e..368334b54c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/Concept.java @@ -14,7 +14,6 @@ import com.bakdata.conquery.models.auth.permissions.ConceptPermission; import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; import com.bakdata.conquery.models.common.CDateSet; -import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.exceptions.ConfigurationException; @@ -89,12 +88,12 @@ public int countElements() { /** * Allows concepts to create their own altered FiltersNode if necessary. */ - public QPNode createConceptQuery(QueryPlanContext context, List> filters, List> aggregators, List> eventDateAggregators, Column validityDateColumn) { + public QPNode createConceptQuery(QueryPlanContext context, List> filters, List> aggregators, List> eventDateAggregators, ValidityDate validityDate) { final QPNode child = filters.isEmpty() && aggregators.isEmpty() ? new Leaf() : FiltersNode.create(filters, aggregators, eventDateAggregators); // Only if a validityDateColumn exists, capsule children in ValidityDateNode - return validityDateColumn != null ? new ValidityDateNode(validityDateColumn, child) : child; + return validityDate != null ? new ValidityDateNode(validityDate, child) : child; } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/ValidityDate.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/ValidityDate.java index 5bdbe1c8bf..762d507209 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/ValidityDate.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/ValidityDate.java @@ -1,10 +1,14 @@ package com.bakdata.conquery.models.datasets.concepts; -import javax.validation.constraints.NotNull; +import javax.annotation.CheckForNull; +import javax.annotation.Nullable; import com.bakdata.conquery.io.jackson.serializer.NsIdRef; +import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Dataset; +import com.bakdata.conquery.models.events.Bucket; +import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.models.identifiable.Labeled; import com.bakdata.conquery.models.identifiable.ids.NamespacedIdentifiable; import com.bakdata.conquery.models.identifiable.ids.specific.ValidityDateId; @@ -15,6 +19,7 @@ import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; +import lombok.ToString; import lombok.extern.slf4j.Slf4j; @Getter @@ -24,39 +29,108 @@ public class ValidityDate extends Labeled implements NamespacedIdentifiable { @NsIdRef - @NotNull + @Nullable private Column column; + @NsIdRef + @Nullable + private Column startColumn; + @NsIdRef + @Nullable + private Column endColumn; @JsonBackReference + @ToString.Exclude @EqualsAndHashCode.Exclude private Connector connector; + public static ValidityDate create(Column column) { + final ValidityDate validityDate = new ValidityDate(); + validityDate.setColumn(column); + return validityDate; + } + + public static ValidityDate create(Column startColumn, Column endColumn) { + final ValidityDate validityDate = new ValidityDate(); + validityDate.setColumn(startColumn); + validityDate.setColumn(endColumn); + return validityDate; + } + @Override public ValidityDateId createId() { return new ValidityDateId(connector.getId(), getName()); } - @JsonIgnore - @ValidationMethod(message = "Column is not of Date or DateRange.") - public boolean isValidValidityDates() { - if (getColumn().getType().isDateCompatible()) { - return true; + @CheckForNull + public CDateRange getValidityDate(int event, Bucket bucket) { + // I spent a lot of time trying to create two classes implementing single/multi-column valditiy dates separately. + // JsonCreator was not happy, and I could not figure out why. This is probably the most performant implementation that's not two classes. + + if (getColumn() != null) { + if (bucket.has(event, getColumn())) { + return bucket.getAsDateRange(event, getColumn()); + } + + return null; + } + + final Column startColumn = getStartColumn(); + final Column endColumn = getEndColumn(); + + final boolean hasStart = bucket.has(event, startColumn); + final boolean hasEnd = bucket.has(event, endColumn); + + if (!hasStart && !hasEnd) { + return null; } - log.error("ValidityDate-Column[{}] is not of type DATE or DATERANGE", getColumn().getId()); - return false; + final int start = hasStart ? bucket.getDate(event, startColumn) : Integer.MIN_VALUE; + final int end = hasEnd ? bucket.getDate(event, endColumn) : Integer.MAX_VALUE; + + return CDateRange.of(start, end); + } + + public boolean containsColumn(Column column) { + return column.equals(getColumn()) || column.equals(getStartColumn()) || column.equals(getEndColumn()); } @JsonIgnore @ValidationMethod(message = "ValidityDate is not for Connectors' Table.") public boolean isForConnectorsTable() { - if (getColumn().getTable().equals(connector.getTable())) { - return true; + final boolean anyColumnNotForConnector = + (startColumn != null && !startColumn.getTable().equals(connector.getTable())) + || (endColumn != null && !endColumn.getTable().equals(connector.getTable())); + + final boolean columnNotForConnector = column != null && !column.getTable().equals(connector.getTable()); + + return !anyColumnNotForConnector && !columnNotForConnector; + } + + @JsonIgnore + @ValidationMethod(message = "Single column date range (set via column) and two column date range (set via startColumn and endColumn) are exclusive.") + public boolean isExclusiveValidityDates() { + if (column == null) { + return startColumn != null && endColumn != null; } + return startColumn == null && endColumn == null; + } - log.error("ValidityDate[{}](Column = `{}`) does not belong to Connector[{}]#Table[{}]", getId(), getColumn().getId(), getId(), connector.getTable().getId()); + @JsonIgnore + @ValidationMethod(message = "Both columns of a two-column validity date have to be of type DATE.") + public boolean isValidTwoColumnValidityDates() { + if (startColumn == null || endColumn == null) { + return true; + } + return startColumn.getType() == MajorTypeId.DATE && endColumn.getType() == MajorTypeId.DATE; + } - return false; + @JsonIgnore + @ValidationMethod(message = "Column is not of type DATE or DATE_RANGE.") + public boolean isValidValidityDatesSingleColumn() { + if (column == null) { + return true; + } + return column.getType().isDateCompatible(); } @JsonIgnore @@ -64,4 +138,5 @@ public boolean isForConnectorsTable() { public Dataset getDataset() { return connector.getDataset(); } -} \ No newline at end of file + +} diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/Bucket.java b/backend/src/main/java/com/bakdata/conquery/models/events/Bucket.java index f7df84d48c..618920194f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/Bucket.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/Bucket.java @@ -17,6 +17,7 @@ import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.Import; import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.models.events.stores.root.BooleanStore; import com.bakdata.conquery.models.events.stores.root.ColumnStore; import com.bakdata.conquery.models.events.stores.root.DateRangeStore; @@ -164,8 +165,14 @@ public CDateRange getDateRange(int event, Column column) { return ((DateRangeStore) getStore(column)).getDateRange(event); } - public boolean eventIsContainedIn(int event, Column column, CDateSet dateRanges) { - return dateRanges.intersects(getAsDateRange(event, column)); + public boolean eventIsContainedIn(int event, ValidityDate validityDate, CDateSet dateRanges) { + final CDateRange dateRange = validityDate.getValidityDate(event, this); + + if (dateRange == null){ + return false; + } + + return dateRanges.intersects(dateRange); } public CDateRange getAsDateRange(int event, Column column) { @@ -181,10 +188,10 @@ public Object createScriptValue(int event, @NotNull Column column) { } public Map calculateMap(int event) { - Map out = new HashMap<>(stores.length); + final Map out = new HashMap<>(stores.length); for (int i = 0; i < stores.length; i++) { - ColumnStore store = stores[i]; + final ColumnStore store = stores[i]; if (!store.has(event)) { continue; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/events/EmptyBucket.java b/backend/src/main/java/com/bakdata/conquery/models/events/EmptyBucket.java index e31cee658a..d820c82fb2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/events/EmptyBucket.java +++ b/backend/src/main/java/com/bakdata/conquery/models/events/EmptyBucket.java @@ -7,6 +7,7 @@ import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.models.events.stores.root.ColumnStore; import lombok.Getter; @@ -25,7 +26,7 @@ public EmptyBucket() { @Override - public boolean eventIsContainedIn(int event, Column column, CDateSet dateRanges) { + public boolean eventIsContainedIn(int event, ValidityDate column, CDateSet dateRanges) { return false; } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutionContext.java b/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutionContext.java index 666962d7e0..f0cb08f30d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutionContext.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/QueryExecutionContext.java @@ -7,10 +7,10 @@ import com.bakdata.conquery.io.storage.ModificationShieldedWorkerStorage; import com.bakdata.conquery.models.common.CDate; import com.bakdata.conquery.models.common.CDateSet; -import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.SecondaryIdDescription; import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.datasets.concepts.Connector; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.BucketManager; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; @@ -36,7 +36,7 @@ public class QueryExecutionContext { private final BucketManager bucketManager; - private Column validityDateColumn; + private ValidityDate validityDateColumn; @NonNull private CDateSet dateRestriction = CDateSet.createFull(); private Connector connector; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/TableExportQueryPlan.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/TableExportQueryPlan.java index 73551b7f07..9288871085 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/TableExportQueryPlan.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/TableExportQueryPlan.java @@ -7,7 +7,9 @@ import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; import com.bakdata.conquery.models.common.CDateSet; +import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.CBlock; import com.bakdata.conquery.models.query.QueryExecutionContext; @@ -80,7 +82,7 @@ public Optional execute(QueryExecutionContext ctx, Entity for (Map.Entry entry : tables.entrySet()) { final CQTable cqTable = entry.getKey(); - final Column validityDateColumn = cqTable.findValidityDateColumn(); + final ValidityDate validityDate = cqTable.findValidityDate(); final QPNode query = entry.getValue(); final Map cblocks = ctx.getBucketManager().getEntityCBlocksForConnector(entity, cqTable.getConnector()); @@ -95,8 +97,8 @@ public Optional execute(QueryExecutionContext ctx, Entity for (int event = start; event < end; event++) { - if (validityDateColumn != null - && !bucket.eventIsContainedIn(event, validityDateColumn, dateRange)) { + if (validityDate != null + && !bucket.eventIsContainedIn(event, validityDate, dateRange)) { continue; } @@ -104,7 +106,7 @@ public Optional execute(QueryExecutionContext ctx, Entity continue; } - final Object[] resultRow = collectRow(totalColumns, cqTable, bucket, event, validityDateColumn, cblocks.get(bucket)); + final Object[] resultRow = collectRow(totalColumns, cqTable, bucket, event, validityDate, cblocks.get(bucket)); results.add(resultRow); } @@ -146,29 +148,40 @@ private boolean isRowIncluded(QPNode query, Bucket bucket, Entity entity, int ev return query.isContained(); } - private Object[] collectRow(int totalColumns, CQTable exportDescription, Bucket bucket, int event, Column validityDateColumn, CBlock cblock) { + private Object[] collectRow(int totalColumns, CQTable exportDescription, Bucket bucket, int event, ValidityDate validityDate, CBlock cblock) { final Object[] entry = new Object[totalColumns]; + + final CDateRange date; + + if(validityDate != null && (date = validityDate.getValidityDate(event, bucket)) != null) { + entry[0] = List.of(date); + } + entry[1] = exportDescription.getConnector().getTable().getLabel(); for (Column column : exportDescription.getConnector().getTable().getColumns()) { - if (!bucket.has(event, column)) { + // ValidityDates are handled separately. + if (validityDate != null && validityDate.containsColumn(column)){ continue; } - final int position = positions.get(column); + if (!positions.containsKey(column)) { + continue; + } - if (column.equals(validityDateColumn)) { - entry[position] = List.of(bucket.getAsDateRange(event, column)); + if (!bucket.has(event, column)) { continue; } + final int position = positions.get(column); + if (!rawConceptValues && column.equals(exportDescription.getConnector().getColumn())) { entry[position] = cblock.getMostSpecificChildLocalId(event); continue; } - + entry[position] = bucket.createScriptValue(event, column); } return entry; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java index 7f10736554..0f3344f648 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDateUnionAggregator.java @@ -3,8 +3,9 @@ import java.util.Set; import com.bakdata.conquery.models.common.CDateSet; -import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; @@ -24,7 +25,7 @@ public class EventDateUnionAggregator extends Aggregator { private final Set requiredTables; - private Column validityDateColumn; + private ValidityDate validityDateColumn; private CDateSet set = CDateSet.createEmpty(); private CDateSet dateRestriction; @@ -41,10 +42,7 @@ public void init(Entity entity, QueryExecutionContext context) { @Override public void nextTable(QueryExecutionContext ctx, Table currentTable) { validityDateColumn = ctx.getValidityDateColumn(); - if (validityDateColumn != null && !validityDateColumn.getType().isDateCompatible()) { - throw new IllegalStateException("The validityDateColumn " + validityDateColumn + " is not a DATE TYPE"); - } - + dateRestriction = ctx.getDateRestriction(); super.nextTable(ctx, currentTable); } @@ -61,10 +59,13 @@ public void acceptEvent(Bucket bucket, int event) { return; } - if (!bucket.has(event, validityDateColumn)) { + final CDateRange dateRange = validityDateColumn.getValidityDate(event, bucket); + + if (dateRange == null){ return; } - set.maskedAdd(bucket.getAsDateRange(event, validityDateColumn), dateRestriction); + + set.maskedAdd(dateRange, dateRestriction); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java index 4a0119f910..5f563588cc 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/EventDurationSumAggregator.java @@ -6,8 +6,8 @@ import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; @@ -26,7 +26,7 @@ public class EventDurationSumAggregator extends Aggregator { @CheckForNull private CDateSet dateRestriction; @CheckForNull - private Column validityDateColumn; + private ValidityDate validityDateColumn; private int realUpperBound; @Override @@ -48,12 +48,12 @@ public void acceptEvent(Bucket bucket, int event) { return; } - if (!bucket.has(event, validityDateColumn)) { + final CDateRange value = validityDateColumn.getValidityDate(event, bucket); + + if (value == null){ return; } - final CDateRange value = bucket.getAsDateRange(event, validityDateColumn); - set.maskedAdd(value, dateRestriction, realUpperBound); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java index f75a71f8d7..4e09852192 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/QuarterAggregator.java @@ -8,8 +8,8 @@ import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.QuarterUtils; import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; @@ -31,7 +31,7 @@ public class QuarterAggregator extends Aggregator { private CDateSet set = CDateSet.createEmpty(); private CDateSet dateRestriction; - private Column column; + private ValidityDate validityDate; private int realUpperBound; @@ -48,19 +48,19 @@ public void init(Entity entity, QueryExecutionContext context) { @Override public void nextTable(QueryExecutionContext ctx, Table currentTable) { - column = ctx.getValidityDateColumn(); + validityDate = ctx.getValidityDateColumn(); dateRestriction = ctx.getDateRestriction(); } @Override public void acceptEvent(Bucket bucket, int event) { - if (getColumn() == null || !bucket.has(event, getColumn())) { + final CDateRange dateRange = validityDate.getValidityDate(event, bucket); + + if (dateRange == null){ return; } - final CDateRange value = bucket.getAsDateRange(event, getColumn()); - - set.maskedAdd(value, dateRestriction, realUpperBound); + set.maskedAdd(dateRange, dateRestriction, realUpperBound); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java index 7723b7cd33..e2ad370f73 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/SpecialDateUnion.java @@ -1,8 +1,9 @@ package com.bakdata.conquery.models.query.queryplan.aggregators.specific; import com.bakdata.conquery.models.common.CDateSet; -import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; @@ -20,7 +21,7 @@ public class SpecialDateUnion extends Aggregator { private CDateSet set = CDateSet.createEmpty(); - private Column currentColumn; + private ValidityDate validityDate; private CDateSet dateRestriction; @@ -31,18 +32,25 @@ public void init(Entity entity, QueryExecutionContext context) { @Override public void nextTable(QueryExecutionContext ctx, Table table) { - currentColumn = ctx.getValidityDateColumn(); + validityDate = ctx.getValidityDateColumn(); dateRestriction = ctx.getDateRestriction(); } @Override public void acceptEvent(Bucket bucket, int event) { - if (currentColumn == null || !bucket.has(event, currentColumn)) { + if (validityDate == null) { set.addAll(dateRestriction); return; } - set.maskedAdd(bucket.getAsDateRange(event, currentColumn), dateRestriction); + final CDateRange dateRange = validityDate.getValidityDate(event, bucket); + + if (dateRange == null){ + set.addAll(dateRestriction); + return; + } + + set.maskedAdd(dateRange, dateRestriction); } /** diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java index 1887a4fbd4..7e70495cc3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/FirstValueAggregator.java @@ -5,6 +5,7 @@ import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; @@ -26,7 +27,7 @@ public class FirstValueAggregator extends SingleColumnAggregator { private int date = CDateRange.POSITIVE_INFINITY; - private Column validityDateColumn; + private ValidityDate validityDate; public FirstValueAggregator(Column column) { super(column); @@ -41,7 +42,7 @@ public void init(Entity entity, QueryExecutionContext context) { @Override public void nextTable(QueryExecutionContext ctx, Table currentTable) { - validityDateColumn = ctx.getValidityDateColumn(); + validityDate = ctx.getValidityDateColumn(); } @Override @@ -50,23 +51,25 @@ public void acceptEvent(Bucket bucket, int event) { return; } - if (validityDateColumn == null) { + if (validityDate == null) { // If there is no validity date, take the first possible value if(selectedBucket == null) { selectedBucket = bucket; selectedEvent = OptionalInt.of(event); } else { - log.trace("There is more than one value for the {}. Choosing the very first one encountered", this.getClass().getSimpleName()); + log.trace("There is more than one value for the {}. Choosing the very first one encountered", getClass().getSimpleName()); } return; } - if(! bucket.has(event, validityDateColumn)) { - // TODO this might be an IllegalState + + final CDateRange dateRange = validityDate.getValidityDate(event, bucket); + + if (dateRange == null){ return; } - int next = bucket.getAsDateRange(event, validityDateColumn).getMinValue(); + final int next = dateRange.getMinValue(); if (next < date) { date = next; @@ -74,7 +77,7 @@ public void acceptEvent(Bucket bucket, int event) { selectedBucket = bucket; } else if (next == date) { - log.trace("There is more than one value for the {}. Choosing the very first one encountered", this.getClass().getSimpleName()); + log.trace("There is more than one value for the {}. Choosing the very first one encountered", getClass().getSimpleName()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java index bee141cbab..0e359c3f98 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/aggregators/specific/value/LastValueAggregator.java @@ -5,6 +5,7 @@ import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.query.QueryExecutionContext; import com.bakdata.conquery.models.query.entity.Entity; @@ -26,7 +27,7 @@ public class LastValueAggregator extends SingleColumnAggregator { private Bucket selectedBucket; private int date; - private Column validityDateColumn; + private ValidityDate validityDateColumn; public LastValueAggregator(Column column) { super(column); @@ -60,14 +61,14 @@ public void acceptEvent(Bucket bucket, int event) { } return; } - - if(! bucket.has(event, validityDateColumn)) { - // TODO this might be an IllegalState + + final CDateRange dateRange = validityDateColumn.getValidityDate(event, bucket); + + if (dateRange == null){ return; } - - int next = bucket.getAsDateRange(event, validityDateColumn).getMaxValue(); + int next = dateRange.getMaxValue(); if (next > date) { date = next; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ValidityDateNode.java b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ValidityDateNode.java index 940912753b..9d29a7f8dd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ValidityDateNode.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/queryplan/specific/ValidityDateNode.java @@ -5,8 +5,8 @@ import com.bakdata.conquery.models.common.CDateSet; import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Table; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.models.events.Bucket; import com.bakdata.conquery.models.events.CBlock; import com.bakdata.conquery.models.query.QueryExecutionContext; @@ -15,32 +15,30 @@ import com.google.common.base.Preconditions; import lombok.ToString; -@ToString(of = "validityDateColumn", callSuper = true) +@ToString(of = "validityDate", callSuper = true) public class ValidityDateNode extends QPChainNode { - private final Column validityDateColumn; + private final ValidityDate validityDate; private transient CDateSet restriction; protected Map preCurrentRow; - public ValidityDateNode(Column validityDateColumn, QPNode child) { + public ValidityDateNode(ValidityDate validityDate, QPNode child) { super(child); - Preconditions.checkNotNull(validityDateColumn, this.getClass().getSimpleName() + " needs a validityDateColumn"); - this.validityDateColumn = validityDateColumn; + Preconditions.checkNotNull(validityDate, this.getClass().getSimpleName() + " needs a validityDate"); + this.validityDate = validityDate; } @Override public void acceptEvent(Bucket bucket, int event) { + //no dateRestriction or event is in date restriction + final boolean contained = bucket.eventIsContainedIn(event, validityDate, context.getDateRestriction()); - //if event has null validityDate cancel - if (!bucket.has(event, validityDateColumn)) { + if (!contained){ return; } - //no dateRestriction or event is in date restriction - if (restriction.isAll() || bucket.eventIsContainedIn(event, validityDateColumn, context.getDateRestriction())) { - getChild().acceptEvent(bucket, event); - } + getChild().acceptEvent(bucket, event); } @Override @@ -59,7 +57,7 @@ public boolean isContained() { @Override public void nextTable(QueryExecutionContext ctx, Table currentTable) { - super.nextTable(ctx.withValidityDateColumn(validityDateColumn), currentTable); + super.nextTable(ctx.withValidityDateColumn(validityDate), currentTable); restriction = ctx.getDateRestriction(); preCurrentRow = ctx.getBucketManager().getEntityCBlocksForConnector(getEntity(), context.getConnector()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/UniqueNamer.java b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/UniqueNamer.java index 5c91e52b56..ce9502d9ea 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/UniqueNamer.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/resultinfo/UniqueNamer.java @@ -1,15 +1,11 @@ package com.bakdata.conquery.models.query.resultinfo; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; import java.util.Objects; import com.bakdata.conquery.models.query.PrintSettings; import com.fasterxml.jackson.annotation.JsonIgnore; import com.google.common.collect.ConcurrentHashMultiset; import com.google.common.collect.Multiset; -import com.google.common.collect.Multisets; import lombok.Getter; import lombok.NonNull; import lombok.RequiredArgsConstructor; @@ -38,7 +34,8 @@ public class UniqueNamer { @NonNull @JsonIgnore public final String getUniqueName(ResultInfo info) { - @NonNull String label = Objects.requireNonNullElse(info.userColumnName(settings), info.defaultColumnName(settings)); + @NonNull + String label = Objects.requireNonNullElse(info.userColumnName(settings), info.defaultColumnName(settings)); // lookup if prefix is needed and computed it if necessary String uniqueName = label; synchronized (ocurrenceCounter) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/ConceptSelects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/ConceptSelects.java index 8e42dde561..2d2d808632 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/ConceptSelects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/ConceptSelects.java @@ -1,11 +1,13 @@ package com.bakdata.conquery.sql.conversion.context.selects; +import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Optional; -import java.util.function.Function; import java.util.stream.Stream; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import com.bakdata.conquery.sql.models.ColumnDateRange; import lombok.Builder; import lombok.Value; import lombok.With; @@ -20,19 +22,26 @@ public class ConceptSelects implements Selects { Field primaryColumn; - Optional> dateRestriction; - Optional> validityDate; + Optional dateRestrictionRange; + Optional validityDate; List> eventSelect; List> eventFilter; List> groupSelect; List> groupFilter; + @Override + public Selects withValidityDate(ColumnDateRange validityDate) { + return this.toBuilder() + .validityDate(Optional.of(validityDate)) + .build(); + } + @Override public ConceptSelects byName(String qualifier) { return builder() .primaryColumn(this.mapFieldToQualifier(qualifier, this.primaryColumn)) - .dateRestriction(this.mapFieldStreamToQualifier(qualifier, this.dateRestriction.stream()).findFirst()) - .validityDate(this.mapFieldStreamToQualifier(qualifier, this.validityDate.stream()).findFirst()) + .dateRestrictionRange(this.dateRestrictionRange.map(dateRestriction -> dateRestriction.qualify(qualifier))) + .validityDate(this.validityDate.map(validityDate -> validityDate.qualify(qualifier))) .eventSelect(this.mapFieldStreamToQualifier(qualifier, this.eventSelect.stream()).toList()) .eventFilter(this.mapFieldStreamToQualifier(qualifier, this.eventFilter.stream()).toList()) .groupSelect(this.mapFieldStreamToQualifier(qualifier, this.groupSelect.stream()).toList()) @@ -51,19 +60,22 @@ public List> all() { private Stream> primaryColumnAndValidityDate() { return Stream.concat( Stream.of(this.primaryColumn), - this.validityDate.stream() + this.validityDate.map(ColumnDateRange::toFields).stream().flatMap(Collection::stream) ); } @Override public List> explicitSelects() { - return Stream.of( - this.dateRestriction.stream(), - this.eventSelect.stream(), - this.eventFilter.stream(), - this.groupSelect.stream(), - this.groupFilter.stream() - ).flatMap(Function.identity()).toList(); + + List> explicitSelects = new ArrayList<>(); + + dateRestrictionRange.ifPresent(columnDateRange -> explicitSelects.addAll(columnDateRange.toFields())); + explicitSelects.addAll(eventSelect); + explicitSelects.addAll(eventFilter); + explicitSelects.addAll(groupSelect); + explicitSelects.addAll(groupFilter); + + return explicitSelects; } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/MergedSelects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/MergedSelects.java index 1ce36e8ae8..7d3547d5b2 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/MergedSelects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/MergedSelects.java @@ -1,10 +1,12 @@ package com.bakdata.conquery.sql.conversion.context.selects; +import java.util.Collection; import java.util.List; import java.util.Optional; import java.util.stream.Stream; import com.bakdata.conquery.sql.conversion.context.step.QueryStep; +import com.bakdata.conquery.sql.models.ColumnDateRange; import lombok.AccessLevel; import lombok.AllArgsConstructor; import lombok.Value; @@ -20,11 +22,13 @@ @AllArgsConstructor(access = AccessLevel.PRIVATE) public class MergedSelects implements Selects { - String PRIMARY_COLUMN_ALIAS = "primary_column"; - + public static final String PRIMARY_COLUMN_NAME = "primary_column"; Field primaryColumn; - Optional> validityDate; + /** + * An aggregated validity date of all validity dates of each {@link QueryStep} passed to the {@link MergedSelects} constructor. + */ + Optional validityDate; /** * A merged list of all select fields, except the primary column and validity date, @@ -35,43 +39,25 @@ public class MergedSelects implements Selects { public MergedSelects(List querySteps) { this.primaryColumn = this.coalescePrimaryColumns(querySteps); - this.validityDate = this.extractValidityDate(querySteps); + this.validityDate = this.extractValidityDates(querySteps); this.mergedSelects = this.mergeSelects(querySteps); } - private Field coalescePrimaryColumns(List querySteps) { - List> primaryColumns = querySteps.stream() - .map(queryStep -> this.mapFieldToQualifier(queryStep.getCteName(), queryStep.getSelects().getPrimaryColumn())) - .toList(); - return DSL.coalesce((Object) primaryColumns.get(0), primaryColumns.subList(1, primaryColumns.size()).toArray()) - .as(PRIMARY_COLUMN_ALIAS); - } - - private Optional> extractValidityDate(List querySteps) { - // TODO: date aggregation... - if (querySteps.isEmpty()) { - return Optional.empty(); - } - QueryStep firstQueryStep = querySteps.get(0); - return this.mapFieldStreamToQualifier(firstQueryStep.getCteName(), firstQueryStep.getSelects().getValidityDate().stream()) - .findFirst(); - } - - private List> mergeSelects(List queriesToJoin) { - return queriesToJoin.stream() - .flatMap(queryStep -> queryStep.getSelects().explicitSelects().stream() - .map(field -> this.mapFieldToQualifier(queryStep.getCteName(), field))) - .toList(); + @Override + public Selects withValidityDate(ColumnDateRange validityDate) { + return new MergedSelects( + this.primaryColumn, + Optional.of(validityDate), + this.mergedSelects + ); } @Override public MergedSelects byName(String qualifier) { return new MergedSelects( this.mapFieldToQualifier(qualifier, this.primaryColumn), - this.mapFieldStreamToQualifier(qualifier, this.validityDate.stream()).findFirst(), - this.mergedSelects.stream() - .map(field -> this.mapFieldToQualifier(qualifier, field)) - .toList() + this.validityDate.map(columnDateRange -> columnDateRange.qualify(qualifier)), + this.mapFieldStreamToQualifier(qualifier, this.mergedSelects.stream()).toList() ); } @@ -83,16 +69,43 @@ public List> all() { ).toList(); } + @Override + public List> explicitSelects() { + return this.mergedSelects; + } + + private Field coalescePrimaryColumns(List querySteps) { + List> primaryColumns = querySteps.stream() + .map(queryStep -> this.mapFieldToQualifier(queryStep.getCteName(), queryStep.getSelects() + .getPrimaryColumn())) + .toList(); + return DSL.coalesce((Object) primaryColumns.get(0), primaryColumns.subList(1, primaryColumns.size()).toArray()) + .as(PRIMARY_COLUMN_NAME); + } + + private Optional extractValidityDates(List querySteps) { + // TODO: date aggregation... + return querySteps.stream() + .filter(queryStep -> queryStep.getSelects().getValidityDate().isPresent()) + .map(queryStep -> { + ColumnDateRange validityDate = queryStep.getSelects().getValidityDate().get(); + return validityDate.qualify(queryStep.getCteName()); + }) + .findFirst(); + } + + private List> mergeSelects(List queriesToJoin) { + return queriesToJoin.stream() + .flatMap(queryStep -> queryStep.getSelects().explicitSelects().stream() + .map(field -> this.mapFieldToQualifier(queryStep.getCteName(), field))) + .toList(); + } + private Stream> primaryColumnAndValidityDate() { return Stream.concat( Stream.of(this.primaryColumn), - this.validityDate.stream() + this.validityDate.map(ColumnDateRange::toFields).stream().flatMap(Collection::stream) ); } - @Override - public List> explicitSelects() { - return this.mergedSelects; - } - } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/Selects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/Selects.java index 6229c247a9..09d7259e25 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/Selects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/Selects.java @@ -4,6 +4,7 @@ import java.util.Optional; import java.util.stream.Stream; +import com.bakdata.conquery.sql.models.ColumnDateRange; import org.jooq.Field; import org.jooq.impl.DSL; @@ -11,7 +12,9 @@ public interface Selects { Field getPrimaryColumn(); - Optional> getValidityDate(); + Optional getValidityDate(); + + Selects withValidityDate(ColumnDateRange validityDate); /** * Returns the selected columns as fully qualified reference. diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQConceptConverter.java index db4e9b43b2..400dac447f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQConceptConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQConceptConverter.java @@ -73,22 +73,38 @@ private QueryStep buildDateRestrictionQueryStep( String conceptLabel, QueryStep previous ) { - if (((ConceptSelects) previous.getSelects()).getDateRestriction().isEmpty()) { + if (((ConceptSelects) previous.getSelects()).getDateRestrictionRange().isEmpty()) { return previous; } ConceptSelects dateRestrictionSelects = this.prepareDateRestrictionSelects(node, previous); - List dateRestriction = this.buildDateRestriction(context, previous); + Condition dateRestriction = this.buildDateRestriction(context, previous); + String dateRestrictionCteName = "concept_%s_date_restriction".formatted(conceptLabel); return QueryStep.builder() - .cteName(createCteName(conceptLabel, "_date_restriction")) + .cteName(dateRestrictionCteName) .fromTable(QueryStep.toTableLike(previous.getCteName())) .selects(dateRestrictionSelects) - .conditions(dateRestriction) + .conditions(List.of(dateRestriction)) .predecessors(List.of(previous)) .build(); } + private ConceptSelects prepareDateRestrictionSelects(CQConcept conceptNode, QueryStep previous) { + ConceptSelects.ConceptSelectsBuilder selectsBuilder = ((ConceptSelects) previous.getQualifiedSelects()).toBuilder(); + selectsBuilder.dateRestrictionRange(Optional.empty()); + if (conceptNode.isExcludeFromTimeAggregation()) { + selectsBuilder.validityDate(Optional.empty()); + } + return selectsBuilder.build(); + } + + private Condition buildDateRestriction(ConversionContext context, QueryStep previous) { + ConceptSelects previousSelects = (ConceptSelects) previous.getSelects(); + return context.getSqlDialect().getFunction() + .dateRestriction(previousSelects.getDateRestrictionRange().get(), previousSelects.getValidityDate().get()); + } + /** * selects: * - all of previous steps @@ -141,27 +157,6 @@ private QueryStep buildEventFilterQueryStep( .build(); } - private ConceptSelects prepareDateRestrictionSelects(CQConcept node, QueryStep previous) { - ConceptSelects.ConceptSelectsBuilder selectsBuilder = ((ConceptSelects) previous.getQualifiedSelects()).toBuilder(); - selectsBuilder.dateRestriction(Optional.empty()); - if (node.isExcludeFromTimeAggregation()) { - selectsBuilder.validityDate(Optional.empty()); - } - return selectsBuilder.build(); - } - - private List buildDateRestriction(ConversionContext context, QueryStep previous) { - return ((ConceptSelects) previous.getSelects()).getDateRestriction() - .map(dateRestrictionColumn -> getDateRestrictionAsCondition(context, previous, dateRestrictionColumn)) - .orElseGet(Collections::emptyList); - } - - private static List getDateRestrictionAsCondition(ConversionContext context, QueryStep previous, Field dateRestrictionColumn) { - return previous.getSelects().getValidityDate().stream() - .map(validityDateColumn -> context.getSqlDialect().getFunction().dateRestriction(dateRestrictionColumn, validityDateColumn)) - .toList(); - } - private ConceptSelects prepareEventSelectSelects( ConversionContext context, CQTable table, diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConceptPreprocessingService.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConceptPreprocessingService.java index 2800d8b30a..94970f4621 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConceptPreprocessingService.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConceptPreprocessingService.java @@ -7,19 +7,17 @@ import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; -import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.sql.conversion.context.ConversionContext; import com.bakdata.conquery.sql.conversion.context.selects.ConceptSelects; import com.bakdata.conquery.sql.conversion.context.step.QueryStep; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.models.ColumnDateRange; import org.jooq.Field; import org.jooq.impl.DSL; public class ConceptPreprocessingService { - private static final String DATE_RESTRICTION_COLUMN_NAME = "date_restriction"; - private static final String VALIDITY_DATE_COLUMN_NAME_SUFFIX = "_validity_date"; private final CQConcept concept; private final ConversionContext context; private final SqlFunctionProvider sqlFunctionProvider; @@ -43,7 +41,7 @@ public QueryStep buildPreprocessingQueryStepForTable(String conceptLabel, CQTabl ConceptSelects.ConceptSelectsBuilder selectsBuilder = ConceptSelects.builder(); selectsBuilder.primaryColumn(DSL.field(context.getConfig().getPrimaryColumn())); - selectsBuilder.dateRestriction(this.getDateRestrictionSelect(table)); + selectsBuilder.dateRestrictionRange(this.getDateRestrictionSelect(table)); selectsBuilder.validityDate(this.getValidityDateSelect(table, conceptLabel)); List> conceptSelectFields = this.getColumnSelectReferences(table); @@ -71,23 +69,18 @@ public QueryStep buildPreprocessingQueryStepForTable(String conceptLabel, CQTabl .build(); } - private Optional> getDateRestrictionSelect(CQTable table) { + private Optional getDateRestrictionSelect(CQTable table) { if (!this.context.dateRestrictionActive() || !this.tableHasValidityDates(table)) { return Optional.empty(); } - CDateRange dateRestrictionRange = this.context.getDateRestrictionRange(); - Field dateRestriction = this.sqlFunctionProvider.daterange(dateRestrictionRange) - .as(DATE_RESTRICTION_COLUMN_NAME); - return Optional.of(dateRestriction); + return Optional.of(sqlFunctionProvider.daterange(context.getDateRestrictionRange())); } - private Optional> getValidityDateSelect(CQTable table, String conceptLabel) { + private Optional getValidityDateSelect(CQTable table, String conceptLabel) { if (!this.validityDateIsRequired(table)) { return Optional.empty(); } - Field validityDateRange = this.sqlFunctionProvider.daterange(table.findValidityDateColumn()) - .as(conceptLabel + VALIDITY_DATE_COLUMN_NAME_SUFFIX); - return Optional.of(validityDateRange); + return Optional.of(sqlFunctionProvider.daterange(table.findValidityDate(), conceptLabel)); } /** diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java index 5ee2069393..5fe2c9620c 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java @@ -1,8 +1,16 @@ package com.bakdata.conquery.sql.conversion.dialect; +import java.sql.Date; +import java.time.temporal.ChronoUnit; +import java.util.Map; + import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; +import com.bakdata.conquery.sql.models.ColumnDateRange; +import org.jetbrains.annotations.NotNull; import org.jooq.Condition; +import org.jooq.DatePart; import org.jooq.Field; import org.jooq.impl.DSL; @@ -13,40 +21,114 @@ */ public class PostgreSqlFunctionProvider implements SqlFunctionProvider { + private static final String INFINITY_DATE_VALUE = "infinity"; + private static final String MINUS_INFINITY_DATE_VALUE = "-infinity"; + + private static final Map DATE_CONVERSION = Map.of( + ChronoUnit.DECADES, DatePart.DECADE, + ChronoUnit.YEARS, DatePart.YEAR, + ChronoUnit.DAYS, DatePart.DAY, + ChronoUnit.MONTHS, DatePart.MONTH, + ChronoUnit.CENTURIES, DatePart.CENTURY + ); + @Override - public Condition dateRestriction(Field dateRestrictionColumn, Field validityDateColumn) { - // the && operator checks if two ranges overlap (see https://www.postgresql.org/docs/15/functions-range.html) - return DSL.condition( - "{0} && {1}", - dateRestrictionColumn, - validityDateColumn - ); + public Condition dateRestriction(ColumnDateRange dateRestriction, ColumnDateRange validityDate) { + if (!validityDate.isSingleColumnRange()) { + throw new UnsupportedOperationException("The validity date range has to be converted to a daterange field in the preprocessing step."); + } + else { + // the && operator checks if two ranges overlap (see https://www.postgresql.org/docs/15/functions-range.html) + return DSL.condition( + "{0} && {1}", + dateRestriction.getRange(), + validityDate.getRange() + ); + } } @Override - public Field daterange(CDateRange dateRestriction) { - return DSL.field( + public ColumnDateRange daterange(CDateRange dateRestriction) { + + String min = MINUS_INFINITY_DATE_VALUE; + String max = INFINITY_DATE_VALUE; + + if (dateRestriction.hasLowerBound()) { + min = dateRestriction.getMin().toString(); + } + if (dateRestriction.hasUpperBound()) { + max = dateRestriction.getMax().toString(); + } + + Field dateRestrictionRange = DSL.field( "daterange({0}::date, {1}::date, '[]')", - DSL.val(dateRestriction.getMin().toString()), - DSL.val(dateRestriction.getMax().toString()) + DSL.val(min), + DSL.val(max) ); + + return ColumnDateRange.of(dateRestrictionRange) + .asDateRestrictionRange(); } @Override - public Field daterange(Column column) { - return switch (column.getType()) { - // if validityDateColumn is a DATE_RANGE we can make use of Postgres' integrated daterange type. - case DATE_RANGE -> DSL.field(column.getName()); - // if the validity date column is not of daterange type, we construct it manually - case DATE -> DSL.field( - "daterange({0}, {0}, '[]')", - DSL.field(column.getName()) - ); - default -> throw new IllegalArgumentException( - "Given column type '%s' can't be converted to a proper date restriction." - .formatted(column.getType()) - ); - }; + public ColumnDateRange daterange(ValidityDate validityDate, String alias) { + + Field dateRange; + + if (validityDate.getEndColumn() != null) { + + Column startColumn = validityDate.getStartColumn(); + Column endColumn = validityDate.getEndColumn(); + + dateRange = daterange(startColumn, endColumn, "[]"); + } + else { + Column column = validityDate.getColumn(); + dateRange = switch (column.getType()) { + // if validityDateColumn is a DATE_RANGE we can make use of Postgres' integrated daterange type. + case DATE_RANGE -> DSL.field(DSL.name(column.getName())); + // if the validity date column is not of daterange type, we construct it manually + case DATE -> daterange(column, column, "[]"); + default -> throw new IllegalArgumentException( + "Given column type '%s' can't be converted to a proper date restriction.".formatted(column.getType()) + ); + }; + } + + return ColumnDateRange.of(dateRange) + .asValidityDateRange(alias); + } + + @Override + public Field daterangeString(ColumnDateRange columnDateRange) { + if (!columnDateRange.isSingleColumnRange()) { + throw new UnsupportedOperationException("All column date ranges should have been converted to single column ranges."); + } + return columnDateRange.getRange(); + } + + @Override + public Field dateDistance(ChronoUnit timeUnit, Column startDateColumn, Date endDateExpression) { + + DatePart datePart = DATE_CONVERSION.get(timeUnit); + if (datePart == null) { + throw new UnsupportedOperationException("Chrono unit %s is not supported".formatted(timeUnit)); + } + + // we can now safely cast to Field of type Date + Field startDate = DSL.field(DSL.name(startDateColumn.getName()), Date.class); + return DSL.dateDiff(datePart, startDate, endDateExpression); + } + + @NotNull + private static Field daterange(Column startColumn, Column endColumn, String bounds) { + return DSL.function( + "daterange", + Object.class, + DSL.field(DSL.name(startColumn.getName())), + DSL.field(DSL.name(endColumn.getName())), + DSL.val(bounds) + ); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index cdfde5aa06..9e5ebab7df 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -1,14 +1,16 @@ package com.bakdata.conquery.sql.conversion.dialect; import java.sql.Date; +import java.time.temporal.ChronoUnit; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.models.events.MajorTypeId; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.sql.conversion.context.step.QueryStep; +import com.bakdata.conquery.sql.models.ColumnDateRange; import org.jooq.Condition; -import org.jooq.DatePart; import org.jooq.Field; +import org.jooq.Name; import org.jooq.Record; import org.jooq.Table; import org.jooq.TableOnConditionStep; @@ -21,39 +23,26 @@ public interface SqlFunctionProvider { String DEFAULT_DATE_FORMAT = "yyyy-mm-dd"; - - Condition dateRestriction(Field dateRestrictionColumn, Field validityDateColumn); - /** - * @return A daterange for a date restriction. + * A date restriction condition is true if holds: + * dateRestrictionStart <= validityDateEnd and dateRestrictionEnd >= validityDateStart */ - Field daterange(CDateRange dateRestriction); + Condition dateRestriction(ColumnDateRange dateRestrictionRange, ColumnDateRange validityFieldRange); - /** - * @return A daterange for an existing column. - */ - Field daterange(Column column); + ColumnDateRange daterange(CDateRange dateRestriction); - default Field toDate(String dateColumn) { - return DSL.toDate(dateColumn, DEFAULT_DATE_FORMAT); - } + ColumnDateRange daterange(ValidityDate validityDate, String conceptLabel); - default Field dateDistance(DatePart timeUnit, Date endDate, Column startDateColumn) { - if (startDateColumn.getType() != MajorTypeId.DATE) { - throw new UnsupportedOperationException("Can't calculate date distance to column of type " - + startDateColumn.getType()); - } - // we can now safely cast to Field of type Date - Field startDate = DSL.field(startDateColumn.getName(), Date.class); - return DSL.dateDiff(timeUnit, startDate, endDate); - } + Field daterangeString(ColumnDateRange columnDateRange); + + Field dateDistance(ChronoUnit datePart, Column startDateColumn, Date endDateExpression); - default Condition in(String columnName, String[] values) { + default Condition in(Name columnName, String[] values) { return DSL.field(columnName) .in(values); } - default Field first(String columnName) { + default Field first(Name columnName) { // TODO: this is just a temporary placeholder return DSL.field(columnName); } @@ -80,4 +69,8 @@ default TableOnConditionStep fullOuterJoin( .on(leftPartPrimaryColumn.eq(rightPartPrimaryColumn)); } + default Field toDate(String dateExpression) { + return DSL.toDate(dateExpression, DEFAULT_DATE_FORMAT); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/FilterConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/FilterConverter.java index 1b645ba0fa..cdc40c3395 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/FilterConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/filter/FilterConverter.java @@ -4,6 +4,8 @@ import com.bakdata.conquery.models.datasets.concepts.filters.SingleColumnFilter; import com.bakdata.conquery.sql.conversion.Converter; import org.jooq.Condition; +import org.jooq.Name; +import org.jooq.impl.DSL; /** * Converts a {@link com.bakdata.conquery.apiv1.query.concept.filter.FilterValue} @@ -13,9 +15,9 @@ */ public interface FilterConverter> extends Converter { - static String getColumnName(FilterValue filter) { + static Name getColumnName(FilterValue filter) { // works for now but we might have to distinguish later if we encounter non-SingleColumnFilters - return ((SingleColumnFilter) filter.getFilter()).getColumn().getName(); + return DSL.name(((SingleColumnFilter) filter.getFilter()).getColumn().getName()); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java index 9dd3367ed8..7a71880816 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/query/ConceptQueryConverter.java @@ -5,19 +5,29 @@ import com.bakdata.conquery.apiv1.query.ConceptQuery; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.context.selects.Selects; import com.bakdata.conquery.sql.conversion.context.step.QueryStep; import com.bakdata.conquery.sql.conversion.context.step.QueryStepTransformer; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import com.bakdata.conquery.sql.models.ColumnDateRange; +import org.jooq.Field; import org.jooq.Record; import org.jooq.Select; public class ConceptQueryConverter implements NodeConverter { + public static final String FINAL_VALIDITY_DATE_COLUMN_NAME = "dates"; private final QueryStepTransformer queryStepTransformer; public ConceptQueryConverter(QueryStepTransformer queryStepTransformer) { this.queryStepTransformer = queryStepTransformer; } + @Override + public Class getConversionClass() { + return ConceptQuery.class; + } + @Override public ConversionContext convert(ConceptQuery node, ConversionContext context) { @@ -27,7 +37,7 @@ public ConversionContext convert(ConceptQuery node, ConversionContext context) { QueryStep preFinalStep = contextAfterConversion.getQuerySteps().iterator().next(); QueryStep finalStep = QueryStep.builder() .cteName(null) // the final QueryStep won't be converted to a CTE - .selects(preFinalStep.getQualifiedSelects()) + .selects(this.toFinalSelects(preFinalStep, context)) .fromTable(QueryStep.toTableLike(preFinalStep.getCteName())) .conditions(preFinalStep.getConditions()) .predecessors(List.of(preFinalStep)) @@ -37,8 +47,22 @@ public ConversionContext convert(ConceptQuery node, ConversionContext context) { return context.withFinalQuery(finalQuery); } - @Override - public Class getConversionClass() { - return ConceptQuery.class; + /** + * @return The final selects containing the final validity date, if present, as a string aggregation field. + */ + private Selects toFinalSelects(QueryStep preFinalStep, ConversionContext context) { + + Selects finalSelects = preFinalStep.getQualifiedSelects(); + + if (finalSelects.getValidityDate().isEmpty()) { + return finalSelects; + } + + SqlFunctionProvider functionProvider = context.getSqlDialect().getFunction(); + Field finalValidityDateSelect = functionProvider.daterangeString(finalSelects.getValidityDate().get()) + .as(FINAL_VALIDITY_DATE_COLUMN_NAME); + + return finalSelects.withValidityDate(ColumnDateRange.of(finalValidityDateSelect)); } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/DateDistanceConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/DateDistanceConverter.java index d70800c39f..52339db897 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/DateDistanceConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/DateDistanceConverter.java @@ -2,27 +2,18 @@ import java.sql.Date; import java.time.LocalDate; -import java.time.temporal.ChronoUnit; -import java.util.Map; import java.util.Objects; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DateDistanceSelect; import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.supplier.DateNowSupplier; -import org.jooq.DatePart; import org.jooq.Field; public class DateDistanceConverter implements SelectConverter { - private static final Map DATE_CONVERSION = Map.of( - ChronoUnit.DECADES, DatePart.DECADE, - ChronoUnit.YEARS, DatePart.YEAR, - ChronoUnit.DAYS, DatePart.DAY, - ChronoUnit.MONTHS, DatePart.MONTH, - ChronoUnit.CENTURIES, DatePart.CENTURY - ); private final DateNowSupplier dateNowSupplier; public DateDistanceConverter(DateNowSupplier dateNowSupplier) { @@ -31,19 +22,18 @@ public DateDistanceConverter(DateNowSupplier dateNowSupplier) { @Override public Field convert(DateDistanceSelect select, ConversionContext context) { - DatePart timeUnit = DATE_CONVERSION.get(select.getTimeUnit()); - if (timeUnit == null) { - throw new UnsupportedOperationException("Chrono unit %s is not supported".formatted(select.getTimeUnit())); - } - Column startDateColumn = select.getColumn(); - Date endDate = getEndDate(context); + Column startDateColumn = select.getColumn(); if (startDateColumn.getType() != MajorTypeId.DATE) { throw new UnsupportedOperationException("Can't calculate date distance to column of type " + startDateColumn.getType()); } - return context.getSqlDialect().getFunction().dateDistance(timeUnit, endDate, startDateColumn) - .as(select.getLabel()); + + SqlFunctionProvider functionProvider = context.getSqlDialect().getFunction(); + Date endDate = getEndDate(context); + + return functionProvider.dateDistance(select.getTimeUnit(), startDateColumn, endDate) + .as(select.getLabel()); } private Date getEndDate(ConversionContext context) { diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/FirstValueConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/FirstValueConverter.java index 50a66ddb03..af1593bf91 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/FirstValueConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/FirstValueConverter.java @@ -4,12 +4,15 @@ import com.bakdata.conquery.sql.conversion.context.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import org.jooq.Field; +import org.jooq.Name; +import org.jooq.impl.DSL; public class FirstValueConverter implements SelectConverter { public Field convert(FirstValueSelect select, ConversionContext context) { SqlFunctionProvider fn = context.getSqlDialect().getFunction(); - return fn.first(select.getColumn().getName()); + Name columnName = DSL.name(select.getColumn().getName()); + return fn.first(columnName); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/models/ColumnDateRange.java b/backend/src/main/java/com/bakdata/conquery/sql/models/ColumnDateRange.java new file mode 100644 index 0000000000..e0faabdf4f --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/models/ColumnDateRange.java @@ -0,0 +1,84 @@ +package com.bakdata.conquery.sql.models; + +import java.util.List; + +import lombok.Getter; +import org.jooq.Field; +import org.jooq.impl.DSL; + +@Getter +public class ColumnDateRange { + + private static final String DATE_RESTRICTION_COLUMN_NAME = "date_restriction"; + private static final String VALIDITY_DATE_COLUMN_NAME_SUFFIX = "_validity_date"; + private static final String START_SUFFIX = "_start"; + private static final String END_SUFFIX = "_end"; + + private final boolean isEmpty; + private final Field range; + private final Field start; + private final Field end; + + private ColumnDateRange(boolean isEmpty, Field range, Field startColumn, Field endColumn) { + this.isEmpty = isEmpty; + this.range = range; + this.start = startColumn; + this.end = endColumn; + } + + public static ColumnDateRange of(Field rangeColumn) { + return new ColumnDateRange(false, rangeColumn, null, null); + } + + public static ColumnDateRange of(Field startColumn, Field endColumn) { + return new ColumnDateRange(true, null, startColumn, endColumn); + } + + public ColumnDateRange asDateRestrictionRange() { + return this.as(DATE_RESTRICTION_COLUMN_NAME); + } + + public ColumnDateRange asValidityDateRange(String alias) { + return this.as(alias + VALIDITY_DATE_COLUMN_NAME_SUFFIX); + } + + /** + * @return True if this {@link ColumnDateRange} consists of only 1 column. + * False if it consists of a start and end field. + */ + public boolean isSingleColumnRange() { + return this.range != null; + } + + public List> toFields() { + if (isSingleColumnRange()) { + return List.of(this.range); + } + return List.of(this.start, this.end); + } + + public ColumnDateRange qualify(String qualifier) { + if (isSingleColumnRange()) { + return ColumnDateRange.of(mapFieldOntoQualifier(getRange(), qualifier)); + } + return ColumnDateRange.of( + mapFieldOntoQualifier(getStart(), qualifier), + mapFieldOntoQualifier(getEnd(), qualifier) + ); + } + + private ColumnDateRange as(String alias) { + if (isSingleColumnRange()) { + return ColumnDateRange.of(this.range.as(alias)); + } + return ColumnDateRange.of( + this.start.as(alias + START_SUFFIX), + this.end.as(alias + END_SUFFIX) + ); + } + + private Field mapFieldOntoQualifier(Field field, String qualifier) { + return DSL.field(DSL.name(qualifier, field.getName())); + } + +} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java index bd2fd0f5bc..d40a970169 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java @@ -143,8 +143,10 @@ public Stream sqlTests(SqlDialect sqlDialect, SqlConnectorConfig sq Stream paths = Files.walk(testRootDir); List dynamicTestStream = paths.filter(path -> !Files.isDirectory(path) && path.toString().endsWith(".json")) - .map(path -> SqlIntegrationTest.fromPath(path, sqlDialect, sqlConfig)) - .map(test -> DynamicTest.dynamicTest(test.getTestSpec().getLabel(), test)).toList(); + .map(path -> SqlIntegrationTest.fromPath(path, sqlDialect, sqlConfig)) + .filter(sqlIntegrationTest -> sqlIntegrationTest.getTestSpec() + .supportsDialects(sqlConfig.getDialect())) + .map(test -> DynamicTest.dynamicTest(test.getTestSpec().getLabel(), test)).toList(); return dynamicTestStream.stream(); } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlIntegrationTestSpec.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlIntegrationTestSpec.java index 83bc210090..ba61fe0173 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlIntegrationTestSpec.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlIntegrationTestSpec.java @@ -14,13 +14,12 @@ import com.bakdata.conquery.integration.json.ConqueryTestSpec; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.Jackson; +import com.bakdata.conquery.models.config.Dialect; import com.bakdata.conquery.models.datasets.Table; import com.bakdata.conquery.models.datasets.concepts.Concept; import com.bakdata.conquery.models.exceptions.JSONException; import com.bakdata.conquery.models.query.results.EntityResult; -import com.bakdata.conquery.models.query.results.SinglelineEntityResult; import com.bakdata.conquery.sql.conquery.SqlManagedQuery; -import com.bakdata.conquery.sql.execution.SqlEntityResult; import com.bakdata.conquery.sql.execution.SqlExecutionResult; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; @@ -41,6 +40,8 @@ public class SqlIntegrationTestSpec extends ConqueryTestSpec supportedDialects; + @NotNull @JsonProperty("query") private JsonNode rawQuery; @@ -65,6 +66,13 @@ public class SqlIntegrationTestSpec extends ConqueryTestSpec validityDates = List.of(val0, val1, val2); connector.setColumn(column); connector.setConcept(concept); diff --git a/backend/src/test/java/com/bakdata/conquery/models/types/SerialisationObjectsUtil.java b/backend/src/test/java/com/bakdata/conquery/models/types/SerialisationObjectsUtil.java index a27126bd39..d2e9409287 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/types/SerialisationObjectsUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/models/types/SerialisationObjectsUtil.java @@ -77,8 +77,7 @@ public static TreeConcept createConcept(CentralRegistry registry, Dataset datase concept.setConnectors(List.of(connector)); - ValidityDate valDate = new ValidityDate(); - valDate.setColumn(dateColumn); + ValidityDate valDate = ValidityDate.create(dateColumn); valDate.setConnector(connector); valDate.setLabel("valLabel"); valDate.setName("valName"); diff --git a/backend/src/test/java/com/bakdata/conquery/util/SerialisationObjectsUtil.java b/backend/src/test/java/com/bakdata/conquery/util/SerialisationObjectsUtil.java index 60a23aebb8..de4137ac1f 100644 --- a/backend/src/test/java/com/bakdata/conquery/util/SerialisationObjectsUtil.java +++ b/backend/src/test/java/com/bakdata/conquery/util/SerialisationObjectsUtil.java @@ -77,8 +77,7 @@ public static TreeConcept createConcept(CentralRegistry registry, Dataset datase concept.setConnectors(List.of(connector)); - ValidityDate valDate = new ValidityDate(); - valDate.setColumn(dateColumn); + ValidityDate valDate = ValidityDate.create(dateColumn); valDate.setConnector(connector); valDate.setLabel("valLabel"); valDate.setName("valName"); diff --git a/backend/src/test/resources/tests/sql/date_restriction/only_max/content.csv b/backend/src/test/resources/tests/sql/date_restriction/only_max/content.csv new file mode 100644 index 0000000000..212025dec4 --- /dev/null +++ b/backend/src/test/resources/tests/sql/date_restriction/only_max/content.csv @@ -0,0 +1,9 @@ +pid,datum,geschlecht +1,2012-01-01,"f" +2,2010-07-15,"m" +3,2012-11-10,"f" +4,2012-11-11,"m" +5,2007-11-11,"" +6,2012-11-11,"" +7,2012-11-11,"mf" +8,2012-11-11,"fm" diff --git a/backend/src/test/resources/tests/sql/date_restriction/only_max/expected.csv b/backend/src/test/resources/tests/sql/date_restriction/only_max/expected.csv new file mode 100644 index 0000000000..c34539ab81 --- /dev/null +++ b/backend/src/test/resources/tests/sql/date_restriction/only_max/expected.csv @@ -0,0 +1,3 @@ +pid,datum +1,"[2012-01-01,2012-01-02)" +3,"[2012-11-10,2012-11-11)" diff --git a/backend/src/test/resources/tests/sql/date_restriction/only_max/only_max.spec.json b/backend/src/test/resources/tests/sql/date_restriction/only_max/only_max.spec.json new file mode 100644 index 0000000000..fbb6d2e1d0 --- /dev/null +++ b/backend/src/test/resources/tests/sql/date_restriction/only_max/only_max.spec.json @@ -0,0 +1,84 @@ +{ + "label": "Date restriction query with simple date validity date and only max date restriction set", + "type": "SQL_TEST", + "expectedCsv": "expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "type": "DATE_RESTRICTION", + "dateRange": { + "max": "2012-12-31" + }, + "child": { + "ids": [ + "geschlecht_select" + ], + "type": "CONCEPT", + "label": "Geschlecht SELECT", + "tables": [ + { + "id": "geschlecht_select.geschlecht_connector", + "filters": [ + { + "filter": "geschlecht_select.geschlecht_connector.geschlecht", + "type": "BIG_MULTI_SELECT", + "value": [ + "f" + ] + } + ] + } + ] + } + } + ] + } + }, + "concepts": [ + { + "label": "geschlecht_select", + "type": "TREE", + "connectors": [ + { + "label": "geschlecht_connector", + "table": "table1", + "validityDates": { + "label": "datum", + "column": "table1.datum" + }, + "filters": { + "label": "geschlecht", + "description": "Geschlecht zur gegebenen Datumseinschränkung", + "column": "table1.geschlecht", + "type": "SELECT" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/date_restriction/simple_date/content.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + }, + { + "name": "geschlecht", + "type": "STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/date_restriction/only_min/content.csv b/backend/src/test/resources/tests/sql/date_restriction/only_min/content.csv new file mode 100644 index 0000000000..212025dec4 --- /dev/null +++ b/backend/src/test/resources/tests/sql/date_restriction/only_min/content.csv @@ -0,0 +1,9 @@ +pid,datum,geschlecht +1,2012-01-01,"f" +2,2010-07-15,"m" +3,2012-11-10,"f" +4,2012-11-11,"m" +5,2007-11-11,"" +6,2012-11-11,"" +7,2012-11-11,"mf" +8,2012-11-11,"fm" diff --git a/backend/src/test/resources/tests/sql/date_restriction/only_min/expected.csv b/backend/src/test/resources/tests/sql/date_restriction/only_min/expected.csv new file mode 100644 index 0000000000..c34539ab81 --- /dev/null +++ b/backend/src/test/resources/tests/sql/date_restriction/only_min/expected.csv @@ -0,0 +1,3 @@ +pid,datum +1,"[2012-01-01,2012-01-02)" +3,"[2012-11-10,2012-11-11)" diff --git a/backend/src/test/resources/tests/sql/date_restriction/only_min/only_min.spec.json b/backend/src/test/resources/tests/sql/date_restriction/only_min/only_min.spec.json new file mode 100644 index 0000000000..76a93dafde --- /dev/null +++ b/backend/src/test/resources/tests/sql/date_restriction/only_min/only_min.spec.json @@ -0,0 +1,84 @@ +{ + "label": "Date restriction query with simple date validity date and only min date restriction set", + "type": "SQL_TEST", + "expectedCsv": "expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "AND", + "children": [ + { + "type": "DATE_RESTRICTION", + "dateRange": { + "min": "2012-01-01" + }, + "child": { + "ids": [ + "geschlecht_select" + ], + "type": "CONCEPT", + "label": "Geschlecht SELECT", + "tables": [ + { + "id": "geschlecht_select.geschlecht_connector", + "filters": [ + { + "filter": "geschlecht_select.geschlecht_connector.geschlecht", + "type": "BIG_MULTI_SELECT", + "value": [ + "f" + ] + } + ] + } + ] + } + } + ] + } + }, + "concepts": [ + { + "label": "geschlecht_select", + "type": "TREE", + "connectors": [ + { + "label": "geschlecht_connector", + "table": "table1", + "validityDates": { + "label": "datum", + "column": "table1.datum" + }, + "filters": { + "label": "geschlecht", + "description": "Geschlecht zur gegebenen Datumseinschränkung", + "column": "table1.geschlecht", + "type": "SELECT" + } + } + ] + } + ], + "content": { + "tables": [ + { + "csv": "tests/sql/date_restriction/simple_date/content.csv", + "name": "table1", + "primaryColumn": { + "name": "pid", + "type": "STRING" + }, + "columns": [ + { + "name": "datum", + "type": "DATE" + }, + { + "name": "geschlecht", + "type": "STRING" + } + ] + } + ] + } +} diff --git a/tutorial/mimic_iii_demo/age_gender.ipynb b/tutorial/mimic_iii_demo/age_gender.ipynb index f1a2f40929..2824cfb129 100644 --- a/tutorial/mimic_iii_demo/age_gender.ipynb +++ b/tutorial/mimic_iii_demo/age_gender.ipynb @@ -346,7 +346,7 @@ "source": [ "## Preprocessing and Upload\n", "\n", - "The next tutorial is to [Preprocess and Upload](./preprocess_and_upload.ipynb) all data and meta data produced from this notebook." + "The next tutorial is to [Preprocess and Upload](./preprocess_and_upload.ipynb) all data and meta data produced from this notebook.\n" ] } ], diff --git a/tutorial/mimic_iii_demo/icd9.ipynb b/tutorial/mimic_iii_demo/icd9.ipynb index 547875081c..69656e305e 100644 --- a/tutorial/mimic_iii_demo/icd9.ipynb +++ b/tutorial/mimic_iii_demo/icd9.ipynb @@ -470,7 +470,7 @@ "source": [ "## Preprocessing and Upload\n", "\n", - "The next tutorial is to [Preprocess and Upload](./preprocess_and_upload.ipynb) all data and meta data produced from this notebook." + "The next tutorial is to [Preprocess and Upload](./preprocess_and_upload.ipynb) all data and meta data produced from this notebook.\n" ] } ], From 16150e0952a342949cbfa651d9c733cfdd779518 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 15 Aug 2023 15:57:23 +0200 Subject: [PATCH 489/679] create a RejectionHandler that blocks the caller --- .../xodus/stores/SerializingStore.java | 9 ++--- .../util/CallerBlocksRejectionHandler.java | 33 +++++++++++++++++++ 2 files changed, 38 insertions(+), 4 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/util/CallerBlocksRejectionHandler.java diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index 46c85356c8..b86de46226 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -21,6 +21,7 @@ import com.bakdata.conquery.io.storage.Store; import com.bakdata.conquery.models.config.XodusStoreFactory; import com.bakdata.conquery.models.exceptions.ValidatorHelper; +import com.bakdata.conquery.util.CallerBlocksRejectionHandler; import com.bakdata.conquery.util.io.FileUtil; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -332,11 +333,11 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { final IterationStatistic result = new IterationStatistic(); final ArrayList unreadables = new ArrayList<>(); - final ThreadPoolExecutor executorService = new ThreadPoolExecutor(5, 5, - 60L, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(5), + final ThreadPoolExecutor executorService = new ThreadPoolExecutor(10, 10, + 0, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(50), Executors.defaultThreadFactory(), - new ThreadPoolExecutor.CallerRunsPolicy() + new CallerBlocksRejectionHandler(TimeUnit.MINUTES.toMillis(5)) ); store.forEach((k, v) -> { diff --git a/backend/src/main/java/com/bakdata/conquery/util/CallerBlocksRejectionHandler.java b/backend/src/main/java/com/bakdata/conquery/util/CallerBlocksRejectionHandler.java new file mode 100644 index 0000000000..b5c8a3355e --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/util/CallerBlocksRejectionHandler.java @@ -0,0 +1,33 @@ +package com.bakdata.conquery.util; + +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import lombok.Data; + +@Data +public class CallerBlocksRejectionHandler implements RejectedExecutionHandler { + + private final long timeoutMillis; + + @Override + public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { + if (executor.isShutdown()){ + return; + } + + try { + final boolean success = executor.getQueue().offer(r, getTimeoutMillis(), TimeUnit.MILLISECONDS); + + if(!success){ + throw new RejectedExecutionException("Could not submit within specified timeout."); + } + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RejectedExecutionException("Thread was interrupted."); + } + } +} From 9b26190aa15f53d630f5e19334f9fe84949170b3 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 15 Aug 2023 16:24:14 +0200 Subject: [PATCH 490/679] reduce core pool size and queue size, also measure time spent waiting --- .../io/storage/xodus/stores/SerializingStore.java | 9 ++++++--- .../conquery/util/CallerBlocksRejectionHandler.java | 10 ++++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index b86de46226..9adf10fd14 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -36,6 +36,7 @@ import lombok.SneakyThrows; import lombok.ToString; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.time.DurationFormatUtils; import org.jetbrains.annotations.NotNull; /** @@ -332,12 +333,13 @@ private static String sanitiseFileName(@NotNull String name) { public IterationStatistic forEach(StoreEntryConsumer consumer) { final IterationStatistic result = new IterationStatistic(); final ArrayList unreadables = new ArrayList<>(); + final CallerBlocksRejectionHandler rejectionHandler = new CallerBlocksRejectionHandler(TimeUnit.MINUTES.toMillis(5)); - final ThreadPoolExecutor executorService = new ThreadPoolExecutor(10, 10, + final ThreadPoolExecutor executorService = new ThreadPoolExecutor(2, 2, 0, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(50), + new ArrayBlockingQueue<>(10), Executors.defaultThreadFactory(), - new CallerBlocksRejectionHandler(TimeUnit.MINUTES.toMillis(5)) + rejectionHandler ); store.forEach((k, v) -> { @@ -390,6 +392,7 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { log.debug("Still waiting for {} to load.", this); } + log.debug("Waited {} on workers.", DurationFormatUtils.formatDurationHMS(rejectionHandler.getWaitedMillis().sum())); // Print some statistics final int total = result.getTotalProcessed(); log.debug( diff --git a/backend/src/main/java/com/bakdata/conquery/util/CallerBlocksRejectionHandler.java b/backend/src/main/java/com/bakdata/conquery/util/CallerBlocksRejectionHandler.java index b5c8a3355e..b6ce4b4556 100644 --- a/backend/src/main/java/com/bakdata/conquery/util/CallerBlocksRejectionHandler.java +++ b/backend/src/main/java/com/bakdata/conquery/util/CallerBlocksRejectionHandler.java @@ -4,6 +4,7 @@ import java.util.concurrent.RejectedExecutionHandler; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.LongAdder; import lombok.Data; @@ -11,17 +12,22 @@ public class CallerBlocksRejectionHandler implements RejectedExecutionHandler { private final long timeoutMillis; + private final LongAdder waitedMillis = new LongAdder(); @Override public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { - if (executor.isShutdown()){ + if (executor.isShutdown()) { return; } try { + long before = System.currentTimeMillis(); final boolean success = executor.getQueue().offer(r, getTimeoutMillis(), TimeUnit.MILLISECONDS); + long after = System.currentTimeMillis(); - if(!success){ + waitedMillis.add(after - before); + + if (!success) { throw new RejectedExecutionException("Could not submit within specified timeout."); } } From 89ad58463f9a3fe21c7fb3c7b14f39cdf93c5190 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Wed, 16 Aug 2023 12:02:08 +0200 Subject: [PATCH 491/679] use absolute position, fix bug with laying a concept within another --- .../DropzoneBetweenElements.tsx | 22 ++++----- .../form-components/DropzoneList.tsx | 46 ++++++++++++------- .../form-concept-group/FormConceptGroup.tsx | 32 ++++++------- 3 files changed, 52 insertions(+), 48 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index d6d9a0511a..4d54e98d08 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -7,6 +7,7 @@ interface Props { onDrop: (item: DroppableObject, monitor: DropTargetMonitor) => void; acceptedDropTypes: string[]; lastElement?: boolean; + top?: number; } const RootHeightBase = 30; @@ -15,13 +16,14 @@ const Root = styled("div")` width: 100%; left: 0; right: 0; - position: relative; + position: absolute; border-radius: ${({ theme }) => theme.borderRadius}; `; -const Line = styled("div")` +const Line = styled("div")<{show:boolean}>` overflow: hidden; display: block; + visibility: ${({show}) => show ? "visible" : "hidden"}; background-color: ${({ theme }) => theme.col.blueGrayDark}; margin: 1px 0; height: ${LineHeight}px; @@ -34,6 +36,7 @@ const DropzoneBetweenElements = < acceptedDropTypes, onDrop, lastElement, + top }: Props) => { const [{ isOver }, addZoneRef] = useDrop({ accept: acceptedDropTypes, @@ -44,24 +47,17 @@ const DropzoneBetweenElements = < }), }); - const rootHeightMultiplier = lastElement ? 0.5 : 1; - const rootDefaultMarginTop = (lastElement ? -15 : -5) - LineHeight; - const rootOverMarginTop = lastElement ? -23 : -10; - const rootDefaultTop = lastElement ? -5 : -10; - const rootOverTop = (lastElement ? -2 : -15) - LineHeight; + const rootHeightMultiplier = lastElement ? 0.7 : 1; return ( <> - {isOver && } + diff --git a/frontend/src/js/external-forms/form-components/DropzoneList.tsx b/frontend/src/js/external-forms/form-components/DropzoneList.tsx index 09e653a93b..2e890ef35e 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneList.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneList.tsx @@ -19,7 +19,7 @@ import DropzoneBetweenElements from "./DropzoneBetweenElements"; const ListItem = styled("div")` position: relative; - padding: 0 5px; + padding: 5px; box-shadow: 0 0 3px 0 rgba(0, 0, 0, 0.1); background-color: white; border-radius: ${({ theme }) => theme.borderRadius}; @@ -36,6 +36,10 @@ const Row = styled("div")` align-items: center; `; +const ConceptContainer = styled("div")` + position: relative; +`; + interface PropsT { className?: string; label?: ReactNode; @@ -78,6 +82,24 @@ const DropzoneList = ( const showDropzone = (items && items.length === 0) || !disallowMultipleColumns; + function genItems(){ + return items.map((item, i) => ( + + {!disallowMultipleColumns && ( + + )} + + onDelete(i)} /> + {item} + + + )) + } + return (
@@ -90,30 +112,20 @@ const DropzoneList = ( {tooltip && } {items && items.length > 0 && ( -
- {items.map((item, i) => ( -
- {!disallowMultipleColumns && ( - - )} - - onDelete(i)} /> - {item} - -
- ))} + <> + { genItems()} + {!disallowMultipleColumns && ( )} -
+ + )}
{showDropzone && onImportLines && ( diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index bf381b6db3..9ad9dfb435 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -87,9 +87,7 @@ interface Props { }) => ReactNode; } -const DropzoneListItem = styled("div")` - margin-top: -20px; -`; +const DropzoneListItem = styled("div")``; const Row = styled("div")` display: flex; @@ -448,29 +446,27 @@ const FormConceptGroup = (props: Props) => { }} onDropConcept={(concept) => { let { valueIdx, conceptIdx } = editedFormQueryNodePosition; + let updatedValue = props.value; if (isMovedObject(concept)) { + const { movedFromFieldName, movedFromAndIdx, movedFromOrIdx } = concept.dragContext; valueIdx = - valueIdx > concept.dragContext.movedFromAndIdx + valueIdx > movedFromAndIdx ? valueIdx - 1 : valueIdx; - if (concept.dragContext.movedFromFieldName === props.fieldName) { - const updatedValue = - props.value[concept.dragContext.movedFromAndIdx].concepts + if (movedFromFieldName === props.fieldName) { + updatedValue = + updatedValue[movedFromAndIdx].concepts .length === 1 ? removeValue( - props.value, - concept.dragContext.movedFromAndIdx, + updatedValue, + movedFromAndIdx, ) : removeConcept( - props.value, - concept.dragContext.movedFromAndIdx, - concept.dragContext.movedFromOrIdx, + updatedValue, + movedFromAndIdx, + movedFromOrIdx, ); - return props.onChange( - setConceptProperties(updatedValue, valueIdx, conceptIdx, { - ids: [...concept.ids, ...editedNode.ids], - }), - ); + setEditedFormQueryNodePosition({ valueIdx, conceptIdx }); } else { if (exists(concept.dragContext.deleteFromOtherField)) { concept.dragContext.deleteFromOtherField(); @@ -478,7 +474,7 @@ const FormConceptGroup = (props: Props) => { } } props.onChange( - setConceptProperties(props.value, valueIdx, conceptIdx, { + setConceptProperties(updatedValue, valueIdx, conceptIdx, { ids: [...concept.ids, ...editedNode.ids], }), ); From 98e19d94e16ae141fdc8119250d4a46778273c8a Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Wed, 16 Aug 2023 12:15:04 +0200 Subject: [PATCH 492/679] formatting --- .../DropzoneBetweenElements.tsx | 11 ++++---- .../form-components/DropzoneList.tsx | 26 +++++++++---------- .../form-concept-group/FormConceptGroup.tsx | 18 +++++-------- 3 files changed, 24 insertions(+), 31 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index 4d54e98d08..7d46cd7cb7 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -20,10 +20,10 @@ const Root = styled("div")` border-radius: ${({ theme }) => theme.borderRadius}; `; -const Line = styled("div")<{show:boolean}>` +const Line = styled("div")<{ show: boolean }>` overflow: hidden; display: block; - visibility: ${({show}) => show ? "visible" : "hidden"}; + visibility: ${({ show }) => (show ? "visible" : "hidden")}; background-color: ${({ theme }) => theme.col.blueGrayDark}; margin: 1px 0; height: ${LineHeight}px; @@ -36,7 +36,7 @@ const DropzoneBetweenElements = < acceptedDropTypes, onDrop, lastElement, - top + top, }: Props) => { const [{ isOver }, addZoneRef] = useDrop({ accept: acceptedDropTypes, @@ -51,12 +51,11 @@ const DropzoneBetweenElements = < return ( <> - + diff --git a/frontend/src/js/external-forms/form-components/DropzoneList.tsx b/frontend/src/js/external-forms/form-components/DropzoneList.tsx index 2e890ef35e..2222525592 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneList.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneList.tsx @@ -82,7 +82,7 @@ const DropzoneList = ( const showDropzone = (items && items.length === 0) || !disallowMultipleColumns; - function genItems(){ + function genItems() { return items.map((item, i) => ( {!disallowMultipleColumns && ( @@ -97,7 +97,7 @@ const DropzoneList = ( {item} - )) + )); } return ( @@ -113,18 +113,18 @@ const DropzoneList = ( {items && items.length > 0 && ( <> - { genItems()} + {genItems()} - - {!disallowMultipleColumns && ( - - )} - + + {!disallowMultipleColumns && ( + + )} + )}
diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 9ad9dfb435..1ef51b1073 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -448,21 +448,15 @@ const FormConceptGroup = (props: Props) => { let { valueIdx, conceptIdx } = editedFormQueryNodePosition; let updatedValue = props.value; if (isMovedObject(concept)) { - const { movedFromFieldName, movedFromAndIdx, movedFromOrIdx } = concept.dragContext; - valueIdx = - valueIdx > movedFromAndIdx - ? valueIdx - 1 - : valueIdx; + const { movedFromFieldName, movedFromAndIdx, movedFromOrIdx } = + concept.dragContext; + valueIdx = valueIdx > movedFromAndIdx ? valueIdx - 1 : valueIdx; if (movedFromFieldName === props.fieldName) { updatedValue = - updatedValue[movedFromAndIdx].concepts - .length === 1 - ? removeValue( - updatedValue, - movedFromAndIdx, - ) + updatedValue[movedFromAndIdx].concepts.length === 1 + ? removeValue(updatedValue, movedFromAndIdx) : removeConcept( - updatedValue, + updatedValue, movedFromAndIdx, movedFromOrIdx, ); From 4de97258140e9b41c988cf0819ec4f6ec4b93a6f Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Wed, 16 Aug 2023 12:17:01 +0200 Subject: [PATCH 493/679] inline --- .../form-components/DropzoneList.tsx | 34 ++++++++----------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneList.tsx b/frontend/src/js/external-forms/form-components/DropzoneList.tsx index 2222525592..7af7f1bf0c 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneList.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneList.tsx @@ -82,24 +82,6 @@ const DropzoneList = ( const showDropzone = (items && items.length === 0) || !disallowMultipleColumns; - function genItems() { - return items.map((item, i) => ( - - {!disallowMultipleColumns && ( - - )} - - onDelete(i)} /> - {item} - - - )); - } - return (
@@ -113,7 +95,21 @@ const DropzoneList = ( {items && items.length > 0 && ( <> - {genItems()} + {items.map((item, i) => ( + + {!disallowMultipleColumns && ( + + )} + + onDelete(i)} /> + {item} + + + ))} {!disallowMultipleColumns && ( From 6b5e7e9b2e07fc5348af24ab436b36b21af1c178 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 16 Aug 2023 12:25:40 +0200 Subject: [PATCH 494/679] log attrs for debugging --- .../java/com/bakdata/conquery/commands/MigrateCommand.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/commands/MigrateCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/MigrateCommand.java index f63c78daf9..e465501fc9 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/MigrateCommand.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/MigrateCommand.java @@ -105,11 +105,13 @@ public void configure(Subparser subparser) { @Override protected void run(io.dropwizard.setup.Environment environment, Namespace namespace, ConqueryConfig configuration) throws Exception { + log.debug("Attrs: {}", namespace.getAttrs()); + final File inStoreDirectory = namespace.get("in"); final File outStoreDirectory = namespace.get("out"); - final boolean inGzip = namespace.getBoolean("in-gzip"); - final boolean outGzip = namespace.getBoolean("out-gzip"); + final boolean inGzip = namespace.get("in-gzip"); + final boolean outGzip = namespace.get("out-gzip"); final long logsize = ((XodusStoreFactory) configuration.getStorage()).getXodus().getLogFileSize().toKilobytes(); From a0c263b7c36262cb216fc7661ea235fa39b48ab8 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 16 Aug 2023 12:59:12 +0200 Subject: [PATCH 495/679] fixes access to attrs --- .../java/com/bakdata/conquery/commands/MigrateCommand.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/commands/MigrateCommand.java b/backend/src/main/java/com/bakdata/conquery/commands/MigrateCommand.java index e465501fc9..5a5484184d 100644 --- a/backend/src/main/java/com/bakdata/conquery/commands/MigrateCommand.java +++ b/backend/src/main/java/com/bakdata/conquery/commands/MigrateCommand.java @@ -105,13 +105,11 @@ public void configure(Subparser subparser) { @Override protected void run(io.dropwizard.setup.Environment environment, Namespace namespace, ConqueryConfig configuration) throws Exception { - log.debug("Attrs: {}", namespace.getAttrs()); - final File inStoreDirectory = namespace.get("in"); final File outStoreDirectory = namespace.get("out"); - final boolean inGzip = namespace.get("in-gzip"); - final boolean outGzip = namespace.get("out-gzip"); + final boolean inGzip = namespace.get("in_gzip"); + final boolean outGzip = namespace.get("out_gzip"); final long logsize = ((XodusStoreFactory) configuration.getStorage()).getXodus().getLogFileSize().toKilobytes(); From 848adeeed788b6b9c706cb31fc78527999f6aaef Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 16 Aug 2023 13:02:56 +0200 Subject: [PATCH 496/679] simplify code in TreeConcept#findMostSpecificChild and improve logging --- .../datasets/concepts/tree/TreeConcept.java | 42 ++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/TreeConcept.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/TreeConcept.java index d500616780..92640e3213 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/TreeConcept.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/tree/TreeConcept.java @@ -152,35 +152,39 @@ private ConceptTreeChild findMostSpecificChild(String stringValue, CalculatedVal continue; } - if (match == null) { - match = n; + if (match != null) { + failed = true; + log.error("Value '{}' matches the two nodes {} and {} in the tree {} (row={}))" + , stringValue, match.getId(), n.getId(), n.getConcept().getId(), rowMap.getValue()); + continue; + } - if (n.getChildIndex() != null) { - ConceptTreeChild specificChild = n.getChildIndex().findMostSpecificChild(stringValue); + match = n; - if (specificChild != null) { - match = specificChild; - } - } + if (n.getChildIndex() == null) { + continue; } - else { - failed = true; - log.error("Value '{}' matches the two nodes {} and {} in the tree {} (row={}))" - , stringValue, match.getLabel(), n.getLabel(), n.getConcept().getLabel(), rowMap.getValue()); - // TODO Why don't we return null here and drop the `failed`-flag? + + final ConceptTreeChild specificChild = n.getChildIndex().findMostSpecificChild(stringValue); + + if (specificChild == null) { + continue; } + + match = specificChild; } if (failed) { return null; } - else if (match != null) { - best = match; - currentList = match.getChildren(); - } - else { - break; + + // Nothing better found below, so return best-so far match + if (match == null) { + return best; } + + best = match; + currentList = match.getChildren(); } return best; } From 8ea002c88457b55c9d36e0b2fbfcea44b96ddc7d Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Wed, 16 Aug 2023 14:11:41 +0200 Subject: [PATCH 497/679] fix highlighting in tooltip --- frontend/src/js/tooltip/Tooltip.tsx | 74 +++++++++++++++++++++++++---- 1 file changed, 64 insertions(+), 10 deletions(-) diff --git a/frontend/src/js/tooltip/Tooltip.tsx b/frontend/src/js/tooltip/Tooltip.tsx index 49bd6f8621..7c1e1b0188 100644 --- a/frontend/src/js/tooltip/Tooltip.tsx +++ b/frontend/src/js/tooltip/Tooltip.tsx @@ -1,9 +1,17 @@ import styled from "@emotion/styled"; import { faThumbtack, IconDefinition } from "@fortawesome/free-solid-svg-icons"; -import { ReactNode } from "react"; +import { + Children, + DetailedHTMLProps, + ElementType, + HTMLAttributes, + ReactElement, + ReactNode, +} from "react"; import Highlighter from "react-highlight-words"; import { useTranslation } from "react-i18next"; import Markdown from "react-markdown"; +import { ReactMarkdownProps } from "react-markdown/lib/complex-types"; import { useDispatch, useSelector } from "react-redux"; import remarkGfm from "remark-gfm"; @@ -158,6 +166,52 @@ const ConceptLabel = ({ ); }; +function isReactElement(element: any): element is ReactElement { + return ( + element && + typeof element === "object" && + element.hasOwnProperty("type") && + element.hasOwnProperty("props") + ); +} + +function highlight( + words: string[], + Element: Omit< + DetailedHTMLProps, HTMLElement>, + "ref" + > & + ReactMarkdownProps, +): ReactElement | null { + let children = Children.map(Element.children, (child): ReactElement => { + if (!child) return <>; + if (typeof child === "string") { + return HighlightedText({ words, text: child }); + } + if (typeof child === "number" || typeof child === "boolean") { + return <>{child}; + } + if (isReactElement(child)) { + if (Array.isArray(Element)) { + return child; + } + let TagName = child.type as ElementType; + return ( + + {highlight(words, child.props.children)} + + ); + } + return <>{child}; + }); + + if (Array.isArray(Element) || !Element.node) { + return <>{children}; + } + let TagName = Element.node?.tagName as ElementType; + return {children}; +} + const Tooltip = () => { const words = useSelector( (state) => state.conceptTrees.search.words || [], @@ -236,15 +290,15 @@ const Tooltip = () => { searchHighlight(node) - } - } + components={{ + // TODO: Won't work anymore with the latest react-markdown, because + // Try to use another package for highlighting that doesn't depend on a string + // or just highlight ourselves + p: (a) => highlight(words, a), + td: (a) => highlight(words, a), + b: (a) => highlight(words, a), + th: (a) => highlight(words, a), + }} > {info.value} From da7c6094b07a6864cf1730edc60627eb6fc87a99 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Wed, 16 Aug 2023 14:27:12 +0200 Subject: [PATCH 498/679] Simplify code --- frontend/src/js/tooltip/Tooltip.tsx | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/frontend/src/js/tooltip/Tooltip.tsx b/frontend/src/js/tooltip/Tooltip.tsx index 7c1e1b0188..d117729f56 100644 --- a/frontend/src/js/tooltip/Tooltip.tsx +++ b/frontend/src/js/tooltip/Tooltip.tsx @@ -6,7 +6,9 @@ import { ElementType, HTMLAttributes, ReactElement, + ReactFragment, ReactNode, + ReactPortal, } from "react"; import Highlighter from "react-highlight-words"; import { useTranslation } from "react-i18next"; @@ -166,9 +168,10 @@ const ConceptLabel = ({ ); }; -function isReactElement(element: any): element is ReactElement { +function isReactElement( + element: ReactFragment | ReactElement | ReactPortal | boolean | number, +): element is ReactElement { return ( - element && typeof element === "object" && element.hasOwnProperty("type") && element.hasOwnProperty("props") @@ -188,13 +191,7 @@ function highlight( if (typeof child === "string") { return HighlightedText({ words, text: child }); } - if (typeof child === "number" || typeof child === "boolean") { - return <>{child}; - } if (isReactElement(child)) { - if (Array.isArray(Element)) { - return child; - } let TagName = child.type as ElementType; return ( From 8dfed4fbe8211cc9def7d55e7b400b2c5a0cf176 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 16 Aug 2023 16:27:24 +0200 Subject: [PATCH 499/679] move multithreaded reading into separate class, hopefully avoiding a lot of references --- .../xodus/stores/SerializingStore.java | 186 ++++++++++++------ 1 file changed, 123 insertions(+), 63 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index 9adf10fd14..24880ff5a9 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -7,10 +7,14 @@ import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Collection; +import java.util.List; import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.function.Supplier; @@ -21,7 +25,6 @@ import com.bakdata.conquery.io.storage.Store; import com.bakdata.conquery.models.config.XodusStoreFactory; import com.bakdata.conquery.models.exceptions.ValidatorHelper; -import com.bakdata.conquery.util.CallerBlocksRejectionHandler; import com.bakdata.conquery.util.io.FileUtil; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; @@ -32,11 +35,11 @@ import jetbrains.exodus.ArrayByteIterable; import jetbrains.exodus.ByteIterable; import lombok.Data; +import lombok.NoArgsConstructor; import lombok.NonNull; import lombok.SneakyThrows; import lombok.ToString; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.time.DurationFormatUtils; import org.jetbrains.annotations.NotNull; /** @@ -333,66 +336,28 @@ private static String sanitiseFileName(@NotNull String name) { public IterationStatistic forEach(StoreEntryConsumer consumer) { final IterationStatistic result = new IterationStatistic(); final ArrayList unreadables = new ArrayList<>(); - final CallerBlocksRejectionHandler rejectionHandler = new CallerBlocksRejectionHandler(TimeUnit.MINUTES.toMillis(5)); - - final ThreadPoolExecutor executorService = new ThreadPoolExecutor(2, 2, - 0, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(10), - Executors.defaultThreadFactory(), - rejectionHandler - ); - - store.forEach((k, v) -> { - executorService.submit(() -> { - - result.incrTotalProcessed(); - - // Try to read the key first - final KEY key = getDeserializedAndDumpFailed( - k, - this::readKey, - () -> new String(k.getBytesUnsafe()), - v, - "Could not parse key [{}]" - ); - if (key == null) { - unreadables.add(k); - result.incrFailedKeys(); - return; - } - // Try to read the value - final VALUE value = getDeserializedAndDumpFailed( - v, - this::readValue, - key::toString, - v, - "Could not parse value for key [{}]" - ); - - if (value == null) { - unreadables.add(k); - result.incrFailedValues(); - return; - } + final int nWorkers = 5; - // Apply the consumer to key and value - try { - consumer.accept(key, value, v.getLength()); - } - catch (Exception e) { - log.warn("Unable to apply for-each consumer on key[{}]", key, e); - } - }); - }); + final BlockingQueue workQueue = new ArrayBlockingQueue<>(nWorkers * 5); + final AtomicBoolean done = new AtomicBoolean(false); + + final ExecutorService executorService = Executors.newFixedThreadPool(nWorkers); + + for (int ignored = 0; ignored < nWorkers; ignored++) { + final Reader reader = new Reader(workQueue, done, consumer, result, unreadables); + executorService.submit(reader::run); + } + + store.forEach((k, v) -> workQueue.add(new Pair(k, v))); executorService.shutdown(); + done.set(true); - while (!executorService.awaitTermination(1, TimeUnit.MINUTES)) { - log.debug("Still waiting for {} to load.", this); + while(!executorService.awaitTermination(30, TimeUnit.SECONDS)){ + log.debug("Still waiting for {} jobs.", workQueue.size()); } - log.debug("Waited {} on workers.", DurationFormatUtils.formatDurationHMS(rejectionHandler.getWaitedMillis().sum())); // Print some statistics final int total = result.getTotalProcessed(); log.debug( @@ -494,22 +459,117 @@ public void close() { store.close(); } - @Data + @NoArgsConstructor public static class IterationStatistic { - private int totalProcessed; - private int failedKeys; - private int failedValues; + private final AtomicInteger totalProcessed = new AtomicInteger(); + private final AtomicInteger failedKeys = new AtomicInteger(); + private final AtomicInteger failedValues = new AtomicInteger(); public void incrTotalProcessed() { - totalProcessed++; + totalProcessed.incrementAndGet(); } public void incrFailedKeys() { - failedKeys++; + failedKeys.incrementAndGet(); } public void incrFailedValues() { - failedValues++; + failedValues.incrementAndGet(); + } + + public void setTotalProcessed(int totalProcessed) { + this.totalProcessed.set(totalProcessed); + } + + public void setFailedKeys(int failedKeys) { + this.failedKeys.set(failedKeys); + } + + public void setFailedValues(int failedValues) { + this.failedValues.set(failedValues); + } + + public int getFailedKeys() { + return failedKeys.get(); + } + + public int getFailedValues() { + return failedValues.get(); + } + + public int getTotalProcessed() { + return totalProcessed.get(); + } + } + + private record Pair(ByteIterable key, ByteIterable value) { + } + + @Data + private class Reader { + private final BlockingQueue queue; + private final AtomicBoolean done; + private final StoreEntryConsumer consumer; + private final IterationStatistic result; + private final List unreadables; + + public void run() { + + while (!done.get() || !queue.isEmpty()) { + try { + final Pair next = queue.poll(100, TimeUnit.MILLISECONDS); + + if (next == null) { + continue; + } + + handle(consumer, result, unreadables, next.key, next.value); + } + catch (Exception exception) { + log.warn("", exception); + } + } + } + + private void handle(StoreEntryConsumer consumer, IterationStatistic result, List unreadables, ByteIterable k, ByteIterable v) { + result.incrTotalProcessed(); + + // Try to read the key first + final KEY key = getDeserializedAndDumpFailed( + k, + SerializingStore.this::readKey, + () -> new String(k.getBytesUnsafe()), + v, + "Could not parse key [{}]" + ); + if (key == null) { + unreadables.add(k); + result.incrFailedKeys(); + return; + } + + // Try to read the value + final VALUE value = getDeserializedAndDumpFailed( + v, + SerializingStore.this::readValue, + key::toString, + v, + "Could not parse value for key [{}]" + ); + + if (value == null) { + unreadables.add(k); + result.incrFailedValues(); + return; + } + + // Apply the consumer to key and value + try { + consumer.accept(key, value, v.getLength()); + } + catch (Exception e) { + log.warn("Unable to apply for-each consumer on key[{}]", key, e); + } } } } From 3230f8c2644164f0f72cfdf92107cdc005fe7469 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 16 Aug 2023 16:30:29 +0200 Subject: [PATCH 500/679] fixes using wrong insertion method --- .../io/storage/xodus/stores/SerializingStore.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index 24880ff5a9..dac11b19b2 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -349,7 +349,14 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { executorService.submit(reader::run); } - store.forEach((k, v) -> workQueue.add(new Pair(k, v))); + store.forEach((k, v) -> { + try { + workQueue.put(new Pair(k, v)); + } + catch (InterruptedException e) { + throw new RuntimeException(e); + } + }); executorService.shutdown(); done.set(true); From 62d7385527066f3fb2858c66d54ad0be5c4b772a Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 16 Aug 2023 16:36:03 +0200 Subject: [PATCH 501/679] increase both workers and buffers per worker --- .../conquery/io/storage/xodus/stores/SerializingStore.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index dac11b19b2..90236e6ac1 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -337,9 +337,9 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { final IterationStatistic result = new IterationStatistic(); final ArrayList unreadables = new ArrayList<>(); - final int nWorkers = 5; + final int nWorkers = 10; - final BlockingQueue workQueue = new ArrayBlockingQueue<>(nWorkers * 5); + final BlockingQueue workQueue = new ArrayBlockingQueue<>(nWorkers * 20); final AtomicBoolean done = new AtomicBoolean(false); final ExecutorService executorService = Executors.newFixedThreadPool(nWorkers); From 102f03225af871fceeac23cbf8272d7b7311ebaf Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Wed, 16 Aug 2023 16:42:46 +0200 Subject: [PATCH 502/679] Allow for li elements --- frontend/src/js/tooltip/Tooltip.tsx | 41 ++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/frontend/src/js/tooltip/Tooltip.tsx b/frontend/src/js/tooltip/Tooltip.tsx index d117729f56..7c782f4229 100644 --- a/frontend/src/js/tooltip/Tooltip.tsx +++ b/frontend/src/js/tooltip/Tooltip.tsx @@ -178,6 +178,16 @@ function isReactElement( ); } +type MarkdownElement = + | ReactFragment + | ReactElement + | ReactPortal + | boolean + | number + | boolean + | string + | null + | undefined; function highlight( words: string[], Element: Omit< @@ -186,7 +196,10 @@ function highlight( > & ReactMarkdownProps, ): ReactElement | null { - let children = Children.map(Element.children, (child): ReactElement => { + if (!Element) { + return Element; + } + const mappingFunction = (child: MarkdownElement): ReactElement => { if (!child) return <>; if (typeof child === "string") { return HighlightedText({ words, text: child }); @@ -200,13 +213,18 @@ function highlight( ); } return <>{child}; - }); + }; + + if (Array.isArray(Element)) { + return <>{Children.map(Element, mappingFunction)}; + } - if (Array.isArray(Element) || !Element.node) { - return <>{children}; + if (typeof Element === "object" && Element.hasOwnProperty("children")) { + let children = Children.map(Element.children, mappingFunction); + let TagName = Element.node?.tagName as ElementType; + return {children}; } - let TagName = Element.node?.tagName as ElementType; - return {children}; + return <>{Element}; } const Tooltip = () => { @@ -291,10 +309,13 @@ const Tooltip = () => { // TODO: Won't work anymore with the latest react-markdown, because // Try to use another package for highlighting that doesn't depend on a string // or just highlight ourselves - p: (a) => highlight(words, a), - td: (a) => highlight(words, a), - b: (a) => highlight(words, a), - th: (a) => highlight(words, a), + p: (el) => highlight(words, el), + td: (el) => highlight(words, el), + b: (el) => highlight(words, el), + th: (el) => highlight(words, el), + i: (el) => highlight(words, el), + ul: (el) => highlight(words, el), + ol: (el) => highlight(words, el), }} > {info.value} From b11ab637245b4eadbf2e18fa2d7c088c3fd0c908 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Wed, 16 Aug 2023 16:48:11 +0200 Subject: [PATCH 503/679] add headers --- frontend/src/js/tooltip/Tooltip.tsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/js/tooltip/Tooltip.tsx b/frontend/src/js/tooltip/Tooltip.tsx index 7c782f4229..15fbe0fb9b 100644 --- a/frontend/src/js/tooltip/Tooltip.tsx +++ b/frontend/src/js/tooltip/Tooltip.tsx @@ -316,6 +316,8 @@ const Tooltip = () => { i: (el) => highlight(words, el), ul: (el) => highlight(words, el), ol: (el) => highlight(words, el), + h1: (el) => highlight(words, el), + h2: (el) => highlight(words, el), }} > {info.value} From 3139b38e35bfb96b95823c1e079999485cd9f9f5 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Wed, 16 Aug 2023 17:04:59 +0200 Subject: [PATCH 504/679] fix regression for tables --- frontend/src/js/tooltip/Tooltip.tsx | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/frontend/src/js/tooltip/Tooltip.tsx b/frontend/src/js/tooltip/Tooltip.tsx index 15fbe0fb9b..28c29c0e7c 100644 --- a/frontend/src/js/tooltip/Tooltip.tsx +++ b/frontend/src/js/tooltip/Tooltip.tsx @@ -219,12 +219,13 @@ function highlight( return <>{Children.map(Element, mappingFunction)}; } - if (typeof Element === "object" && Element.hasOwnProperty("children")) { - let children = Children.map(Element.children, mappingFunction); - let TagName = Element.node?.tagName as ElementType; - return {children}; - } - return <>{Element}; + let children = + typeof Element === "object" && Element.hasOwnProperty("children") + ? Children.map(Element.children, mappingFunction) + : Element.children; + + let TagName = Element.node?.tagName as ElementType; + return {children}; } const Tooltip = () => { @@ -306,9 +307,6 @@ const Tooltip = () => { highlight(words, el), td: (el) => highlight(words, el), b: (el) => highlight(words, el), From 2f51ab2e4520f0db2d96404e505217b935042e69 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 16 Aug 2023 18:03:26 +0200 Subject: [PATCH 505/679] properly parametrize nWorkers and buffer size, and cleanup --- .../io/storage/xodus/stores/BigStore.java | 9 +-- .../xodus/stores/KeyIncludingStore.java | 9 +-- .../xodus/stores/SerializingStore.java | 52 ++++++++++------ .../models/auth/apitoken/TokenStorage.java | 4 +- .../auth/basic/LocalAuthenticationRealm.java | 2 +- .../models/config/XodusStoreFactory.java | 61 ++++++++++++------- .../io/storage/xodus/stores/BigStoreTest.java | 4 +- .../stores/SerializingStoreDumpTest.java | 2 +- 8 files changed, 85 insertions(+), 58 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java index 556c561b6d..8fbc3b38b1 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java @@ -13,6 +13,7 @@ import java.util.Iterator; import java.util.List; import java.util.UUID; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import java.util.stream.Stream; @@ -62,7 +63,7 @@ public BigStore(XodusStoreFactory config, StoreInfo storeInfo, Consumer storeCloseHook, Consumer storeRemoveHook, - ObjectMapper mapper) { + ObjectMapper mapper, int nWorkers, int bufferPerWorker) { this.storeInfo = storeInfo; // Recommendation by the author of Xodus is to have logFileSize at least be 4 times the biggest file size. @@ -77,7 +78,7 @@ public BigStore(XodusStoreFactory config, BigStoreMetaKeys.class, config.isValidateOnWrite(), config.isRemoveUnreadableFromStore(), - config.getUnreadableDataDumpDirectory() + config.getUnreadableDataDumpDirectory(), nWorkers, bufferPerWorker ); @@ -90,7 +91,7 @@ public BigStore(XodusStoreFactory config, byte[].class, config.isValidateOnWrite(), config.isRemoveUnreadableFromStore(), - config.getUnreadableDataDumpDirectory() + config.getUnreadableDataDumpDirectory(), nWorkers, bufferPerWorker ); @@ -158,7 +159,7 @@ public Collection getAll() { @Override public Collection getAllKeys() { - List out = new ArrayList<>(); + Collection out = new ConcurrentLinkedQueue<>(); // has to be concurrent because forEach is concurrent. metaStore.forEach((key, value, size) -> out.add(key)); return out; } diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/KeyIncludingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/KeyIncludingStore.java index 9d1dbdf6ca..585eb6460d 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/KeyIncludingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/KeyIncludingStore.java @@ -1,11 +1,10 @@ package com.bakdata.conquery.io.storage.xodus.stores; -import com.bakdata.conquery.io.storage.Store; - import java.io.Closeable; import java.io.IOException; import java.util.Collection; -import java.util.function.Consumer; + +import com.bakdata.conquery.io.storage.Store; public abstract class KeyIncludingStore implements Closeable { @@ -26,9 +25,7 @@ public VALUE get(KEY key) { return store.get(key); } - public void forEach(Consumer consumer) { - store.forEach((key, value, size) -> consumer.accept(value)); - } + public void update(VALUE value) { updated(value); diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index 90236e6ac1..6de77f12ce 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -5,18 +5,18 @@ import java.io.PrintStream; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; -import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.function.Supplier; +import java.util.stream.IntStream; import javax.validation.Validator; @@ -106,6 +106,8 @@ public class SerializingStore implements Store { private final boolean removeUnreadablesFromUnderlyingStore; private final ObjectMapper objectMapper; + private final int nWorkers; + private final int bufferPerWorker; public , CLASS_V extends Class> SerializingStore(XodusStore store, Validator validator, @@ -114,7 +116,7 @@ public , CLASS_V extends Class> SerializingSto CLASS_V valueType, boolean validateOnWrite, boolean removeUnreadableFromStore, - File unreadableDataDumpDirectory) { + File unreadableDataDumpDirectory, int nWorkers, int bufferPerWorker) { this.store = store; this.validator = validator; this.validateOnWrite = validateOnWrite; @@ -135,6 +137,9 @@ public , CLASS_V extends Class> SerializingSto unreadableValuesDumpDir = unreadableDataDumpDirectory; + this.nWorkers = nWorkers; + this.bufferPerWorker = bufferPerWorker; + if (shouldDumpUnreadables()) { if (!unreadableValuesDumpDir.exists() && !unreadableValuesDumpDir.mkdirs()) { throw new IllegalStateException("Could not create dump directory: " + unreadableValuesDumpDir); @@ -247,8 +252,6 @@ private static void dumpToFile(@NonNull ByteIterable obj, @NonNull String keyOfD throw new IllegalStateException("Could not create `%s`.".formatted(dumpfile.getParentFile())); } - //TODO FK: dump in a separate thread so we are not blocking the reader thread. - // Write json try { log.info("Dumping value of key {} to {} (because it cannot be deserialized anymore).", keyOfDump, dumpfile.getCanonicalPath()); @@ -330,36 +333,38 @@ private static String sanitiseFileName(@NotNull String name) { * Iterates a given consumer over the entries of this store. * Depending on the {@link XodusStoreFactory} corrupt entries may be dump to a file and/or removed from the store. * These entries are not submitted to the consumer. + * + * @implNote This method is concurrent! */ @SneakyThrows @Override public IterationStatistic forEach(StoreEntryConsumer consumer) { final IterationStatistic result = new IterationStatistic(); - final ArrayList unreadables = new ArrayList<>(); - - final int nWorkers = 10; + final Collection unreadables = new ConcurrentLinkedQueue<>(); - final BlockingQueue workQueue = new ArrayBlockingQueue<>(nWorkers * 20); - final AtomicBoolean done = new AtomicBoolean(false); + // Some magic number of buffering per worker, that isn't so high, that we fill up RAM with useless stuff, but have enough data to keep the workers occupied. + final BlockingQueue workQueue = new ArrayBlockingQueue<>(nWorkers * bufferPerWorker); final ExecutorService executorService = Executors.newFixedThreadPool(nWorkers); - for (int ignored = 0; ignored < nWorkers; ignored++) { - final Reader reader = new Reader(workQueue, done, consumer, result, unreadables); - executorService.submit(reader::run); - } + final List readers = IntStream.range(0, nWorkers) + .mapToObj(ignored -> new Reader(workQueue, consumer, result, unreadables)) + .peek(reader -> executorService.submit(reader::run)) + .toList(); + // We read in single thread, and deserialise and dispatch in multiple threads. store.forEach((k, v) -> { try { workQueue.put(new Pair(k, v)); } catch (InterruptedException e) { + //TODO wat do? throw new RuntimeException(e); } }); executorService.shutdown(); - done.set(true); + readers.forEach(Reader::finish); while(!executorService.awaitTermination(30, TimeUnit.SECONDS)){ log.debug("Still waiting for {} jobs.", workQueue.size()); @@ -466,8 +471,10 @@ public void close() { store.close(); } + @NoArgsConstructor public static class IterationStatistic { + //TODO move into reader? private final AtomicInteger totalProcessed = new AtomicInteger(); private final AtomicInteger failedKeys = new AtomicInteger(); private final AtomicInteger failedValues = new AtomicInteger(); @@ -515,16 +522,16 @@ private record Pair(ByteIterable key, ByteIterable value) { @Data private class Reader { private final BlockingQueue queue; - private final AtomicBoolean done; + private boolean done = false; private final StoreEntryConsumer consumer; private final IterationStatistic result; - private final List unreadables; + private final Collection unreadables; public void run() { - while (!done.get() || !queue.isEmpty()) { + while (!done || !queue.isEmpty()) { try { - final Pair next = queue.poll(100, TimeUnit.MILLISECONDS); + final Pair next = queue.poll(1, TimeUnit.SECONDS); if (next == null) { continue; @@ -533,12 +540,17 @@ public void run() { handle(consumer, result, unreadables, next.key, next.value); } catch (Exception exception) { + //TODO probably split for InterrupedException? No idea how to handle that though log.warn("", exception); } } } - private void handle(StoreEntryConsumer consumer, IterationStatistic result, List unreadables, ByteIterable k, ByteIterable v) { + public void finish() { + done = true; + } + + private void handle(StoreEntryConsumer consumer, IterationStatistic result, Collection unreadables, ByteIterable k, ByteIterable v) { result.incrTotalProcessed(); // Try to read the key first diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/apitoken/TokenStorage.java b/backend/src/main/java/com/bakdata/conquery/models/auth/apitoken/TokenStorage.java index c42b24ec0c..9d4ec129d7 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/apitoken/TokenStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/apitoken/TokenStorage.java @@ -72,7 +72,7 @@ public void start(){ ApiTokenData.class, true, false, - null + null, 1, 100 )); openStoresInEnv.add(data); @@ -90,7 +90,7 @@ public void start(){ ApiTokenData.MetaData.class, true, false, - null + null, 1, 100 )); openStoresInEnv.add(meta); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/LocalAuthenticationRealm.java b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/LocalAuthenticationRealm.java index 2632931fc8..6fca172e9b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/LocalAuthenticationRealm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/LocalAuthenticationRealm.java @@ -108,7 +108,7 @@ protected void onInit() { PasswordHasher.HashedEntry.class, false, true, - null + null, 1, 100 )); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index 437ba9c2b9..068ca1363d 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -16,6 +16,7 @@ import javax.annotation.Nullable; import javax.validation.Valid; import javax.validation.Validator; +import javax.validation.constraints.Min; import javax.validation.constraints.NotNull; import com.bakdata.conquery.io.cps.CPSType; @@ -125,32 +126,45 @@ public class XodusStoreFactory implements StoreFactory { private Path directory = Path.of("storage"); - private boolean validateOnWrite = false; + private boolean validateOnWrite; @NotNull @Valid private XodusConfig xodus = new XodusConfig(); - private boolean useWeakDictionaryCaching = false; + /** + * Number of threads reading from XoduStore. + * @implNote it's always only one thread reading from disk, dispatching to multiple reader threads. + */ + @Min(1) + private int readerWorkers = 10; + + /** + * How many slots of buffering to use before the IO thread is put to sleep. + */ + @Min(1) + private int bufferPerWorker = 20; + + private boolean useWeakDictionaryCaching; @NotNull private Duration weakCacheDuration = Duration.hours(48); /** * Flag for the {@link SerializingStore} whether to delete values from the underlying store, that cannot be mapped to an object anymore. */ - private boolean removeUnreadableFromStore = false; + private boolean removeUnreadableFromStore; /** * When set, all values that could not be deserialized from the persistent store, are dump into individual files. */ @Nullable - private File unreadableDataDumpDirectory = null; + private File unreadableDataDumpDirectory; /** * If set, an environment will not be loaded if it misses a required store. * If not set, the environment is loaded and the application needs to create the store. * This is useful if a new version introduces a new store, but will also alter the environment upon reading. */ - private boolean loadEnvironmentWithMissingStores = false; + private boolean loadEnvironmentWithMissingStores; @JsonIgnore private transient Validator validator; @@ -175,13 +189,13 @@ public Collection discoverWorkerStorages() { private List loadNamespacedStores(String prefix, Function creator, Set storesToTest) { - File baseDir = getDirectory().toFile(); + final File baseDir = getDirectory().toFile(); if (baseDir.mkdirs()) { log.warn("Had to create Storage Dir at `{}`", baseDir); } - List storages = new ArrayList<>(); + final List storages = new ArrayList<>(); for (File directory : Objects.requireNonNull(baseDir.listFiles((file, name) -> file.isDirectory() && name.startsWith(prefix)))) { @@ -194,7 +208,7 @@ private List loadNamespacedStores(String prefix continue; } - T namespacedStorage = creator.apply(name); + final T namespacedStorage = creator.apply(name); storages.add(namespacedStorage); } @@ -203,8 +217,8 @@ private List loadNamespacedStores(String prefix } private boolean environmentHasStores(File pathName, Set storesToTest) { - Environment env = findEnvironment(pathName); - boolean exists = env.computeInTransaction(t -> { + final Environment env = findEnvironment(pathName); + final boolean exists = env.computeInTransaction(t -> { final List allStoreNames = env.getAllStoreNames(t); final boolean complete = new HashSet<>(allStoreNames).containsAll(storesToTest); if (complete) { @@ -273,7 +287,7 @@ public IdentifiableStore createDictionaryStore(CentralRegistry centr DICTIONARIES.storeInfo(), this::closeStore, this::removeStore, - centralRegistry.injectIntoNew(objectMapper) + centralRegistry.injectIntoNew(objectMapper), getReaderWorkers(), getBufferPerWorker() ); openStoresInEnv.put(bigStore.getDataXodusStore().getEnvironment(), bigStore.getDataXodusStore()); openStoresInEnv.put(bigStore.getMetaXodusStore().getEnvironment(), bigStore.getMetaXodusStore()); @@ -316,7 +330,7 @@ public SingletonStore createIdMappingStore(String pathName, ObjectM synchronized (openStoresInEnv) { final BigStore bigStore = - new BigStore<>(this, validator, environment, ID_MAPPING.storeInfo(), this::closeStore, this::removeStore, objectMapper); + new BigStore<>(this, validator, environment, ID_MAPPING.storeInfo(), this::closeStore, this::removeStore, objectMapper, 10, 20); openStoresInEnv.put(bigStore.getDataXodusStore().getEnvironment(), bigStore.getDataXodusStore()); openStoresInEnv.put(bigStore.getMetaXodusStore().getEnvironment(), bigStore.getMetaXodusStore()); @@ -396,14 +410,14 @@ private Environment findEnvironment(@NonNull File path) { } private Environment findEnvironment(String pathName) { - File path = getStorageDir(pathName); + final File path = getStorageDir(pathName); return findEnvironment(path); } private void closeStore(XodusStore store) { - Environment env = store.getEnvironment(); + final Environment env = store.getEnvironment(); synchronized (openStoresInEnv) { - Collection stores = openStoresInEnv.get(env); + final Collection stores = openStoresInEnv.get(env); stores.remove(store); log.info("Closed XodusStore: {}", store); @@ -427,9 +441,9 @@ private void closeEnvironment(Environment env) { } private void removeStore(XodusStore store) { - Environment env = store.getEnvironment(); + final Environment env = store.getEnvironment(); synchronized (openStoresInEnv){ - Collection stores = openStoresInEnv.get(env); + final Collection stores = openStoresInEnv.get(env); stores.remove(store); @@ -467,9 +481,11 @@ public Store createStore(Environment environment, Valid if(openStoresInEnv.get(environment).stream().map(XodusStore::getName).anyMatch(name -> storeInfo.getName().equals(name))){ throw new IllegalStateException("Attempted to open an already opened store:" + storeInfo.getName()); } - final XodusStore store = - new XodusStore(environment, storeInfo.getName(), this::closeStore, this::removeStore); + + final XodusStore store = new XodusStore(environment, storeInfo.getName(), this::closeStore, this::removeStore); + openStoresInEnv.put(environment, store); + return new CachedStore<>( new SerializingStore<>( store, @@ -477,9 +493,10 @@ public Store createStore(Environment environment, Valid objectMapper, storeInfo.getKeyType(), storeInfo.getValueType(), - this.isValidateOnWrite(), - this.isRemoveUnreadableFromStore(), - this.getUnreadableDataDumpDirectory() + isValidateOnWrite(), + isRemoveUnreadableFromStore(), + getUnreadableDataDumpDirectory(), + getReaderWorkers(), getBufferPerWorker() )); } } diff --git a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/BigStoreTest.java b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/BigStoreTest.java index f27b1a1aa6..be92d2e953 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/BigStoreTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/BigStoreTest.java @@ -64,7 +64,7 @@ public void destroy() throws IOException { public void testFull() throws JSONException, IOException { BigStore store = new BigStore<>(new XodusStoreFactory(), Validators.newValidator(), env, - StoreMappings.DICTIONARIES.storeInfo(), (e) -> {}, (e) -> {}, MAPPER + StoreMappings.DICTIONARIES.storeInfo(), (e) -> {}, (e) -> {}, MAPPER, 10, 20 ); @@ -106,7 +106,7 @@ public void testFull() throws JSONException, IOException { @Test public void testEmpty() throws JSONException, IOException { BigStore store = new BigStore<>(new XodusStoreFactory(), Validators.newValidator(), env, - StoreMappings.DICTIONARIES.storeInfo(), (e) -> {}, (e) -> {}, MAPPER + StoreMappings.DICTIONARIES.storeInfo(), (e) -> {}, (e) -> {}, MAPPER, 10, 20 ); store.setChunkByteSize(Ints.checkedCast(DataSize.megabytes(1).toBytes())); diff --git a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java index e06b241ebb..5d92970875 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java @@ -72,7 +72,7 @@ private SerializingStore createSerializedStore(XodusSto storeId.getValueType(), config.isValidateOnWrite(), config.isRemoveUnreadableFromStore(), - config.getUnreadableDataDumpDirectory() + config.getUnreadableDataDumpDirectory(), 1, 1 ); } From 079e9c2eccb5b856fa1b199e8a62dd243ec0c05f Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Wed, 16 Aug 2023 18:27:12 +0200 Subject: [PATCH 506/679] Improve select validation in concept list fields --- frontend/src/js/external-forms/form/Field.tsx | 20 +++++-- frontend/src/js/external-forms/validators.ts | 53 ++++++++++++++++--- frontend/src/js/model/select.ts | 14 ++++- .../src/js/query-node-editor/NodeSelects.tsx | 5 +- .../src/js/query-node-editor/TableSelects.tsx | 11 ++-- .../InputSelect/SelectListOption.tsx | 10 ++-- frontend/src/localization/de.json | 3 +- frontend/src/localization/en.json | 3 +- 8 files changed, 89 insertions(+), 30 deletions(-) diff --git a/frontend/src/js/external-forms/form/Field.tsx b/frontend/src/js/external-forms/form/Field.tsx index c39a051775..48776d37c1 100644 --- a/frontend/src/js/external-forms/form/Field.tsx +++ b/frontend/src/js/external-forms/form/Field.tsx @@ -73,12 +73,23 @@ type Props = T & { noContainer?: boolean; noLabel?: boolean; }; -const FieldContainer = styled("div")<{ noLabel?: boolean }>` +const FieldContainer = styled("div")<{ noLabel?: boolean; hasError?: boolean }>` + display: flex; + flex-direction: column; + gap: 5px; padding: ${({ noLabel }) => (noLabel ? "7px 10px" : "2px 10px 7px")}; background-color: white; border-radius: ${({ theme }) => theme.borderRadius}; - border: 1px solid ${({ theme }) => theme.col.grayLight}; + border: 1px solid + ${({ theme, hasError }) => (hasError ? theme.col.red : theme.col.grayLight)}; `; + +const ErrorContainer = styled("div")` + color: ${({ theme }) => theme.col.red}; + font-weight: 700; + font-size: ${({ theme }) => theme.font.sm}; +`; + const ConnectedField = ({ children, control, @@ -89,7 +100,7 @@ const ConnectedField = ({ ...props }: Props) => { const { t } = useTranslation(); - const { field } = useController({ + const { field, fieldState } = useController({ name: formField.name, defaultValue, control, @@ -105,8 +116,9 @@ const ConnectedField = ({ return noContainer ? (
{children({ ...field, ...props })}
) : ( - + {children({ ...field, ...props })} + {fieldState.error?.message} ); }; diff --git a/frontend/src/js/external-forms/validators.ts b/frontend/src/js/external-forms/validators.ts index 50abbdc670..b70169a8ce 100644 --- a/frontend/src/js/external-forms/validators.ts +++ b/frontend/src/js/external-forms/validators.ts @@ -1,8 +1,16 @@ import { TFunction } from "i18next"; import { isEmpty } from "../common/helpers/commonHelper"; +import { exists } from "../common/helpers/exists"; +import { isValidSelect } from "../model/select"; -import { CheckboxField, Field, FormField } from "./config-types"; +import { + CheckboxField, + ConceptListField, + Field, + FormField, +} from "./config-types"; +import { FormConceptGroupT } from "./form-concept-group/formConceptGroupState"; export const validateRequired = (t: TFunction, value: any): string | null => { return isEmpty(value) ? t("externalForms.formValidation.isRequired") : null; @@ -57,15 +65,48 @@ export const validateConceptGroupFilled = ( : null; }; +const validateRestrictedSelects = ( + t: TFunction, + value: FormConceptGroupT[], + field: ConceptListField, +) => { + if (!value || value.length === 0) return null; + + const { allowlistedSelects, blocklistedSelects } = field; + + const hasAllowlistedSelects = (allowlistedSelects?.length || 0) > 0; + const hasBlocklistedSelects = (blocklistedSelects?.length || 0) > 0; + + if (hasAllowlistedSelects || hasBlocklistedSelects) { + const validSelects = value + .flatMap((v) => v.concepts) + .filter(exists) + .flatMap((c) => { + const tableSelects = c.tables.flatMap((t) => t.selects); + + return [...c.selects, ...tableSelects].filter( + isValidSelect({ allowlistedSelects, blocklistedSelects }), + ); + }); + + if (validSelects.length === 0) { + return t("externalForms.formValidation.validSelectRequired"); + } + } + + return null; +}; + +// TODO: Refactor using generics to try and tie the `field` to its `value` const DEFAULT_VALIDATION_BY_TYPE: Record< FormField["type"], - null | ((t: TFunction, value: any) => string | null) + null | ((t: TFunction, value: any, field: any) => string | null) > = { STRING: null, TEXTAREA: null, NUMBER: null, CHECKBOX: null, - CONCEPT_LIST: null, + CONCEPT_LIST: validateRestrictedSelects, RESULT_GROUP: null, SELECT: null, TABS: null, @@ -86,7 +127,7 @@ function getNotEmptyValidation(fieldType: string) { } } -function getPossibleValidations(fieldType: string) { +function getConfigurableValidations(fieldType: string) { return { NOT_EMPTY: getNotEmptyValidation(fieldType), GREATER_THAN_ZERO: validatePositive, @@ -108,7 +149,7 @@ export function getErrorForField( ) { const defaultValidation = DEFAULT_VALIDATION_BY_TYPE[field.type]; - let error = defaultValidation ? defaultValidation(t, value) : null; + let error = defaultValidation ? defaultValidation(t, value, field) : null; if ( isFieldWithValidations(field) && @@ -116,7 +157,7 @@ export function getErrorForField( field.validations.length > 0 ) { for (let validation of field.validations) { - const validateFn = getPossibleValidations(field.type)[validation]; + const validateFn = getConfigurableValidations(field.type)[validation]; if (validateFn) { error = error || validateFn(t, value); diff --git a/frontend/src/js/model/select.ts b/frontend/src/js/model/select.ts index b8029a1820..3030d8432b 100644 --- a/frontend/src/js/model/select.ts +++ b/frontend/src/js/model/select.ts @@ -63,10 +63,20 @@ export function selectIsWithinTypes( ); } +interface AllowBlocklistedSelects { + blocklistedSelects?: SelectorResultType[]; + allowlistedSelects?: SelectorResultType[]; +} + export const isSelectDisabled = ( select: SelectorT, - blocklistedSelects?: SelectorResultType[], - allowlistedSelects?: SelectorResultType[], + { blocklistedSelects, allowlistedSelects }: AllowBlocklistedSelects, ) => (!!allowlistedSelects && !selectIsWithinTypes(select, allowlistedSelects)) || (!!blocklistedSelects && selectIsWithinTypes(select, blocklistedSelects)); + +export const isValidSelect = + ({ blocklistedSelects, allowlistedSelects }: AllowBlocklistedSelects) => + (select: SelectedSelectorT) => + !!select.selected && + !isSelectDisabled(select, { blocklistedSelects, allowlistedSelects }); diff --git a/frontend/src/js/query-node-editor/NodeSelects.tsx b/frontend/src/js/query-node-editor/NodeSelects.tsx index f117d16369..a5a8f880b9 100644 --- a/frontend/src/js/query-node-editor/NodeSelects.tsx +++ b/frontend/src/js/query-node-editor/NodeSelects.tsx @@ -27,11 +27,10 @@ const NodeSelects = ({ sortSelects(selects).map((select) => ({ value: select.id, label: select.label, - disabled: isSelectDisabled( - select, + disabled: isSelectDisabled(select, { blocklistedSelects, allowlistedSelects, - ), + }), })), [selects, allowlistedSelects, blocklistedSelects], ); diff --git a/frontend/src/js/query-node-editor/TableSelects.tsx b/frontend/src/js/query-node-editor/TableSelects.tsx index b7fd814e5c..89a2a09f67 100644 --- a/frontend/src/js/query-node-editor/TableSelects.tsx +++ b/frontend/src/js/query-node-editor/TableSelects.tsx @@ -1,7 +1,7 @@ import { useMemo } from "react"; import type { SelectOptionT, SelectorResultType } from "../api/types"; -import { isSelectDisabled, sortSelects } from "../model/select"; +import { isSelectDisabled, isValidSelect, sortSelects } from "../model/select"; import { SelectedSelectorT } from "../standard-query-editor/types"; import InputMultiSelect from "../ui-components/InputMultiSelect/InputMultiSelect"; @@ -24,19 +24,18 @@ const TableSelects = ({ return sortSelects(selects).map((select) => ({ value: select.id, label: select.label, - disabled: isSelectDisabled( - select, + disabled: isSelectDisabled(select, { blocklistedSelects, allowlistedSelects, - ), + }), })); }, [selects, allowlistedSelects, blocklistedSelects]); const value = useMemo(() => { return selects - .filter(({ selected }) => !!selected) + .filter(isValidSelect({ blocklistedSelects, allowlistedSelects })) .map(({ id, label }) => ({ value: id, label: label })); - }, [selects]); + }, [selects, allowlistedSelects, blocklistedSelects]); return (
diff --git a/frontend/src/js/ui-components/InputSelect/SelectListOption.tsx b/frontend/src/js/ui-components/InputSelect/SelectListOption.tsx index 0d975ee031..c884228a0c 100644 --- a/frontend/src/js/ui-components/InputSelect/SelectListOption.tsx +++ b/frontend/src/js/ui-components/InputSelect/SelectListOption.tsx @@ -20,12 +20,8 @@ const Container = styled("div")` background-color: ${theme.col.blueGrayVeryLight}; `}; - ${({ disabled }) => - disabled && - css` - opacity: 0.5; - cursor: not-allowed; - `}; + opacity: ${({ disabled }) => (disabled ? 0.4 : 1)}; + cursor: ${({ disabled }) => (disabled ? "not-allowed" : "pointer")}; /* to style react-markdown */ p { @@ -47,7 +43,7 @@ const SelectListOption = forwardRef( const label = option.label || String(option.value); return ( - + {option.displayLabel ? ( option.displayLabel ) : ( diff --git a/frontend/src/localization/de.json b/frontend/src/localization/de.json index 82e79e2581..e5bbafb6ba 100644 --- a/frontend/src/localization/de.json +++ b/frontend/src/localization/de.json @@ -269,7 +269,8 @@ "formValidation": { "mustBePositiveNumber": "Muss positive Zahl sein", "isRequired": "Erforderlich", - "invalidDateRange": "Enddatum liegt vor Startdatum" + "invalidDateRange": "Enddatum liegt vor Startdatum", + "validSelectRequired": "Kompatibler Ausgabewert erforderlich" }, "default": { "conceptDropzoneLabel": "Füge ein Konzept oder eine Konzeptliste hinzu", diff --git a/frontend/src/localization/en.json b/frontend/src/localization/en.json index 0c207597fe..be8312517c 100644 --- a/frontend/src/localization/en.json +++ b/frontend/src/localization/en.json @@ -269,7 +269,8 @@ "formValidation": { "mustBePositiveNumber": "Must be a positive number", "isRequired": "Required", - "invalidDateRange": "End date is smaller than start date" + "invalidDateRange": "End date is smaller than start date", + "validSelectRequired": "Valid select required" }, "default": { "conceptDropzoneLabel": "Add a concept or a concept list", From 71b8aa78ff745c37123fc82efa459c1592d94aa1 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 17 Aug 2023 09:37:56 +0200 Subject: [PATCH 507/679] use ExecutorService like a grownup --- .../xodus/stores/SerializingStore.java | 137 ++++++------------ 1 file changed, 43 insertions(+), 94 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index 6de77f12ce..b6acc83351 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -6,17 +6,13 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.Collection; -import java.util.List; import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.function.Supplier; -import java.util.stream.IntStream; import javax.validation.Validator; @@ -34,7 +30,6 @@ import com.google.common.base.Throwables; import jetbrains.exodus.ArrayByteIterable; import jetbrains.exodus.ByteIterable; -import lombok.Data; import lombok.NoArgsConstructor; import lombok.NonNull; import lombok.SneakyThrows; @@ -343,31 +338,21 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { final Collection unreadables = new ConcurrentLinkedQueue<>(); // Some magic number of buffering per worker, that isn't so high, that we fill up RAM with useless stuff, but have enough data to keep the workers occupied. - final BlockingQueue workQueue = new ArrayBlockingQueue<>(nWorkers * bufferPerWorker); - final ExecutorService executorService = Executors.newFixedThreadPool(nWorkers); - - final List readers = IntStream.range(0, nWorkers) - .mapToObj(ignored -> new Reader(workQueue, consumer, result, unreadables)) - .peek(reader -> executorService.submit(reader::run)) - .toList(); + final ThreadPoolExecutor executorService = new ThreadPoolExecutor( + nWorkers, nWorkers, + 0, TimeUnit.SECONDS, + new ArrayBlockingQueue<>(nWorkers * bufferPerWorker), + new ThreadPoolExecutor.CallerRunsPolicy() + ); // We read in single thread, and deserialise and dispatch in multiple threads. - store.forEach((k, v) -> { - try { - workQueue.put(new Pair(k, v)); - } - catch (InterruptedException e) { - //TODO wat do? - throw new RuntimeException(e); - } - }); + store.forEach((k, v) -> executorService.submit(() -> handle(consumer, result, unreadables, k, v))); executorService.shutdown(); - readers.forEach(Reader::finish); while(!executorService.awaitTermination(30, TimeUnit.SECONDS)){ - log.debug("Still waiting for {} jobs.", workQueue.size()); + log.debug("Still waiting for {} jobs.", executorService.getQueue().size()); } // Print some statistics @@ -474,7 +459,6 @@ public void close() { @NoArgsConstructor public static class IterationStatistic { - //TODO move into reader? private final AtomicInteger totalProcessed = new AtomicInteger(); private final AtomicInteger failedKeys = new AtomicInteger(); private final AtomicInteger failedValues = new AtomicInteger(); @@ -516,79 +500,44 @@ public int getTotalProcessed() { } } - private record Pair(ByteIterable key, ByteIterable value) { - } - - @Data - private class Reader { - private final BlockingQueue queue; - private boolean done = false; - private final StoreEntryConsumer consumer; - private final IterationStatistic result; - private final Collection unreadables; - - public void run() { - - while (!done || !queue.isEmpty()) { - try { - final Pair next = queue.poll(1, TimeUnit.SECONDS); - - if (next == null) { - continue; - } - - handle(consumer, result, unreadables, next.key, next.value); - } - catch (Exception exception) { - //TODO probably split for InterrupedException? No idea how to handle that though - log.warn("", exception); - } - } - } - - public void finish() { - done = true; + private void handle(StoreEntryConsumer consumer, IterationStatistic result, Collection unreadables, ByteIterable k, ByteIterable v) { + result.incrTotalProcessed(); + + // Try to read the key first + final KEY key = getDeserializedAndDumpFailed( + k, + SerializingStore.this::readKey, + () -> new String(k.getBytesUnsafe()), + v, + "Could not parse key [{}]" + ); + if (key == null) { + unreadables.add(k); + result.incrFailedKeys(); + return; } - private void handle(StoreEntryConsumer consumer, IterationStatistic result, Collection unreadables, ByteIterable k, ByteIterable v) { - result.incrTotalProcessed(); + // Try to read the value + final VALUE value = getDeserializedAndDumpFailed( + v, + SerializingStore.this::readValue, + key::toString, + v, + "Could not parse value for key [{}]" + ); - // Try to read the key first - final KEY key = getDeserializedAndDumpFailed( - k, - SerializingStore.this::readKey, - () -> new String(k.getBytesUnsafe()), - v, - "Could not parse key [{}]" - ); - if (key == null) { - unreadables.add(k); - result.incrFailedKeys(); - return; - } - - // Try to read the value - final VALUE value = getDeserializedAndDumpFailed( - v, - SerializingStore.this::readValue, - key::toString, - v, - "Could not parse value for key [{}]" - ); - - if (value == null) { - unreadables.add(k); - result.incrFailedValues(); - return; - } + if (value == null) { + unreadables.add(k); + result.incrFailedValues(); + return; + } - // Apply the consumer to key and value - try { - consumer.accept(key, value, v.getLength()); - } - catch (Exception e) { - log.warn("Unable to apply for-each consumer on key[{}]", key, e); - } + // Apply the consumer to key and value + try { + consumer.accept(key, value, v.getLength()); + } + catch (Exception e) { + log.warn("Unable to apply for-each consumer on key[{}]", key, e); } } } From c9582cbad8fb563bbdef71f00a4bea00642c122a Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 17 Aug 2023 10:29:53 +0200 Subject: [PATCH 508/679] cleanup --- .../xodus/stores/SerializingStore.java | 21 +++++----- .../util/CallerBlocksRejectionHandler.java | 39 ------------------- 2 files changed, 10 insertions(+), 50 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/util/CallerBlocksRejectionHandler.java diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index b6acc83351..5ef1333e7a 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -337,8 +337,6 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { final IterationStatistic result = new IterationStatistic(); final Collection unreadables = new ConcurrentLinkedQueue<>(); - // Some magic number of buffering per worker, that isn't so high, that we fill up RAM with useless stuff, but have enough data to keep the workers occupied. - final ThreadPoolExecutor executorService = new ThreadPoolExecutor( nWorkers, nWorkers, 0, TimeUnit.SECONDS, @@ -357,16 +355,17 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { // Print some statistics final int total = result.getTotalProcessed(); + log.debug( - String.format( - "While processing store %s:\n\tEntries processed:\t%d\n\tKey read failure:\t%d (%.2f%%)\n\tValue read failure:\t%d (%.2f%%)", - store.getName(), - total, - result.getFailedKeys(), - total > 0 ? (float) result.getFailedKeys() / total * 100 : 0, - result.getFailedValues(), - total > 0 ? (float) result.getFailedValues() / total * 100 : 0 - )); + "While processing store %s:\n\tEntries processed:\t%d\n\tKey read failure:\t%d (%.2f%%)\n\tValue read failure:\t%d (%.2f%%)" + .formatted( + store.getName(), + total, + result.getFailedKeys(), + total > 0 ? (float) result.getFailedKeys() / total * 100 : 0, + result.getFailedValues(), + total > 0 ? (float) result.getFailedValues() / total * 100 : 0 + )); // Remove corrupted entries from the store if configured so if (removeUnreadablesFromUnderlyingStore) { diff --git a/backend/src/main/java/com/bakdata/conquery/util/CallerBlocksRejectionHandler.java b/backend/src/main/java/com/bakdata/conquery/util/CallerBlocksRejectionHandler.java deleted file mode 100644 index b6ce4b4556..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/util/CallerBlocksRejectionHandler.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.bakdata.conquery.util; - -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.RejectedExecutionHandler; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.LongAdder; - -import lombok.Data; - -@Data -public class CallerBlocksRejectionHandler implements RejectedExecutionHandler { - - private final long timeoutMillis; - private final LongAdder waitedMillis = new LongAdder(); - - @Override - public void rejectedExecution(Runnable r, ThreadPoolExecutor executor) { - if (executor.isShutdown()) { - return; - } - - try { - long before = System.currentTimeMillis(); - final boolean success = executor.getQueue().offer(r, getTimeoutMillis(), TimeUnit.MILLISECONDS); - long after = System.currentTimeMillis(); - - waitedMillis.add(after - before); - - if (!success) { - throw new RejectedExecutionException("Could not submit within specified timeout."); - } - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - throw new RejectedExecutionException("Thread was interrupted."); - } - } -} From 1ccddc3946f01fc88622c94159f56cc778ea3777 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 17 Aug 2023 10:46:12 +0200 Subject: [PATCH 509/679] fixes comparator of IterationStatistic --- .../io/storage/xodus/stores/SerializingStore.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index 5ef1333e7a..a788bef874 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -30,6 +30,8 @@ import com.google.common.base.Throwables; import jetbrains.exodus.ArrayByteIterable; import jetbrains.exodus.ByteIterable; +import lombok.Data; +import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import lombok.NonNull; import lombok.SneakyThrows; @@ -457,9 +459,14 @@ public void close() { @NoArgsConstructor + @EqualsAndHashCode + @Data public static class IterationStatistic { + @EqualsAndHashCode.Exclude private final AtomicInteger totalProcessed = new AtomicInteger(); + @EqualsAndHashCode.Exclude private final AtomicInteger failedKeys = new AtomicInteger(); + @EqualsAndHashCode.Exclude private final AtomicInteger failedValues = new AtomicInteger(); public void incrTotalProcessed() { @@ -486,14 +493,17 @@ public void setFailedValues(int failedValues) { this.failedValues.set(failedValues); } + @EqualsAndHashCode.Include public int getFailedKeys() { return failedKeys.get(); } + @EqualsAndHashCode.Include public int getFailedValues() { return failedValues.get(); } + @EqualsAndHashCode.Include public int getTotalProcessed() { return totalProcessed.get(); } From 7df676153c24283495db5128efe015f3b7d7300b Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Thu, 17 Aug 2023 11:16:13 +0200 Subject: [PATCH 510/679] change root to Dropzone component -> extend dropzone component to allow for hovering --- .../DropzoneBetweenElements.tsx | 65 +++++++++---------- .../form-components/DropzoneList.tsx | 5 +- .../form-concept-group/FormConceptGroup.tsx | 6 +- frontend/src/js/ui-components/Dropzone.tsx | 8 ++- 4 files changed, 43 insertions(+), 41 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index 7d46cd7cb7..937d2ac68c 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -1,24 +1,19 @@ import styled from "@emotion/styled"; -import { DropTargetMonitor, useDrop } from "react-dnd"; +import { useState } from "react"; +import { DropTargetMonitor } from "react-dnd"; -import { PossibleDroppableObject } from "../../ui-components/Dropzone"; +import Dropzone, { + PossibleDroppableObject, +} from "../../ui-components/Dropzone"; -interface Props { - onDrop: (item: DroppableObject, monitor: DropTargetMonitor) => void; +interface Props { + onDrop: (props: PossibleDroppableObject, monitor: DropTargetMonitor) => void; acceptedDropTypes: string[]; - lastElement?: boolean; - top?: number; + top: number; + height: number; } -const RootHeightBase = 30; const LineHeight = 3; -const Root = styled("div")` - width: 100%; - left: 0; - right: 0; - position: absolute; - border-radius: ${({ theme }) => theme.borderRadius}; -`; const Line = styled("div")<{ show: boolean }>` overflow: hidden; @@ -30,35 +25,33 @@ const Line = styled("div")<{ show: boolean }>` border-radius: 2px; `; -const DropzoneBetweenElements = < - DroppableObject extends PossibleDroppableObject, ->({ +const SxDropzone = styled(Dropzone)<{ height: number; top: number }>` + height: ${({ height }) => height}px; + top: ${({ top }) => top}px; + position: absolute; + background-color: transparent; +`; + +const DropzoneBetweenElements = ({ acceptedDropTypes, onDrop, - lastElement, + height, top, -}: Props) => { - const [{ isOver }, addZoneRef] = useDrop({ - accept: acceptedDropTypes, - drop: onDrop, - collect: (monitor) => ({ - isOver: monitor.isOver(), - isDroppable: monitor.canDrop(), - }), - }); - - const rootHeightMultiplier = lastElement ? 0.7 : 1; +}: Props) => { + let [isOver, setIsOver] = useState(false); return ( <> - + ); }; diff --git a/frontend/src/js/external-forms/form-components/DropzoneList.tsx b/frontend/src/js/external-forms/form-components/DropzoneList.tsx index 7af7f1bf0c..8bf451dc93 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneList.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneList.tsx @@ -58,7 +58,7 @@ interface PropsT { onImportLines: (lines: string[]) => void; dropBetween: ( i: number, - ) => (item: DroppableObject, monitor: DropTargetMonitor) => void; + ) => (item: PossibleDroppableObject, monitor: DropTargetMonitor) => void; } const DropzoneList = ( @@ -102,6 +102,7 @@ const DropzoneList = ( acceptedDropTypes={acceptedDropTypes} onDrop={dropBetween(i)} top={-15} + height={30} /> )} @@ -117,7 +118,7 @@ const DropzoneList = ( acceptedDropTypes={acceptedDropTypes} onDrop={dropBetween(items.length)} top={-20} - lastElement + height={15} /> )} diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 1ef51b1073..7bbb28ba88 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -16,7 +16,7 @@ import { nodeHasNonDefaultSettings, } from "../../model/node"; import type { DragItemConceptTreeNode } from "../../standard-query-editor/types"; -import { isMovedObject } from "../../ui-components/Dropzone"; +import { PossibleDroppableObject, isMovedObject } from "../../ui-components/Dropzone"; import DropzoneWithFileInput, { DragItemFile, } from "../../ui-components/DropzoneWithFileInput"; @@ -206,7 +206,9 @@ const FormConceptGroup = (props: Props) => { : props.attributeDropzoneText } dropBetween={(i: number) => { - return (item: DragItemConceptTreeNode) => { + return (item: PossibleDroppableObject) => { + if (item.type !== DNDType.CONCEPT_TREE_NODE)return; + if (props.isValidConcept && !props.isValidConcept(item)) return null; diff --git a/frontend/src/js/ui-components/Dropzone.tsx b/frontend/src/js/ui-components/Dropzone.tsx index 9662a49891..c83df6cd46 100644 --- a/frontend/src/js/ui-components/Dropzone.tsx +++ b/frontend/src/js/ui-components/Dropzone.tsx @@ -1,5 +1,5 @@ import styled from "@emotion/styled"; -import { ForwardedRef, forwardRef, ReactNode } from "react"; +import { ForwardedRef, forwardRef, ReactNode, useEffect } from "react"; import { DropTargetMonitor, useDrop } from "react-dnd"; import { DNDType } from "../common/constants/dndTypes"; @@ -67,6 +67,7 @@ export interface DropzoneProps { canDrop?: (props: DroppableObject, monitor: DropTargetMonitor) => boolean; onClick?: () => void; children?: (args: ChildArgs) => ReactNode; + setIsOver?: (state: boolean) => void; } export type PossibleDroppableObject = @@ -107,6 +108,7 @@ const Dropzone = ( onClick, invisible, children, + setIsOver, }: DropzoneProps, ref?: ForwardedRef, ) => { @@ -126,6 +128,10 @@ const Dropzone = ( }), }); + useEffect(() => { + if (setIsOver) setIsOver(isOver); + }, [isOver, setIsOver]); + return ( { From 0beaeb3f511259382dcc8c264f25800e6b92be27 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Thu, 17 Aug 2023 11:18:31 +0200 Subject: [PATCH 511/679] formatting --- .../form-components/DropzoneBetweenElements.tsx | 2 +- .../form-concept-group/FormConceptGroup.tsx | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index 937d2ac68c..5ddbc9cd01 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -6,7 +6,7 @@ import Dropzone, { PossibleDroppableObject, } from "../../ui-components/Dropzone"; -interface Props { +interface Props { onDrop: (props: PossibleDroppableObject, monitor: DropTargetMonitor) => void; acceptedDropTypes: string[]; top: number; diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 7bbb28ba88..82de595e0e 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -16,7 +16,10 @@ import { nodeHasNonDefaultSettings, } from "../../model/node"; import type { DragItemConceptTreeNode } from "../../standard-query-editor/types"; -import { PossibleDroppableObject, isMovedObject } from "../../ui-components/Dropzone"; +import { + PossibleDroppableObject, + isMovedObject, +} from "../../ui-components/Dropzone"; import DropzoneWithFileInput, { DragItemFile, } from "../../ui-components/DropzoneWithFileInput"; @@ -207,8 +210,8 @@ const FormConceptGroup = (props: Props) => { } dropBetween={(i: number) => { return (item: PossibleDroppableObject) => { - if (item.type !== DNDType.CONCEPT_TREE_NODE)return; - + if (item.type !== DNDType.CONCEPT_TREE_NODE) return; + if (props.isValidConcept && !props.isValidConcept(item)) return null; From b08dc1e684fb9fce8fa730acfa6995f03fda59c6 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 17 Aug 2023 14:03:47 +0200 Subject: [PATCH 512/679] Update backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java Co-authored-by: Torben Meyer --- .../com/bakdata/conquery/models/config/XodusStoreFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index 068ca1363d..480918b7f2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -330,7 +330,7 @@ public SingletonStore createIdMappingStore(String pathName, ObjectM synchronized (openStoresInEnv) { final BigStore bigStore = - new BigStore<>(this, validator, environment, ID_MAPPING.storeInfo(), this::closeStore, this::removeStore, objectMapper, 10, 20); + new BigStore<>(this, validator, environment, ID_MAPPING.storeInfo(), this::closeStore, this::removeStore, objectMapper, getReaderWorkers(), getBufferPerWorker()); openStoresInEnv.put(bigStore.getDataXodusStore().getEnvironment(), bigStore.getDataXodusStore()); openStoresInEnv.put(bigStore.getMetaXodusStore().getEnvironment(), bigStore.getMetaXodusStore()); From ad3add127e6eb3e55567d23c0657b9873d2dd55f Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 17 Aug 2023 16:43:44 +0200 Subject: [PATCH 513/679] Move to a global ExecutorService in XodusStoreFactory --- .../io/storage/xodus/stores/BigStore.java | 7 +- .../xodus/stores/SerializingStore.java | 89 ++++++++++--------- .../models/auth/apitoken/TokenStorage.java | 5 +- .../auth/basic/LocalAuthenticationRealm.java | 3 +- .../models/config/XodusStoreFactory.java | 28 +++++- .../io/storage/xodus/stores/BigStoreTest.java | 5 +- .../stores/SerializingStoreDumpTest.java | 3 +- 7 files changed, 87 insertions(+), 53 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java index 8fbc3b38b1..4beb3633c0 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/BigStore.java @@ -14,6 +14,7 @@ import java.util.List; import java.util.UUID; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Consumer; import java.util.stream.Stream; @@ -63,7 +64,7 @@ public BigStore(XodusStoreFactory config, StoreInfo storeInfo, Consumer storeCloseHook, Consumer storeRemoveHook, - ObjectMapper mapper, int nWorkers, int bufferPerWorker) { + ObjectMapper mapper, ExecutorService executorService) { this.storeInfo = storeInfo; // Recommendation by the author of Xodus is to have logFileSize at least be 4 times the biggest file size. @@ -78,7 +79,7 @@ public BigStore(XodusStoreFactory config, BigStoreMetaKeys.class, config.isValidateOnWrite(), config.isRemoveUnreadableFromStore(), - config.getUnreadableDataDumpDirectory(), nWorkers, bufferPerWorker + config.getUnreadableDataDumpDirectory(), executorService ); @@ -91,7 +92,7 @@ public BigStore(XodusStoreFactory config, byte[].class, config.isValidateOnWrite(), config.isRemoveUnreadableFromStore(), - config.getUnreadableDataDumpDirectory(), nWorkers, bufferPerWorker + config.getUnreadableDataDumpDirectory(), executorService ); diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index a788bef874..82ebdd7efc 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -6,12 +6,16 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.Collection; -import java.util.concurrent.ArrayBlockingQueue; +import java.util.List; +import java.util.Objects; +import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; +import java.util.function.Predicate; import java.util.function.Supplier; import javax.validation.Validator; @@ -28,6 +32,10 @@ import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectWriter; import com.google.common.base.Throwables; +import com.google.common.util.concurrent.Futures; +import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.ListeningExecutorService; +import com.google.common.util.concurrent.MoreExecutors; import jetbrains.exodus.ArrayByteIterable; import jetbrains.exodus.ByteIterable; import lombok.Data; @@ -38,6 +46,7 @@ import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.TestOnly; /** * Key-value-store from {@link KEY} type values to {@link VALUE} values. ACID consistent, stored on disk using {@link jetbrains.exodus.env.Store} via {@link XodusStore}. @@ -62,7 +71,7 @@ public class SerializingStore implements Store { /** * Deserializer for keys */ - private final ThreadLocal keyReader; + private final ObjectReader keyReader; /** * Serializer for values @@ -72,7 +81,7 @@ public class SerializingStore implements Store { /** * Deserializer for values */ - private final ThreadLocal valueReader; + private final ObjectReader valueReader; /** * Optional validator used for serialization. @@ -103,8 +112,7 @@ public class SerializingStore implements Store { private final boolean removeUnreadablesFromUnderlyingStore; private final ObjectMapper objectMapper; - private final int nWorkers; - private final int bufferPerWorker; + private final ExecutorService executor; public , CLASS_V extends Class> SerializingStore(XodusStore store, Validator validator, @@ -113,7 +121,7 @@ public , CLASS_V extends Class> SerializingSto CLASS_V valueType, boolean validateOnWrite, boolean removeUnreadableFromStore, - File unreadableDataDumpDirectory, int nWorkers, int bufferPerWorker) { + File unreadableDataDumpDirectory, ExecutorService executorService) { this.store = store; this.validator = validator; this.validateOnWrite = validateOnWrite; @@ -124,22 +132,21 @@ public , CLASS_V extends Class> SerializingSto valueWriter = objectMapper.writerFor(this.valueType); - valueReader = ThreadLocal.withInitial(() -> objectMapper.readerFor(this.valueType)); + valueReader = objectMapper.readerFor(this.valueType); keyWriter = objectMapper.writerFor(keyType); - keyReader = ThreadLocal.withInitial(() -> objectMapper.readerFor(keyType)); + keyReader = objectMapper.readerFor(keyType); removeUnreadablesFromUnderlyingStore = removeUnreadableFromStore; unreadableValuesDumpDir = unreadableDataDumpDirectory; - this.nWorkers = nWorkers; - this.bufferPerWorker = bufferPerWorker; + executor = executorService; if (shouldDumpUnreadables()) { if (!unreadableValuesDumpDir.exists() && !unreadableValuesDumpDir.mkdirs()) { - throw new IllegalStateException("Could not create dump directory: " + unreadableValuesDumpDir); + throw new IllegalStateException("Could not create dump directory: %s".formatted(unreadableValuesDumpDir)); } else if (!unreadableValuesDumpDir.isDirectory()) { throw new IllegalArgumentException(String.format("The provided path points to an existing file which is not a directory. Was: %s", unreadableValuesDumpDir.getAbsolutePath())); @@ -154,7 +161,7 @@ private boolean shouldDumpUnreadables() { @Override public void add(KEY key, VALUE value) { if (!valueType.isInstance(value)) { - throw new IllegalStateException("The element " + value + " is not of the required type " + valueType); + throw new IllegalStateException("The element %s is not of the required type %s".formatted(value, valueType)); } if (validateOnWrite) { ValidatorHelper.failOnError(log, validator.validate(value)); @@ -190,7 +197,7 @@ private ByteIterable write(Object obj, ObjectWriter writer) { return new ArrayByteIterable(bytes); } catch (JsonProcessingException e) { - throw new RuntimeException("Failed to write " + obj, e); + throw new RuntimeException("Failed to write %s".formatted(obj), e); } } @@ -223,7 +230,7 @@ public VALUE get(KEY key) { * Deserialize value with {@code valueReader}. */ private VALUE readValue(ByteIterable value) { - return read(valueReader.get(), value); + return read(valueReader, value); } /** @@ -286,7 +293,7 @@ private T read(ObjectReader reader, ByteIterable obj) { return reader.readValue(obj.getBytesUnsafe(), 0, obj.getLength()); } catch (IOException e) { - throw new RuntimeException("Failed to read " + JacksonUtil.toJsonDebug(obj.getBytesUnsafe()), e); + throw new RuntimeException("Failed to read %s".formatted(JacksonUtil.toJsonDebug(obj.getBytesUnsafe())), e); } } @@ -337,24 +344,25 @@ private static String sanitiseFileName(@NotNull String name) { @Override public IterationStatistic forEach(StoreEntryConsumer consumer) { final IterationStatistic result = new IterationStatistic(); - final Collection unreadables = new ConcurrentLinkedQueue<>(); - final ThreadPoolExecutor executorService = new ThreadPoolExecutor( - nWorkers, nWorkers, - 0, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(nWorkers * bufferPerWorker), - new ThreadPoolExecutor.CallerRunsPolicy() - ); + final ListeningExecutorService executorService = MoreExecutors.listeningDecorator(executor); + + final Queue> jobs = new ConcurrentLinkedQueue<>(); - // We read in single thread, and deserialise and dispatch in multiple threads. - store.forEach((k, v) -> executorService.submit(() -> handle(consumer, result, unreadables, k, v))); + // We read in single thread, and deserialize and dispatch in multiple threads. + store.forEach((k, v) -> jobs.add(executorService.submit(() -> handle(consumer, result, k, v)))); - executorService.shutdown(); + final ListenableFuture> allJobs = Futures.allAsList(jobs); - while(!executorService.awaitTermination(30, TimeUnit.SECONDS)){ - log.debug("Still waiting for {} jobs.", executorService.getQueue().size()); + while(allJobs.get(30, TimeUnit.SECONDS) == null){ + log.debug("Still waiting for {} jobs.", jobs.stream().filter(Predicate.not(Future::isDone)).count()); } + final List unreadables = allJobs.get() + .stream() + .filter(Objects::nonNull) + .toList(); + // Print some statistics final int total = result.getTotalProcessed(); @@ -407,13 +415,13 @@ private TYPE getDeserializedAndDumpFailed(ByteIterable serial, Function consumer, IterationStatistic result, Collection unreadables, ByteIterable k, ByteIterable v) { + private ByteIterable handle(StoreEntryConsumer consumer, IterationStatistic result, ByteIterable keyRaw, ByteIterable v) { result.incrTotalProcessed(); // Try to read the key first final KEY key = getDeserializedAndDumpFailed( - k, + keyRaw, SerializingStore.this::readKey, - () -> new String(k.getBytesUnsafe()), + () -> new String(keyRaw.getBytesUnsafe()), v, "Could not parse key [{}]" ); if (key == null) { - unreadables.add(k); result.incrFailedKeys(); - return; + return keyRaw; } // Try to read the value @@ -536,9 +544,8 @@ private void handle(StoreEntryConsumer consumer, IterationStatistic ); if (value == null) { - unreadables.add(k); result.incrFailedValues(); - return; + return keyRaw; } // Apply the consumer to key and value @@ -548,5 +555,7 @@ private void handle(StoreEntryConsumer consumer, IterationStatistic catch (Exception e) { log.warn("Unable to apply for-each consumer on key[{}]", key, e); } + + return null; } } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/apitoken/TokenStorage.java b/backend/src/main/java/com/bakdata/conquery/models/auth/apitoken/TokenStorage.java index 9d4ec129d7..86b34d5267 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/apitoken/TokenStorage.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/apitoken/TokenStorage.java @@ -8,6 +8,7 @@ import java.util.List; import java.util.Optional; import java.util.UUID; +import java.util.concurrent.Executors; import javax.validation.Validator; @@ -72,7 +73,7 @@ public void start(){ ApiTokenData.class, true, false, - null, 1, 100 + null, Executors.newSingleThreadExecutor() )); openStoresInEnv.add(data); @@ -90,7 +91,7 @@ public void start(){ ApiTokenData.MetaData.class, true, false, - null, 1, 100 + null, Executors.newSingleThreadExecutor() )); openStoresInEnv.add(meta); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/LocalAuthenticationRealm.java b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/LocalAuthenticationRealm.java index 6fca172e9b..ed2e0eb49b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/auth/basic/LocalAuthenticationRealm.java +++ b/backend/src/main/java/com/bakdata/conquery/models/auth/basic/LocalAuthenticationRealm.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.util.List; import java.util.Optional; +import java.util.concurrent.Executors; import javax.validation.Validator; @@ -108,7 +109,7 @@ protected void onInit() { PasswordHasher.HashedEntry.class, false, true, - null, 1, 100 + null, Executors.newSingleThreadExecutor() )); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java index 480918b7f2..4e28a84f8c 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/XodusStoreFactory.java @@ -11,6 +11,10 @@ import java.util.List; import java.util.Objects; import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.function.Function; import javax.annotation.Nullable; @@ -136,7 +140,7 @@ public class XodusStoreFactory implements StoreFactory { * @implNote it's always only one thread reading from disk, dispatching to multiple reader threads. */ @Min(1) - private int readerWorkers = 10; + private int readerWorkers = Runtime.getRuntime().availableProcessors(); /** * How many slots of buffering to use before the IO thread is put to sleep. @@ -144,6 +148,22 @@ public class XodusStoreFactory implements StoreFactory { @Min(1) private int bufferPerWorker = 20; + @JsonIgnore + private ExecutorService readerExecutorService; + + public ExecutorService getReaderExecutorService() { + if (readerExecutorService == null){ + readerExecutorService = new ThreadPoolExecutor( + 1, getReaderWorkers(), + 5, TimeUnit.MINUTES, + new ArrayBlockingQueue<>(getReaderWorkers() * getBufferPerWorker()), + new ThreadPoolExecutor.CallerRunsPolicy() + ); + } + + return readerExecutorService; + } + private boolean useWeakDictionaryCaching; @NotNull private Duration weakCacheDuration = Duration.hours(48); @@ -287,7 +307,7 @@ public IdentifiableStore createDictionaryStore(CentralRegistry centr DICTIONARIES.storeInfo(), this::closeStore, this::removeStore, - centralRegistry.injectIntoNew(objectMapper), getReaderWorkers(), getBufferPerWorker() + centralRegistry.injectIntoNew(objectMapper), getReaderExecutorService() ); openStoresInEnv.put(bigStore.getDataXodusStore().getEnvironment(), bigStore.getDataXodusStore()); openStoresInEnv.put(bigStore.getMetaXodusStore().getEnvironment(), bigStore.getMetaXodusStore()); @@ -330,7 +350,7 @@ public SingletonStore createIdMappingStore(String pathName, ObjectM synchronized (openStoresInEnv) { final BigStore bigStore = - new BigStore<>(this, validator, environment, ID_MAPPING.storeInfo(), this::closeStore, this::removeStore, objectMapper, getReaderWorkers(), getBufferPerWorker()); + new BigStore<>(this, validator, environment, ID_MAPPING.storeInfo(), this::closeStore, this::removeStore, objectMapper, getReaderExecutorService()); openStoresInEnv.put(bigStore.getDataXodusStore().getEnvironment(), bigStore.getDataXodusStore()); openStoresInEnv.put(bigStore.getMetaXodusStore().getEnvironment(), bigStore.getMetaXodusStore()); @@ -496,7 +516,7 @@ public Store createStore(Environment environment, Valid isValidateOnWrite(), isRemoveUnreadableFromStore(), getUnreadableDataDumpDirectory(), - getReaderWorkers(), getBufferPerWorker() + getReaderExecutorService() )); } } diff --git a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/BigStoreTest.java b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/BigStoreTest.java index be92d2e953..4ce7bb25d7 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/BigStoreTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/BigStoreTest.java @@ -7,6 +7,7 @@ import java.io.IOException; import java.io.SequenceInputStream; import java.nio.file.Files; +import java.util.concurrent.Executors; import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.io.storage.StoreMappings; @@ -64,7 +65,7 @@ public void destroy() throws IOException { public void testFull() throws JSONException, IOException { BigStore store = new BigStore<>(new XodusStoreFactory(), Validators.newValidator(), env, - StoreMappings.DICTIONARIES.storeInfo(), (e) -> {}, (e) -> {}, MAPPER, 10, 20 + StoreMappings.DICTIONARIES.storeInfo(), (e) -> {}, (e) -> {}, MAPPER, Executors.newSingleThreadExecutor() ); @@ -106,7 +107,7 @@ public void testFull() throws JSONException, IOException { @Test public void testEmpty() throws JSONException, IOException { BigStore store = new BigStore<>(new XodusStoreFactory(), Validators.newValidator(), env, - StoreMappings.DICTIONARIES.storeInfo(), (e) -> {}, (e) -> {}, MAPPER, 10, 20 + StoreMappings.DICTIONARIES.storeInfo(), (e) -> {}, (e) -> {}, MAPPER, Executors.newSingleThreadExecutor() ); store.setChunkByteSize(Ints.checkedCast(DataSize.megabytes(1).toBytes())); diff --git a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java index 5d92970875..d55eb144ed 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStoreDumpTest.java @@ -4,6 +4,7 @@ import java.io.File; import java.io.IOException; +import java.util.concurrent.Executors; import javax.validation.Validator; @@ -72,7 +73,7 @@ private SerializingStore createSerializedStore(XodusSto storeId.getValueType(), config.isValidateOnWrite(), config.isRemoveUnreadableFromStore(), - config.getUnreadableDataDumpDirectory(), 1, 1 + config.getUnreadableDataDumpDirectory(), Executors.newSingleThreadExecutor() ); } From ab753486a1ecc2aaac3b65db55f519829db4af98 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 17 Aug 2023 17:06:09 +0200 Subject: [PATCH 514/679] cleanup --- .../io/storage/xodus/stores/SerializingStore.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index 82ebdd7efc..ec11f0fa6b 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -310,7 +310,6 @@ public static File makeDumpFileName(@NotNull String keyOfDump, @NotNull File unr .resolve(storeName) .resolve(sanitiseFileName(keyOfDump) + "." + DUMP_FILE_EXTENSION) .toFile(); - } /** @@ -326,7 +325,6 @@ public static File makeExceptionFileName(@NotNull String keyOfDump, @NotNull Fil .resolve(storeName) .resolve(sanitiseFileName(keyOfDump) + "." + EXCEPTION_FILE_EXTENSION) .toFile(); - } private static String sanitiseFileName(@NotNull String name) { @@ -518,7 +516,7 @@ public int getTotalProcessed() { } } - private ByteIterable handle(StoreEntryConsumer consumer, IterationStatistic result, ByteIterable keyRaw, ByteIterable v) { + private ByteIterable handle(StoreEntryConsumer consumer, IterationStatistic result, ByteIterable keyRaw, ByteIterable valueRaw) { result.incrTotalProcessed(); // Try to read the key first @@ -526,7 +524,7 @@ private ByteIterable handle(StoreEntryConsumer consumer, IterationSt keyRaw, SerializingStore.this::readKey, () -> new String(keyRaw.getBytesUnsafe()), - v, + valueRaw, "Could not parse key [{}]" ); if (key == null) { @@ -536,10 +534,10 @@ private ByteIterable handle(StoreEntryConsumer consumer, IterationSt // Try to read the value final VALUE value = getDeserializedAndDumpFailed( - v, + valueRaw, SerializingStore.this::readValue, key::toString, - v, + valueRaw, "Could not parse value for key [{}]" ); @@ -550,7 +548,7 @@ private ByteIterable handle(StoreEntryConsumer consumer, IterationSt // Apply the consumer to key and value try { - consumer.accept(key, value, v.getLength()); + consumer.accept(key, value, valueRaw.getLength()); } catch (Exception e) { log.warn("Unable to apply for-each consumer on key[{}]", key, e); From 09b039ff1c002b44c31cf3aeff02c0814b82e79c Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Thu, 17 Aug 2023 17:57:18 +0200 Subject: [PATCH 515/679] fixes duplicate and faulty implementation of dumpToFile --- .../xodus/stores/SerializingStore.java | 57 +++---------------- 1 file changed, 8 insertions(+), 49 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index 6b1a1095d1..c4d2591d71 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -192,48 +192,6 @@ private static String sanitiseFileName(@NotNull String name) { return FileUtil.SAVE_FILENAME_REPLACEMENT_MATCHER.matcher(name).replaceAll("_"); } - /** - * Dumps the content of an unreadable value to a file as a json (it tries to parse it as an object and than tries to dump it as a json). - * - * @param obj The object to dump. - * @param keyOfDump The key under which the unreadable value is accessible. It is used for the file name. - * @param reason The exception causing us to dump the file - * @param unreadableDumpDir The director to dump to. The method assumes that the directory exists and is okay to write to. - * @param storeName The name of the store which is also used in the dump file name. - */ - private static void dumpToFile(@NonNull ByteIterable obj, @NonNull String keyOfDump, Exception reason, @NonNull File unreadableDumpDir, String storeName, ObjectMapper objectMapper) { - // Create dump filehandle - final File dumpfile = makeDumpFileName(keyOfDump, unreadableDumpDir, storeName); - final File exceptionFileName = makeExceptionFileName(keyOfDump, unreadableDumpDir, storeName); - - if (dumpfile.exists() || exceptionFileName.exists()) { - log.trace("Abort dumping of file {} because it already exists.", dumpfile); - return; - } - - if (!dumpfile.getParentFile().exists() && !dumpfile.getParentFile().mkdirs()) { - throw new IllegalStateException("Could not create `%s`.".formatted(dumpfile.getParentFile())); - } - - // Write json - try { - log.info("Dumping value of key {} to {} (because it cannot be deserialized anymore).", keyOfDump, dumpfile.getCanonicalPath()); - - final JsonNode dump = objectMapper.readerFor(JsonNode.class).readValue(obj.getBytesUnsafe(), 0, obj.getLength()); - Jackson.MAPPER.writer().writeValue(dumpfile, dump); - } - catch (IOException e) { - log.error("Failed to dump unreadable value of key `{}` to file `{}`", keyOfDump, dumpfile, e); - } - - try (PrintStream out = new PrintStream(exceptionFileName)) { - reason.printStackTrace(out); - } - catch (IOException e) { - log.error("Failed to dump exception for `{}` to file `{}`.", keyOfDump, exceptionFileName, e); - } - - } @Override public void add(KEY key, VALUE value) { @@ -324,7 +282,7 @@ private VALUE readValue(ByteIterable value) { * @param unreadableDumpDir The director to dump to. The method assumes that the directory exists and is okay to write to. * @param storeName The name of the store which is also used in the dump file name. */ - private static void dumpToFile(@NonNull byte[] gzippedObj, @NonNull String keyOfDump, Exception reason, @NonNull File unreadableDumpDir, String storeName, ObjectMapper objectMapper) { + private static void dumpToFile(byte[] gzippedObj, @NonNull String keyOfDump, Exception reason, @NonNull File unreadableDumpDir, String storeName, ObjectMapper objectMapper) { // Create dump filehandle final File dumpfile = makeDumpFileName(keyOfDump, unreadableDumpDir, storeName); final File exceptionFileName = makeExceptionFileName(keyOfDump, unreadableDumpDir, storeName); @@ -338,13 +296,11 @@ private static void dumpToFile(@NonNull byte[] gzippedObj, @NonNull String keyOf throw new IllegalStateException("Could not create `%s`.".formatted(dumpfile.getParentFile())); } - //TODO FK: dump in a separate thread so we are not blocking the reader thread. - // Write json try { log.info("Dumping value of key {} to {} (because it cannot be deserialized anymore).", keyOfDump, dumpfile.getCanonicalPath()); - final JsonNode dump = objectMapper.readerFor(JsonNode.class).readValue(debugUnGzip(gzippedObj)); + final JsonNode dump = objectMapper.readerFor(JsonNode.class).readValue(new GZIPInputStream(new ByteArrayInputStream(gzippedObj))); Jackson.MAPPER.writer().writeValue(dumpfile, dump); } catch (IOException e) { @@ -411,11 +367,14 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { final ListenableFuture> allJobs = Futures.allAsList(jobs); - while (allJobs.get(30, TimeUnit.SECONDS) == null) { + + List maybeFailed; + + while ((maybeFailed = allJobs.get(30, TimeUnit.SECONDS)) == null) { log.debug("Still waiting for {} jobs.", jobs.stream().filter(Predicate.not(Future::isDone)).count()); } - final List unreadables = allJobs.get().stream().filter(Objects::nonNull).toList(); + final List unreadables = maybeFailed.stream().filter(Objects::nonNull).toList(); // Print some statistics final int total = result.getTotalProcessed(); @@ -491,7 +450,7 @@ private TYPE getDeserializedAndDumpFailed(ByteIterable serial, Function Date: Mon, 21 Aug 2023 15:28:15 +0200 Subject: [PATCH 516/679] test if the query result contains dates --- .../apiv1/execution/FullExecutionStatus.java | 2 ++ .../models/execution/ManagedExecution.java | 33 ++++++++++++++++--- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/execution/FullExecutionStatus.java b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/FullExecutionStatus.java index 17c058884d..32d9d45c38 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/execution/FullExecutionStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/FullExecutionStatus.java @@ -46,6 +46,8 @@ public class FullExecutionStatus extends ExecutionStatus { */ private boolean canExpand; + private boolean containsDates; + /** * Is set to the query description if the user can expand all included concepts. */ diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java index 3dae68245c..fb709ad704 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java @@ -5,6 +5,7 @@ import java.time.ZoneId; import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Set; import java.util.UUID; @@ -20,6 +21,9 @@ import com.bakdata.conquery.apiv1.execution.FullExecutionStatus; import com.bakdata.conquery.apiv1.execution.OverviewExecutionStatus; import com.bakdata.conquery.apiv1.query.QueryDescription; +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import com.bakdata.conquery.apiv1.query.concept.specific.external.CQExternal; +import com.bakdata.conquery.apiv1.query.concept.specific.external.DateFormat; import com.bakdata.conquery.io.cps.CPSBase; import com.bakdata.conquery.io.jackson.serializer.MetaIdRef; import com.bakdata.conquery.io.jackson.serializer.NsIdRef; @@ -41,7 +45,6 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.query.PrintSettings; import com.bakdata.conquery.models.query.Visitable; -import com.bakdata.conquery.models.worker.DistributedNamespace; import com.bakdata.conquery.models.worker.Namespace; import com.bakdata.conquery.util.QueryUtils; import com.bakdata.conquery.util.QueryUtils.NamespacedIdentifiableCollector; @@ -347,6 +350,30 @@ protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject */ protected void setAdditionalFieldsForStatusWithSource(Subject subject, FullExecutionStatus status) { QueryDescription query = getSubmitted(); + + status.setCanExpand(canSubjectExpand(subject, query)); + + status.setContainsDates(containsDates(query)); + + status.setQuery(canSubjectExpand(subject, query) ? getSubmitted() : null); + } + + private boolean containsDates(QueryDescription query) { + return Visitable.stream(query) + .anyMatch(visitable -> { + if (visitable instanceof CQConcept cqConcept) { + return cqConcept.isAggregateEventDates(); + } + + if (visitable instanceof CQExternal external) { + return Arrays.stream(DateFormat.values()).anyMatch(external.getFormat()::contains); + } + + return false; + }); + } + + private boolean canSubjectExpand(Subject subject, QueryDescription query) { NamespacedIdentifiableCollector namespacesIdCollector = new NamespacedIdentifiableCollector(); query.visit(namespacesIdCollector); @@ -358,9 +385,7 @@ protected void setAdditionalFieldsForStatusWithSource(Subject subject, FullExecu .collect(Collectors.toSet()); boolean canExpand = subject.isPermittedAll(concepts, Ability.READ); - - status.setCanExpand(canExpand); - status.setQuery(canExpand ? getSubmitted() : null); + return canExpand; } @JsonIgnore From 1ec0dcb0e2c4d58c67de3ba0c7b7fc2f15d13b77 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 22 Aug 2023 16:06:51 +0200 Subject: [PATCH 517/679] Validate form right away, show no red --- frontend/src/js/external-forms/FormsTab.tsx | 13 +++++++++++++ frontend/src/js/external-forms/form/Field.tsx | 5 +++-- frontend/src/js/query-runner/QueryRunner.tsx | 19 ++++++++----------- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/frontend/src/js/external-forms/FormsTab.tsx b/frontend/src/js/external-forms/FormsTab.tsx index 1a7f93aa9c..5ffd4cdb17 100644 --- a/frontend/src/js/external-forms/FormsTab.tsx +++ b/frontend/src/js/external-forms/FormsTab.tsx @@ -94,8 +94,18 @@ const useInitializeForm = ({ mode: "onChange", }); + useEffect( + function triggerValidationInitially() { + methods.trigger(); + }, + [methods, config], + ); + const onReset = useCallback(() => { methods.reset(defaultValues); + // Because for some reason, running this in the same tick doesn't work + // Asked about it: https://github.com/orgs/react-hook-form/discussions/10823 + setTimeout(() => methods.trigger(), 0); }, [methods, defaultValues]); const onResetActiveForm = useCallback(() => { @@ -103,6 +113,9 @@ const useInitializeForm = ({ ...methods.getValues(), ...defaultValues, }); + // Because for some reason, running this in the same tick doesn't work + // Asked about it: https://github.com/orgs/react-hook-form/discussions/10823 + setTimeout(() => methods.trigger(), 0); }, [methods, defaultValues]); return { methods, config, datasetOptions, onReset, onResetActiveForm }; diff --git a/frontend/src/js/external-forms/form/Field.tsx b/frontend/src/js/external-forms/form/Field.tsx index 48776d37c1..ad657d4e2f 100644 --- a/frontend/src/js/external-forms/form/Field.tsx +++ b/frontend/src/js/external-forms/form/Field.tsx @@ -81,11 +81,12 @@ const FieldContainer = styled("div")<{ noLabel?: boolean; hasError?: boolean }>` background-color: white; border-radius: ${({ theme }) => theme.borderRadius}; border: 1px solid - ${({ theme, hasError }) => (hasError ? theme.col.red : theme.col.grayLight)}; + ${({ theme, hasError }) => + hasError ? theme.col.blueGrayDark : theme.col.grayLight}; `; const ErrorContainer = styled("div")` - color: ${({ theme }) => theme.col.red}; + color: ${({ theme }) => theme.col.blueGrayDark}; font-weight: 700; font-size: ${({ theme }) => theme.font.sm}; `; diff --git a/frontend/src/js/query-runner/QueryRunner.tsx b/frontend/src/js/query-runner/QueryRunner.tsx index 7b01ef1918..296b46be41 100644 --- a/frontend/src/js/query-runner/QueryRunner.tsx +++ b/frontend/src/js/query-runner/QueryRunner.tsx @@ -1,5 +1,4 @@ import styled from "@emotion/styled"; -import { FC } from "react"; import { useHotkeys } from "react-hotkeys-hook"; import { exists } from "../common/helpers/exists"; @@ -37,22 +36,20 @@ const LoadingGroup = styled("div")` justify-content: flex-end; `; -interface PropsT { - queryRunner?: QueryRunnerStateT; - isQueryRunning: boolean; - disabled: boolean; - buttonTooltip?: string; - startQuery: () => void; - stopQuery: () => void; -} - -const QueryRunner: FC = ({ +const QueryRunner = ({ queryRunner, startQuery, stopQuery, buttonTooltip, isQueryRunning, disabled, +}: { + queryRunner?: QueryRunnerStateT; + isQueryRunning: boolean; + disabled: boolean; + buttonTooltip?: string; + startQuery: () => void; + stopQuery: () => void; }) => { const btnAction = isQueryRunning ? stopQuery : startQuery; const isStartStopLoading = From e55f0262fcf1fd8582f298cad3d281e9d516fb73 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Wed, 23 Aug 2023 12:03:02 +0200 Subject: [PATCH 518/679] Upgrade react-hook-form to fix async trigger --- frontend/package.json | 2 +- frontend/src/js/external-forms/FormsTab.tsx | 8 ++------ frontend/yarn.lock | 8 ++++---- 3 files changed, 7 insertions(+), 11 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index d886016692..7a622bc734 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -64,7 +64,7 @@ "react-dom": "^18.1.0", "react-error-boundary": "^3.1.4", "react-highlight-words": "^0.20.0", - "react-hook-form": "^7.43.5", + "react-hook-form": "^7.45.4", "react-hotkeys-hook": "^4.3.8", "react-i18next": "^12.2.0", "react-list": "^0.8.16", diff --git a/frontend/src/js/external-forms/FormsTab.tsx b/frontend/src/js/external-forms/FormsTab.tsx index 5ffd4cdb17..b7ffa118aa 100644 --- a/frontend/src/js/external-forms/FormsTab.tsx +++ b/frontend/src/js/external-forms/FormsTab.tsx @@ -103,9 +103,7 @@ const useInitializeForm = ({ const onReset = useCallback(() => { methods.reset(defaultValues); - // Because for some reason, running this in the same tick doesn't work - // Asked about it: https://github.com/orgs/react-hook-form/discussions/10823 - setTimeout(() => methods.trigger(), 0); + methods.trigger(); }, [methods, defaultValues]); const onResetActiveForm = useCallback(() => { @@ -113,9 +111,7 @@ const useInitializeForm = ({ ...methods.getValues(), ...defaultValues, }); - // Because for some reason, running this in the same tick doesn't work - // Asked about it: https://github.com/orgs/react-hook-form/discussions/10823 - setTimeout(() => methods.trigger(), 0); + methods.trigger(); }, [methods, defaultValues]); return { methods, config, datasetOptions, onReset, onResetActiveForm }; diff --git a/frontend/yarn.lock b/frontend/yarn.lock index ae9546cbe7..825ff7e73f 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -8534,10 +8534,10 @@ react-highlight-words@^0.20.0: memoize-one "^4.0.0" prop-types "^15.5.8" -react-hook-form@^7.43.5: - version "7.43.5" - resolved "https://registry.yarnpkg.com/react-hook-form/-/react-hook-form-7.43.5.tgz#b320405594f1506d8d57b954383166d4ff563778" - integrity sha512-YcaXhuFHoOPipu5pC7ckxrLrialiOcU91pKu8P+isAcXZyMgByUK9PkI9j5fENO4+6XU5PwWXRGMIFlk9u9UBQ== +react-hook-form@^7.45.4: + version "7.45.4" + resolved "https://registry.yarnpkg.com/react-hook-form/-/react-hook-form-7.45.4.tgz#73d228b704026ae95d7e5f7b207a681b173ec62a" + integrity sha512-HGDV1JOOBPZj10LB3+OZgfDBTn+IeEsNOKiq/cxbQAIbKaiJUe/KV8DBUzsx0Gx/7IG/orWqRRm736JwOfUSWQ== react-hotkeys-hook@^4.3.8: version "4.3.8" From 9422cd5fd27bc12ce8ee94b6e04f24cf7f463bcf Mon Sep 17 00:00:00 2001 From: Jonas Arnhold Date: Wed, 23 Aug 2023 16:54:31 +0200 Subject: [PATCH 519/679] Add HANA support (#109) (#3154) --- .github/workflows/test_backend.yml | 2 +- backend/pom.xml | 5 + .../mode/local/LocalManagerProvider.java | 16 +- .../conquery/models/config/Dialect.java | 3 +- .../models/config/SqlConnectorConfig.java | 1 + .../conquery/sql/DslContextFactory.java | 9 +- .../conversion/context/ConversionContext.java | 17 +- .../context/selects/ConceptSelects.java | 2 +- .../context/selects/MergedSelects.java | 19 +- .../conversion/context/selects/Selects.java | 5 +- .../conversion/context/step/QueryStep.java | 6 +- .../cqelement/CQConceptConverter.java | 203 ------------------ .../cqelement/concept/CQConceptConverter.java | 76 +++++++ .../cqelement/concept/ConceptQueryStep.java | 40 ++++ .../concept/DateRestrictionQueryStep.java | 44 ++++ .../concept/EventFilterQueryStep.java | 39 ++++ .../cqelement/concept/EventSelectStep.java | 39 ++++ .../concept/FinalConceptQueryStep.java | 23 ++ .../PreprocessingQueryStep.java} | 80 +++---- .../cqelement/concept/StepContext.java | 22 ++ .../conversion/dialect/HanaSqlDialect.java | 46 ++++ .../dialect/HanaSqlFunctionProvider.java | 134 ++++++++++++ .../dialect/PostgreSqlFunctionProvider.java | 57 ++--- .../sql/conversion/dialect/SqlDialect.java | 16 +- .../dialect/SqlFunctionProvider.java | 5 +- .../select/DateDistanceConverter.java | 20 +- .../select/FirstValueConverter.java | 4 +- .../sql/execution/SqlExecutionService.java | 5 +- .../conquery/sql/models/ColumnDateRange.java | 24 ++- .../integration/IntegrationTests.java | 4 +- .../integration/sql/CsvTableImporter.java | 65 ++++-- .../sql/SqlIntegrationTestSpec.java | 8 +- .../sql/TestPostgreSqlDialect.java | 35 --- .../sql/dialect/HanaSqlIntegrationTests.java | 168 +++++++++++++++ .../sql/dialect/MockDateNowSupplier.java | 14 ++ .../PostgreSqlIntegrationTests.java | 47 ++-- .../sql/dialect/TestContextProvider.java | 11 + .../sql/testcontainer/hana/HanaContainer.java | 71 ++++++ .../tests/sql/and/different_concept/and.json | 13 +- .../sql/and/different_concept/content_1.csv | 26 +-- .../sql/and/different_concept/expected.csv | 8 +- .../{and.json => and_same_concept.json} | 0 .../date_restriction_date_column/content.csv | 18 +- .../date_restriction_date_column.json | 11 +- .../date_restriction_date_range.json | 85 -------- .../content.csv | 0 .../daterange_column.spec.json | 83 +++++++ .../expected.csv | 2 +- .../tests/sql/filter/number/content.csv | 26 +-- .../tests/sql/filter/number/number.spec.json | 4 - .../sql/filter/number_only_max/content.csv | 26 +-- .../number_only_max/number_only_max.spec.json | 4 - .../sql/filter/number_only_min/content.csv | 26 +-- .../number_only_min/number_only_min.spec.json | 4 - .../tests/sql/filter/select/content.csv | 18 +- .../tests/sql/filter/select/select.spec.json | 4 - .../sql/or/different_concept/content_1.csv | 26 +-- .../sql/or/different_concept/expected.csv | 2 +- .../sql/or/different_concept/or.spec.json | 11 +- ...or.spec.json => or_same_concept.spec.json} | 0 .../selects/date_distance/months/content.csv | 2 +- .../selects/date_distance/months/expected.csv | 2 +- .../selects/date_distance/years/expected.csv | 2 +- executable/pom.xml | 9 +- 64 files changed, 1169 insertions(+), 628 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQConceptConverter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptQueryStep.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/DateRestrictionQueryStep.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterQueryStep.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventSelectStep.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FinalConceptQueryStep.java rename backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/{ConceptPreprocessingService.java => concept/PreprocessingQueryStep.java} (50%) create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/StepContext.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlDialect.java create mode 100644 backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java delete mode 100644 backend/src/test/java/com/bakdata/conquery/integration/sql/TestPostgreSqlDialect.java create mode 100644 backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java create mode 100644 backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/MockDateNowSupplier.java rename backend/src/test/java/com/bakdata/conquery/integration/sql/{ => dialect}/PostgreSqlIntegrationTests.java (67%) create mode 100644 backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestContextProvider.java create mode 100644 backend/src/test/java/com/bakdata/conquery/integration/sql/testcontainer/hana/HanaContainer.java rename backend/src/test/resources/tests/sql/and/same_concept/{and.json => and_same_concept.json} (100%) delete mode 100644 backend/src/test/resources/tests/sql/date_restriction/daterange/date_restriction_date_range.json rename backend/src/test/resources/tests/sql/date_restriction/{daterange => postgres_daterange}/content.csv (100%) create mode 100644 backend/src/test/resources/tests/sql/date_restriction/postgres_daterange/daterange_column.spec.json rename backend/src/test/resources/tests/sql/date_restriction/{daterange => postgres_daterange}/expected.csv (68%) rename backend/src/test/resources/tests/sql/or/same_concept/{or.spec.json => or_same_concept.spec.json} (100%) diff --git a/.github/workflows/test_backend.yml b/.github/workflows/test_backend.yml index bcb2ce27c5..f250dfcc49 100644 --- a/.github/workflows/test_backend.yml +++ b/.github/workflows/test_backend.yml @@ -14,7 +14,7 @@ on: jobs: test: runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 20 steps: - name: Cache local Maven repository uses: actions/cache@v2 diff --git a/backend/pom.xml b/backend/pom.xml index c2cdef06ad..5f4b400d24 100644 --- a/backend/pom.xml +++ b/backend/pom.xml @@ -379,5 +379,10 @@ 1.17.6 test + + com.sap.cloud.db.jdbc + ngdbc + 2.17.10 + diff --git a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java index 794df98a87..eb76f7ff68 100644 --- a/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/mode/local/LocalManagerProvider.java @@ -10,22 +10,34 @@ import com.bakdata.conquery.mode.ManagerProvider; import com.bakdata.conquery.mode.NamespaceHandler; import com.bakdata.conquery.models.config.ConqueryConfig; +import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.models.worker.DatasetRegistry; import com.bakdata.conquery.models.worker.LocalNamespace; import com.bakdata.conquery.models.worker.ShardNodeInformation; import com.bakdata.conquery.sql.DslContextFactory; import com.bakdata.conquery.sql.SqlContext; +import com.bakdata.conquery.sql.conversion.dialect.HanaSqlDialect; import com.bakdata.conquery.sql.conversion.dialect.PostgreSqlDialect; +import com.bakdata.conquery.sql.conversion.dialect.SqlDialect; import io.dropwizard.setup.Environment; +import org.jooq.DSLContext; public class LocalManagerProvider implements ManagerProvider { private static final Supplier> EMPTY_NODE_PROVIDER = Collections::emptyList; public DelegateManager provideManager(ConqueryConfig config, Environment environment) { + InternalObjectMapperCreator creator = ManagerProvider.newInternalObjectMapperCreator(config, environment.getValidator()); - // todo(tm): proper injection - SqlContext sqlContext = new SqlContext(config.getSqlConnectorConfig(), new PostgreSqlDialect(DslContextFactory.create(config.getSqlConnectorConfig()))); + + SqlConnectorConfig sqlConnectorConfig = config.getSqlConnectorConfig(); + DSLContext dslContext = DslContextFactory.create(sqlConnectorConfig); + SqlDialect sqlDialect = switch (sqlConnectorConfig.getDialect()) { + case POSTGRESQL -> new PostgreSqlDialect(dslContext); + case HANA -> new HanaSqlDialect(dslContext); + }; + SqlContext sqlContext = new SqlContext(sqlConnectorConfig, sqlDialect); + NamespaceHandler namespaceHandler = new LocalNamespaceHandler(config, creator, sqlContext); DatasetRegistry datasetRegistry = ManagerProvider.createDatasetRegistry(namespaceHandler, config, creator); creator.init(datasetRegistry); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java b/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java index 2ec655aea9..b57931bbac 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/Dialect.java @@ -6,7 +6,8 @@ @Getter public enum Dialect { - POSTGRESQL(SQLDialect.POSTGRES); + POSTGRESQL(SQLDialect.POSTGRES), + HANA(SQLDialect.DEFAULT); private final SQLDialect jooqDialect; diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java index 857018cada..e6e83b723b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/SqlConnectorConfig.java @@ -25,5 +25,6 @@ public class SqlConnectorConfig { private String databasePassword; private String jdbcConnectionUrl; + private String primaryColumn = "pid"; } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java b/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java index 16bfe54ecd..980aa597bb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/DslContextFactory.java @@ -6,6 +6,7 @@ import com.zaxxer.hikari.HikariConfig; import com.zaxxer.hikari.HikariDataSource; import org.jooq.DSLContext; +import org.jooq.conf.RenderQuotedNames; import org.jooq.conf.Settings; import org.jooq.impl.DSL; @@ -19,10 +20,16 @@ public static DSLContext create(SqlConnectorConfig config) { DataSource dataSource = new HikariDataSource(hikariConfig); + Settings settings = new Settings() + .withRenderFormatted(config.isWithPrettyPrinting()) + // enforces all identifiers to be quoted if not explicitly unquoted via DSL.unquotedName() + // to prevent any lowercase/uppercase SQL dialect specific identifier naming issues + .withRenderQuotedNames(RenderQuotedNames.EXPLICIT_DEFAULT_QUOTED); + return DSL.using( dataSource, config.getDialect().getJooqDialect(), - new Settings().withRenderFormatted(config.isWithPrettyPrinting()) + settings ); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/ConversionContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/ConversionContext.java index b07383dacf..cdd73d7f8f 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/ConversionContext.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/ConversionContext.java @@ -1,5 +1,7 @@ package com.bakdata.conquery.sql.conversion.context; +import java.util.List; + import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.config.SqlConnectorConfig; import com.bakdata.conquery.sql.conversion.NodeConverterService; @@ -12,8 +14,6 @@ import org.jooq.Record; import org.jooq.Select; -import java.util.List; - @Value @With @Builder(toBuilder = true) @@ -25,20 +25,23 @@ public class ConversionContext { @Singular List querySteps; Select finalQuery; - boolean negation; CDateRange dateRestrictionRange; int queryStepCounter; - + boolean negation; + boolean isGroupBy; public boolean dateRestrictionActive() { return this.dateRestrictionRange != null; } + /** + * Adds a converted {@link QueryStep} to the list of query steps of this {@link ConversionContext} and increments its conceptCounter by 1. + */ public ConversionContext withQueryStep(QueryStep queryStep) { return this.toBuilder() - .queryStep(queryStep) - .queryStepCounter(queryStepCounter + 1) - .build(); + .queryStep(queryStep) + .queryStepCounter(queryStepCounter + 1) + .build(); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/ConceptSelects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/ConceptSelects.java index 2d2d808632..9e398edad3 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/ConceptSelects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/ConceptSelects.java @@ -37,7 +37,7 @@ public Selects withValidityDate(ColumnDateRange validityDate) { } @Override - public ConceptSelects byName(String qualifier) { + public ConceptSelects qualifiedWith(String qualifier) { return builder() .primaryColumn(this.mapFieldToQualifier(qualifier, this.primaryColumn)) .dateRestrictionRange(this.dateRestrictionRange.map(dateRestriction -> dateRestriction.qualify(qualifier))) diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/MergedSelects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/MergedSelects.java index 7d3547d5b2..461b44a631 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/MergedSelects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/MergedSelects.java @@ -53,11 +53,11 @@ public Selects withValidityDate(ColumnDateRange validityDate) { } @Override - public MergedSelects byName(String qualifier) { + public MergedSelects qualifiedWith(String cteName) { return new MergedSelects( - this.mapFieldToQualifier(qualifier, this.primaryColumn), - this.validityDate.map(columnDateRange -> columnDateRange.qualify(qualifier)), - this.mapFieldStreamToQualifier(qualifier, this.mergedSelects.stream()).toList() + this.mapFieldToQualifier(cteName, this.primaryColumn), + this.validityDate.map(columnDateRange -> columnDateRange.qualify(cteName)), + this.mapFieldStreamToQualifier(cteName, this.mergedSelects.stream()).toList() ); } @@ -76,8 +76,7 @@ public List> explicitSelects() { private Field coalescePrimaryColumns(List querySteps) { List> primaryColumns = querySteps.stream() - .map(queryStep -> this.mapFieldToQualifier(queryStep.getCteName(), queryStep.getSelects() - .getPrimaryColumn())) + .map(queryStep -> queryStep.getQualifiedSelects().getPrimaryColumn()) .toList(); return DSL.coalesce((Object) primaryColumns.get(0), primaryColumns.subList(1, primaryColumns.size()).toArray()) .as(PRIMARY_COLUMN_NAME); @@ -87,17 +86,13 @@ private Optional extractValidityDates(List queryStep // TODO: date aggregation... return querySteps.stream() .filter(queryStep -> queryStep.getSelects().getValidityDate().isPresent()) - .map(queryStep -> { - ColumnDateRange validityDate = queryStep.getSelects().getValidityDate().get(); - return validityDate.qualify(queryStep.getCteName()); - }) + .map(queryStep -> queryStep.getQualifiedSelects().getValidityDate().get()) .findFirst(); } private List> mergeSelects(List queriesToJoin) { return queriesToJoin.stream() - .flatMap(queryStep -> queryStep.getSelects().explicitSelects().stream() - .map(field -> this.mapFieldToQualifier(queryStep.getCteName(), field))) + .flatMap(queryStep -> queryStep.getQualifiedSelects().explicitSelects().stream()) .toList(); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/Selects.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/Selects.java index 09d7259e25..30d15b17b7 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/Selects.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/selects/Selects.java @@ -23,7 +23,7 @@ public interface Selects { * @return selects as fully qualified reference * @see Selects#mapFieldToQualifier(String, Field) */ - Selects byName(String qualifier); + Selects qualifiedWith(String qualifier); /** * @return A list of all select fields including the primary column and validity date. @@ -55,9 +55,6 @@ default Stream> mapFieldStreamToQualifier(String qualifier, Stream *

* This function maps the select {@code c1 - c2 as c} to {@code t1.c}. * - * @param qualifier - * @param field - * @return */ default Field mapFieldToQualifier(String qualifier, Field field) { return DSL.field(DSL.name(qualifier, field.getName())); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStep.java index 9d72ec56bd..83ed4298ff 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStep.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/context/step/QueryStep.java @@ -1,5 +1,6 @@ package com.bakdata.conquery.sql.conversion.context.step; +import java.util.Collections; import java.util.List; import com.bakdata.conquery.sql.conversion.context.selects.Selects; @@ -20,7 +21,8 @@ public class QueryStep { String cteName; Selects selects; TableLike fromTable; - List conditions; + @Builder.Default + List conditions = Collections.emptyList(); /** * The CTEs referenced by this QueryStep */ @@ -34,7 +36,7 @@ public static TableLike toTableLike(String fromTableName) { * @return All selects re-mapped to a qualifier, which is the cteName of this QueryStep. */ public Selects getQualifiedSelects() { - return this.selects.byName(this.cteName); + return this.selects.qualifiedWith(this.cteName); } } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQConceptConverter.java deleted file mode 100644 index 400dac447f..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/CQConceptConverter.java +++ /dev/null @@ -1,203 +0,0 @@ -package com.bakdata.conquery.sql.conversion.cqelement; - -import java.util.Collections; -import java.util.List; -import java.util.Locale; -import java.util.Optional; - -import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; -import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; -import com.bakdata.conquery.sql.conversion.NodeConverter; -import com.bakdata.conquery.sql.conversion.context.ConversionContext; -import com.bakdata.conquery.sql.conversion.context.selects.ConceptSelects; -import com.bakdata.conquery.sql.conversion.context.step.QueryStep; -import com.bakdata.conquery.sql.conversion.filter.FilterConverterService; -import com.bakdata.conquery.sql.conversion.select.SelectConverterService; -import org.jooq.Condition; -import org.jooq.Field; - -public class CQConceptConverter implements NodeConverter { - - private final FilterConverterService filterConverterService; - private final SelectConverterService selectConverterService; - - public CQConceptConverter(FilterConverterService filterConverterService, SelectConverterService selectConverterService) { - this.filterConverterService = filterConverterService; - this.selectConverterService = selectConverterService; - } - - @Override - public Class getConversionClass() { - return CQConcept.class; - } - - @Override - public ConversionContext convert(CQConcept node, ConversionContext context) { - - if (node.getTables().size() > 1) { - throw new UnsupportedOperationException("Can't handle concepts with multiple tables for now."); - } - - ConceptPreprocessingService preprocessingService = new ConceptPreprocessingService(node, context); - CQTable table = node.getTables().get(0); - String conceptLabel = this.getConceptLabel(node, context); - - QueryStep preprocessingStep = preprocessingService.buildPreprocessingQueryStepForTable(conceptLabel, table); - QueryStep dateRestriction = this.buildDateRestrictionQueryStep(context, node, conceptLabel, preprocessingStep); - QueryStep eventSelect = this.buildEventSelectQueryStep(context, table, conceptLabel, dateRestriction); - QueryStep eventFilter = this.buildEventFilterQueryStep(context, table, conceptLabel, eventSelect); - QueryStep finalStep = this.buildFinalQueryStep(conceptLabel, eventFilter); - - return context.withQueryStep(finalStep); - } - - private String getConceptLabel(CQConcept node, ConversionContext context) { - // only relevant for debugging purposes as it will be part of the generated SQL query - // we prefix each cte name of a concept with an incrementing counter to prevent naming collisions if the same concept is selected multiple times - return "%s_%s".formatted( - context.getQueryStepCounter(), - node.getUserOrDefaultLabel(Locale.ENGLISH) - .toLowerCase() - .replace(' ', '_') - .replaceAll("\\s", "_") - ); - } - - /** - * selects: - * - all of previous step - */ - private QueryStep buildDateRestrictionQueryStep( - ConversionContext context, - CQConcept node, - String conceptLabel, - QueryStep previous - ) { - if (((ConceptSelects) previous.getSelects()).getDateRestrictionRange().isEmpty()) { - return previous; - } - - ConceptSelects dateRestrictionSelects = this.prepareDateRestrictionSelects(node, previous); - Condition dateRestriction = this.buildDateRestriction(context, previous); - String dateRestrictionCteName = "concept_%s_date_restriction".formatted(conceptLabel); - - return QueryStep.builder() - .cteName(dateRestrictionCteName) - .fromTable(QueryStep.toTableLike(previous.getCteName())) - .selects(dateRestrictionSelects) - .conditions(List.of(dateRestriction)) - .predecessors(List.of(previous)) - .build(); - } - - private ConceptSelects prepareDateRestrictionSelects(CQConcept conceptNode, QueryStep previous) { - ConceptSelects.ConceptSelectsBuilder selectsBuilder = ((ConceptSelects) previous.getQualifiedSelects()).toBuilder(); - selectsBuilder.dateRestrictionRange(Optional.empty()); - if (conceptNode.isExcludeFromTimeAggregation()) { - selectsBuilder.validityDate(Optional.empty()); - } - return selectsBuilder.build(); - } - - private Condition buildDateRestriction(ConversionContext context, QueryStep previous) { - ConceptSelects previousSelects = (ConceptSelects) previous.getSelects(); - return context.getSqlDialect().getFunction() - .dateRestriction(previousSelects.getDateRestrictionRange().get(), previousSelects.getValidityDate().get()); - } - - /** - * selects: - * - all of previous steps - * - transformed columns with selects - */ - private QueryStep buildEventSelectQueryStep( - ConversionContext context, - CQTable table, - String conceptLabel, QueryStep previous - ) { - if (table.getSelects().isEmpty()) { - return previous; - } - - ConceptSelects eventSelectSelects = this.prepareEventSelectSelects(context, table, previous); - - return QueryStep.builder() - .cteName(createCteName(conceptLabel, "_event_select")) - .fromTable(QueryStep.toTableLike(previous.getCteName())) - .selects(eventSelectSelects) - .conditions(Collections.emptyList()) - .predecessors(List.of(previous)) - .build(); - } - - /** - * selects: - * - all of previous step - * - remove filter - */ - private QueryStep buildEventFilterQueryStep( - ConversionContext context, - CQTable table, - String conceptLabel, - QueryStep previous - ) { - if (table.getFilters().isEmpty()) { - return previous; - } - - ConceptSelects eventFilterSelects = this.prepareEventFilterSelects(previous); - List eventFilterConditions = this.buildEventFilterConditions(context, table); - - return QueryStep.builder() - .cteName(createCteName(conceptLabel, "_event_filter")) - .fromTable(QueryStep.toTableLike(previous.getCteName())) - .selects(eventFilterSelects) - .conditions(eventFilterConditions) - .predecessors(List.of(previous)) - .build(); - } - - private ConceptSelects prepareEventSelectSelects( - ConversionContext context, - CQTable table, - QueryStep previous - ) { - return ((ConceptSelects) previous.getQualifiedSelects()).withEventSelect(this.getEventSelects(context, table)); - } - - private ConceptSelects prepareEventFilterSelects(QueryStep previous) { - return ((ConceptSelects) previous.getQualifiedSelects()).withEventFilter(Collections.emptyList()); - } - - private List buildEventFilterConditions(ConversionContext context, CQTable table) { - return table.getFilters().stream() - .map(filterValue -> this.filterConverterService.convert(filterValue, context)) - .toList(); - } - - private List> getEventSelects(ConversionContext context, CQTable table) { - return table.getSelects().stream() - .map(select -> (Field) this.selectConverterService.convert(select, context)) - .toList(); - } - - /** - * selects: - * - all of previous step - */ - private QueryStep buildFinalQueryStep(String conceptLabel, QueryStep previous) { - ConceptSelects finalSelects = ((ConceptSelects) previous.getQualifiedSelects()); - return QueryStep.builder() - .cteName(createCteName(conceptLabel, "")) - .fromTable(QueryStep.toTableLike(previous.getCteName())) - .selects(finalSelects) - .conditions(Collections.emptyList()) - .predecessors(List.of(previous)) - .build(); - } - - private static String createCteName(String conceptLabel, String suffix) { - return "concept_%s%s".formatted(conceptLabel, suffix); - } - -} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java new file mode 100644 index 0000000000..64e05c8cfd --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/CQConceptConverter.java @@ -0,0 +1,76 @@ +package com.bakdata.conquery.sql.conversion.cqelement.concept; + +import java.util.List; +import java.util.Locale; +import java.util.Optional; + +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.context.selects.ConceptSelects; +import com.bakdata.conquery.sql.conversion.context.step.QueryStep; +import com.bakdata.conquery.sql.conversion.filter.FilterConverterService; +import com.bakdata.conquery.sql.conversion.select.SelectConverterService; + +public class CQConceptConverter implements NodeConverter { + + private final List querySteps; + + public CQConceptConverter(FilterConverterService filterConverterService, SelectConverterService selectConverterService) { + this.querySteps = List.of( + new PreprocessingQueryStep(), + new DateRestrictionQueryStep(), + new EventSelectStep(selectConverterService), + new EventFilterQueryStep(filterConverterService), + new FinalConceptQueryStep() + ); + } + + @Override + public Class getConversionClass() { + return CQConcept.class; + } + + @Override + public ConversionContext convert(CQConcept node, ConversionContext context) { + + if (node.getTables().size() > 1) { + throw new UnsupportedOperationException("Can't handle concepts with multiple tables for now."); + } + + StepContext stepContext = StepContext.builder() + .context(context) + .node(node) + .table(node.getTables().get(0)) + .conceptLabel(this.getConceptLabel(node, context)) + .sqlFunctions(context.getSqlDialect().getFunction()) + .build(); + + for (ConceptQueryStep queryStep : this.querySteps) { + Optional convert = queryStep.convert(stepContext); + if (convert.isEmpty()) { + continue; + } + stepContext = stepContext.toBuilder() + .previous(convert.get()) + .previousSelects((ConceptSelects) convert.get().getQualifiedSelects()) + .build(); + } + + return context.withQueryStep(stepContext.getPrevious()); + } + + private String getConceptLabel(CQConcept node, ConversionContext context) { + // only relevant for debugging purposes as it will be part of the generated SQL query + // we prefix each cte name of a concept with an incrementing counter to prevent naming collisions if the same concept is selected multiple times + return "%s_%s".formatted( + context.getQueryStepCounter(), + node.getUserOrDefaultLabel(Locale.ENGLISH) + .toLowerCase() + .replace(' ', '_') + .replaceAll("\\s", "_") + ); + } + + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptQueryStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptQueryStep.java new file mode 100644 index 0000000000..d0175c8523 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/ConceptQueryStep.java @@ -0,0 +1,40 @@ +package com.bakdata.conquery.sql.conversion.cqelement.concept; + +import java.util.Collections; +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.sql.conversion.context.step.QueryStep; + +abstract class ConceptQueryStep { + + public Optional convert(StepContext context) { + if (!canConvert(context)) { + return Optional.empty(); + } + + QueryStep.QueryStepBuilder queryStepBuilder = this.convertStep(context).cteName(createCteName(context)); + + if (context.getPrevious() != null) { + queryStepBuilder.predecessors(List.of(context.getPrevious())) + .fromTable(QueryStep.toTableLike(context.getPrevious().getCteName())); + } + else { + queryStepBuilder.predecessors(Collections.emptyList()) + .fromTable(QueryStep.toTableLike(context.getTable().getConnector().getTable().getName())); + } + return Optional.of(queryStepBuilder.build()); + + } + + abstract boolean canConvert(StepContext stepContext); + + abstract QueryStep.QueryStepBuilder convertStep(StepContext stepContext); + + abstract String nameSuffix(); + + private String createCteName(StepContext stepContext) { + return "concept_%s%s".formatted(stepContext.getConceptLabel(), nameSuffix()); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/DateRestrictionQueryStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/DateRestrictionQueryStep.java new file mode 100644 index 0000000000..c41c4bea7c --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/DateRestrictionQueryStep.java @@ -0,0 +1,44 @@ +package com.bakdata.conquery.sql.conversion.cqelement.concept; + +import java.util.List; +import java.util.Optional; + +import com.bakdata.conquery.sql.conversion.context.selects.ConceptSelects; +import com.bakdata.conquery.sql.conversion.context.step.QueryStep; +import org.jooq.Condition; + +class DateRestrictionQueryStep extends ConceptQueryStep { + + @Override + public boolean canConvert(StepContext stepContext) { + return stepContext.getPreviousSelects().getDateRestrictionRange().isPresent(); + } + + @Override + public QueryStep.QueryStepBuilder convertStep(StepContext stepContext) { + ConceptSelects dateRestrictionSelects = this.prepareDateRestrictionSelects(stepContext); + Condition dateRestriction = stepContext.getSqlFunctions().dateRestriction( + stepContext.getPreviousSelects().getDateRestrictionRange().get(), + stepContext.getPreviousSelects().getValidityDate().get() + ); + + return QueryStep.builder() + .selects(dateRestrictionSelects) + .conditions(List.of(dateRestriction)); + } + + @Override + public String nameSuffix() { + return "_date_restriction"; + } + + private ConceptSelects prepareDateRestrictionSelects(final StepContext stepContext) { + ConceptSelects.ConceptSelectsBuilder selectsBuilder = stepContext.getPreviousSelects().toBuilder(); + selectsBuilder.dateRestrictionRange(Optional.empty()); + if (stepContext.getNode().isExcludeFromTimeAggregation()) { + selectsBuilder.validityDate(Optional.empty()); + } + return selectsBuilder.build(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterQueryStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterQueryStep.java new file mode 100644 index 0000000000..71b0b80a01 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventFilterQueryStep.java @@ -0,0 +1,39 @@ +package com.bakdata.conquery.sql.conversion.cqelement.concept; + +import java.util.Collections; +import java.util.List; + +import com.bakdata.conquery.sql.conversion.context.selects.ConceptSelects; +import com.bakdata.conquery.sql.conversion.context.step.QueryStep; +import com.bakdata.conquery.sql.conversion.filter.FilterConverterService; +import org.jooq.Condition; + +public class EventFilterQueryStep extends ConceptQueryStep { + + private final FilterConverterService filterConverterService; + + public EventFilterQueryStep(FilterConverterService filterConverterService) { + this.filterConverterService = filterConverterService; + } + + @Override + public boolean canConvert(StepContext stepContext) { + return !stepContext.getTable().getFilters().isEmpty(); + } + + @Override + public QueryStep.QueryStepBuilder convertStep(StepContext stepContext) { + + ConceptSelects eventFilterSelects = stepContext.getPreviousSelects().withEventFilter(Collections.emptyList()); + List eventFilterConditions = stepContext.getTable().getFilters().stream() + .map(filterValue -> this.filterConverterService.convert(filterValue, stepContext.getContext())) + .toList(); + return QueryStep.builder().selects(eventFilterSelects).conditions(eventFilterConditions); + } + + @Override + public String nameSuffix() { + return "_event_filter"; + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventSelectStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventSelectStep.java new file mode 100644 index 0000000000..c0ceb15299 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/EventSelectStep.java @@ -0,0 +1,39 @@ +package com.bakdata.conquery.sql.conversion.cqelement.concept; + +import java.util.List; +import java.util.stream.Stream; + +import com.bakdata.conquery.sql.conversion.context.step.QueryStep; +import com.bakdata.conquery.sql.conversion.select.SelectConverterService; +import org.jooq.Field; + +class EventSelectStep extends ConceptQueryStep { + + private final SelectConverterService selectConverterService; + + EventSelectStep(SelectConverterService selectConverterService) { + this.selectConverterService = selectConverterService; + } + + @Override + public boolean canConvert(StepContext stepContext) { + return !stepContext.getTable().getSelects().isEmpty() || !stepContext.getNode().getSelects().isEmpty(); + } + + @Override + public QueryStep.QueryStepBuilder convertStep(StepContext stepContext) { + return QueryStep.builder().selects(stepContext.getPreviousSelects().withEventSelect(this.getEventSelects(stepContext))); + } + + @Override + public String nameSuffix() { + return "_event_select"; + } + + @SuppressWarnings("unchecked") + private List> getEventSelects(StepContext stepContext) { + return Stream.concat(stepContext.getTable().getSelects().stream(), stepContext.getNode().getSelects().stream()) + .map(select -> (Field) this.selectConverterService.convert(select, stepContext.getContext())) + .toList(); + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FinalConceptQueryStep.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FinalConceptQueryStep.java new file mode 100644 index 0000000000..a58e7e1c91 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/FinalConceptQueryStep.java @@ -0,0 +1,23 @@ +package com.bakdata.conquery.sql.conversion.cqelement.concept; + +import com.bakdata.conquery.sql.conversion.context.selects.Selects; +import com.bakdata.conquery.sql.conversion.context.step.QueryStep; + +class FinalConceptQueryStep extends ConceptQueryStep { + + @Override + public boolean canConvert(StepContext stepContext) { + return true; + } + + @Override + public QueryStep.QueryStepBuilder convertStep(StepContext stepContext) { + Selects finalSelects = stepContext.getPrevious().getQualifiedSelects(); + return QueryStep.builder().selects(finalSelects); + } + + @Override + public String nameSuffix() { + return ""; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConceptPreprocessingService.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingQueryStep.java similarity index 50% rename from backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConceptPreprocessingService.java rename to backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingQueryStep.java index 94970f4621..ce6b9875e4 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/ConceptPreprocessingService.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/PreprocessingQueryStep.java @@ -1,4 +1,4 @@ -package com.bakdata.conquery.sql.conversion.cqelement; +package com.bakdata.conquery.sql.conversion.cqelement.concept; import java.util.Collections; import java.util.List; @@ -6,43 +6,28 @@ import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; -import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; import com.bakdata.conquery.models.datasets.Column; -import com.bakdata.conquery.sql.conversion.context.ConversionContext; import com.bakdata.conquery.sql.conversion.context.selects.ConceptSelects; import com.bakdata.conquery.sql.conversion.context.step.QueryStep; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.models.ColumnDateRange; import org.jooq.Field; import org.jooq.impl.DSL; -public class ConceptPreprocessingService { +class PreprocessingQueryStep extends ConceptQueryStep { - private final CQConcept concept; - private final ConversionContext context; - private final SqlFunctionProvider sqlFunctionProvider; - - public ConceptPreprocessingService(CQConcept concept, ConversionContext context) { - this.concept = concept; - this.context = context; - this.sqlFunctionProvider = this.context.getSqlDialect().getFunction(); + public boolean canConvert(StepContext stepContext) { + // We always apply preprocessing to select the required columns + return true; } - /** - * selects: - * - (primary column) - * - date restriction - * - validity date - * - any filter (group/event) - * - any select (group/event) - */ - public QueryStep buildPreprocessingQueryStepForTable(String conceptLabel, CQTable table) { + public QueryStep.QueryStepBuilder convertStep(StepContext stepContext) { + CQTable table = stepContext.getTable(); ConceptSelects.ConceptSelectsBuilder selectsBuilder = ConceptSelects.builder(); - selectsBuilder.primaryColumn(DSL.field(context.getConfig().getPrimaryColumn())); - selectsBuilder.dateRestrictionRange(this.getDateRestrictionSelect(table)); - selectsBuilder.validityDate(this.getValidityDateSelect(table, conceptLabel)); + selectsBuilder.primaryColumn(DSL.field(DSL.name(stepContext.getContext().getConfig().getPrimaryColumn()))) + .dateRestrictionRange(this.getDateRestrictionSelect(stepContext)) + .validityDate(this.getValidityDateSelect(stepContext)); List> conceptSelectFields = this.getColumnSelectReferences(table); List> conceptFilterFields = this.getColumnFilterReferences(table); @@ -53,43 +38,46 @@ public QueryStep buildPreprocessingQueryStepForTable(String conceptLabel, CQTabl .filter(field -> !conceptSelectFields.contains(field)) .toList(); - selectsBuilder.eventSelect(conceptSelectFields); - selectsBuilder.eventFilter(deduplicatedFilterFields); + selectsBuilder.eventSelect(conceptSelectFields). + eventFilter(deduplicatedFilterFields); // not part of preprocessing yet selectsBuilder.groupSelect(Collections.emptyList()) .groupFilter(Collections.emptyList()); return QueryStep.builder() - .cteName(this.getPreprocessingStepLabel(conceptLabel)) - .fromTable(QueryStep.toTableLike(this.getFromTableName(table))) .selects(selectsBuilder.build()) .conditions(Collections.emptyList()) - .predecessors(Collections.emptyList()) - .build(); + .predecessors(Collections.emptyList()); + } + + @Override + public String nameSuffix() { + return "_preprocessing"; } - private Optional getDateRestrictionSelect(CQTable table) { - if (!this.context.dateRestrictionActive() || !this.tableHasValidityDates(table)) { + private Optional getDateRestrictionSelect(final StepContext stepContext) { + if (!stepContext.getContext().dateRestrictionActive() || !this.tableHasValidityDates(stepContext.getTable())) { return Optional.empty(); } - return Optional.of(sqlFunctionProvider.daterange(context.getDateRestrictionRange())); + ColumnDateRange dateRestriction = stepContext.getContext().getSqlDialect().getFunction().daterange(stepContext.getContext().getDateRestrictionRange()); + return Optional.of(dateRestriction); } - private Optional getValidityDateSelect(CQTable table, String conceptLabel) { - if (!this.validityDateIsRequired(table)) { + private Optional getValidityDateSelect(final StepContext stepContext) { + if (!this.validityDateIsRequired(stepContext)) { return Optional.empty(); } - return Optional.of(sqlFunctionProvider.daterange(table.findValidityDate(), conceptLabel)); + return Optional.of(stepContext.getSqlFunctions().daterange(stepContext.getTable().findValidityDate(), stepContext.getConceptLabel())); } /** * @return True, if a date restriction is active and the node is not excluded from time aggregation * OR there is no date restriction, but still existing validity dates which are included in time aggregation. */ - private boolean validityDateIsRequired(CQTable table) { - return this.tableHasValidityDates(table) - && !this.concept.isExcludeFromTimeAggregation(); + private boolean validityDateIsRequired(final StepContext stepContext) { + return this.tableHasValidityDates(stepContext.getTable()) + && !stepContext.getNode().isExcludeFromTimeAggregation(); } private boolean tableHasValidityDates(CQTable table) { @@ -111,19 +99,9 @@ private List> getColumnFilterReferences(CQTable table) { .toList(); } - private String getFromTableName(CQTable table) { - return table.getConnector() - .getTable() - .getName(); - } private Field mapColumnOntoTable(Column column, CQTable table) { - return DSL.field(DSL.name(this.getFromTableName(table), column.getName())); - } - - private String getPreprocessingStepLabel(String conceptLabel) { - return "concept_%s_preprocessing".formatted(conceptLabel); - } + return DSL.field(DSL.name(table.getConnector().getTable().getName(), column.getName()));} } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/StepContext.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/StepContext.java new file mode 100644 index 0000000000..92f242c5bc --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/cqelement/concept/StepContext.java @@ -0,0 +1,22 @@ +package com.bakdata.conquery.sql.conversion.cqelement.concept; + +import com.bakdata.conquery.apiv1.query.concept.filter.CQTable; +import com.bakdata.conquery.apiv1.query.concept.specific.CQConcept; +import com.bakdata.conquery.sql.conversion.context.ConversionContext; +import com.bakdata.conquery.sql.conversion.context.selects.ConceptSelects; +import com.bakdata.conquery.sql.conversion.context.step.QueryStep; +import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; +import lombok.Builder; +import lombok.Value; + +@Value +@Builder(toBuilder = true) +class StepContext { + ConversionContext context; + SqlFunctionProvider sqlFunctions; + CQConcept node; + CQTable table; + String conceptLabel; + QueryStep previous; + ConceptSelects previousSelects; +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlDialect.java new file mode 100644 index 0000000000..41887c2a2c --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlDialect.java @@ -0,0 +1,46 @@ +package com.bakdata.conquery.sql.conversion.dialect; + +import java.util.List; + +import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; +import com.bakdata.conquery.models.datasets.concepts.select.Select; +import com.bakdata.conquery.models.query.Visitable; +import com.bakdata.conquery.sql.conversion.NodeConverter; +import com.bakdata.conquery.sql.conversion.filter.FilterConverter; +import com.bakdata.conquery.sql.conversion.select.SelectConverter; +import org.jooq.DSLContext; + +public class HanaSqlDialect implements SqlDialect { + + private final DSLContext dslContext; + + public HanaSqlDialect(DSLContext dslContext) { + this.dslContext = dslContext; + } + + @Override + public DSLContext getDSLContext() { + return this.dslContext; + } + + @Override + public List> getNodeConverters() { + return getDefaultNodeConverters(); + } + + @Override + public List>> getFilterConverters() { + return getDefaultFilterConverters(); + } + + @Override + public List> getSelectConverters() { + return getDefaultSelectConverters(); + } + + @Override + public SqlFunctionProvider getFunction() { + return new HanaSqlFunctionProvider(); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java new file mode 100644 index 0000000000..13a7682aca --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/HanaSqlFunctionProvider.java @@ -0,0 +1,134 @@ +package com.bakdata.conquery.sql.conversion.dialect; + +import java.sql.Date; +import java.time.temporal.ChronoUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import com.bakdata.conquery.models.common.daterange.CDateRange; +import com.bakdata.conquery.models.datasets.Column; +import com.bakdata.conquery.models.datasets.concepts.ValidityDate; +import com.bakdata.conquery.sql.models.ColumnDateRange; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.Name; +import org.jooq.impl.DSL; + +public class HanaSqlFunctionProvider implements SqlFunctionProvider { + + private static final String INFINITY_DATE_VALUE = "9999-12-31"; + private static final String MINUS_INFINITY_DATE_VALUE = "0001-01-01"; + + @Override + public Condition dateRestriction(ColumnDateRange dateRestriction, ColumnDateRange validityDate) { + + if (dateRestriction.isSingleColumnRange() || validityDate.isSingleColumnRange()) { + throw new UnsupportedOperationException("HANA does not support single column ranges."); + } + + Condition dateRestrictionStartsBeforeDate = dateRestriction.getStart().lessOrEqual(validityDate.getEnd()); + Condition dateRestrictionEndsAfterDate = dateRestriction.getEnd().greaterOrEqual(validityDate.getStart()); + + return DSL.condition(dateRestrictionStartsBeforeDate.and(dateRestrictionEndsAfterDate)); + } + + @Override + public ColumnDateRange daterange(CDateRange dateRestriction) { + + String startDateExpression = MINUS_INFINITY_DATE_VALUE; + String endDateExpression = INFINITY_DATE_VALUE; + + if (dateRestriction.hasLowerBound()) { + startDateExpression = dateRestriction.getMin().toString(); + } + if (dateRestriction.hasUpperBound()) { + endDateExpression = dateRestriction.getMax().toString(); + } + + return ColumnDateRange.of(toDateField(startDateExpression), toDateField(endDateExpression)) + .asDateRestrictionRange(); + } + + @Override + public ColumnDateRange daterange(ValidityDate validityDate, String conceptLabel) { + + Column startColumn; + Column endColumn; + + if (validityDate.getEndColumn() != null) { + startColumn = validityDate.getStartColumn(); + endColumn = validityDate.getEndColumn(); + } + else { + startColumn = validityDate.getColumn(); + endColumn = validityDate.getColumn(); + } + + // when aggregating date ranges, we want to treat the last day of the range as excluded, + // so when using the date value of the end column, we add +1 day as end of the date range + Field rangeStart = DSL.field(DSL.name(startColumn.getName()), Date.class); + Field rangeEnd = addDay(endColumn); + + return ColumnDateRange.of(rangeStart, rangeEnd) + .asValidityDateRange(conceptLabel); + } + + @Override + public Field daterangeString(ColumnDateRange columnDateRange) { + + if (columnDateRange.isSingleColumnRange()) { + throw new UnsupportedOperationException("HANA does not support single-column date ranges."); + } + + String datesConcatenated = Stream.of(columnDateRange.getStart(), columnDateRange.getEnd()) + .map(" || %s || "::formatted) + .collect(Collectors.joining(" ',' ", "'['", "')'")); + + return DSL.field(datesConcatenated); + } + + @Override + public Field dateDistance(ChronoUnit timeUnit, Name startDateColumnName, Date endDateExpression) { + + String betweenFunction = switch (timeUnit) { + case DAYS -> "DAYS_BETWEEN"; + case MONTHS -> "MONTHS_BETWEEN"; + case YEARS, DECADES, CENTURIES -> "YEARS_BETWEEN"; + default -> throw new UnsupportedOperationException("Given ChronoUnit %s is not supported."); + }; + + Field startDate = DSL.field(startDateColumnName, Date.class); + Field endDate = toDateField(endDateExpression.toString()); + Field dateDistance = DSL.function(betweenFunction, Integer.class, startDate, endDate); + + // HANA does not support decades or centuries directly + dateDistance = switch (timeUnit) { + case DECADES -> dateDistance.divide(10); + case CENTURIES -> dateDistance.divide(100); + default -> dateDistance; + }; + + // otherwise HANA would return floating point numbers for date distances + return dateDistance.cast(Integer.class); + } + + @Override + public Field toDateField(String dateExpression) { + return DSL.function( + "TO_DATE", + Date.class, + DSL.val(dateExpression), + DSL.val(DEFAULT_DATE_FORMAT) + ); + } + + private Field addDay(Column dateColumn) { + return DSL.function( + "ADD_DAYS", + Date.class, + DSL.field(DSL.name(dateColumn.getName())), + DSL.val(1) + ); + } + +} diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java index 5fe2c9620c..60925126bb 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/PostgreSqlFunctionProvider.java @@ -2,16 +2,15 @@ import java.sql.Date; import java.time.temporal.ChronoUnit; -import java.util.Map; import com.bakdata.conquery.models.common.daterange.CDateRange; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.sql.models.ColumnDateRange; -import org.jetbrains.annotations.NotNull; import org.jooq.Condition; import org.jooq.DatePart; import org.jooq.Field; +import org.jooq.Name; import org.jooq.impl.DSL; /** @@ -24,14 +23,6 @@ public class PostgreSqlFunctionProvider implements SqlFunctionProvider { private static final String INFINITY_DATE_VALUE = "infinity"; private static final String MINUS_INFINITY_DATE_VALUE = "-infinity"; - private static final Map DATE_CONVERSION = Map.of( - ChronoUnit.DECADES, DatePart.DECADE, - ChronoUnit.YEARS, DatePart.YEAR, - ChronoUnit.DAYS, DatePart.DAY, - ChronoUnit.MONTHS, DatePart.MONTH, - ChronoUnit.CENTURIES, DatePart.CENTURY - ); - @Override public Condition dateRestriction(ColumnDateRange dateRestriction, ColumnDateRange validityDate) { if (!validityDate.isSingleColumnRange()) { @@ -50,20 +41,20 @@ public Condition dateRestriction(ColumnDateRange dateRestriction, ColumnDateRang @Override public ColumnDateRange daterange(CDateRange dateRestriction) { - String min = MINUS_INFINITY_DATE_VALUE; - String max = INFINITY_DATE_VALUE; + String startDateExpression = MINUS_INFINITY_DATE_VALUE; + String endDateExpression = INFINITY_DATE_VALUE; if (dateRestriction.hasLowerBound()) { - min = dateRestriction.getMin().toString(); + startDateExpression = dateRestriction.getMin().toString(); } if (dateRestriction.hasUpperBound()) { - max = dateRestriction.getMax().toString(); + endDateExpression = dateRestriction.getMax().toString(); } Field dateRestrictionRange = DSL.field( "daterange({0}::date, {1}::date, '[]')", - DSL.val(min), - DSL.val(max) + DSL.val(startDateExpression), + DSL.val(endDateExpression) ); return ColumnDateRange.of(dateRestrictionRange) @@ -108,20 +99,28 @@ public Field daterangeString(ColumnDateRange columnDateRange) { } @Override - public Field dateDistance(ChronoUnit timeUnit, Column startDateColumn, Date endDateExpression) { + public Field dateDistance(ChronoUnit timeUnit, Name startDateColumnName, Date endDateExpression) { + + Field startDate = DSL.field(startDateColumnName, Date.class); + Field endDate = toDateField(endDateExpression.toString()); - DatePart datePart = DATE_CONVERSION.get(timeUnit); - if (datePart == null) { - throw new UnsupportedOperationException("Chrono unit %s is not supported".formatted(timeUnit)); + if (timeUnit == ChronoUnit.DAYS) { + return endDate.minus(startDate).coerce(Integer.class); } - // we can now safely cast to Field of type Date - Field startDate = DSL.field(DSL.name(startDateColumn.getName()), Date.class); - return DSL.dateDiff(datePart, startDate, endDateExpression); + Field age = DSL.function("AGE", Object.class, endDate, startDate); + + return switch (timeUnit) { + case MONTHS -> extract(DatePart.YEAR, age).multiply(12) + .plus(extract(DatePart.MONTH, age)); + case YEARS -> extract(DatePart.YEAR, age); + case DECADES -> extract(DatePart.DECADE, age); + case CENTURIES -> extract(DatePart.CENTURY, age); + default -> throw new UnsupportedOperationException("Given ChronoUnit %s is not supported."); + }; } - @NotNull - private static Field daterange(Column startColumn, Column endColumn, String bounds) { + private Field daterange(Column startColumn, Column endColumn, String bounds) { return DSL.function( "daterange", Object.class, @@ -131,4 +130,12 @@ private static Field daterange(Column startColumn, Column endColumn, Str ); } + private Field extract(DatePart datePart, Field timeInterval) { + return DSL.function( + "EXTRACT", + Integer.class, + DSL.inlined(DSL.field("%s FROM %s".formatted(datePart, timeInterval))) + ); + } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java index 8ee2a08256..6bdba63b99 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlDialect.java @@ -1,12 +1,21 @@ package com.bakdata.conquery.sql.conversion.dialect; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + import com.bakdata.conquery.apiv1.query.concept.filter.FilterValue; import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.query.Visitable; import com.bakdata.conquery.sql.conversion.Converter; import com.bakdata.conquery.sql.conversion.NodeConverter; import com.bakdata.conquery.sql.conversion.context.step.QueryStepTransformer; -import com.bakdata.conquery.sql.conversion.cqelement.*; +import com.bakdata.conquery.sql.conversion.cqelement.CQAndConverter; +import com.bakdata.conquery.sql.conversion.cqelement.CQDateRestrictionConverter; +import com.bakdata.conquery.sql.conversion.cqelement.CQNegationConverter; +import com.bakdata.conquery.sql.conversion.cqelement.CQOrConverter; +import com.bakdata.conquery.sql.conversion.cqelement.concept.CQConceptConverter; import com.bakdata.conquery.sql.conversion.filter.FilterConverter; import com.bakdata.conquery.sql.conversion.filter.FilterConverterService; import com.bakdata.conquery.sql.conversion.filter.MultiSelectConverter; @@ -19,11 +28,6 @@ import com.bakdata.conquery.sql.conversion.supplier.SystemDateNowSupplier; import org.jooq.DSLContext; -import java.util.List; -import java.util.Map; -import java.util.function.Function; -import java.util.stream.Collectors; - public interface SqlDialect { SqlFunctionProvider getFunction(); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java index 9e5ebab7df..3a0b305139 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/dialect/SqlFunctionProvider.java @@ -4,7 +4,6 @@ import java.time.temporal.ChronoUnit; import com.bakdata.conquery.models.common.daterange.CDateRange; -import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.ValidityDate; import com.bakdata.conquery.sql.conversion.context.step.QueryStep; import com.bakdata.conquery.sql.models.ColumnDateRange; @@ -35,7 +34,7 @@ public interface SqlFunctionProvider { Field daterangeString(ColumnDateRange columnDateRange); - Field dateDistance(ChronoUnit datePart, Column startDateColumn, Date endDateExpression); + Field dateDistance(ChronoUnit datePart, Name startDateColumn, Date endDateExpression); default Condition in(Name columnName, String[] values) { return DSL.field(columnName) @@ -69,7 +68,7 @@ default TableOnConditionStep fullOuterJoin( .on(leftPartPrimaryColumn.eq(rightPartPrimaryColumn)); } - default Field toDate(String dateExpression) { + default Field toDateField(String dateExpression) { return DSL.toDate(dateExpression, DEFAULT_DATE_FORMAT); } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/DateDistanceConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/DateDistanceConverter.java index 52339db897..c2f90372f8 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/DateDistanceConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/DateDistanceConverter.java @@ -2,15 +2,15 @@ import java.sql.Date; import java.time.LocalDate; +import java.time.temporal.ChronoUnit; import java.util.Objects; -import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.select.connector.specific.DateDistanceSelect; -import com.bakdata.conquery.models.events.MajorTypeId; import com.bakdata.conquery.sql.conversion.context.ConversionContext; -import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import com.bakdata.conquery.sql.conversion.supplier.DateNowSupplier; import org.jooq.Field; +import org.jooq.Name; +import org.jooq.impl.DSL; public class DateDistanceConverter implements SelectConverter { @@ -23,17 +23,12 @@ public DateDistanceConverter(DateNowSupplier dateNowSupplier) { @Override public Field convert(DateDistanceSelect select, ConversionContext context) { - Column startDateColumn = select.getColumn(); - if (startDateColumn.getType() != MajorTypeId.DATE) { - throw new UnsupportedOperationException("Can't calculate date distance to column of type " - + startDateColumn.getType()); - } - - SqlFunctionProvider functionProvider = context.getSqlDialect().getFunction(); + ChronoUnit timeUnit = select.getTimeUnit(); + Name startDateColumnName = DSL.name(select.getColumn().getName()); Date endDate = getEndDate(context); - return functionProvider.dateDistance(select.getTimeUnit(), startDateColumn, endDate) - .as(select.getLabel()); + return context.getSqlDialect().getFunction().dateDistance(timeUnit, startDateColumnName, endDate) + .as(select.getLabel()); } private Date getEndDate(ConversionContext context) { @@ -53,4 +48,5 @@ private Date getEndDate(ConversionContext context) { public Class getConversionClass() { return DateDistanceSelect.class; } + } diff --git a/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/FirstValueConverter.java b/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/FirstValueConverter.java index af1593bf91..f91c5e4d8a 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/FirstValueConverter.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/conversion/select/FirstValueConverter.java @@ -4,15 +4,13 @@ import com.bakdata.conquery.sql.conversion.context.ConversionContext; import com.bakdata.conquery.sql.conversion.dialect.SqlFunctionProvider; import org.jooq.Field; -import org.jooq.Name; import org.jooq.impl.DSL; public class FirstValueConverter implements SelectConverter { public Field convert(FirstValueSelect select, ConversionContext context) { SqlFunctionProvider fn = context.getSqlDialect().getFunction(); - Name columnName = DSL.name(select.getColumn().getName()); - return fn.first(columnName); + return fn.first(DSL.name(select.getColumn().getName())); } @Override diff --git a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java index 5bea87bd0d..e847611ad5 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/execution/SqlExecutionService.java @@ -10,7 +10,6 @@ import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.query.results.EntityResult; -import com.bakdata.conquery.models.query.results.SinglelineEntityResult; import com.bakdata.conquery.sql.conquery.SqlManagedQuery; import com.google.common.base.Stopwatch; import lombok.RequiredArgsConstructor; @@ -33,8 +32,10 @@ public SqlExecutionResult execute(SqlManagedQuery sqlQuery) { private SqlExecutionResult createStatementAndExecute(SqlManagedQuery sqlQuery, Connection connection) { + String sqlString = sqlQuery.getSqlQuery().getSqlString(); + log.debug("Executing query: \n{}", sqlString); try (Statement statement = connection.createStatement(); - ResultSet resultSet = statement.executeQuery(sqlQuery.getSqlQuery().getSqlString())) { + ResultSet resultSet = statement.executeQuery(sqlString)) { int columnCount = resultSet.getMetaData().getColumnCount(); List columnNames = this.getColumnNames(resultSet, columnCount); List resultTable = this.createResultTable(resultSet, columnCount); diff --git a/backend/src/main/java/com/bakdata/conquery/sql/models/ColumnDateRange.java b/backend/src/main/java/com/bakdata/conquery/sql/models/ColumnDateRange.java index e0faabdf4f..6e1f832c18 100644 --- a/backend/src/main/java/com/bakdata/conquery/sql/models/ColumnDateRange.java +++ b/backend/src/main/java/com/bakdata/conquery/sql/models/ColumnDateRange.java @@ -1,6 +1,8 @@ package com.bakdata.conquery.sql.models; +import java.sql.Date; import java.util.List; +import java.util.stream.Stream; import lombok.Getter; import org.jooq.Field; @@ -16,10 +18,10 @@ public class ColumnDateRange { private final boolean isEmpty; private final Field range; - private final Field start; - private final Field end; + private final Field start; + private final Field end; - private ColumnDateRange(boolean isEmpty, Field range, Field startColumn, Field endColumn) { + private ColumnDateRange(boolean isEmpty, Field range, Field startColumn, Field endColumn) { this.isEmpty = isEmpty; this.range = range; this.start = startColumn; @@ -30,7 +32,7 @@ public static ColumnDateRange of(Field rangeColumn) { return new ColumnDateRange(false, rangeColumn, null, null); } - public static ColumnDateRange of(Field startColumn, Field endColumn) { + public static ColumnDateRange of(Field startColumn, Field endColumn) { return new ColumnDateRange(true, null, startColumn, endColumn); } @@ -54,16 +56,18 @@ public List> toFields() { if (isSingleColumnRange()) { return List.of(this.range); } - return List.of(this.start, this.end); + return Stream.of(this.start, this.end) + .map(dateField -> dateField.coerce(Object.class)) + .toList(); } public ColumnDateRange qualify(String qualifier) { if (isSingleColumnRange()) { - return ColumnDateRange.of(mapFieldOntoQualifier(getRange(), qualifier)); + return ColumnDateRange.of(mapFieldOntoQualifier(getRange(), Object.class, qualifier)); } return ColumnDateRange.of( - mapFieldOntoQualifier(getStart(), qualifier), - mapFieldOntoQualifier(getEnd(), qualifier) + mapFieldOntoQualifier(getStart(), Date.class, qualifier), + mapFieldOntoQualifier(getEnd(), Date.class, qualifier) ); } @@ -77,8 +81,8 @@ private ColumnDateRange as(String alias) { ); } - private Field mapFieldOntoQualifier(Field field, String qualifier) { - return DSL.field(DSL.name(qualifier, field.getName())); + private Field mapFieldOntoQualifier(Field field, Class fieldType, String qualifier) { + return DSL.field(DSL.name(qualifier, field.getName()), fieldType); } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java index d40a970169..72072df65e 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/IntegrationTests.java @@ -54,6 +54,9 @@ public class IntegrationTests { private static final ObjectWriter CONFIG_WRITER; static { + + SharedMetricRegistries.setDefault("test"); + final ObjectMapper mapper = Jackson.MAPPER.copy(); MAPPER = mapper.setConfig(mapper.getDeserializationConfig().withView(View.Persistence.class)) @@ -135,7 +138,6 @@ public Stream programmaticTests() { @SneakyThrows public Stream sqlTests(SqlDialect sqlDialect, SqlConnectorConfig sqlConfig) { - SharedMetricRegistries.setDefault("test"); final Path testRootDir = Path.of(Objects.requireNonNullElse( System.getenv(TestTags.SQL_BACKEND_TEST_DIRECTORY_ENVIRONMENT_VARIABLE), SqlIntegrationTest.SQL_TEST_DIR diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java index 78a3d72366..3af71cc613 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/CsvTableImporter.java @@ -1,12 +1,12 @@ package com.bakdata.conquery.integration.sql; - import java.io.IOException; import java.math.BigDecimal; import java.nio.file.Files; import java.nio.file.Path; import java.sql.Connection; import java.sql.Date; +import java.sql.SQLException; import java.sql.Statement; import java.util.ArrayList; import java.util.Arrays; @@ -26,6 +26,7 @@ import com.google.common.base.Strings; import com.univocity.parsers.csv.CsvParser; import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; import org.jooq.DataType; import org.jooq.Field; @@ -38,8 +39,10 @@ import org.jooq.impl.SQLDataType; import org.jooq.postgres.extensions.types.DateRange; +@Slf4j public class CsvTableImporter { + private static final int DEFAULT_VARCHAR_LENGTH = 25; // HANA will use 1 as default otherwise private final DSLContext dslContext; private final DateRangeParser dateRangeParser; private final CsvParser csvReader; @@ -56,30 +59,17 @@ public CsvTableImporter(DSLContext dslContext) { */ public void importTableIntoDatabase(RequiredTable requiredTable) { - Table table = DSL.table(requiredTable.getName()); + Table table = DSL.table(DSL.name(requiredTable.getName())); List allRequiredColumns = this.getAllRequiredColumns(requiredTable); List> columns = this.createFieldsForColumns(allRequiredColumns); List content = this.getTablesContentFromCSV(requiredTable.getCsv(), allRequiredColumns); - // because we currently won't shut down the container between the testcases, we drop tables upfront if they - // exist to ensure consistency if table names of different testcases are the same - String dropTableStatement = dslContext.dropTableIfExists(table) - .getSQL(ParamType.INLINED); - - String createTableStatement = dslContext.createTable(table) - .columns(columns) - .getSQL(ParamType.INLINED); - - String insertIntoTableStatement = dslContext.insertInto(table, columns) - .valuesOfRows(content) - .getSQL(ParamType.INLINED); - - // we directly use JDBC because JOOQ can't cope with PostgreSQL custom types + // we directly use JDBC because JOOQ can't cope with some custom types like daterange dslContext.connection((Connection connection) -> { try (Statement statement = connection.createStatement()) { - statement.execute(dropTableStatement); - statement.execute(createTableStatement); - statement.execute(insertIntoTableStatement); + dropTable(table, statement); + createTable(table, columns, statement); + insertValuesIntoTable(table, columns, content, statement); } }); } @@ -94,6 +84,36 @@ public List readExpectedEntities(Path csv) throws IOException { return results; } + private void insertValuesIntoTable(Table table, List> columns, List content, Statement statement) throws SQLException { + for (RowN rowN : content) { + // e.g. HANA does not support bulk insert, so we insert row by row + String insertRowStatement = dslContext.insertInto(table, columns) + .values(rowN) + .getSQL(ParamType.INLINED); + log.info("Inserting into table: {}", insertRowStatement); + statement.execute(insertRowStatement); + } + } + + private void createTable(Table table, List> columns, Statement statement) throws SQLException { + String createTableStatement = dslContext.createTable(table) + .columns(columns) + .getSQL(ParamType.INLINED); + log.info("Creating table: {}", createTableStatement); + statement.execute(createTableStatement); + } + + private void dropTable(Table table, Statement statement) { + try { + // DROP TABLE IF EXISTS is not supported in HANA, we just ignore possible errors if the table does not exist + String dropTableStatement = dslContext.dropTable(table) + .getSQL(ParamType.INLINED); + statement.execute(dropTableStatement); + } + catch (SQLException e) { + log.info("Dropping table {} failed.", table.getName(), e); + } + } private List> createFieldsForColumns(List requiredColumns) { return requiredColumns.stream() @@ -110,15 +130,16 @@ private List getAllRequiredColumns(RequiredTable table) { private Field createField(RequiredColumn requiredColumn) { DataType dataType = switch (requiredColumn.getType()) { - case STRING -> SQLDataType.VARCHAR; + case STRING -> SQLDataType.VARCHAR(DEFAULT_VARCHAR_LENGTH); case INTEGER -> SQLDataType.INTEGER; case BOOLEAN -> SQLDataType.BOOLEAN; - case REAL -> SQLDataType.REAL; + // TODO: temporary workaround until we cast ResultSet elements back + case REAL -> SQLDataType.DECIMAL(10,2); case DECIMAL, MONEY -> SQLDataType.DECIMAL; case DATE -> SQLDataType.DATE; case DATE_RANGE -> new BuiltInDataType<>(DateRange.class, "daterange"); }; - return DSL.field(requiredColumn.getName(), dataType); + return DSL.field(DSL.name(requiredColumn.getName()), dataType); } @SneakyThrows diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlIntegrationTestSpec.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlIntegrationTestSpec.java index ba61fe0173..c6340f3227 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlIntegrationTestSpec.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/SqlIntegrationTestSpec.java @@ -94,13 +94,16 @@ public void executeTest(SqlStandaloneSupport support) throws IOException { SqlManagedQuery managedQuery = support.getExecutionManager() .runQuery(support.getNamespace(), getQuery(), support.getTestUser(), support.getDataset(), support.getConfig(), false); - log.info("Execute query: \n{}", managedQuery.getSqlQuery().getSqlString()); SqlExecutionResult result = managedQuery.getResult(); List resultCsv = result.getTable(); + Path expectedCsvFile = this.specDir.resolve(this.expectedCsv); List expectedCsv = support.getTableImporter().readExpectedEntities(expectedCsvFile); - Assertions.assertThat(resultCsv).usingRecursiveFieldByFieldElementComparator().containsExactlyElementsOf(expectedCsv); + + Assertions.assertThat(resultCsv) + .usingRecursiveFieldByFieldElementComparatorIgnoringFields("entityId") + .containsExactlyInAnyOrderElementsOf(expectedCsv); } @Override @@ -128,5 +131,4 @@ private void importConcepts(SqlStandaloneSupport support) throws IOException, JS } } - } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/TestPostgreSqlDialect.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/TestPostgreSqlDialect.java deleted file mode 100644 index 09ee227eb3..0000000000 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/TestPostgreSqlDialect.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.bakdata.conquery.integration.sql; - -import com.bakdata.conquery.models.datasets.concepts.select.Select; -import com.bakdata.conquery.sql.conversion.select.SelectConverter; -import com.bakdata.conquery.sql.conversion.select.DateDistanceConverter; -import com.bakdata.conquery.sql.conversion.dialect.PostgreSqlDialect; -import com.bakdata.conquery.sql.conversion.supplier.DateNowSupplier; -import org.jooq.DSLContext; - -import java.time.LocalDate; -import java.util.List; - -public class TestPostgreSqlDialect extends PostgreSqlDialect { - - public TestPostgreSqlDialect(DSLContext dslContext) { - super(dslContext); - } - - @Override - public List> getSelectConverters() { - return this.customizeSelectConverters(List.of( - new DateDistanceConverter(new MockDateNowSupplier()) - )); - } - - private class MockDateNowSupplier implements DateNowSupplier { - - @Override - public LocalDate getLocalDateNow() { - return LocalDate.parse("2023-03-28"); - } - - } - -} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java new file mode 100644 index 0000000000..8da1528554 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/HanaSqlIntegrationTests.java @@ -0,0 +1,168 @@ +package com.bakdata.conquery.integration.sql.dialect; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.PosixFilePermission; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Stream; + +import com.bakdata.conquery.TestTags; +import com.bakdata.conquery.integration.IntegrationTests; +import com.bakdata.conquery.integration.sql.testcontainer.hana.HanaContainer; +import com.bakdata.conquery.models.config.Dialect; +import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.models.datasets.concepts.select.Select; +import com.bakdata.conquery.sql.DslContextFactory; +import com.bakdata.conquery.sql.conversion.dialect.HanaSqlDialect; +import com.bakdata.conquery.sql.conversion.select.DateDistanceConverter; +import com.bakdata.conquery.sql.conversion.select.SelectConverter; +import com.google.common.base.Strings; +import lombok.Getter; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.jooq.DSLContext; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DynamicTest; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.TestFactory; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.utility.DockerImageName; + +@Slf4j +public class HanaSqlIntegrationTests extends IntegrationTests { + + private final static DockerImageName HANA_IMAGE = DockerImageName.parse("saplabs/hanaexpress:2.00.061.00.20220519.1"); + private static final Path TMP_HANA_MOUNT_DIR = Paths.get("/tmp/data/hana"); + private static boolean useLocalHanaDb = true; + + static { + final String USE_LOCAL_HANA_DB = System.getenv("USE_LOCAL_HANA_DB"); + if (!Strings.isNullOrEmpty(USE_LOCAL_HANA_DB)) { + useLocalHanaDb = Boolean.parseBoolean(USE_LOCAL_HANA_DB); + } + } + + public HanaSqlIntegrationTests() { + super("tests/", "com.bakdata.conquery.integration"); + } + + @TestFactory + @Tag(TestTags.INTEGRATION_SQL_BACKEND) + public Stream sqlBackendTests() { + + TestContextProvider provider = useLocalHanaDb + ? new HanaTestcontainerContextProvider() + : new RemoteHanaContextProvider(); + + log.info("Running HANA tests with %s.".formatted(provider.getClass().getSimpleName())); + + DSLContext dslContext = provider.getDslContext(); + SqlConnectorConfig config = provider.getSqlConnectorConfig(); + + return super.sqlTests(new TestHanaDialect(dslContext), config); + } + + @SneakyThrows + @BeforeAll + public static void prepareTmpHanaDir() { + + if (!useLocalHanaDb) { + return; + } + + Path masterPasswordFile = TMP_HANA_MOUNT_DIR.resolve("password.json"); + String content = "{\"master_password\":\"%s\"}".formatted(HanaContainer.DEFAULT_MASTER_PASSWORD); + + Files.createDirectories(TMP_HANA_MOUNT_DIR); + Files.write(masterPasswordFile, content.getBytes(), StandardOpenOption.CREATE, StandardOpenOption.WRITE); + Files.setPosixFilePermissions(TMP_HANA_MOUNT_DIR, Set.of(PosixFilePermission.values())); + } + + @SneakyThrows + @AfterAll + public static void tearDownClass() { + if (!Files.exists(TMP_HANA_MOUNT_DIR)) { + return; + } + try (Stream walk = Files.walk(TMP_HANA_MOUNT_DIR)) { + walk.sorted((p1, p2) -> - p1.compareTo(p2)) + .map(Path::toFile) + .forEach(File::delete); + } + } + + private static class TestHanaDialect extends HanaSqlDialect { + + public TestHanaDialect(DSLContext dslContext) { + super(dslContext); + } + + @Override + public List> getSelectConverters() { + return this.customizeSelectConverters(List.of( + new DateDistanceConverter(new MockDateNowSupplier()) + )); + } + + } + + @Getter + private static class HanaTestcontainerContextProvider implements TestContextProvider { + + private final DSLContext dslContext; + private final SqlConnectorConfig sqlConnectorConfig; + + @Container + private final HanaContainer hanaContainer; + + public HanaTestcontainerContextProvider() { + this.hanaContainer = new HanaContainer<>(HANA_IMAGE) + .withFileSystemBind(TMP_HANA_MOUNT_DIR.toString(), "/home/secrets"); + this.hanaContainer.start(); + + this.sqlConnectorConfig = SqlConnectorConfig.builder() + .dialect(Dialect.HANA) + .jdbcConnectionUrl(hanaContainer.getJdbcUrl()) + .databaseUsername(hanaContainer.getUsername()) + .databasePassword(hanaContainer.getPassword()) + .withPrettyPrinting(true) + .primaryColumn("pid") + .build(); + this.dslContext = DslContextFactory.create(sqlConnectorConfig); + } + + } + + @Getter + private static class RemoteHanaContextProvider implements TestContextProvider { + + private final static String PORT = Objects.requireNonNullElse(System.getenv("CONQUERY_SQL_PORT"), "39041"); + private final static String HOST = System.getenv("CONQUERY_SQL_DB"); + private final static String CONNECTION_URL = "jdbc:sap://%s:%s/databaseName=HXE&encrypt=true&validateCertificate=false".formatted(HOST, PORT); + private final static String USERNAME = System.getenv("CONQUERY_SQL_USER"); + private final static String PASSWORD = System.getenv("CONQUERY_SQL_PASSWORD"); + private final DSLContext dslContext; + private final SqlConnectorConfig sqlConnectorConfig; + + public RemoteHanaContextProvider() { + this.sqlConnectorConfig = SqlConnectorConfig.builder() + .enabled(true) + .dialect(Dialect.HANA) + .withPrettyPrinting(true) + .jdbcConnectionUrl(CONNECTION_URL) + .databaseUsername(USERNAME) + .databasePassword(PASSWORD) + .primaryColumn("pid") + .build(); + this.dslContext = DslContextFactory.create(sqlConnectorConfig); + } + + } + +} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/MockDateNowSupplier.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/MockDateNowSupplier.java new file mode 100644 index 0000000000..47a510d588 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/MockDateNowSupplier.java @@ -0,0 +1,14 @@ +package com.bakdata.conquery.integration.sql.dialect; + +import java.time.LocalDate; + +import com.bakdata.conquery.sql.conversion.supplier.DateNowSupplier; + +public class MockDateNowSupplier implements DateNowSupplier { + + @Override + public LocalDate getLocalDateNow() { + return LocalDate.parse("2023-03-28"); + } + +} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/PostgreSqlIntegrationTests.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java similarity index 67% rename from backend/src/test/java/com/bakdata/conquery/integration/sql/PostgreSqlIntegrationTests.java rename to backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java index 2e83b9b7e6..d4804c3076 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/sql/PostgreSqlIntegrationTests.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/PostgreSqlIntegrationTests.java @@ -1,7 +1,6 @@ -package com.bakdata.conquery.integration.sql; - -import static org.assertj.core.api.Assertions.assertThat; +package com.bakdata.conquery.integration.sql.dialect; +import java.util.List; import java.util.stream.Stream; import com.bakdata.conquery.TestTags; @@ -9,11 +8,15 @@ import com.bakdata.conquery.integration.IntegrationTests; import com.bakdata.conquery.models.config.Dialect; import com.bakdata.conquery.models.config.SqlConnectorConfig; +import com.bakdata.conquery.models.datasets.concepts.select.Select; import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.i18n.I18n; import com.bakdata.conquery.sql.DslContextFactory; import com.bakdata.conquery.sql.SqlQuery; import com.bakdata.conquery.sql.conquery.SqlManagedQuery; +import com.bakdata.conquery.sql.conversion.dialect.PostgreSqlDialect; +import com.bakdata.conquery.sql.conversion.select.DateDistanceConverter; +import com.bakdata.conquery.sql.conversion.select.SelectConverter; import com.bakdata.conquery.sql.execution.SqlExecutionService; import lombok.extern.slf4j.Slf4j; import org.assertj.core.api.Assertions; @@ -33,9 +36,9 @@ public class PostgreSqlIntegrationTests extends IntegrationTests { private static final DockerImageName postgreSqlImageName = DockerImageName.parse("postgres:alpine3.17"); - private static final String databaseName = "test"; - private static final String username = "user"; - private static final String password = "pass"; + private static final String DATABASE_NAME = "test"; + private static final String USERNAME = "user"; + private static final String PASSWORD = "pass"; private static DSLContext dslContext; private static SqlConnectorConfig sqlConfig; @@ -44,20 +47,20 @@ public PostgreSqlIntegrationTests() { } @Container - private static final PostgreSQLContainer postgresqlContainer = new PostgreSQLContainer<>(postgreSqlImageName) - .withDatabaseName(databaseName) - .withUsername(username) - .withPassword(password); + private static final PostgreSQLContainer POSTGRESQL_CONTAINER = new PostgreSQLContainer<>(postgreSqlImageName) + .withDatabaseName(DATABASE_NAME) + .withUsername(USERNAME) + .withPassword(PASSWORD); @BeforeAll static void before() { - postgresqlContainer.start(); + POSTGRESQL_CONTAINER.start(); sqlConfig = SqlConnectorConfig.builder() .dialect(Dialect.POSTGRESQL) - .jdbcConnectionUrl(postgresqlContainer.getJdbcUrl()) - .databaseUsername(username) - .databasePassword(password) + .jdbcConnectionUrl(POSTGRESQL_CONTAINER.getJdbcUrl()) + .databaseUsername(USERNAME) + .databasePassword(PASSWORD) .withPrettyPrinting(true) .primaryColumn("pid") .build(); @@ -77,7 +80,7 @@ public void shouldThrowException() { SqlManagedQuery emptyQuery = new SqlManagedQuery(new ConceptQuery(), null, null, null, new SqlQuery("")); Assertions.assertThatThrownBy(() -> executionService.execute(emptyQuery)) .isInstanceOf(ConqueryError.SqlError.class) - .hasMessageContaining("Something went wrong while querying the database: $org.postgresql.util.PSQLException"); + .hasMessageContaining("$org.postgresql.util.PSQLException"); } @@ -87,5 +90,19 @@ public Stream sqlBackendTests() { return super.sqlTests(new TestPostgreSqlDialect(dslContext), sqlConfig); } + private static class TestPostgreSqlDialect extends PostgreSqlDialect { + + public TestPostgreSqlDialect(DSLContext dslContext) { + super(dslContext); + } + + @Override + public List> getSelectConverters() { + return this.customizeSelectConverters(List.of( + new DateDistanceConverter(new MockDateNowSupplier()) + )); + } + + } } diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestContextProvider.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestContextProvider.java new file mode 100644 index 0000000000..a4971a7ff5 --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/dialect/TestContextProvider.java @@ -0,0 +1,11 @@ +package com.bakdata.conquery.integration.sql.dialect; + +import com.bakdata.conquery.models.config.SqlConnectorConfig; +import org.jooq.DSLContext; + +public interface TestContextProvider { + + SqlConnectorConfig getSqlConnectorConfig(); + DSLContext getDslContext(); + +} diff --git a/backend/src/test/java/com/bakdata/conquery/integration/sql/testcontainer/hana/HanaContainer.java b/backend/src/test/java/com/bakdata/conquery/integration/sql/testcontainer/hana/HanaContainer.java new file mode 100644 index 0000000000..b60a476ffe --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/integration/sql/testcontainer/hana/HanaContainer.java @@ -0,0 +1,71 @@ +package com.bakdata.conquery.integration.sql.testcontainer.hana; + +import java.time.Duration; + +import org.testcontainers.containers.JdbcDatabaseContainer; +import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy; +import org.testcontainers.utility.DockerImageName; + +public class HanaContainer> extends JdbcDatabaseContainer { + + public static final Integer DEFAULT_TENANT_HANA_PORT = 39041; + public static final String DEFAULT_MASTER_PASSWORD = "HXEHana1"; + private static final String USERNAME = "SYSTEM"; + private static final String DATABASE_NAME = "HXE"; + + public HanaContainer(DockerImageName dockerImageName) { + super(dockerImageName); + setWaitStrategy( + new LogMessageWaitStrategy() + .withRegEx(".*Startup finished.*\\s") + .withStartupTimeout(Duration.ofMinutes(10)) + ); + addExposedPort(DEFAULT_TENANT_HANA_PORT); + setCommand(composeHanaArgs()); + } + + @Override + public String getDriverClassName() { + return "com.sap.cloud.db.jdbc"; + } + + @Override + public String getJdbcUrl() { + return "jdbc:sap://%s:%s/?databaseName=%s&encrypt=true&validateCertificate=false".formatted( + getHost(), + getMappedPort(DEFAULT_TENANT_HANA_PORT), + DATABASE_NAME + ); + } + + @Override + public String getUsername() { + return USERNAME; + } + + @Override + public String getPassword() { + return DEFAULT_MASTER_PASSWORD; + } + + @Override + public String getDatabaseName() { + return DATABASE_NAME; + } + + @Override + protected String getTestQueryString() { + return "SELECT 1"; + } + + @Override + protected void waitUntilContainerStarted() { + getWaitStrategy().waitUntilReady(this); + } + + private String composeHanaArgs() { + return "--agree-to-sap-license " + + "--passwords-url file:///home/secrets/password.json"; + } + +} diff --git a/backend/src/test/resources/tests/sql/and/different_concept/and.json b/backend/src/test/resources/tests/sql/and/different_concept/and.json index 9e855976df..67b6bdf742 100644 --- a/backend/src/test/resources/tests/sql/and/different_concept/and.json +++ b/backend/src/test/resources/tests/sql/and/different_concept/and.json @@ -22,7 +22,7 @@ "type": "REAL_RANGE", "value": { "min": 0, - "max": 1 + "max": 1.0 } } ], @@ -93,7 +93,8 @@ "table": "table1", "validityDates": { "label": "datum", - "column": "table1.datum" + "startColumn": "table1.datum_start", + "endColumn": "table1.datum_end" }, "filters": { "label": "value", @@ -175,8 +176,12 @@ "type": "REAL" }, { - "name": "datum", - "type": "DATE_RANGE" + "name": "datum_start", + "type": "DATE" + }, + { + "name": "datum_end", + "type": "DATE" } ] }, diff --git a/backend/src/test/resources/tests/sql/and/different_concept/content_1.csv b/backend/src/test/resources/tests/sql/and/different_concept/content_1.csv index 1851eed139..244295a111 100644 --- a/backend/src/test/resources/tests/sql/and/different_concept/content_1.csv +++ b/backend/src/test/resources/tests/sql/and/different_concept/content_1.csv @@ -1,13 +1,13 @@ -pid,value,datum -1,1,"2014-06-30/2015-06-30" -2,1.01,"2014-06-30/2015-06-30" -1,1,"2015-02-03/2015-06-30" -1,0.5,"2014-06-30/2015-06-30" -3,0.5,"2014-04-30/2014-06-30" -4,1,"2014-06-30/2015-06-30" -5,0.5,"2014-04-30/2014-06-30" -5,1,"2014-06-30/2015-06-30" -6,1,"2014-04-30/2014-06-30" -7,1,"2014-02-05/2014-02-20" -8,1,"2014-04-30/2014-06-30" -7,-1,"2014-06-30/2015-06-30" +pid,value,datum_start,datum_end +1,1,2014-06-30,2015-06-30 +2,1.01,2014-06-30,2015-06-30 +1,1,2015-02-03,2015-06-30 +1,0.5,2014-06-30,2015-06-30 +3,0.5,2014-04-30,2014-06-30 +4,1,2014-06-30,2015-06-30 +5,0.5,2014-04-30,2014-06-30 +5,1,2014-06-30,2015-06-30 +6,1,2014-04-30,2014-06-30 +7,1,2014-02-05,2014-02-20 +8,1,2014-04-30,2014-06-30 +7,-1,2014-06-30,2015-06-30 diff --git a/backend/src/test/resources/tests/sql/and/different_concept/expected.csv b/backend/src/test/resources/tests/sql/and/different_concept/expected.csv index beeae56d79..c1885bafaf 100644 --- a/backend/src/test/resources/tests/sql/and/different_concept/expected.csv +++ b/backend/src/test/resources/tests/sql/and/different_concept/expected.csv @@ -1,4 +1,4 @@ -pid,datum,value,geschlecht,language -1,"[2014-06-30,2015-06-30)",1,f,de -1,"[2015-02-03,2015-06-30)",1,f,de -1,"[2014-06-30,2015-06-30)",0.5,f,de +pid,validity_date_1value,geschlecht,language +1,"[2014-06-30,2015-07-01)",1.00,f,de +1,"[2015-02-03,2015-07-01)",1.00,f,de +1,"[2014-06-30,2015-07-01)",0.50,f,de diff --git a/backend/src/test/resources/tests/sql/and/same_concept/and.json b/backend/src/test/resources/tests/sql/and/same_concept/and_same_concept.json similarity index 100% rename from backend/src/test/resources/tests/sql/and/same_concept/and.json rename to backend/src/test/resources/tests/sql/and/same_concept/and_same_concept.json diff --git a/backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/content.csv b/backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/content.csv index 6f280c47ef..69f66cbc3e 100644 --- a/backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/content.csv +++ b/backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/content.csv @@ -1,9 +1,9 @@ -pid,datum,datum_alt,geschlecht -1,"2012-06-30/2015-06-30",2012-01-01,"f" -2,"2012-06-30/2015-06-30",2010-07-15,"m" -3,"2012-02-03/2012-06-30",2012-11-10,"f" -4,"2010-06-30/2015-06-30",2012-11-11,"m" -5,"2011-04-30/2014-06-30",2007-11-11,"" -6,"2015-06-30/2016-06-30",2012-11-11,"" -7,"2014-04-30/2015-06-30",2012-11-11,"mf" -8,"2012-04-30/2014-06-30",2012-11-11,"fm" +pid,datum_start,datum_end,datum_alt,geschlecht +1,2012-06-30,2015-06-30,2012-01-01,"f" +2,2012-06-30,2015-06-30,2010-07-15,"m" +3,2012-02-03,2012-06-30,2012-11-10,"f" +4,2010-06-30,2015-06-30,2012-11-11,"m" +5,2011-04-30,2014-06-30,2007-11-11,"" +6,2015-06-30,2016-06-30,2012-11-11,"" +7,2014-04-30,2015-06-30,2012-11-11,"mf" +8,2012-04-30,2014-06-30,2012-11-11,"fm" diff --git a/backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/date_restriction_date_column.json b/backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/date_restriction_date_column.json index cecd7086e0..08897212ea 100644 --- a/backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/date_restriction_date_column.json +++ b/backend/src/test/resources/tests/sql/date_restriction/date_restriction_date_column/date_restriction_date_column.json @@ -52,7 +52,8 @@ "validityDates": [ { "label": "datum", - "column": "table1.datum" + "startColumn": "table1.datum_start", + "endColumn": "table1.datum_end" }, { "label": "datum_alt", @@ -80,8 +81,12 @@ }, "columns": [ { - "name": "datum", - "type": "DATE_RANGE" + "name": "datum_start", + "type": "DATE" + }, + { + "name": "datum_end", + "type": "DATE" }, { "name": "datum_alt", diff --git a/backend/src/test/resources/tests/sql/date_restriction/daterange/date_restriction_date_range.json b/backend/src/test/resources/tests/sql/date_restriction/daterange/date_restriction_date_range.json deleted file mode 100644 index 4942aacbfa..0000000000 --- a/backend/src/test/resources/tests/sql/date_restriction/daterange/date_restriction_date_range.json +++ /dev/null @@ -1,85 +0,0 @@ -{ - "label": "Date restriction query with daterange validity date", - "type": "SQL_TEST", - "expectedCsv": "expected.csv", - "query": { - "type": "CONCEPT_QUERY", - "root": { - "type": "AND", - "children": [ - { - "type": "DATE_RESTRICTION", - "dateRange": { - "min": "2012-01-01", - "max": "2012-12-31" - }, - "child": { - "ids": [ - "geschlecht_select" - ], - "type": "CONCEPT", - "label": "Geschlecht SELECT", - "tables": [ - { - "id": "geschlecht_select.geschlecht_connector", - "filters": [ - { - "filter": "geschlecht_select.geschlecht_connector.geschlecht", - "type": "BIG_MULTI_SELECT", - "value": [ - "f" - ] - } - ] - } - ] - } - } - ] - } - }, - "concepts": [ - { - "label": "geschlecht_select", - "type": "TREE", - "connectors": [ - { - "label": "geschlecht_connector", - "table": "table1", - "validityDates": { - "label": "datum", - "column": "table1.datum" - }, - "filters": { - "label": "geschlecht", - "description": "Geschlecht zur gegebenen Datumseinschränkung", - "column": "table1.geschlecht", - "type": "SELECT" - } - } - ] - } - ], - "content": { - "tables": [ - { - "csv": "tests/sql/date_restriction/daterange/content.csv", - "name": "table1", - "primaryColumn": { - "name": "pid", - "type": "STRING" - }, - "columns": [ - { - "name": "datum", - "type": "DATE_RANGE" - }, - { - "name": "geschlecht", - "type": "STRING" - } - ] - } - ] - } -} diff --git a/backend/src/test/resources/tests/sql/date_restriction/daterange/content.csv b/backend/src/test/resources/tests/sql/date_restriction/postgres_daterange/content.csv similarity index 100% rename from backend/src/test/resources/tests/sql/date_restriction/daterange/content.csv rename to backend/src/test/resources/tests/sql/date_restriction/postgres_daterange/content.csv diff --git a/backend/src/test/resources/tests/sql/date_restriction/postgres_daterange/daterange_column.spec.json b/backend/src/test/resources/tests/sql/date_restriction/postgres_daterange/daterange_column.spec.json new file mode 100644 index 0000000000..62965cefe5 --- /dev/null +++ b/backend/src/test/resources/tests/sql/date_restriction/postgres_daterange/daterange_column.spec.json @@ -0,0 +1,83 @@ +{ + "type": "SQL_TEST", + "supportedDialects": [ + "POSTGRESQL" + ], + "label": "Date restriction with a daterange column validity date (PostgreSQL only)", + "expectedCsv": "expected.csv", + "query": { + "type": "CONCEPT_QUERY", + "root": { + "type": "DATE_RESTRICTION", + "dateRange": { + "min": "2012-01-01", + "max": "2012-12-31" + }, + "child": { + "ids": [ + "geschlecht_select" + ], + "type": "CONCEPT", + "label": "Geschlecht SELECT", + "tables": [ + { + "id": "geschlecht_select.geschlecht_connector", + "filters": [ + { + "filter": "geschlecht_select.geschlecht_connector.geschlecht", + "type": "BIG_MULTI_SELECT", + "value": [ + "f" + ] + } + ] + } + ] + } + } + }, + "concepts":[ + { + "label":"geschlecht_select", + "type":"TREE", + "connectors":[ + { + "label":"geschlecht_connector", + "table":"table1", + "validityDates":{ + "label":"datum", + "column":"table1.datum" + }, + "filters":{ + "label":"geschlecht", + "description":"Geschlecht zur gegebenen Datumseinschränkung", + "column":"table1.geschlecht", + "type":"SELECT" + } + } + ] + } + ], + "content":{ + "tables":[ + { + "csv":"tests/sql/date_restriction/postgres_daterange/content.csv", + "name":"table1", + "primaryColumn":{ + "name":"pid", + "type":"STRING" + }, + "columns":[ + { + "name":"datum", + "type":"DATE_RANGE" + }, + { + "name":"geschlecht", + "type":"STRING" + } + ] + } + ] + } +} diff --git a/backend/src/test/resources/tests/sql/date_restriction/daterange/expected.csv b/backend/src/test/resources/tests/sql/date_restriction/postgres_daterange/expected.csv similarity index 68% rename from backend/src/test/resources/tests/sql/date_restriction/daterange/expected.csv rename to backend/src/test/resources/tests/sql/date_restriction/postgres_daterange/expected.csv index a8bff966b3..777e8d3e55 100644 --- a/backend/src/test/resources/tests/sql/date_restriction/daterange/expected.csv +++ b/backend/src/test/resources/tests/sql/date_restriction/postgres_daterange/expected.csv @@ -1,3 +1,3 @@ -pid,datum +pid,datum_start,datum_end 1,"[2012-06-30,2015-06-30)" 3,"[2012-02-03,2012-06-30)" diff --git a/backend/src/test/resources/tests/sql/filter/number/content.csv b/backend/src/test/resources/tests/sql/filter/number/content.csv index 1851eed139..5ceffe16ca 100644 --- a/backend/src/test/resources/tests/sql/filter/number/content.csv +++ b/backend/src/test/resources/tests/sql/filter/number/content.csv @@ -1,13 +1,13 @@ -pid,value,datum -1,1,"2014-06-30/2015-06-30" -2,1.01,"2014-06-30/2015-06-30" -1,1,"2015-02-03/2015-06-30" -1,0.5,"2014-06-30/2015-06-30" -3,0.5,"2014-04-30/2014-06-30" -4,1,"2014-06-30/2015-06-30" -5,0.5,"2014-04-30/2014-06-30" -5,1,"2014-06-30/2015-06-30" -6,1,"2014-04-30/2014-06-30" -7,1,"2014-02-05/2014-02-20" -8,1,"2014-04-30/2014-06-30" -7,-1,"2014-06-30/2015-06-30" +pid,value +1,1 +2,1.01 +1,1 +1,0.5 +3,0.5 +4,1 +5,0.5 +5,1 +6,1 +7,1 +8,1 +7,-1 diff --git a/backend/src/test/resources/tests/sql/filter/number/number.spec.json b/backend/src/test/resources/tests/sql/filter/number/number.spec.json index 46dd0f5425..e480a59934 100644 --- a/backend/src/test/resources/tests/sql/filter/number/number.spec.json +++ b/backend/src/test/resources/tests/sql/filter/number/number.spec.json @@ -63,10 +63,6 @@ { "name": "value", "type": "REAL" - }, - { - "name": "datum", - "type": "DATE_RANGE" } ] } diff --git a/backend/src/test/resources/tests/sql/filter/number_only_max/content.csv b/backend/src/test/resources/tests/sql/filter/number_only_max/content.csv index 1851eed139..5ceffe16ca 100644 --- a/backend/src/test/resources/tests/sql/filter/number_only_max/content.csv +++ b/backend/src/test/resources/tests/sql/filter/number_only_max/content.csv @@ -1,13 +1,13 @@ -pid,value,datum -1,1,"2014-06-30/2015-06-30" -2,1.01,"2014-06-30/2015-06-30" -1,1,"2015-02-03/2015-06-30" -1,0.5,"2014-06-30/2015-06-30" -3,0.5,"2014-04-30/2014-06-30" -4,1,"2014-06-30/2015-06-30" -5,0.5,"2014-04-30/2014-06-30" -5,1,"2014-06-30/2015-06-30" -6,1,"2014-04-30/2014-06-30" -7,1,"2014-02-05/2014-02-20" -8,1,"2014-04-30/2014-06-30" -7,-1,"2014-06-30/2015-06-30" +pid,value +1,1 +2,1.01 +1,1 +1,0.5 +3,0.5 +4,1 +5,0.5 +5,1 +6,1 +7,1 +8,1 +7,-1 diff --git a/backend/src/test/resources/tests/sql/filter/number_only_max/number_only_max.spec.json b/backend/src/test/resources/tests/sql/filter/number_only_max/number_only_max.spec.json index 1b49d2b5aa..3f9f991d8a 100644 --- a/backend/src/test/resources/tests/sql/filter/number_only_max/number_only_max.spec.json +++ b/backend/src/test/resources/tests/sql/filter/number_only_max/number_only_max.spec.json @@ -62,10 +62,6 @@ { "name": "value", "type": "REAL" - }, - { - "name": "datum", - "type": "DATE_RANGE" } ] } diff --git a/backend/src/test/resources/tests/sql/filter/number_only_min/content.csv b/backend/src/test/resources/tests/sql/filter/number_only_min/content.csv index 1851eed139..5ceffe16ca 100644 --- a/backend/src/test/resources/tests/sql/filter/number_only_min/content.csv +++ b/backend/src/test/resources/tests/sql/filter/number_only_min/content.csv @@ -1,13 +1,13 @@ -pid,value,datum -1,1,"2014-06-30/2015-06-30" -2,1.01,"2014-06-30/2015-06-30" -1,1,"2015-02-03/2015-06-30" -1,0.5,"2014-06-30/2015-06-30" -3,0.5,"2014-04-30/2014-06-30" -4,1,"2014-06-30/2015-06-30" -5,0.5,"2014-04-30/2014-06-30" -5,1,"2014-06-30/2015-06-30" -6,1,"2014-04-30/2014-06-30" -7,1,"2014-02-05/2014-02-20" -8,1,"2014-04-30/2014-06-30" -7,-1,"2014-06-30/2015-06-30" +pid,value +1,1 +2,1.01 +1,1 +1,0.5 +3,0.5 +4,1 +5,0.5 +5,1 +6,1 +7,1 +8,1 +7,-1 diff --git a/backend/src/test/resources/tests/sql/filter/number_only_min/number_only_min.spec.json b/backend/src/test/resources/tests/sql/filter/number_only_min/number_only_min.spec.json index 918c2d521d..c95f1d2670 100644 --- a/backend/src/test/resources/tests/sql/filter/number_only_min/number_only_min.spec.json +++ b/backend/src/test/resources/tests/sql/filter/number_only_min/number_only_min.spec.json @@ -62,10 +62,6 @@ { "name": "value", "type": "REAL" - }, - { - "name": "datum", - "type": "DATE_RANGE" } ] } diff --git a/backend/src/test/resources/tests/sql/filter/select/content.csv b/backend/src/test/resources/tests/sql/filter/select/content.csv index db93b08bd4..7d719b498c 100644 --- a/backend/src/test/resources/tests/sql/filter/select/content.csv +++ b/backend/src/test/resources/tests/sql/filter/select/content.csv @@ -1,9 +1,9 @@ -pid,datum,geschlecht -1,2012-01-01,"f" -2,2010-07-15,"m" -3,2013-11-10,"f" -4,2012-11-11,"m" -5,2007-11-11,"" -6,2012-11-11,"" -7,2012-11-11,"mf" -8,2012-11-11,"fm" +pid,geschlecht +1,"f" +2,"m" +3,"f" +4,"m" +5,"" +6,"" +7,"mf" +8,"fm" diff --git a/backend/src/test/resources/tests/sql/filter/select/select.spec.json b/backend/src/test/resources/tests/sql/filter/select/select.spec.json index 11ae8586f4..c545967747 100644 --- a/backend/src/test/resources/tests/sql/filter/select/select.spec.json +++ b/backend/src/test/resources/tests/sql/filter/select/select.spec.json @@ -59,10 +59,6 @@ "type":"STRING" }, "columns":[ - { - "name":"datum", - "type":"DATE" - }, { "name":"geschlecht", "type":"STRING" diff --git a/backend/src/test/resources/tests/sql/or/different_concept/content_1.csv b/backend/src/test/resources/tests/sql/or/different_concept/content_1.csv index 1851eed139..244295a111 100644 --- a/backend/src/test/resources/tests/sql/or/different_concept/content_1.csv +++ b/backend/src/test/resources/tests/sql/or/different_concept/content_1.csv @@ -1,13 +1,13 @@ -pid,value,datum -1,1,"2014-06-30/2015-06-30" -2,1.01,"2014-06-30/2015-06-30" -1,1,"2015-02-03/2015-06-30" -1,0.5,"2014-06-30/2015-06-30" -3,0.5,"2014-04-30/2014-06-30" -4,1,"2014-06-30/2015-06-30" -5,0.5,"2014-04-30/2014-06-30" -5,1,"2014-06-30/2015-06-30" -6,1,"2014-04-30/2014-06-30" -7,1,"2014-02-05/2014-02-20" -8,1,"2014-04-30/2014-06-30" -7,-1,"2014-06-30/2015-06-30" +pid,value,datum_start,datum_end +1,1,2014-06-30,2015-06-30 +2,1.01,2014-06-30,2015-06-30 +1,1,2015-02-03,2015-06-30 +1,0.5,2014-06-30,2015-06-30 +3,0.5,2014-04-30,2014-06-30 +4,1,2014-06-30,2015-06-30 +5,0.5,2014-04-30,2014-06-30 +5,1,2014-06-30,2015-06-30 +6,1,2014-04-30,2014-06-30 +7,1,2014-02-05,2014-02-20 +8,1,2014-04-30,2014-06-30 +7,-1,2014-06-30,2015-06-30 diff --git a/backend/src/test/resources/tests/sql/or/different_concept/expected.csv b/backend/src/test/resources/tests/sql/or/different_concept/expected.csv index dea40ae400..e5155afc92 100644 --- a/backend/src/test/resources/tests/sql/or/different_concept/expected.csv +++ b/backend/src/test/resources/tests/sql/or/different_concept/expected.csv @@ -1,4 +1,4 @@ pid,validity_date_1,,value,geschlecht,language -7,"[2014-06-30,2015-06-30)",-1,mf, +7,"[2014-06-30,2015-07-01)",-1.00,mf, 8,,,,fr 2,,,,fr diff --git a/backend/src/test/resources/tests/sql/or/different_concept/or.spec.json b/backend/src/test/resources/tests/sql/or/different_concept/or.spec.json index 4fd7f736e4..0e29374a60 100644 --- a/backend/src/test/resources/tests/sql/or/different_concept/or.spec.json +++ b/backend/src/test/resources/tests/sql/or/different_concept/or.spec.json @@ -93,7 +93,8 @@ "table": "table1", "validityDates": { "label": "datum", - "column": "table1.datum" + "startColumn": "table1.datum_start", + "endColumn": "table1.datum_end" }, "filters": { "label": "value", @@ -175,8 +176,12 @@ "type": "REAL" }, { - "name": "datum", - "type": "DATE_RANGE" + "name": "datum_start", + "type": "DATE" + }, + { + "name": "datum_end", + "type": "DATE" } ] }, diff --git a/backend/src/test/resources/tests/sql/or/same_concept/or.spec.json b/backend/src/test/resources/tests/sql/or/same_concept/or_same_concept.spec.json similarity index 100% rename from backend/src/test/resources/tests/sql/or/same_concept/or.spec.json rename to backend/src/test/resources/tests/sql/or/same_concept/or_same_concept.spec.json diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/months/content.csv b/backend/src/test/resources/tests/sql/selects/date_distance/months/content.csv index c2d4f04aef..2dfbea2947 100644 --- a/backend/src/test/resources/tests/sql/selects/date_distance/months/content.csv +++ b/backend/src/test/resources/tests/sql/selects/date_distance/months/content.csv @@ -1,5 +1,5 @@ pid,datum,geschlecht -1,2012-01-01,"f" +1,2012-01-29,"f" 2,2010-07-15,"m" 3,2010-11-10,"f" 4,2013-11-11,"m" diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/months/expected.csv b/backend/src/test/resources/tests/sql/selects/date_distance/months/expected.csv index d0212b39b1..62ac211ec8 100644 --- a/backend/src/test/resources/tests/sql/selects/date_distance/months/expected.csv +++ b/backend/src/test/resources/tests/sql/selects/date_distance/months/expected.csv @@ -1,3 +1,3 @@ pid,date_distance_months -1,134 +1,133 3,148 diff --git a/backend/src/test/resources/tests/sql/selects/date_distance/years/expected.csv b/backend/src/test/resources/tests/sql/selects/date_distance/years/expected.csv index c0ec9df19d..41a6149bff 100644 --- a/backend/src/test/resources/tests/sql/selects/date_distance/years/expected.csv +++ b/backend/src/test/resources/tests/sql/selects/date_distance/years/expected.csv @@ -1,3 +1,3 @@ pid,date_distance_years 1,11 -3,13 +3,12 diff --git a/executable/pom.xml b/executable/pom.xml index ca58ce7c88..12762e1c6c 100644 --- a/executable/pom.xml +++ b/executable/pom.xml @@ -68,9 +68,12 @@ - com.bakdata.conquery.ConqueryServer - + implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> + com.bakdata.conquery.ConqueryServer + + true + + From 4a189290c04a625d84303e03498e36dd972da369 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 28 Aug 2023 11:30:53 +0200 Subject: [PATCH 520/679] Refactor line to be in Dropzone, minor changes for code cleanliness --- .../DropzoneBetweenElements.tsx | 26 +++++++++---------- .../form-components/DropzoneList.tsx | 5 ++-- .../form-concept-group/FormConceptGroup.tsx | 5 +++- frontend/src/js/ui-components/Dropzone.tsx | 6 ----- 4 files changed, 19 insertions(+), 23 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index 5ddbc9cd01..2e939ef1d4 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -1,5 +1,4 @@ import styled from "@emotion/styled"; -import { useState } from "react"; import { DropTargetMonitor } from "react-dnd"; import Dropzone, { @@ -13,16 +12,17 @@ interface Props { height: number; } -const LineHeight = 3; +const Root = styled("div")` + display: flex; + height: 4px; +`; -const Line = styled("div")<{ show: boolean }>` - overflow: hidden; - display: block; - visibility: ${({ show }) => (show ? "visible" : "hidden")}; +const Line = styled("div")` background-color: ${({ theme }) => theme.col.blueGrayDark}; margin: 1px 0; - height: ${LineHeight}px; + height: 4px; border-radius: 2px; + flex-grow: 1; `; const SxDropzone = styled(Dropzone)<{ height: number; top: number }>` @@ -38,21 +38,19 @@ const DropzoneBetweenElements = ({ height, top, }: Props) => { - let [isOver, setIsOver] = useState(false); - return ( - <> - + - + > + {({ isOver }) => isOver && } + + ); }; diff --git a/frontend/src/js/external-forms/form-components/DropzoneList.tsx b/frontend/src/js/external-forms/form-components/DropzoneList.tsx index 8bf451dc93..05fd3c5751 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneList.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneList.tsx @@ -23,6 +23,7 @@ const ListItem = styled("div")` box-shadow: 0 0 3px 0 rgba(0, 0, 0, 0.1); background-color: white; border-radius: ${({ theme }) => theme.borderRadius}; + margin-bottom: 5px; `; const StyledIconButton = styled(IconButton)` @@ -117,8 +118,8 @@ const DropzoneList = ( )} diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 82de595e0e..0c30a5fe49 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -225,7 +225,10 @@ const FormConceptGroup = (props: Props) => { item.dragContext; if (movedFromFieldName === props.fieldName) { - if (i > movedFromAndIdx && movedFromOrIdx === 0) { + const willConceptMoveUp = + i > movedFromAndIdx && + props.value[movedFromAndIdx].concepts.length == 1; + if (willConceptMoveUp) { insertIndex = i - 1; } newPropsValue = diff --git a/frontend/src/js/ui-components/Dropzone.tsx b/frontend/src/js/ui-components/Dropzone.tsx index c83df6cd46..1b4a42b45e 100644 --- a/frontend/src/js/ui-components/Dropzone.tsx +++ b/frontend/src/js/ui-components/Dropzone.tsx @@ -67,7 +67,6 @@ export interface DropzoneProps { canDrop?: (props: DroppableObject, monitor: DropTargetMonitor) => boolean; onClick?: () => void; children?: (args: ChildArgs) => ReactNode; - setIsOver?: (state: boolean) => void; } export type PossibleDroppableObject = @@ -108,7 +107,6 @@ const Dropzone = ( onClick, invisible, children, - setIsOver, }: DropzoneProps, ref?: ForwardedRef, ) => { @@ -128,10 +126,6 @@ const Dropzone = ( }), }); - useEffect(() => { - if (setIsOver) setIsOver(isOver); - }, [isOver, setIsOver]); - return ( { From 9475ac1399c580b9420241f1e8fd33984122b2a4 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 28 Aug 2023 11:34:42 +0200 Subject: [PATCH 521/679] small lint changes --- .../js/external-forms/form-concept-group/FormConceptGroup.tsx | 2 +- frontend/src/js/ui-components/Dropzone.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 0c30a5fe49..6171a402ff 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -227,7 +227,7 @@ const FormConceptGroup = (props: Props) => { if (movedFromFieldName === props.fieldName) { const willConceptMoveUp = i > movedFromAndIdx && - props.value[movedFromAndIdx].concepts.length == 1; + props.value[movedFromAndIdx].concepts.length === 1; if (willConceptMoveUp) { insertIndex = i - 1; } diff --git a/frontend/src/js/ui-components/Dropzone.tsx b/frontend/src/js/ui-components/Dropzone.tsx index 1b4a42b45e..9662a49891 100644 --- a/frontend/src/js/ui-components/Dropzone.tsx +++ b/frontend/src/js/ui-components/Dropzone.tsx @@ -1,5 +1,5 @@ import styled from "@emotion/styled"; -import { ForwardedRef, forwardRef, ReactNode, useEffect } from "react"; +import { ForwardedRef, forwardRef, ReactNode } from "react"; import { DropTargetMonitor, useDrop } from "react-dnd"; import { DNDType } from "../common/constants/dndTypes"; From b45c424488a45222aa0ba48e5774acc1cec425ad Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 28 Aug 2023 12:21:46 +0200 Subject: [PATCH 522/679] Simplify css and DropzoneBetweenElements --- .../DropzoneBetweenElements.tsx | 55 ++++++++----------- .../form-components/DropzoneList.tsx | 5 -- 2 files changed, 22 insertions(+), 38 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index 2e939ef1d4..07fa20463e 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -5,52 +5,41 @@ import Dropzone, { PossibleDroppableObject, } from "../../ui-components/Dropzone"; -interface Props { - onDrop: (props: PossibleDroppableObject, monitor: DropTargetMonitor) => void; - acceptedDropTypes: string[]; - top: number; - height: number; -} - -const Root = styled("div")` - display: flex; - height: 4px; -`; +const LINE_HEIGHT = 4; const Line = styled("div")` background-color: ${({ theme }) => theme.col.blueGrayDark}; - margin: 1px 0; - height: 4px; - border-radius: 2px; - flex-grow: 1; + height: ${LINE_HEIGHT}px; + width: 100%; + border-radius: ${({ theme }) => theme.borderRadius}; `; -const SxDropzone = styled(Dropzone)<{ height: number; top: number }>` - height: ${({ height }) => height}px; - top: ${({ top }) => top}px; +const SxDropzone = styled(Dropzone)` + height: 30px; position: absolute; + top: 0; + left: 0; + transform: translateY(calc(-50% - ${LINE_HEIGHT / 2}px)); + z-index: 10; background-color: transparent; `; const DropzoneBetweenElements = ({ acceptedDropTypes, onDrop, - height, - top, -}: Props) => { +}: { + onDrop: (props: PossibleDroppableObject, monitor: DropTargetMonitor) => void; + acceptedDropTypes: string[]; +}) => { return ( - - - {({ isOver }) => isOver && } - - + + {({ isOver }) => isOver && } + ); }; diff --git a/frontend/src/js/external-forms/form-components/DropzoneList.tsx b/frontend/src/js/external-forms/form-components/DropzoneList.tsx index 05fd3c5751..00d5b86052 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneList.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneList.tsx @@ -102,8 +102,6 @@ const DropzoneList = ( )} @@ -112,14 +110,11 @@ const DropzoneList = ( ))} - {!disallowMultipleColumns && ( )} From f1cfc82c41e2126f15663890b111541c51e27386 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 28 Aug 2023 12:59:01 +0200 Subject: [PATCH 523/679] use mark directive --- frontend/package.json | 1 + frontend/src/js/tooltip/Tooltip.tsx | 93 +++-------------------------- frontend/yarn.lock | 28 +++++++++ 3 files changed, 37 insertions(+), 85 deletions(-) diff --git a/frontend/package.json b/frontend/package.json index d886016692..958226c031 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -76,6 +76,7 @@ "react-window": "^1.8.6", "redux": "^4.1.2", "redux-devtools-extension": "^2.13.9", + "remark-flexible-markers": "^1.0.3", "remark-gfm": "^3.0.1", "resize-observer-polyfill": "^1.5.1", "typesafe-actions": "^5.1.0", diff --git a/frontend/src/js/tooltip/Tooltip.tsx b/frontend/src/js/tooltip/Tooltip.tsx index 28c29c0e7c..572dbae924 100644 --- a/frontend/src/js/tooltip/Tooltip.tsx +++ b/frontend/src/js/tooltip/Tooltip.tsx @@ -1,20 +1,11 @@ import styled from "@emotion/styled"; import { faThumbtack, IconDefinition } from "@fortawesome/free-solid-svg-icons"; -import { - Children, - DetailedHTMLProps, - ElementType, - HTMLAttributes, - ReactElement, - ReactFragment, - ReactNode, - ReactPortal, -} from "react"; +import { ReactNode } from "react"; import Highlighter from "react-highlight-words"; import { useTranslation } from "react-i18next"; import Markdown from "react-markdown"; -import { ReactMarkdownProps } from "react-markdown/lib/complex-types"; import { useDispatch, useSelector } from "react-redux"; +import remarkFlexibleMarkers from "remark-flexible-markers"; import remarkGfm from "remark-gfm"; import type { StateT } from "../app/reducers"; @@ -168,65 +159,10 @@ const ConceptLabel = ({ ); }; -function isReactElement( - element: ReactFragment | ReactElement | ReactPortal | boolean | number, -): element is ReactElement { - return ( - typeof element === "object" && - element.hasOwnProperty("type") && - element.hasOwnProperty("props") - ); -} - -type MarkdownElement = - | ReactFragment - | ReactElement - | ReactPortal - | boolean - | number - | boolean - | string - | null - | undefined; -function highlight( - words: string[], - Element: Omit< - DetailedHTMLProps, HTMLElement>, - "ref" - > & - ReactMarkdownProps, -): ReactElement | null { - if (!Element) { - return Element; - } - const mappingFunction = (child: MarkdownElement): ReactElement => { - if (!child) return <>; - if (typeof child === "string") { - return HighlightedText({ words, text: child }); - } - if (isReactElement(child)) { - let TagName = child.type as ElementType; - return ( - - {highlight(words, child.props.children)} - - ); - } - return <>{child}; - }; - - if (Array.isArray(Element)) { - return <>{Children.map(Element, mappingFunction)}; - } - - let children = - typeof Element === "object" && Element.hasOwnProperty("children") - ? Children.map(Element.children, mappingFunction) - : Element.children; - - let TagName = Element.node?.tagName as ElementType; - return {children}; -} +const mark = (text: string, words: string[]): string => { + const regex = new RegExp(words.join("|"), "gi"); + return text.replace(regex, "==$&=="); +}; const Tooltip = () => { const words = useSelector( @@ -304,21 +240,8 @@ const Tooltip = () => { - highlight(words, el), - td: (el) => highlight(words, el), - b: (el) => highlight(words, el), - th: (el) => highlight(words, el), - i: (el) => highlight(words, el), - ul: (el) => highlight(words, el), - ol: (el) => highlight(words, el), - h1: (el) => highlight(words, el), - h2: (el) => highlight(words, el), - }} - > - {info.value} + + {mark(info.value, words)} ))} diff --git a/frontend/yarn.lock b/frontend/yarn.lock index ae9546cbe7..4137d6297a 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -3113,6 +3113,13 @@ dependencies: "@types/unist" "*" +"@types/mdast@^3.0.10": + version "3.0.12" + resolved "https://registry.yarnpkg.com/@types/mdast/-/mdast-3.0.12.tgz#beeb511b977c875a5b0cc92eab6fcac2f0895514" + integrity sha512-DT+iNIRNX884cx0/Q1ja7NyUPpZuv0KPyL5rGNxm1WC1OtHstl7n4Jb7nk+xacNShQMbczJjt8uFzznpp6kYBg== + dependencies: + "@types/unist" "^2" + "@types/mdx@^2.0.0": version "2.0.3" resolved "https://registry.yarnpkg.com/@types/mdx/-/mdx-2.0.3.tgz#43fd32414f17fcbeced3578109a6edd877a2d96e" @@ -3317,6 +3324,11 @@ resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.6.tgz#250a7b16c3b91f672a24552ec64678eeb1d3a08d" integrity sha512-PBjIUxZHOuj0R15/xuwJYjFi+KZdNFrehocChv4g5hu6aFroHue8m0lBP0POdK2nKzbw0cgV1mws8+V/JAcEkQ== +"@types/unist@^2": + version "2.0.7" + resolved "https://registry.yarnpkg.com/@types/unist/-/unist-2.0.7.tgz#5b06ad6894b236a1d2bd6b2f07850ca5c59cf4d6" + integrity sha512-cputDpIbFgLUaGQn6Vqg3/YsJwxUwHLO13v3i5ouxT4lat0khip9AEWxtERujXV9wxIB1EyF97BSJFt6vpdI8g== + "@types/use-sync-external-store@^0.0.3": version "0.0.3" resolved "https://registry.yarnpkg.com/@types/use-sync-external-store/-/use-sync-external-store-0.0.3.tgz#b6725d5f4af24ace33b36fafd295136e75509f43" @@ -8849,6 +8861,15 @@ remark-external-links@^8.0.0: space-separated-tokens "^1.0.0" unist-util-visit "^2.0.0" +remark-flexible-markers@^1.0.3: + version "1.0.3" + resolved "https://registry.yarnpkg.com/remark-flexible-markers/-/remark-flexible-markers-1.0.3.tgz#50914f5cac13da45b7d0ccb9a517b6c3cc8c85dc" + integrity sha512-O3aXLFXbPZRS9lZfTxqzgG0sknqgMdz0pfqp4vx5cofjlPfSJVyGMfTB5jxIdum4XzhN0FVtRBB+ksZocX168w== + dependencies: + "@types/mdast" "^3.0.10" + unist-builder "^3.0.1" + unist-util-visit "^4.0.0" + remark-gfm@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/remark-gfm/-/remark-gfm-3.0.1.tgz#0b180f095e3036545e9dddac0e8df3fa5cfee54f" @@ -9822,6 +9843,13 @@ unique-string@^2.0.0: dependencies: crypto-random-string "^2.0.0" +unist-builder@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/unist-builder/-/unist-builder-3.0.1.tgz#258b89dcadd3c973656b2327b347863556907f58" + integrity sha512-gnpOw7DIpCA0vpr6NqdPvTWnlPTApCTRzr+38E6hCWx3rz/cjo83SsKIlS1Z+L5ttScQ2AwutNnb8+tAvpb6qQ== + dependencies: + "@types/unist" "^2.0.0" + unist-util-generated@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/unist-util-generated/-/unist-util-generated-2.0.1.tgz#e37c50af35d3ed185ac6ceacb6ca0afb28a85cae" From c762a1b8faf44fd979988ee3f5bdb6f360b816b2 Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 28 Aug 2023 14:32:42 +0200 Subject: [PATCH 524/679] Fix last dropzone overlapping --- .../form-components/DropzoneBetweenElements.tsx | 5 ++++- .../src/js/external-forms/form-components/DropzoneList.tsx | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx index 07fa20463e..6642ee3f51 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneBetweenElements.tsx @@ -20,19 +20,22 @@ const SxDropzone = styled(Dropzone)` top: 0; left: 0; transform: translateY(calc(-50% - ${LINE_HEIGHT / 2}px)); - z-index: 10; + z-index: 1; background-color: transparent; `; const DropzoneBetweenElements = ({ acceptedDropTypes, onDrop, + className, }: { onDrop: (props: PossibleDroppableObject, monitor: DropTargetMonitor) => void; acceptedDropTypes: string[]; + className?: string; }) => { return ( { className?: string; @@ -112,7 +115,7 @@ const DropzoneList = ( ))} {!disallowMultipleColumns && ( - From 67f0c5ceecbbfb1eebaea22a7cb5a0bed390c81c Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 28 Aug 2023 14:53:47 +0200 Subject: [PATCH 525/679] code cleanup --- .../form-concept-group/FormConceptGroup.tsx | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 6171a402ff..8821c30203 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -210,7 +210,7 @@ const FormConceptGroup = (props: Props) => { } dropBetween={(i: number) => { return (item: PossibleDroppableObject) => { - if (item.type !== DNDType.CONCEPT_TREE_NODE) return; + if (item.type !== DNDType.CONCEPT_TREE_NODE) return null; if (props.isValidConcept && !props.isValidConcept(item)) return null; @@ -225,10 +225,10 @@ const FormConceptGroup = (props: Props) => { item.dragContext; if (movedFromFieldName === props.fieldName) { - const willConceptMoveUp = + const willConceptMoveDown = i > movedFromAndIdx && props.value[movedFromAndIdx].concepts.length === 1; - if (willConceptMoveUp) { + if (willConceptMoveDown) { insertIndex = i - 1; } newPropsValue = @@ -458,7 +458,15 @@ const FormConceptGroup = (props: Props) => { if (isMovedObject(concept)) { const { movedFromFieldName, movedFromAndIdx, movedFromOrIdx } = concept.dragContext; - valueIdx = valueIdx > movedFromAndIdx ? valueIdx - 1 : valueIdx; + + // If the concept is moved from the same field and the concept is the only one + // in the value the index of the selected concept might change after the drop + const willSelectedConceptIndexChange = + valueIdx > movedFromAndIdx && + props.value[movedFromOrIdx].concepts.length === 1; + valueIdx = willSelectedConceptIndexChange + ? valueIdx - 1 + : valueIdx; if (movedFromFieldName === props.fieldName) { updatedValue = updatedValue[movedFromAndIdx].concepts.length === 1 From af52afc9bb28da39abfc947105664887b6791a77 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 28 Aug 2023 15:11:20 +0200 Subject: [PATCH 526/679] useMemo on regex --- frontend/src/js/tooltip/Tooltip.tsx | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/frontend/src/js/tooltip/Tooltip.tsx b/frontend/src/js/tooltip/Tooltip.tsx index 572dbae924..345ad29d76 100644 --- a/frontend/src/js/tooltip/Tooltip.tsx +++ b/frontend/src/js/tooltip/Tooltip.tsx @@ -1,6 +1,6 @@ import styled from "@emotion/styled"; import { faThumbtack, IconDefinition } from "@fortawesome/free-solid-svg-icons"; -import { ReactNode } from "react"; +import { ReactNode, useMemo } from "react"; import Highlighter from "react-highlight-words"; import { useTranslation } from "react-i18next"; import Markdown from "react-markdown"; @@ -159,8 +159,7 @@ const ConceptLabel = ({ ); }; -const mark = (text: string, words: string[]): string => { - const regex = new RegExp(words.join("|"), "gi"); +const mark = (text: string, regex: RegExp): string => { return text.replace(regex, "==$&=="); }; @@ -188,6 +187,8 @@ const Tooltip = () => { (state) => state.tooltip.toggleAdditionalInfos, ); + const highlightRegex = useMemo(() => new RegExp(words.join("|"), "gi"), [words]); + const dispatch = useDispatch(); const onToggleAdditionalInfos = () => dispatch(toggleInfos()); @@ -241,7 +242,7 @@ const Tooltip = () => { - {mark(info.value, words)} + {mark(info.value, highlightRegex)} ))} From b29ce846a6f10cfca7cd59d516223f2fdcdaf8bf Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 28 Aug 2023 15:17:02 +0200 Subject: [PATCH 527/679] format, and fix small bug --- frontend/src/js/tooltip/Tooltip.tsx | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/src/js/tooltip/Tooltip.tsx b/frontend/src/js/tooltip/Tooltip.tsx index 345ad29d76..f79c507bea 100644 --- a/frontend/src/js/tooltip/Tooltip.tsx +++ b/frontend/src/js/tooltip/Tooltip.tsx @@ -160,6 +160,7 @@ const ConceptLabel = ({ }; const mark = (text: string, regex: RegExp): string => { + if (!regex) return text; return text.replace(regex, "==$&=="); }; @@ -187,7 +188,10 @@ const Tooltip = () => { (state) => state.tooltip.toggleAdditionalInfos, ); - const highlightRegex = useMemo(() => new RegExp(words.join("|"), "gi"), [words]); + const highlightRegex = useMemo( + () => words.length > 0 ? new RegExp(words.join("|"), "gi") : null, + [words], + ); const dispatch = useDispatch(); const onToggleAdditionalInfos = () => dispatch(toggleInfos()); From 4c074095decb8749ab534cc99d43993addaf390d Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 28 Aug 2023 15:27:03 +0200 Subject: [PATCH 528/679] fix small error --- frontend/src/js/tooltip/Tooltip.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frontend/src/js/tooltip/Tooltip.tsx b/frontend/src/js/tooltip/Tooltip.tsx index f79c507bea..ba370e7345 100644 --- a/frontend/src/js/tooltip/Tooltip.tsx +++ b/frontend/src/js/tooltip/Tooltip.tsx @@ -159,7 +159,7 @@ const ConceptLabel = ({ ); }; -const mark = (text: string, regex: RegExp): string => { +const mark = (text: string, regex: RegExp | null): string => { if (!regex) return text; return text.replace(regex, "==$&=="); }; @@ -189,7 +189,7 @@ const Tooltip = () => { ); const highlightRegex = useMemo( - () => words.length > 0 ? new RegExp(words.join("|"), "gi") : null, + () => (words.length > 0 ? new RegExp(words.join("|"), "gi") : null), [words], ); From 351353383b38a523837608b9064304c3a53d3204 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 28 Aug 2023 16:58:59 +0200 Subject: [PATCH 529/679] filter out words with 0 length for regex -> leading to invalid regex --- frontend/src/js/tooltip/Tooltip.tsx | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/frontend/src/js/tooltip/Tooltip.tsx b/frontend/src/js/tooltip/Tooltip.tsx index ba370e7345..bfbf5247a4 100644 --- a/frontend/src/js/tooltip/Tooltip.tsx +++ b/frontend/src/js/tooltip/Tooltip.tsx @@ -188,10 +188,12 @@ const Tooltip = () => { (state) => state.tooltip.toggleAdditionalInfos, ); - const highlightRegex = useMemo( - () => (words.length > 0 ? new RegExp(words.join("|"), "gi") : null), - [words], - ); + const highlightRegex = useMemo(() => { + console.log("words", words); + return words.length > 0 + ? new RegExp(words.filter((word) => word.length > 0).join("|"), "gi") + : null; + }, [words]); const dispatch = useDispatch(); const onToggleAdditionalInfos = () => dispatch(toggleInfos()); From 3b3ec99cd6f9e47a1b0469dd23d79c57d905ec72 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Mon, 28 Aug 2023 17:00:18 +0200 Subject: [PATCH 530/679] remove log --- frontend/src/js/tooltip/Tooltip.tsx | 1 - 1 file changed, 1 deletion(-) diff --git a/frontend/src/js/tooltip/Tooltip.tsx b/frontend/src/js/tooltip/Tooltip.tsx index bfbf5247a4..2e8c0fe511 100644 --- a/frontend/src/js/tooltip/Tooltip.tsx +++ b/frontend/src/js/tooltip/Tooltip.tsx @@ -189,7 +189,6 @@ const Tooltip = () => { ); const highlightRegex = useMemo(() => { - console.log("words", words); return words.length > 0 ? new RegExp(words.filter((word) => word.length > 0).join("|"), "gi") : null; From e6dc0a3b99d94dbab91450aee6e6b8a84488fdf1 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Tue, 29 Aug 2023 10:37:39 +0200 Subject: [PATCH 531/679] Change top for last and first DropzoneBetweenElements and change timing for hover again --- .../external-forms/form-components/DropzoneList.tsx | 13 ++++++++++--- .../js/small-tab-navigation/HoverNavigatable.tsx | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneList.tsx b/frontend/src/js/external-forms/form-components/DropzoneList.tsx index 2aa8d41181..048ec99144 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneList.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneList.tsx @@ -40,8 +40,14 @@ const Row = styled("div")` const ConceptContainer = styled("div")` position: relative; `; -const SxDropzoneBetweenElements = styled(DropzoneBetweenElements)` + +const SxDropzoneBetweenElements = styled(DropzoneBetweenElements)<{index: number}>` + ${({ index }) => index === 0 ? "top: 3px;" : ""} +`; + +const SxLastDropzoneBetweenElements = styled(DropzoneBetweenElements)` height: 15px; + top: -5px; `; interface PropsT { @@ -102,9 +108,10 @@ const DropzoneList = ( {items.map((item, i) => ( {!disallowMultipleColumns && ( - )} @@ -115,7 +122,7 @@ const DropzoneList = ( ))} {!disallowMultipleColumns && ( - diff --git a/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx b/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx index 1f70870c3b..f09417cb02 100644 --- a/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx +++ b/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx @@ -33,7 +33,7 @@ const Root = styled("div")<{ `; // estimated to feel responsive, but not too quick -const TIME_UNTIL_NAVIGATE = 1000; +const TIME_UNTIL_NAVIGATE = 1400; export const HoverNavigatable = ({ triggerNavigate, From b1b24ee2a3f73692aa70635aac35b9880e9d7d58 Mon Sep 17 00:00:00 2001 From: Fabian Blank Date: Tue, 29 Aug 2023 11:33:48 +0200 Subject: [PATCH 532/679] Formatting --- .../src/js/external-forms/form-components/DropzoneList.tsx | 6 ++++-- frontend/src/js/small-tab-navigation/HoverNavigatable.tsx | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/frontend/src/js/external-forms/form-components/DropzoneList.tsx b/frontend/src/js/external-forms/form-components/DropzoneList.tsx index 048ec99144..953d16e1e2 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneList.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneList.tsx @@ -41,8 +41,10 @@ const ConceptContainer = styled("div")` position: relative; `; -const SxDropzoneBetweenElements = styled(DropzoneBetweenElements)<{index: number}>` - ${({ index }) => index === 0 ? "top: 3px;" : ""} +const SxDropzoneBetweenElements = styled(DropzoneBetweenElements)<{ + index: number; +}>` + ${({ index }) => (index === 0 ? "top: 3px;" : "")} `; const SxLastDropzoneBetweenElements = styled(DropzoneBetweenElements)` diff --git a/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx b/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx index f09417cb02..0be4464475 100644 --- a/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx +++ b/frontend/src/js/small-tab-navigation/HoverNavigatable.tsx @@ -33,7 +33,7 @@ const Root = styled("div")<{ `; // estimated to feel responsive, but not too quick -const TIME_UNTIL_NAVIGATE = 1400; +const TIME_UNTIL_NAVIGATE = 1300; export const HoverNavigatable = ({ triggerNavigate, From 078a89d7cd84d08f35e797f00799dad0e227eadc Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Wed, 30 Aug 2023 12:54:25 +0200 Subject: [PATCH 533/679] adds config option to always allow custom input in SELECT-style Filters --- .../query/concept/filter/FilterValue.java | 2 +- .../models/config/FrontendConfig.java | 5 +++ .../concepts/FrontEndConceptBuilder.java | 35 ++++++++++--------- .../datasets/concepts/filters/Filter.java | 25 ++++++------- .../filters/specific/CountFilter.java | 3 +- .../filters/specific/CountQuartersFilter.java | 3 +- .../filters/specific/DateDistanceFilter.java | 3 +- .../filters/specific/DurationSumFilter.java | 3 +- .../concepts/filters/specific/FlagFilter.java | 3 +- .../filters/specific/NumberFilter.java | 3 +- .../filters/specific/PrefixTextFilter.java | 3 +- .../specific/QuartersInYearFilter.java | 3 +- .../filters/specific/SelectFilter.java | 5 +-- .../concepts/filters/specific/SumFilter.java | 3 +- .../resources/api/ConceptsProcessor.java | 7 ++-- .../integration/json/filter/FilterTest.java | 2 +- .../concepts/filters/TestGroupFilter.java | 3 +- .../frontend/FilterSearchItemTest.java | 3 +- 18 files changed, 69 insertions(+), 45 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java index f3d2a1de35..11bd78c789 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/query/concept/filter/FilterValue.java @@ -191,7 +191,7 @@ public void resolve(QueryResolveContext context) { * Values of group filters can have an arbitrary format which is set by the filter itself. * Hence, we treat the value for the filter as Object.class. *

- * The resolved filter instructs the frontend on how to render and serialize the filter value using the {@link Filter#createFrontendConfig()} method. The filter must implement {@link GroupFilter} and provide the type information of the value to correctly deserialize the received object. + * The resolved filter instructs the frontend on how to render and serialize the filter value using the {@link Filter#createFrontendConfig(com.bakdata.conquery.models.config.ConqueryConfig)} method. The filter must implement {@link GroupFilter} and provide the type information of the value to correctly deserialize the received object. */ public static class GroupFilterDeserializer extends StdDeserializer { private final NsIdReferenceDeserializer> nsIdDeserializer = new NsIdReferenceDeserializer<>(Filter.class, null, FilterId.class); diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java index 1bbfcddd56..e2883326f2 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/FrontendConfig.java @@ -60,6 +60,11 @@ public class FrontendConfig { @Email private String contactEmail; + /** + * If true, users are always allowed to add custom values into SelectFilter input fields. + */ + private boolean alwaysAllowCreateValue = false; + @Data public static class CurrencyConfig { diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java index fee4d75bda..d89c285838 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/FrontEndConceptBuilder.java @@ -23,6 +23,7 @@ import com.bakdata.conquery.io.storage.NamespaceStorage; import com.bakdata.conquery.models.auth.entities.Subject; import com.bakdata.conquery.models.auth.permissions.Ability; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.datasets.concepts.select.Select; @@ -35,17 +36,19 @@ import com.bakdata.conquery.models.identifiable.ids.specific.ConceptId; import com.bakdata.conquery.models.identifiable.ids.specific.ConceptTreeChildId; import com.bakdata.conquery.models.identifiable.ids.specific.StructureNodeId; -import lombok.AllArgsConstructor; +import lombok.Data; import lombok.extern.slf4j.Slf4j; /** * This class constructs the concept tree as it is presented to the front end. */ -@AllArgsConstructor +@Data @Slf4j public class FrontEndConceptBuilder { - public static FrontendRoot createRoot(NamespaceStorage storage, Subject subject) { + private final ConqueryConfig conqueryConfig; + + public FrontendRoot createRoot(NamespaceStorage storage, Subject subject) { final FrontendRoot root = new FrontendRoot(); final Map, FrontendNode> roots = root.getConcepts(); @@ -95,7 +98,7 @@ public static FrontendRoot createRoot(NamespaceStorage storage, Subject subject) return root; } - private static FrontendNode createConceptRoot(Concept concept, StructureNode[] structureNodes) { + private FrontendNode createConceptRoot(Concept concept, StructureNode[] structureNodes) { final MatchingStats matchingStats = concept.getMatchingStats(); @@ -121,8 +124,8 @@ private static FrontendNode createConceptRoot(Concept concept, StructureNode[ .flatMap(Collection::stream) .findAny() .isEmpty()) - .selects(concept.getSelects().stream().map(FrontEndConceptBuilder::createSelect).collect(Collectors.toList())) - .tables(concept.getConnectors().stream().map(FrontEndConceptBuilder::createTable).collect(Collectors.toList())) + .selects(concept.getSelects().stream().map(this::createSelect).collect(Collectors.toList())) + .tables(concept.getConnectors().stream().map(this::createTable).collect(Collectors.toList())) .build(); if (concept instanceof ConceptTreeNode tree && tree.getChildren() != null) { @@ -132,7 +135,7 @@ private static FrontendNode createConceptRoot(Concept concept, StructureNode[ } @Nullable - private static FrontendNode createStructureNode(StructureNode structureNode, Map, FrontendNode> roots) { + private FrontendNode createStructureNode(StructureNode structureNode, Map, FrontendNode> roots) { final List unstructured = new ArrayList<>(); for (ConceptId id : structureNode.getContainedRoots()) { if (!roots.containsKey(id)) { @@ -158,7 +161,7 @@ private static FrontendNode createStructureNode(StructureNode structureNode, Map .build(); } - public static FrontendSelect createSelect(Select select) { + public FrontendSelect createSelect(Select select) { return FrontendSelect.builder() .id(select.getId()) .label(select.getLabel()) @@ -168,7 +171,7 @@ public static FrontendSelect createSelect(Select select) { .build(); } - public static FrontendTable createTable(Connector con) { + public FrontendTable createTable(Connector con) { final FrontendTable result = FrontendTable.builder() @@ -176,8 +179,8 @@ public static FrontendTable createTable(Connector con) { .connectorId(con.getId()) .label(con.getLabel()) .isDefault(con.isDefault()) - .filters(con.collectAllFilters().stream().map(FrontEndConceptBuilder::createFilter).collect(Collectors.toList())) - .selects(con.getSelects().stream().map(FrontEndConceptBuilder::createSelect).collect(Collectors.toList())) + .filters(con.collectAllFilters().stream().map(this::createFilter).collect(Collectors.toList())) + .selects(con.getSelects().stream().map(this::createSelect).collect(Collectors.toList())) .supportedSecondaryIds(Arrays.stream(con.getTable().getColumns()) .map(Column::getSecondaryId) .filter(Objects::nonNull) @@ -199,16 +202,16 @@ public static FrontendTable createTable(Connector con) { return result; } - public static FrontendFilterConfiguration.Top createFilter(Filter filter) { + public FrontendFilterConfiguration.Top createFilter(Filter filter) { try { - return filter.createFrontendConfig(); + return filter.createFrontendConfig(conqueryConfig); } catch (ConceptConfigurationException e) { throw new IllegalStateException(e); } } - private static FrontendNode createCTNode(ConceptElement ce) { + private FrontendNode createCTNode(ConceptElement ce) { final MatchingStats matchingStats = ce.getMatchingStats(); FrontendNode.FrontendNodeBuilder nodeBuilder = FrontendNode.builder() .active(null) @@ -248,13 +251,13 @@ private static FrontendNode createCTNode(ConceptElement ce) { return n; } - public static FrontendList createTreeMap(Concept concept) { + public FrontendList createTreeMap(Concept concept) { final FrontendList map = new FrontendList(); fillTreeMap(concept, map); return map; } - private static void fillTreeMap(ConceptElement ce, FrontendList map) { + private void fillTreeMap(ConceptElement ce, FrontendList map) { map.add(ce.getId(), createCTNode(ce)); if (ce instanceof ConceptTreeNode && ((ConceptTreeNode) ce).getChildren() != null) { for (ConceptTreeChild c : ((ConceptTreeNode) ce).getChildren()) { diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/Filter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/Filter.java index 4447e29f9f..c9949d085e 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/Filter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/Filter.java @@ -4,6 +4,7 @@ import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration; import com.bakdata.conquery.io.cps.CPSBase; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.concepts.Connector; @@ -52,21 +53,21 @@ public Dataset getDataset() { return getConnector().getDataset(); } - public FrontendFilterConfiguration.Top createFrontendConfig() throws ConceptConfigurationException { - FrontendFilterConfiguration.Top f = FrontendFilterConfiguration.Top.builder() - .id(getId()) - .label(getLabel()) - .tooltip(getTooltip()) - .unit(getUnit()) - .allowDropFile(getAllowDropFile()) - .pattern(getPattern()) - .defaultValue(getDefaultValue()) - .build(); - configureFrontend(f); + public FrontendFilterConfiguration.Top createFrontendConfig(ConqueryConfig conqueryConfig) throws ConceptConfigurationException { + final FrontendFilterConfiguration.Top f = FrontendFilterConfiguration.Top.builder() + .id(getId()) + .label(getLabel()) + .tooltip(getTooltip()) + .unit(getUnit()) + .allowDropFile(getAllowDropFile()) + .pattern(getPattern()) + .defaultValue(getDefaultValue()) + .build(); + configureFrontend(f, conqueryConfig); return f; } - protected abstract void configureFrontend(FrontendFilterConfiguration.Top f) throws ConceptConfigurationException; + protected abstract void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig conqueryConfig) throws ConceptConfigurationException; @JsonIgnore public abstract List getRequiredColumns(); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java index 845648ad90..e45f0d7c9b 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountFilter.java @@ -12,6 +12,7 @@ import com.bakdata.conquery.io.jackson.serializer.NsIdRef; import com.bakdata.conquery.io.jackson.serializer.NsIdRefCollection; import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.query.filter.RangeFilterNode; @@ -36,7 +37,7 @@ public class CountFilter extends Filter { private boolean distinct; @Override - public void configureFrontend(FrontendFilterConfiguration.Top f) { + public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig conqueryConfig) { f.setType(FrontendFilterType.Fields.INTEGER_RANGE); f.setMin(1); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountQuartersFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountQuartersFilter.java index b0997ffc3c..41d42f9ae3 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountQuartersFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/CountQuartersFilter.java @@ -6,6 +6,7 @@ import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.datasets.concepts.filters.SingleColumnFilter; import com.bakdata.conquery.models.events.MajorTypeId; @@ -26,7 +27,7 @@ public EnumSet getAcceptedColumnTypes() { } @Override - public void configureFrontend(FrontendFilterConfiguration.Top f) { + public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig conqueryConfig) { f.setType(FrontendFilterType.Fields.INTEGER_RANGE); f.setMin(1); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java index c811b797af..ecad117745 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DateDistanceFilter.java @@ -9,6 +9,7 @@ import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.datasets.concepts.filters.SingleColumnFilter; import com.bakdata.conquery.models.events.MajorTypeId; @@ -35,7 +36,7 @@ public EnumSet getAcceptedColumnTypes() { } @Override - public void configureFrontend(FrontendFilterConfiguration.Top f) throws ConceptConfigurationException { + public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig conqueryConfig) throws ConceptConfigurationException { if (getColumn().getType() != MajorTypeId.DATE) { throw new ConceptConfigurationException(getConnector(), "DATE_DISTANCE filter is incompatible with columns of type " + getColumn().getType()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DurationSumFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DurationSumFilter.java index 09087ee70d..cfc180d3ff 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DurationSumFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/DurationSumFilter.java @@ -6,6 +6,7 @@ import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.datasets.concepts.filters.SingleColumnFilter; import com.bakdata.conquery.models.events.MajorTypeId; @@ -29,7 +30,7 @@ public EnumSet getAcceptedColumnTypes() { } @Override - public void configureFrontend(FrontendFilterConfiguration.Top f) throws ConceptConfigurationException { + public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig conqueryConfig) throws ConceptConfigurationException { if (getColumn().getType() != MajorTypeId.DATE_RANGE) { throw new ConceptConfigurationException(getConnector(), "DURATION_SUM filter is incompatible with columns of type " + getColumn().getType()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java index 324ecc22da..0a5c2598ba 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/FlagFilter.java @@ -11,6 +11,7 @@ import com.bakdata.conquery.apiv1.frontend.FrontendValue; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.serializer.NsIdRefCollection; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.error.ConqueryError; @@ -39,7 +40,7 @@ public class FlagFilter extends Filter { private final Map flags; @Override - protected void configureFrontend(FrontendFilterConfiguration.Top f) throws ConceptConfigurationException { + protected void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig conqueryConfig) throws ConceptConfigurationException { f.setType(FrontendFilterType.Fields.MULTI_SELECT); f.setOptions(flags.keySet().stream().map(key -> new FrontendValue(key, key)).toList()); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java index 0040baab68..1894783631 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/NumberFilter.java @@ -7,6 +7,7 @@ import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.common.IRange; import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.datasets.concepts.filters.SingleColumnFilter; import com.bakdata.conquery.models.exceptions.ConceptConfigurationException; @@ -29,7 +30,7 @@ public class NumberFilter> extends SingleColumnFilter { @Override - public void configureFrontend(FrontendFilterConfiguration.Top f) throws ConceptConfigurationException { + public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig conqueryConfig) throws ConceptConfigurationException { final String type = switch (getColumn().getType()) { case MONEY -> FrontendFilterType.Fields.MONEY_RANGE; case INTEGER -> FrontendFilterType.Fields.INTEGER_RANGE; diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/PrefixTextFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/PrefixTextFilter.java index bd5e83db3e..02480ea143 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/PrefixTextFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/PrefixTextFilter.java @@ -5,6 +5,7 @@ import com.bakdata.conquery.apiv1.frontend.FrontendFilterConfiguration; import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; import com.bakdata.conquery.io.cps.CPSType; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.datasets.concepts.filters.SingleColumnFilter; import com.bakdata.conquery.models.events.MajorTypeId; @@ -20,7 +21,7 @@ public class PrefixTextFilter extends SingleColumnFilter { @Override - public void configureFrontend(FrontendFilterConfiguration.Top f) { + public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig conqueryConfig) { f.setType(FrontendFilterType.Fields.STRING); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/QuartersInYearFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/QuartersInYearFilter.java index 11bc8c36fa..1dac22cb0a 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/QuartersInYearFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/QuartersInYearFilter.java @@ -6,6 +6,7 @@ import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.datasets.concepts.filters.SingleColumnFilter; import com.bakdata.conquery.models.events.MajorTypeId; @@ -24,7 +25,7 @@ public EnumSet getAcceptedColumnTypes() { } @Override - public void configureFrontend(FrontendFilterConfiguration.Top f) { + public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig conqueryConfig) { f.setType(FrontendFilterType.Fields.INTEGER_RANGE); f.setMin(1); f.setMax(4); diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java index 8a547e5308..947edb2fcd 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SelectFilter.java @@ -12,6 +12,7 @@ import com.bakdata.conquery.io.jackson.View; import com.bakdata.conquery.io.jackson.serializer.NsIdRef; import com.bakdata.conquery.io.storage.NamespaceStorage; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.config.IndexConfig; import com.bakdata.conquery.models.datasets.concepts.Searchable; import com.bakdata.conquery.models.datasets.concepts.filters.SingleColumnFilter; @@ -56,12 +57,12 @@ public EnumSet getAcceptedColumnTypes() { } @Override - public void configureFrontend(FrontendFilterConfiguration.Top f) throws ConceptConfigurationException { + public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig conqueryConfig) throws ConceptConfigurationException { f.setTemplate(getTemplate()); f.setType(getFilterType()); // If either not searches are available or all are disabled, we allow users to supply their own values - f.setCreatable(getSearchReferences().stream().noneMatch(Predicate.not(Searchable::isSearchDisabled))); + f.setCreatable(conqueryConfig.getFrontend().isAlwaysAllowCreateValue() || getSearchReferences().stream().noneMatch(Predicate.not(Searchable::isSearchDisabled))); f.setOptions(collectLabels()); } diff --git a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java index c89db064ba..643e8e8807 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java +++ b/backend/src/main/java/com/bakdata/conquery/models/datasets/concepts/filters/specific/SumFilter.java @@ -14,6 +14,7 @@ import com.bakdata.conquery.io.jackson.serializer.NsIdRefCollection; import com.bakdata.conquery.models.common.IRange; import com.bakdata.conquery.models.common.Range; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.concepts.filters.Filter; import com.bakdata.conquery.models.events.MajorTypeId; @@ -57,7 +58,7 @@ public class SumFilter> extends Filter private List distinctByColumn = Collections.emptyList(); @Override - public void configureFrontend(FrontendFilterConfiguration.Top f) throws ConceptConfigurationException { + public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig conqueryConfig) throws ConceptConfigurationException { final String type = switch (getColumn().getType()) { case MONEY -> FrontendFilterType.Fields.MONEY_RANGE; case INTEGER -> FrontendFilterType.Fields.INTEGER_RANGE; diff --git a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java index ab877f9b41..d2c1ebac6b 100644 --- a/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/resources/api/ConceptsProcessor.java @@ -65,11 +65,14 @@ public class ConceptsProcessor { private final ConqueryConfig config; + @Getter(lazy = true) + private final FrontEndConceptBuilder frontEndConceptBuilder = new FrontEndConceptBuilder(getConfig()); + private final LoadingCache, FrontendList> nodeCache = CacheBuilder.newBuilder().softValues().expireAfterWrite(10, TimeUnit.MINUTES).build(new CacheLoader<>() { @Override public FrontendList load(Concept concept) { - return FrontEndConceptBuilder.createTreeMap(concept); + return getFrontEndConceptBuilder().createTreeMap(concept); } }); @@ -106,7 +109,7 @@ public CursorAndLength load(Searchable searchable) { public FrontendRoot getRoot(NamespaceStorage storage, Subject subject) { - final FrontendRoot root = FrontEndConceptBuilder.createRoot(storage, subject); + final FrontendRoot root = getFrontEndConceptBuilder().createRoot(storage, subject); // Report Violation ValidatorHelper.createViolationsString(validator.validate(root), log.isTraceEnabled()).ifPresent(log::warn); diff --git a/backend/src/test/java/com/bakdata/conquery/integration/json/filter/FilterTest.java b/backend/src/test/java/com/bakdata/conquery/integration/json/filter/FilterTest.java index 063f4f3e81..2bdfbb7708 100644 --- a/backend/src/test/java/com/bakdata/conquery/integration/json/filter/FilterTest.java +++ b/backend/src/test/java/com/bakdata/conquery/integration/json/filter/FilterTest.java @@ -172,7 +172,7 @@ public Query getQuery() { @Override public void executeTest(StandaloneSupport standaloneSupport) throws IOException { try { - final FrontendFilterConfiguration.Top actual = connector.getFilters().iterator().next().createFrontendConfig(); + final FrontendFilterConfiguration.Top actual = connector.getFilters().iterator().next().createFrontendConfig(standaloneSupport.getConfig()); if (expectedFrontendConfig != null) { log.info("Checking actual FrontendConfig: {}", actual); diff --git a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/filters/TestGroupFilter.java b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/filters/TestGroupFilter.java index ddccaf3428..bcdd7a33e6 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/filters/TestGroupFilter.java +++ b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/filters/TestGroupFilter.java @@ -12,6 +12,7 @@ import com.bakdata.conquery.apiv1.frontend.FrontendFilterType; import com.bakdata.conquery.io.cps.CPSType; import com.bakdata.conquery.io.jackson.View; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.concepts.filters.specific.QueryContextResolvable; import com.bakdata.conquery.models.query.QueryResolveContext; import com.bakdata.conquery.models.query.filter.event.MultiSelectFilterNode; @@ -27,7 +28,7 @@ public class TestGroupFilter extends SingleColumnFilter implements GroupFilter { @Override - public void configureFrontend(FrontendFilterConfiguration.Top f) { + public void configureFrontend(FrontendFilterConfiguration.Top f, ConqueryConfig conqueryConfig) { f.setType(FrontendFilterType.Fields.GROUP); f.setFilters(getFEFilter()); } diff --git a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/frontend/FilterSearchItemTest.java b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/frontend/FilterSearchItemTest.java index 76d7fe7b25..d6b263ec37 100644 --- a/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/frontend/FilterSearchItemTest.java +++ b/backend/src/test/java/com/bakdata/conquery/models/datasets/concepts/frontend/FilterSearchItemTest.java @@ -6,6 +6,7 @@ import com.bakdata.conquery.apiv1.frontend.FrontendTable; import com.bakdata.conquery.apiv1.frontend.FrontendValue; +import com.bakdata.conquery.models.config.ConqueryConfig; import com.bakdata.conquery.models.datasets.Column; import com.bakdata.conquery.models.datasets.Dataset; import com.bakdata.conquery.models.datasets.Table; @@ -67,7 +68,7 @@ public void sortedValidityDates() { connector.setColumn(column); connector.setConcept(concept); connector.setValidityDates(validityDates); - FrontendTable feTable = FrontEndConceptBuilder.createTable(connector); + FrontendTable feTable = new FrontEndConceptBuilder(new ConqueryConfig()).createTable(connector); assertThat(feTable.getDateColumn().getOptions()).containsExactly( new FrontendValue(val0.getId().toString(), "val0"), From 9666c0e4414e316ad2128e66bc2369eba115aca2 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Mon, 4 Sep 2023 16:22:28 +0200 Subject: [PATCH 534/679] remove easily broken trace logging --- .../conquery/io/mina/BinaryJacksonCoder.java | 29 +++++-------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/mina/BinaryJacksonCoder.java b/backend/src/main/java/com/bakdata/conquery/io/mina/BinaryJacksonCoder.java index d71b715036..130d05f1ee 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/mina/BinaryJacksonCoder.java +++ b/backend/src/main/java/com/bakdata/conquery/io/mina/BinaryJacksonCoder.java @@ -1,11 +1,7 @@ package com.bakdata.conquery.io.mina; -import java.io.File; -import java.util.UUID; - import javax.validation.Validator; -import com.bakdata.conquery.io.jackson.Jackson; import com.bakdata.conquery.models.exceptions.ValidatorHelper; import com.bakdata.conquery.models.messages.network.NetworkMessage; import com.bakdata.conquery.models.worker.IdResolveContext; @@ -14,7 +10,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectReader; import com.fasterxml.jackson.databind.ObjectWriter; -import com.fasterxml.jackson.databind.SerializationFeature; import lombok.extern.slf4j.Slf4j; @Slf4j @@ -26,34 +21,26 @@ public class BinaryJacksonCoder implements CQCoder> { public BinaryJacksonCoder(IdResolveContext datasets, Validator validator, ObjectMapper objectMapper) { this.validator = validator; - this.writer = objectMapper - .writerFor(NetworkMessage.class); - this.reader = datasets - .injectIntoNew(objectMapper.readerFor(NetworkMessage.class)) - .without(Feature.AUTO_CLOSE_SOURCE); + writer = objectMapper.writerFor(NetworkMessage.class); + reader = datasets.injectIntoNew(objectMapper.readerFor(NetworkMessage.class)).without(Feature.AUTO_CLOSE_SOURCE); } @Override public Chunkable encode(NetworkMessage message) throws Exception { ValidatorHelper.failOnError(log, validator.validate(message)); - UUID id = message.getMessageId(); - Chunkable chunkable = new Chunkable(id, writer, message); - if(log.isTraceEnabled()) { - Jackson.MAPPER.writerFor(NetworkMessage.class).with(SerializationFeature.INDENT_OUTPUT).writeValue(new File("dumps/out_"+id+".json"), message); - } - return chunkable; + return new Chunkable(message.getMessageId(), writer, message); } @Override public NetworkMessage decode(ChunkedMessage message) throws Exception { - try(EndCheckableInputStream is = message.createInputStream()) { - Object obj = reader.readValue(is); - if(!is.isAtEnd()) { - throw new IllegalStateException("After reading the JSON message "+obj+" the buffer has still bytes available"); + try (EndCheckableInputStream is = message.createInputStream()) { + final Object obj = reader.readValue(is); + if (!is.isAtEnd()) { + throw new IllegalStateException("After reading the JSON message " + obj + " the buffer has still bytes available"); } ValidatorHelper.failOnError(log, validator.validate(obj)); - return (NetworkMessage)obj; + return (NetworkMessage) obj; } } } From 92d8adbe47f376923d33a81ea3f3510dec7784f2 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Mon, 4 Sep 2023 16:59:22 +0200 Subject: [PATCH 535/679] compute containsDates at submission, or fullOverview, but never juts overview --- .../apiv1/execution/ExecutionStatus.java | 2 ++ .../apiv1/execution/FullExecutionStatus.java | 1 - .../models/execution/ManagedExecution.java | 19 +++++++++++++------ .../conquery/models/query/ManagedQuery.java | 12 +++++------- 4 files changed, 20 insertions(+), 14 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/execution/ExecutionStatus.java b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/ExecutionStatus.java index 4aec080f22..51f83c56f6 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/execution/ExecutionStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/ExecutionStatus.java @@ -44,6 +44,8 @@ public abstract class ExecutionStatus { private String queryType; private SecondaryIdDescriptionId secondaryId; + private boolean containsDates; + /** * The urls under from which the result of the execution can be downloaded as soon as it finished successfully. diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/execution/FullExecutionStatus.java b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/FullExecutionStatus.java index 32d9d45c38..2984e904e0 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/execution/FullExecutionStatus.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/execution/FullExecutionStatus.java @@ -46,7 +46,6 @@ public class FullExecutionStatus extends ExecutionStatus { */ private boolean canExpand; - private boolean containsDates; /** * Is set to the query description if the user can expand all included concepts. diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java index fb709ad704..c29184634f 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java +++ b/backend/src/main/java/com/bakdata/conquery/models/execution/ManagedExecution.java @@ -94,6 +94,8 @@ public abstract class ManagedExecution extends IdentifiableImpl { if (visitable instanceof CQConcept cqConcept) { @@ -373,7 +381,7 @@ private boolean containsDates(QueryDescription query) { }); } - private boolean canSubjectExpand(Subject subject, QueryDescription query) { + private static boolean canSubjectExpand(Subject subject, QueryDescription query) { NamespacedIdentifiableCollector namespacesIdCollector = new NamespacedIdentifiableCollector(); query.visit(namespacesIdCollector); @@ -401,9 +409,8 @@ public boolean isReadyToDownload() { @JsonIgnore public String getLabelWithoutAutoLabelSuffix() { - int idx; + final int idx; if (label != null && (idx = label.lastIndexOf(AUTO_LABEL_SUFFIX)) != -1) { - return label.substring(0, idx); } return label; diff --git a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java index 37cc0e5e63..46da29ffe1 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java +++ b/backend/src/main/java/com/bakdata/conquery/models/query/ManagedQuery.java @@ -61,12 +61,12 @@ public class ManagedQuery extends ManagedExecution implements SingleTableResult, */ private Long lastResultCount; - //TODO this can actually be known ahead and reduced to speedup queries. @JsonIgnore private transient Set involvedWorkers; @JsonIgnore private transient List columnDescriptions; + protected ManagedQuery(@JacksonInject(useInput = OptBoolean.FALSE) MetaStorage storage) { super(storage); } @@ -78,9 +78,7 @@ public ManagedQuery(Query query, User owner, Dataset submittedDataset, MetaStora @Override protected void doInitExecutable() { - query.resolve(new QueryResolveContext(getNamespace(), getConfig(), getStorage(), null)); - } @Override @@ -154,15 +152,15 @@ protected void setAdditionalFieldsForStatusWithColumnDescription(Subject subject */ public List generateColumnDescriptions() { Preconditions.checkArgument(isInitialized(), "The execution must have been initialized first"); - List columnDescriptions = new ArrayList<>(); + final List columnDescriptions = new ArrayList<>(); final Locale locale = I18n.LOCALE.get(); - PrintSettings settings = new PrintSettings(true, locale, getNamespace(), getConfig(), null); + final PrintSettings settings = new PrintSettings(true, locale, getNamespace(), getConfig(), null); - UniqueNamer uniqNamer = new UniqueNamer(settings); + final UniqueNamer uniqNamer = new UniqueNamer(settings); - // First add the id columns to the descriptor list. The are the first columns + // First add the id columns to the descriptor list. These are always the first columns for (ResultInfo header : getConfig().getIdColumns().getIdResultInfos()) { columnDescriptions.add(ColumnDescriptor.builder() .label(uniqNamer.getUniqueName(header)) From a593c2e003cbbe0e34175ceb2565be61db120b75 Mon Sep 17 00:00:00 2001 From: Marco Korinth Date: Thu, 31 Aug 2023 14:06:17 +0200 Subject: [PATCH 536/679] fix: datepicker blocks input field --- frontend/src/js/common/helpers/mergeRefs.ts | 17 ++++ .../ui-components/InputDate/CustomHeader.tsx | 2 +- .../js/ui-components/InputDate/InputDate.tsx | 84 +++++++++---------- .../src/js/ui-components/InputDateRange.tsx | 5 +- 4 files changed, 59 insertions(+), 49 deletions(-) create mode 100644 frontend/src/js/common/helpers/mergeRefs.ts diff --git a/frontend/src/js/common/helpers/mergeRefs.ts b/frontend/src/js/common/helpers/mergeRefs.ts new file mode 100644 index 0000000000..20b0e38f17 --- /dev/null +++ b/frontend/src/js/common/helpers/mergeRefs.ts @@ -0,0 +1,17 @@ +type Mutable = { + -readonly [k in keyof T]: T[k]; +}; + +export const mergeRefs = + (...refs: React.Ref[]) => + (value: T): void => { + for (let i = 0; i < refs.length; i += 1) { + const ref = refs[i]; + + if (typeof ref === "function") { + ref(value); + } else if (ref) { + (ref as Mutable>).current = value; + } + } + }; diff --git a/frontend/src/js/ui-components/InputDate/CustomHeader.tsx b/frontend/src/js/ui-components/InputDate/CustomHeader.tsx index 83b0b594c0..310df8480b 100644 --- a/frontend/src/js/ui-components/InputDate/CustomHeader.tsx +++ b/frontend/src/js/ui-components/InputDate/CustomHeader.tsx @@ -102,7 +102,7 @@ const YearMonthSelect = ({ value: i, })); - const [yearSelectOpen, setYearSelectOpen] = useState(false); + const [yearSelectOpen, setYearSelectOpen] = useState(true); const [monthSelectOpen, setMonthSelectOpen] = useState(false); const handleClick = () => { if (yearSelectOpen || monthSelectOpen) { diff --git a/frontend/src/js/ui-components/InputDate/InputDate.tsx b/frontend/src/js/ui-components/InputDate/InputDate.tsx index 00e57bc56b..7b4d987856 100644 --- a/frontend/src/js/ui-components/InputDate/InputDate.tsx +++ b/frontend/src/js/ui-components/InputDate/InputDate.tsx @@ -1,9 +1,12 @@ import styled from "@emotion/styled"; -import { createElement, forwardRef, useRef, useState } from "react"; +import { faCalendar } from "@fortawesome/free-regular-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { createElement, forwardRef, useRef } from "react"; import ReactDatePicker from "react-datepicker"; import "react-datepicker/dist/react-datepicker.css"; import { formatDate, parseDate } from "../../common/helpers/dateHelper"; +import { mergeRefs } from "../../common/helpers/mergeRefs"; import BaseInput, { Props as BaseInputProps } from "../BaseInput"; import { CustomHeader } from "./CustomHeader"; @@ -19,11 +22,25 @@ const Root = styled("div")` } .react-datepicker-popper[data-placement^="bottom"] { padding-top: 4px; - transform: translate3d(0, 32px, 0) !important; } .react-datepicker-popper[data-placement^="top"] { padding-bottom: 0; - transform: translate3d(0, 32px, 0) !important; + } +`; + +const CalendarIcon = styled(FontAwesomeIcon)` + position: absolute; + width: 16px; + height: 16px; + top: calc(50% - 8px); + left: 5px; + cursor: pointer; + color: ${({ theme }) => theme.col.black}; +`; + +const StyledBaseInput = styled(BaseInput)` + input { + padding-left: 22px; } `; @@ -45,28 +62,12 @@ type Props = Omit & { onCalendarSelect?: (val: string) => void; }; -// TODO: Remove this once we have solved -// - that the date picker overlays other fields in forms -const TEMPORARILY_DISABLED_DATE_PICKER = true; - -const InputDate = forwardRef( +const InputDate = forwardRef( ( - { - className, - value, - dateFormat, - onChange, - onCalendarSelect, - onFocus, - onBlur, - onClick, - ...props - }, + { className, value, dateFormat, onChange, onCalendarSelect, ...props }, ref, ) => { const datePickerRef = useRef(null); - const [hasFocus, setHasFocus] = useState(false); - const [focusBlocked, setFocusBlocked] = useState(false); return ( ( if (e.key === "Escape") datePickerRef.current?.setOpen(false); }} > - { onChange(val as string); }} - onFocus={(e) => { - if (focusBlocked) { - e.currentTarget.blur(); - setFocusBlocked(false); - } else { - onFocus?.(e); - setHasFocus(true); - datePickerRef.current?.setOpen(true); - } - }} - onBlur={(e) => { - onBlur?.(e); - setHasFocus(false); - }} - onClick={(e) => { - onClick?.(e); - if (hasFocus) { - datePickerRef.current?.setOpen(true); - } - }} inputProps={{ ...props?.inputProps, onKeyPress: (e) => { @@ -111,8 +91,12 @@ const InputDate = forwardRef( }, }} /> + datePickerRef.current?.setOpen(true)} + /> { if (!val) { @@ -122,7 +106,6 @@ const InputDate = forwardRef( const selectedDate = formatDate(val, dateFormat); onChange(selectedDate); onCalendarSelect?.(selectedDate); - setFocusBlocked(true); datePickerRef.current?.setOpen(false); }} onClickOutside={() => datePickerRef.current?.setOpen(false)} @@ -130,7 +113,16 @@ const InputDate = forwardRef( customInput={createElement(HiddenInput)} calendarContainer={StyledCalendar} calendarStartDay={1} - disabled={TEMPORARILY_DISABLED_DATE_PICKER} + popperProps={{ + modifiers: [ + { + name: "preventOverflow", + options: { + mainAxis: false, + }, + }, + ], + }} /> ); diff --git a/frontend/src/js/ui-components/InputDateRange.tsx b/frontend/src/js/ui-components/InputDateRange.tsx index e58a00745e..33d2bdb747 100644 --- a/frontend/src/js/ui-components/InputDateRange.tsx +++ b/frontend/src/js/ui-components/InputDateRange.tsx @@ -2,6 +2,7 @@ import { css } from "@emotion/react"; import styled from "@emotion/styled"; import { faCalendar } from "@fortawesome/free-regular-svg-icons"; import { FC, ReactNode, createRef, useMemo } from "react"; +import ReactDatePicker from "react-datepicker"; import { useTranslation } from "react-i18next"; import { IndexPrefix } from "../common/components/IndexPrefix"; @@ -166,7 +167,7 @@ const InputDateRange: FC = ({ const min = getDisplayDate("min", value, displayDateFormat); const max = getDisplayDate("max", value, displayDateFormat); - const maxRef = createRef(); + const maxRef = createRef(); const isMinValid = exists(value.min && parseDate(min, displayDateFormat)); const isMaxValid = exists(value.max && parseDate(max, displayDateFormat)); @@ -213,7 +214,7 @@ const InputDateRange: FC = ({ onChange={(val) => onChangeRaw("min", val as string, displayDateFormat) } - onCalendarSelect={() => maxRef.current?.focus()} + onCalendarSelect={() => maxRef.current?.setOpen(true)} onBlur={(e) => applyDate("min", e.target.value, displayDateFormat)} inputProps={{ autoFocus, From 383f61b8960216e599c540e94e21e9ecc3c64dec Mon Sep 17 00:00:00 2001 From: Marco Korinth Date: Tue, 5 Sep 2023 08:00:58 +0200 Subject: [PATCH 537/679] refactor: use IconButton for Datepicker icon and implement mergeRefs dependency --- frontend/package.json | 1 + frontend/src/js/common/helpers/mergeRefs.ts | 17 ----------------- .../js/ui-components/InputDate/InputDate.tsx | 14 +++++++------- frontend/yarn.lock | 5 +++++ 4 files changed, 13 insertions(+), 24 deletions(-) delete mode 100644 frontend/src/js/common/helpers/mergeRefs.ts diff --git a/frontend/package.json b/frontend/package.json index 742c47ff5f..5bef82954a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -69,6 +69,7 @@ "react-i18next": "^12.2.0", "react-list": "^0.8.16", "react-markdown": "^8.0.0", + "react-merge-refs": "^2.0.2", "react-number-format": "^5.1.4", "react-redux": "^8.0.5", "react-router-dom": "^6.9.0", diff --git a/frontend/src/js/common/helpers/mergeRefs.ts b/frontend/src/js/common/helpers/mergeRefs.ts deleted file mode 100644 index 20b0e38f17..0000000000 --- a/frontend/src/js/common/helpers/mergeRefs.ts +++ /dev/null @@ -1,17 +0,0 @@ -type Mutable = { - -readonly [k in keyof T]: T[k]; -}; - -export const mergeRefs = - (...refs: React.Ref[]) => - (value: T): void => { - for (let i = 0; i < refs.length; i += 1) { - const ref = refs[i]; - - if (typeof ref === "function") { - ref(value); - } else if (ref) { - (ref as Mutable>).current = value; - } - } - }; diff --git a/frontend/src/js/ui-components/InputDate/InputDate.tsx b/frontend/src/js/ui-components/InputDate/InputDate.tsx index 7b4d987856..950ceac5c7 100644 --- a/frontend/src/js/ui-components/InputDate/InputDate.tsx +++ b/frontend/src/js/ui-components/InputDate/InputDate.tsx @@ -1,12 +1,12 @@ import styled from "@emotion/styled"; import { faCalendar } from "@fortawesome/free-regular-svg-icons"; -import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { createElement, forwardRef, useRef } from "react"; import ReactDatePicker from "react-datepicker"; import "react-datepicker/dist/react-datepicker.css"; +import { mergeRefs } from "react-merge-refs"; +import IconButton from "../../button/IconButton"; import { formatDate, parseDate } from "../../common/helpers/dateHelper"; -import { mergeRefs } from "../../common/helpers/mergeRefs"; import BaseInput, { Props as BaseInputProps } from "../BaseInput"; import { CustomHeader } from "./CustomHeader"; @@ -28,19 +28,18 @@ const Root = styled("div")` } `; -const CalendarIcon = styled(FontAwesomeIcon)` +const CalendarIcon = styled(IconButton)` position: absolute; width: 16px; height: 16px; top: calc(50% - 8px); left: 5px; - cursor: pointer; - color: ${({ theme }) => theme.col.black}; + padding: 0; `; const StyledBaseInput = styled(BaseInput)` input { - padding-left: 22px; + padding-left: 28px; } `; @@ -93,10 +92,11 @@ const InputDate = forwardRef( /> datePickerRef.current?.setOpen(true)} /> { if (!val) { diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 10880d182f..d65beaf493 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -8622,6 +8622,11 @@ react-markdown@^8.0.0: unist-util-visit "^4.0.0" vfile "^5.0.0" +react-merge-refs@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/react-merge-refs/-/react-merge-refs-2.0.2.tgz#73f576111124897dec4ea56035a97e199e8cb377" + integrity sha512-V5BGTwGa2r+/t0A/BZMS6L7VPXY0CU8xtAhkT3XUoI1WJJhhtvulvoiZkJ5Jt9YAW23m4xFWmhQ+C5HwjtTFhQ== + react-number-format@^5.1.4: version "5.1.4" resolved "https://registry.yarnpkg.com/react-number-format/-/react-number-format-5.1.4.tgz#23057d94a4f1b08e12ee41328e86be929b60a791" From fbc52dd996d0194b11c1bedf96594037c3165065 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 5 Sep 2023 10:53:32 +0200 Subject: [PATCH 538/679] cleanup Shareable code --- .../conquery/models/execution/Shareable.java | 45 +++++++++---------- 1 file changed, 22 insertions(+), 23 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/models/execution/Shareable.java b/backend/src/main/java/com/bakdata/conquery/models/execution/Shareable.java index 922fdf81ef..b626b88ddb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/execution/Shareable.java +++ b/backend/src/main/java/com/bakdata/conquery/models/execution/Shareable.java @@ -31,42 +31,41 @@ public interface Shareable extends Authorized { */ void setShared(boolean shared); - default , S extends Identifiable & Shareable & Authorized> Consumer sharer( - MetaStorage storage, - Subject subject) { + default , S extends Identifiable & Shareable & Authorized> Consumer sharer(MetaStorage storage, Subject subject) { if (!(this instanceof Identifiable)) { log.warn("Cannot share {} ({}) because it does not implement Identifiable", this.getClass(), this.toString()); return QueryUtils.getNoOpEntryPoint(); } - return (patch) -> { - if (patch != null && patch.getGroups() != null) { - S shareable = (S) this; - // Collect groups that do not have access to this instance and remove their probable permission - for (Group group1 : storage.getAllGroups()) { - if (patch.getGroups().contains(group1.getId())) { - continue; - } - log.trace("User {} unshares instance {} ({}) from owner {}.", subject, shareable.getClass().getSimpleName(), shareable.getId(), group1); + return (patch) -> { + if (patch == null || patch.getGroups() == null) { + return; + } - group1.removePermission(shareable.createPermission(AbilitySets.SHAREHOLDER)); + final S shareable = (S) this; + // Collect groups that do not have access to this instance and remove their probable permission + for (Group group : storage.getAllGroups()) { + if (patch.getGroups().contains(group.getId())) { + continue; } + log.trace("User {} unshares instance {} ({}) from owner {}.", subject, shareable.getClass().getSimpleName(), shareable.getId(), group); - if(!patch.getGroups().isEmpty()) { - // Resolve the provided groups - Set groups = patch.getGroups().stream().map(storage::getGroup).collect(Collectors.toSet()); + group.removePermission(shareable.createPermission(AbilitySets.SHAREHOLDER)); + } - for(Group group : groups) { - ConqueryPermission sharePermission = shareable.createPermission(AbilitySets.SHAREHOLDER); - group.addPermission(sharePermission); - log.trace("User {} shares instance {} ({}). Adding permission {} to owner {}.", subject, shareable.getClass().getSimpleName(), shareable.getId(), sharePermission, group); - } - } + // Resolve the provided groups + final Set groups = patch.getGroups().stream().map(storage::getGroup).collect(Collectors.toSet()); + + for(Group group : groups) { + final ConqueryPermission sharePermission = shareable.createPermission(AbilitySets.SHAREHOLDER); + group.addPermission(sharePermission); - this.setShared(!patch.getGroups().isEmpty()); + log.trace("User {} shares instance {} ({}). Adding permission {} to owner {}.", subject, shareable.getClass().getSimpleName(), shareable.getId(), sharePermission, group); } + + setShared(!patch.getGroups().isEmpty()); }; } From 293ce64020a7aac42e6f0e34f75478dcd8c3e109 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 5 Sep 2023 11:12:02 +0200 Subject: [PATCH 539/679] Share sub-executions when sharing executions --- .../bakdata/conquery/apiv1/QueryProcessor.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index 119650f5dc..bd94e84acc 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -312,6 +312,21 @@ public void patchQuery(Subject subject, ManagedExecution execution, MetaDataPatc log.info("Patching {} ({}) with patch: {}", execution.getClass().getSimpleName(), execution, patch); + // If the patch shares the execution, we also share all subQueries + if (patch.getGroups() != null && !patch.getGroups().isEmpty()) { + final MetaDataPatch sharePatch = MetaDataPatch.builder() + .groups(patch.getGroups()) + .build(); + + for (ManagedExecutionId managedExecutionId : execution.getSubmitted().collectRequiredQueries()) { + final ManagedExecution subQuery = storage.getExecution(managedExecutionId); + + subject.authorize(subQuery, Ability.READ); + + patchQuery(subject, subQuery, sharePatch); + } + } + patch.applyTo(execution, storage, subject); storage.updateExecution(execution); } From 2f35c67598322ce9920cc4a8c82921af43e4661b Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 5 Sep 2023 13:49:45 +0200 Subject: [PATCH 540/679] Make all non-required field errors red --- frontend/src/js/external-forms/form/Field.tsx | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/frontend/src/js/external-forms/form/Field.tsx b/frontend/src/js/external-forms/form/Field.tsx index ad657d4e2f..7741bd2f38 100644 --- a/frontend/src/js/external-forms/form/Field.tsx +++ b/frontend/src/js/external-forms/form/Field.tsx @@ -73,7 +73,11 @@ type Props = T & { noContainer?: boolean; noLabel?: boolean; }; -const FieldContainer = styled("div")<{ noLabel?: boolean; hasError?: boolean }>` +const FieldContainer = styled("div")<{ + noLabel?: boolean; + hasError?: boolean; + red?: boolean; +}>` display: flex; flex-direction: column; gap: 5px; @@ -81,12 +85,16 @@ const FieldContainer = styled("div")<{ noLabel?: boolean; hasError?: boolean }>` background-color: white; border-radius: ${({ theme }) => theme.borderRadius}; border: 1px solid - ${({ theme, hasError }) => - hasError ? theme.col.blueGrayDark : theme.col.grayLight}; + ${({ theme, hasError, red }) => + hasError + ? red + ? theme.col.red + : theme.col.blueGrayDark + : theme.col.grayLight}; `; -const ErrorContainer = styled("div")` - color: ${({ theme }) => theme.col.blueGrayDark}; +const ErrorContainer = styled("div")<{ red?: boolean }>` + color: ${({ theme, red }) => (red ? theme.col.red : theme.col.blueGrayDark)}; font-weight: 700; font-size: ${({ theme }) => theme.font.sm}; `; @@ -114,12 +122,21 @@ const ConnectedField = ({ // TODO: REFINE COLORS // const color = useColorByField(formField.type); + const requiredMsg = t("externalForms.formValidation.isRequired"); + const isRedError = fieldState.error?.message !== requiredMsg; + return noContainer ? (

{children({ ...field, ...props })}
) : ( - + {children({ ...field, ...props })} - {fieldState.error?.message} + + {fieldState.error?.message} + ); }; From ddd85b1d275c8491a4c9964eb45e23e9b8133d62 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 5 Sep 2023 15:28:29 +0200 Subject: [PATCH 541/679] find groups execution is already shared with --- .../conquery/apiv1/QueryProcessor.java | 34 ++++++++++++++++--- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java index bd94e84acc..2f38a056ae 100644 --- a/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java +++ b/backend/src/main/java/com/bakdata/conquery/apiv1/QueryProcessor.java @@ -6,10 +6,12 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.function.Consumer; import java.util.function.Predicate; @@ -48,6 +50,7 @@ import com.bakdata.conquery.models.auth.entities.Subject; import com.bakdata.conquery.models.auth.entities.User; import com.bakdata.conquery.models.auth.permissions.Ability; +import com.bakdata.conquery.models.auth.permissions.ConqueryPermission; import com.bakdata.conquery.models.common.Range; import com.bakdata.conquery.models.config.ColumnConfig; import com.bakdata.conquery.models.config.ConqueryConfig; @@ -58,6 +61,7 @@ import com.bakdata.conquery.models.error.ConqueryError; import com.bakdata.conquery.models.execution.ExecutionState; import com.bakdata.conquery.models.execution.ManagedExecution; +import com.bakdata.conquery.models.identifiable.ids.specific.GroupId; import com.bakdata.conquery.models.identifiable.ids.specific.ManagedExecutionId; import com.bakdata.conquery.models.identifiable.mapping.IdPrinter; import com.bakdata.conquery.models.query.ExecutionManager; @@ -314,14 +318,36 @@ public void patchQuery(Subject subject, ManagedExecution execution, MetaDataPatc // If the patch shares the execution, we also share all subQueries if (patch.getGroups() != null && !patch.getGroups().isEmpty()) { - final MetaDataPatch sharePatch = MetaDataPatch.builder() - .groups(patch.getGroups()) - .build(); + for (ManagedExecutionId managedExecutionId : execution.getSubmitted().collectRequiredQueries()) { final ManagedExecution subQuery = storage.getExecution(managedExecutionId); - subject.authorize(subQuery, Ability.READ); + if (!subject.isPermitted(subQuery, Ability.READ)) { + log.warn("Not sharing {} as User {} is not allowed to see it themselves.", subQuery.getId(), subject); + continue; + } + + final ConqueryPermission canReadQuery = subQuery.createPermission(Set.of(Ability.READ)); + + final Set groupsToShareWith = new HashSet<>(patch.getGroups()); + + // Find all groups the query is already shared with, so we do not remove them, as patch is absolute + for (Group group : storage.getAllGroups()) { + if (groupsToShareWith.contains(group.getId())){ + continue; + } + + final Set effectivePermissions = group.getEffectivePermissions(); + + if(effectivePermissions.stream().anyMatch(perm -> perm.implies(canReadQuery))) { + groupsToShareWith.add(group.getId()); + } + } + + final MetaDataPatch sharePatch = MetaDataPatch.builder() + .groups(new ArrayList<>(groupsToShareWith)) + .build(); patchQuery(subject, subQuery, sharePatch); } From 94b1ca4dc4c0c9edb22bb07fb02e0e713afa8692 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 5 Sep 2023 17:16:44 +0200 Subject: [PATCH 542/679] fixes not respecting TimeoutExceptions --- .../xodus/stores/SerializingStore.java | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index c4d2591d71..b4f271060b 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -10,13 +10,16 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.Collection; +import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.Function; import java.util.function.Predicate; @@ -47,7 +50,6 @@ import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; import lombok.NonNull; -import lombok.SneakyThrows; import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.jetbrains.annotations.NotNull; @@ -353,7 +355,6 @@ private static byte[] debugUnGzip(byte[] bytes) throws IOException { * * @implNote This method is concurrent! */ - @SneakyThrows @Override public IterationStatistic forEach(StoreEntryConsumer consumer) { final IterationStatistic result = new IterationStatistic(); @@ -368,11 +369,24 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { final ListenableFuture> allJobs = Futures.allAsList(jobs); - List maybeFailed; + List maybeFailed = Collections.emptyList(); + + do { + try { + maybeFailed = allJobs.get(30, TimeUnit.SECONDS); + } + catch (InterruptedException e) { + Thread.interrupted(); + log.debug("Thread was interrupted."); + } + catch (ExecutionException e) { + throw new RuntimeException(e); + } + catch (TimeoutException e) { + log.debug("Still waiting for {} jobs.", jobs.stream().filter(Predicate.not(Future::isDone)).count()); + } + } while (!allJobs.isDone()); - while ((maybeFailed = allJobs.get(30, TimeUnit.SECONDS)) == null) { - log.debug("Still waiting for {} jobs.", jobs.stream().filter(Predicate.not(Future::isDone)).count()); - } final List unreadables = maybeFailed.stream().filter(Objects::nonNull).toList(); From 2286fe81614b03a8daa6b737082ec9406596f548 Mon Sep 17 00:00:00 2001 From: awildturtok <1553491+awildturtok@users.noreply.github.com> Date: Tue, 5 Sep 2023 20:10:20 +0200 Subject: [PATCH 543/679] fixes handling of errors inside Futures --- .../xodus/stores/SerializingStore.java | 37 +++++++++++-------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java index b4f271060b..bffb1a5426 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java +++ b/backend/src/main/java/com/bakdata/conquery/io/storage/xodus/stores/SerializingStore.java @@ -295,6 +295,7 @@ private static void dumpToFile(byte[] gzippedObj, @NonNull String keyOfDump, Exc } if (!dumpfile.getParentFile().exists() && !dumpfile.getParentFile().mkdirs()) { + //TODO this seems to occur sometimes, is it maybe just a race condition? throw new IllegalStateException("Could not create `%s`.".formatted(dumpfile.getParentFile())); } @@ -412,24 +413,28 @@ public IterationStatistic forEach(StoreEntryConsumer consumer) { } private ByteIterable handle(StoreEntryConsumer consumer, IterationStatistic result, ByteIterable keyRaw, ByteIterable valueRaw) { - result.incrTotalProcessed(); - - // Try to read the key first - final KEY - key = - getDeserializedAndDumpFailed(keyRaw, SerializingStore.this::readKey, () -> new String(keyRaw.getBytesUnsafe()), valueRaw, "Could not parse key [{}]"); - if (key == null) { - result.incrFailedKeys(); - return keyRaw; - } + final KEY key; + final VALUE value; - // Try to read the value - final VALUE - value = - getDeserializedAndDumpFailed(valueRaw, SerializingStore.this::readValue, key::toString, valueRaw, "Could not parse value for key [{}]"); + try { + result.incrTotalProcessed(); - if (value == null) { - result.incrFailedValues(); + // Try to read the key first + key = getDeserializedAndDumpFailed(keyRaw, SerializingStore.this::readKey, () -> new String(keyRaw.getBytesUnsafe()), valueRaw, "Could not parse key [{}]"); + if (key == null) { + result.incrFailedKeys(); + return keyRaw; + } + + // Try to read the value + value = getDeserializedAndDumpFailed(valueRaw, SerializingStore.this::readValue, key::toString, valueRaw, "Could not parse value for key [{}]"); + + if (value == null) { + result.incrFailedValues(); + return keyRaw; + } + }catch(Exception e){ + log.error("Failed processing key/value", e); return keyRaw; } From e7d133b846d17d6b0965d207218c7fd2ab2a6e5e Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Tue, 5 Sep 2023 10:19:21 +0200 Subject: [PATCH 544/679] Increase clickable area of Calendar Button and center it --- .../src/js/ui-components/InputDate/InputDate.tsx | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/frontend/src/js/ui-components/InputDate/InputDate.tsx b/frontend/src/js/ui-components/InputDate/InputDate.tsx index 950ceac5c7..6d035fc919 100644 --- a/frontend/src/js/ui-components/InputDate/InputDate.tsx +++ b/frontend/src/js/ui-components/InputDate/InputDate.tsx @@ -28,13 +28,11 @@ const Root = styled("div")` } `; -const CalendarIcon = styled(IconButton)` +const CalendarButton = styled(IconButton)` position: absolute; - width: 16px; - height: 16px; - top: calc(50% - 8px); - left: 5px; - padding: 0; + left: 0; + top: 0; + padding: 8px 10px; `; const StyledBaseInput = styled(BaseInput)` @@ -90,9 +88,8 @@ const InputDate = forwardRef( }, }} /> - datePickerRef.current?.setOpen(true)} /> Date: Wed, 6 Sep 2023 11:41:32 +0200 Subject: [PATCH 545/679] Reduce ChunkWriter#bufferSize to 2MB as 32MB is a massive waste of Memory --- .../src/main/java/com/bakdata/conquery/io/mina/ChunkWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/src/main/java/com/bakdata/conquery/io/mina/ChunkWriter.java b/backend/src/main/java/com/bakdata/conquery/io/mina/ChunkWriter.java index fdf9095dbe..44d1cd462d 100644 --- a/backend/src/main/java/com/bakdata/conquery/io/mina/ChunkWriter.java +++ b/backend/src/main/java/com/bakdata/conquery/io/mina/ChunkWriter.java @@ -26,7 +26,7 @@ public class ChunkWriter extends ProtocolEncoderAdapter { @Getter @Setter - private int bufferSize = Ints.checkedCast(Size.megabytes(32).toBytes()); + private int bufferSize = Ints.checkedCast(Size.megabytes(2).toBytes()); private final SoftPool bufferPool = new SoftPool<>(() -> IoBuffer.allocate(bufferSize)); @SuppressWarnings("rawtypes") private final CQCoder coder; From e4991f35f721687241755afabe061c8504e84cab Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 11 Sep 2023 16:10:09 +0200 Subject: [PATCH 546/679] Add no dates in results indicator --- frontend/src/js/api/types.ts | 1 + .../js/previous-queries/list/ProjectItem.tsx | 45 ++++++++++++++----- .../list/ProjectItemDragContainer.tsx | 17 +++---- .../js/previous-queries/list/ProjectItems.tsx | 16 +++---- .../previous-queries/list/ProjectItemsTab.tsx | 4 +- .../src/js/previous-queries/list/reducer.ts | 1 + frontend/src/localization/de.json | 3 +- frontend/src/localization/en.json | 3 +- 8 files changed, 57 insertions(+), 33 deletions(-) diff --git a/frontend/src/js/api/types.ts b/frontend/src/js/api/types.ts index a9f45f45f8..83e76d48c2 100644 --- a/frontend/src/js/api/types.ts +++ b/frontend/src/js/api/types.ts @@ -414,6 +414,7 @@ export interface GetQueryResponseDoneT { columnDescriptions: ColumnDescription[] | null; queryType: "CONCEPT_QUERY" | "SECONDARY_ID_QUERY"; requiredTime: number; // In ms, unused at the moment + containsDates: boolean; } export interface GetQueryRunningResponseT { diff --git a/frontend/src/js/previous-queries/list/ProjectItem.tsx b/frontend/src/js/previous-queries/list/ProjectItem.tsx index b586985b12..a8bb14e9be 100644 --- a/frontend/src/js/previous-queries/list/ProjectItem.tsx +++ b/frontend/src/js/previous-queries/list/ProjectItem.tsx @@ -1,5 +1,8 @@ import styled from "@emotion/styled"; -import { faUser as faUserRegular } from "@fortawesome/free-regular-svg-icons"; +import { + faCalendar, + faUser as faUserRegular, +} from "@fortawesome/free-regular-svg-icons"; import { faFolder as faFolderRegular } from "@fortawesome/free-regular-svg-icons"; import { faFolder, @@ -18,6 +21,7 @@ import IconButton from "../../button/IconButton"; import { formatDate } from "../../common/helpers/dateHelper"; import { exists } from "../../common/helpers/exists"; import { useFormLabelByType } from "../../external-forms/stateSelectors"; +import FaIcon from "../../icon/FaIcon"; import FormSymbol from "../../symbols/FormSymbol"; import QuerySymbol from "../../symbols/QuerySymbol"; import WithTooltip from "../../tooltip/WithTooltip"; @@ -129,6 +133,16 @@ const SxDownloadButton = styled(DownloadButton)` } `; +const Row = styled("div")` + display: flex; + align-items: flex-start; + gap: 5px; +`; + +const SxFaIcon = styled(FaIcon)` + opacity: 0.7; +`; + const FoldersButton = styled(IconButton)` margin-right: 10px; `; @@ -220,17 +234,24 @@ const ProjectItem = forwardRef< /> {!isFormConfig(item) && item.resultUrls.length > 0 ? ( - - - {topLeftLabel} - - + + + + {topLeftLabel} + + + {!item.containsDates && ( + + + + )} + ) : ( {topLeftLabel} )} diff --git a/frontend/src/js/previous-queries/list/ProjectItemDragContainer.tsx b/frontend/src/js/previous-queries/list/ProjectItemDragContainer.tsx index 5e0d23b550..89bdbb8f8a 100644 --- a/frontend/src/js/previous-queries/list/ProjectItemDragContainer.tsx +++ b/frontend/src/js/previous-queries/list/ProjectItemDragContainer.tsx @@ -1,4 +1,4 @@ -import { FC, useRef } from "react"; +import { useRef } from "react"; import { useDrag } from "react-dnd"; import { getWidthAndHeight } from "../../app/DndProvider"; @@ -10,19 +10,20 @@ import ProjectItem, { ProjectItemT } from "./ProjectItem"; import { isFormConfig } from "./helpers"; import { PreviousQueryT } from "./reducer"; -interface PropsT { - item: ProjectItemT; - onIndicateShare: () => void; - onIndicateEditFolders: () => void; -} - const getDragType = (item: PreviousQueryT) => { return item.queryType === "CONCEPT_QUERY" ? DNDType.PREVIOUS_QUERY : DNDType.PREVIOUS_SECONDARY_ID_QUERY; }; -const ProjectItemDragContainer: FC = ({ item, ...props }) => { +const ProjectItemDragContainer = ({ + item, + ...props +}: { + item: ProjectItemT; + onIndicateShare: () => void; + onIndicateEditFolders: () => void; +}) => { const ref = useRef(null); const dragItemBase = { diff --git a/frontend/src/js/previous-queries/list/ProjectItems.tsx b/frontend/src/js/previous-queries/list/ProjectItems.tsx index 3e3652d0b7..64db88bce8 100644 --- a/frontend/src/js/previous-queries/list/ProjectItems.tsx +++ b/frontend/src/js/previous-queries/list/ProjectItems.tsx @@ -4,7 +4,6 @@ import { useState, useCallback, useLayoutEffect, - FC, useEffect, } from "react"; import { FixedSizeList } from "react-window"; @@ -17,11 +16,6 @@ import type { ProjectItemT } from "./ProjectItem"; import ProjectItemDragContainer from "./ProjectItemDragContainer"; import ShareProjectItemModal from "./ShareProjectItemModal"; -interface PropsT { - datasetId: DatasetT["id"] | null; - items: ProjectItemT[]; -} - const ROW_SIZE = 62; const ROOT_PADDING_Y = 4; @@ -31,7 +25,13 @@ const Root = styled("div")` padding: ${ROOT_PADDING_Y}px 0; `; -const ProjectItems: FC = ({ datasetId, items }) => { +export const ProjectItems = ({ + datasetId, + items, +}: { + items: ProjectItemT[]; + datasetId: DatasetT["id"] | null; +}) => { const [itemToShare, setItemToShare] = useState(null); const [itemToEditFolders, setItemToEditFolders] = useState(null); @@ -124,5 +124,3 @@ const ProjectItems: FC = ({ datasetId, items }) => { ); }; - -export default ProjectItems; diff --git a/frontend/src/js/previous-queries/list/ProjectItemsTab.tsx b/frontend/src/js/previous-queries/list/ProjectItemsTab.tsx index d7d5ffa657..737a530f42 100644 --- a/frontend/src/js/previous-queries/list/ProjectItemsTab.tsx +++ b/frontend/src/js/previous-queries/list/ProjectItemsTab.tsx @@ -22,7 +22,7 @@ import UploadQueryResults from "../upload/UploadQueryResults"; import Folders from "./Folders"; import FoldersToggleButton from "./FoldersToggleButton"; import { ProjectItemT } from "./ProjectItem"; -import PreviousQueries from "./ProjectItems"; +import { ProjectItems } from "./ProjectItems"; import { useLoadFormConfigs, useLoadQueries } from "./actions"; import type { FormConfigT, PreviousQueryT } from "./reducer"; import { selectPreviousQueries } from "./selector"; @@ -148,7 +148,7 @@ const ProjectItemsTab = ({ datasetId }: PropsT) => { )} - + diff --git a/frontend/src/js/previous-queries/list/reducer.ts b/frontend/src/js/previous-queries/list/reducer.ts index b7799584c4..a934f1e9ea 100644 --- a/frontend/src/js/previous-queries/list/reducer.ts +++ b/frontend/src/js/previous-queries/list/reducer.ts @@ -53,6 +53,7 @@ export interface PreviousQueryT { groups?: UserGroupIdT[]; queryType: "CONCEPT_QUERY" | "SECONDARY_ID_QUERY"; secondaryId?: string | null; + containsDates: boolean; } export interface PreviousQueriesStateT { diff --git a/frontend/src/localization/de.json b/frontend/src/localization/de.json index e5bbafb6ba..edb69201e0 100644 --- a/frontend/src/localization/de.json +++ b/frontend/src/localization/de.json @@ -231,7 +231,8 @@ "shareError": "Konnte Anfrage nicht freigeben", "deleteError": "Konnte Anfrage nicht löschen", "editFolders": "Ordner bearbeiten", - "deleteNow": "Anfrage jetzt löschen" + "deleteNow": "Anfrage jetzt löschen", + "hasNoDates": "Keine Datumswerte in den Ergebnissen vorhanden." }, "projectItemsFilter": { "all": "Alle", diff --git a/frontend/src/localization/en.json b/frontend/src/localization/en.json index be8312517c..42c2fea340 100644 --- a/frontend/src/localization/en.json +++ b/frontend/src/localization/en.json @@ -231,7 +231,8 @@ "shareError": "Could not share query", "deleteError": "Could not delete query", "editFolders": "Edit folders", - "deleteNow": "Delete query now" + "deleteNow": "Delete query now", + "hasNoDates": "No date values in the results." }, "projectItemsFilter": { "all": "All", From 4bc292a063847001f0f163430a60d850e39a3b0f Mon Sep 17 00:00:00 2001 From: Kai Rollmann Date: Mon, 11 Sep 2023 16:22:53 +0200 Subject: [PATCH 547/679] Stop showing whether a field is optional in all form fields --- .../form-components/DropzoneList.tsx | 10 +-- .../form-concept-group/FormConceptGroup.tsx | 2 - .../form-query-dropzone/FormQueryDropzone.tsx | 4 -- frontend/src/js/external-forms/form/Field.tsx | 68 +++++-------------- frontend/src/js/external-forms/form/Form.tsx | 4 +- frontend/src/js/external-forms/helper.ts | 9 --- .../src/js/ui-components/InputDateRange.tsx | 6 +- .../ui-components/InputPlain/InputPlain.tsx | 3 - .../ui-components/InputSelect/InputSelect.tsx | 3 - .../InputTextarea/InputTextarea.tsx | 13 +--- frontend/src/js/ui-components/Labeled.tsx | 4 -- frontend/src/js/ui-components/Optional.tsx | 18 ----- frontend/src/localization/de.json | 1 - frontend/src/localization/en.json | 1 - 14 files changed, 20 insertions(+), 126 deletions(-) delete mode 100644 frontend/src/js/ui-components/Optional.tsx diff --git a/frontend/src/js/external-forms/form-components/DropzoneList.tsx b/frontend/src/js/external-forms/form-components/DropzoneList.tsx index 953d16e1e2..304bfd07e3 100644 --- a/frontend/src/js/external-forms/form-components/DropzoneList.tsx +++ b/frontend/src/js/external-forms/form-components/DropzoneList.tsx @@ -13,7 +13,6 @@ import DropzoneWithFileInput, { DragItemFile, } from "../../ui-components/DropzoneWithFileInput"; import Label from "../../ui-components/Label"; -import Optional from "../../ui-components/Optional"; import DropzoneBetweenElements from "./DropzoneBetweenElements"; @@ -56,7 +55,6 @@ interface PropsT { className?: string; label?: ReactNode; tooltip?: string; - optional?: boolean; dropzoneChildren: (args: ChildArgs) => ReactNode; items: ReactNode[]; acceptedDropTypes: string[]; @@ -78,7 +76,6 @@ const DropzoneList = ( className, label, tooltip, - optional, dropzoneChildren, items, acceptedDropTypes, @@ -97,12 +94,7 @@ const DropzoneList = ( return (
- {label && ( - - )} + {label && } {tooltip && } {items && items.length > 0 && ( diff --git a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx index 8821c30203..5b27a4c58b 100644 --- a/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx +++ b/frontend/src/js/external-forms/form-concept-group/FormConceptGroup.tsx @@ -70,7 +70,6 @@ interface Props { tooltip?: string; newValue: FormConceptGroupT; isSingle?: boolean; - optional?: boolean; disallowMultipleColumns?: boolean; blocklistedTables?: string[]; allowlistedTables?: string[]; @@ -189,7 +188,6 @@ const FormConceptGroup = (props: Props) => { */ ref={dropzoneRef} tooltip={props.tooltip} - optional={props.optional} label={ <> {props.label} diff --git a/frontend/src/js/external-forms/form-query-dropzone/FormQueryDropzone.tsx b/frontend/src/js/external-forms/form-query-dropzone/FormQueryDropzone.tsx index 831017a533..67a3dfbbe5 100644 --- a/frontend/src/js/external-forms/form-query-dropzone/FormQueryDropzone.tsx +++ b/frontend/src/js/external-forms/form-query-dropzone/FormQueryDropzone.tsx @@ -7,7 +7,6 @@ import type { DragItemQuery } from "../../standard-query-editor/types"; import InfoTooltip from "../../tooltip/InfoTooltip"; import Dropzone from "../../ui-components/Dropzone"; import Label from "../../ui-components/Label"; -import Optional from "../../ui-components/Optional"; import ValidatedFormQueryResult from "./ValidatedFormQueryResult"; @@ -23,7 +22,6 @@ const DROP_TYPES = [ interface PropsT { label: string; tooltip?: string; - optional?: boolean; dropzoneText: string; className?: string; value: DragItemQuery | null; @@ -33,7 +31,6 @@ interface PropsT { const FormQueryDropzone: FC = ({ label, tooltip, - optional, dropzoneText, className, value, @@ -60,7 +57,6 @@ const FormQueryDropzone: FC = ({ return (
diff --git a/frontend/src/js/external-forms/form/Field.tsx b/frontend/src/js/external-forms/form/Field.tsx index 7741bd2f38..81663890a5 100644 --- a/frontend/src/js/external-forms/form/Field.tsx +++ b/frontend/src/js/external-forms/form/Field.tsx @@ -33,12 +33,7 @@ import FormConceptGroup from "../form-concept-group/FormConceptGroup"; import type { FormConceptGroupT } from "../form-concept-group/formConceptGroupState"; import FormQueryDropzone from "../form-query-dropzone/FormQueryDropzone"; import FormTabNavigation from "../form-tab-navigation/FormTabNavigation"; -import { - getFieldKey, - getInitialValue, - isFormField, - isOptionalField, -} from "../helper"; +import { getFieldKey, getInitialValue, isFormField } from "../helper"; import { getErrorForField } from "../validators"; import type { DynamicFormValues } from "./Form"; @@ -165,35 +160,28 @@ const NestedFields = styled("div")` border-radius: ${({ theme }) => theme.borderRadius}; `; -interface PropsT { +const setValueConfig = { + shouldValidate: true, + shouldDirty: true, + shouldTouch: true, +}; + +const Field = ({ + field, + ...commonProps +}: { formType: string; h1Index?: number; field: GeneralField; locale: Language; availableDatasets: SelectOptionT[]; - optional?: boolean; register: UseFormRegister; setValue: UseFormSetValue; control: Control; -} - -const setValueConfig = { - shouldValidate: true, - shouldDirty: true, - shouldTouch: true, -}; - -const Field = ({ field, ...commonProps }: PropsT) => { +}) => { const datasetId = useDatasetId(); - const { - formType, - h1Index, - optional, - locale, - availableDatasets, - setValue, - control, - } = commonProps; + const { formType, h1Index, locale, availableDatasets, setValue, control } = + commonProps; const { t } = useTranslation(); const defaultValue = @@ -238,7 +226,6 @@ const Field = ({ field, ...commonProps }: PropsT) => { value={fieldProps.value as string} onChange={(value) => setValue(field.name, value, setValueConfig)} tooltip={field.tooltip ? field.tooltip[locale] : undefined} - optional={optional} /> )} @@ -263,7 +250,6 @@ const Field = ({ field, ...commonProps }: PropsT) => { setValue(field.name, value, setValueConfig); }} tooltip={field.tooltip ? field.tooltip[locale] : undefined} - optional={optional} /> )} @@ -292,7 +278,6 @@ const Field = ({ field, ...commonProps }: PropsT) => { max: field.max, }} tooltip={field.tooltip ? field.tooltip[locale] : undefined} - optional={optional} /> )} @@ -310,7 +295,6 @@ const Field = ({ field, ...commonProps }: PropsT) => { inline={true} label={field.label[locale]} tooltip={field.tooltip ? field.tooltip[locale] : undefined} - optional={optional} value={fieldProps.value as DateStringMinMax} onChange={(value) => setValue(field.name, value, setValueConfig) @@ -332,7 +316,6 @@ const Field = ({ field, ...commonProps }: PropsT) => { label={field.label[locale] || ""} dropzoneText={field.dropzoneLabel[locale] || ""} tooltip={field.tooltip ? field.tooltip[locale] : undefined} - optional={optional} value={fieldProps.value as DragItemQuery} onChange={(value) => setValue(field.name, value, setValueConfig)} /> @@ -374,7 +357,6 @@ const Field = ({ field, ...commonProps }: PropsT) => { label={field.label[locale]} options={options} tooltip={field.tooltip ? field.tooltip[locale] : undefined} - optional={optional} value={fieldProps.value as SelectOptionT | null} onChange={(value) => setValue(field.name, value, setValueConfig)} /> @@ -400,7 +382,6 @@ const Field = ({ field, ...commonProps }: PropsT) => { label={field.label[locale]} options={availableDatasets} tooltip={field.tooltip ? field.tooltip[locale] : undefined} - optional={optional} value={fieldProps.value as SelectOptionT | null} onChange={(value) => setValue(field.name, value, setValueConfig) @@ -428,16 +409,8 @@ const Field = ({ field, ...commonProps }: PropsT) => { > {field.fields.map((f, i) => { const key = getFieldKey(formType, f, i); - const nestedFieldOptional = isOptionalField(f); - return ( - - ); + return ; })} @@ -472,16 +445,8 @@ const Field = ({ field, ...commonProps }: PropsT) => { {tabToShow.fields.map((f, i) => { const key = getFieldKey(formType, f, i); - const nestedFieldOptional = isOptionalField(f); - return ( - - ); + return ; })} ) : ( @@ -524,7 +489,6 @@ const Field = ({ field, ...commonProps }: PropsT) => { blocklistedSelects={field.blocklistedSelects} allowlistedSelects={field.allowlistedSelects} defaults={field.defaults} - optional={optional} isValidConcept={(item) => !nodeIsInvalid( item, diff --git a/frontend/src/js/external-forms/form/Form.tsx b/frontend/src/js/external-forms/form/Form.tsx index 28be70f607..05d8a16c30 100644 --- a/frontend/src/js/external-forms/form/Form.tsx +++ b/frontend/src/js/external-forms/form/Form.tsx @@ -6,7 +6,7 @@ import type { SelectOptionT } from "../../api/types"; import { useActiveLang } from "../../localization/useActiveLang"; import FormHeader from "../FormHeader"; import type { Form as FormType } from "../config-types"; -import { getFieldKey, getH1Index, isOptionalField } from "../helper"; +import { getFieldKey, getH1Index } from "../helper"; import Field from "./Field"; @@ -44,7 +44,6 @@ const Form = memo(({ config, datasetOptions, methods }: Props) => { )} {config.fields.map((field, i) => { const key = getFieldKey(config.type, field, i); - const optional = isOptionalField(field); const h1Index = getH1Index(config.fields, field); return ( @@ -58,7 +57,6 @@ const Form = memo(({ config, datasetOptions, methods }: Props) => { setValue={methods.setValue} availableDatasets={datasetOptions} locale={activeLang} - optional={optional} /> ); })} diff --git a/frontend/src/js/external-forms/helper.ts b/frontend/src/js/external-forms/helper.ts index 0efdfc901a..c1ff843e84 100644 --- a/frontend/src/js/external-forms/helper.ts +++ b/frontend/src/js/external-forms/helper.ts @@ -42,15 +42,6 @@ export const getH1Index = (fields: GeneralField[], field: GeneralField) => { return h1Fields.indexOf(field); }; -export const isOptionalField = (field: GeneralField) => { - return ( - isFormField(field) && - (!("validations" in field) || - ("validations" in field && - (!field.validations || !field.validations.includes("NOT_EMPTY")))) - ); -}; - export const isFormField = (field: GeneralField): field is FormField => { return !nonFormFieldTypes.has(field.type); }; diff --git a/frontend/src/js/ui-components/InputDateRange.tsx b/frontend/src/js/ui-components/InputDateRange.tsx index 33d2bdb747..9327482ad1 100644 --- a/frontend/src/js/ui-components/InputDateRange.tsx +++ b/frontend/src/js/ui-components/InputDateRange.tsx @@ -20,7 +20,6 @@ import InfoTooltip from "../tooltip/InfoTooltip"; import InputDate from "./InputDate/InputDate"; import Label from "./Label"; import Labeled from "./Labeled"; -import Optional from "./Optional"; const Root = styled("div")<{ center?: boolean }>` text-align: ${({ center }) => (center ? "center" : "left")}; @@ -88,7 +87,6 @@ interface PropsT { center?: boolean; autoFocus?: boolean; tooltip?: string; - optional?: boolean; value: DateStringMinMax; onChange: (value: DateStringMinMax) => void; } @@ -115,7 +113,6 @@ const InputDateRange: FC = ({ labelSuffix, value, onChange, - optional, tooltip, }) => { const { t } = useTranslation(); @@ -179,7 +176,6 @@ const InputDateRange: FC = ({ {exists(indexPrefix) && # {indexPrefix}} - {optional && } {label} = ({ {labelSuffix && labelSuffix} ); - }, [t, label, labelSuffix, large, optional, tooltip, indexPrefix]); + }, [t, label, labelSuffix, large, tooltip, indexPrefix]); return ( diff --git a/frontend/src/js/ui-components/InputPlain/InputPlain.tsx b/frontend/src/js/ui-components/InputPlain/InputPlain.tsx index 2fb4df3375..7bedb969b8 100644 --- a/frontend/src/js/ui-components/InputPlain/InputPlain.tsx +++ b/frontend/src/js/ui-components/InputPlain/InputPlain.tsx @@ -17,7 +17,6 @@ const SxBaseInput = styled(BaseInput)<{ fullWidth?: boolean }>` interface Props { label: string; indexPrefix?: number; - optional?: boolean; inputType?: string; money?: boolean; className?: string; @@ -44,7 +43,6 @@ const InputPlain = forwardRef( large, indexPrefix, tooltip, - optional, inputType = "text", money, placeholder, @@ -65,7 +63,6 @@ const InputPlain = forwardRef( largeLabel={large} indexPrefix={indexPrefix} tooltip={tooltip} - optional={optional} > void; sortOptions?: (a: SelectOptionT, b: SelectOptionT, query: string) => number; }) => { @@ -347,7 +345,6 @@ const InputSelect = ({ } indexPrefix={indexPrefix} className={className} - optional={optional} > {Select} diff --git a/frontend/src/js/ui-components/InputTextarea/InputTextarea.tsx b/frontend/src/js/ui-components/InputTextarea/InputTextarea.tsx index 80980be2c6..ba97a43f0c 100644 --- a/frontend/src/js/ui-components/InputTextarea/InputTextarea.tsx +++ b/frontend/src/js/ui-components/InputTextarea/InputTextarea.tsx @@ -35,7 +35,6 @@ interface OtherProps { fullWidth?: boolean; indexPrefix?: number; tooltip?: string; - optional?: boolean; onChange: (value: string | null) => void; } @@ -49,16 +48,7 @@ export const InputTextarea = forwardRef< InputTextareaProps & OtherProps >( ( - { - label, - className, - fullWidth, - indexPrefix, - tooltip, - optional, - onChange, - ...props - }, + { label, className, fullWidth, indexPrefix, tooltip, onChange, ...props }, ref, ) => { const { t } = useTranslation(); @@ -70,7 +60,6 @@ export const InputTextarea = forwardRef< className={className} fullWidth tooltip={tooltip} - optional={optional} >