From 8d8ca047f5f98ba1cc5fbf1f8eaec9a02f29be65 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:29:19 +0100 Subject: [PATCH 1/2] Use dw DataSize for validation --- .../conquery/models/config/ClusterConfig.java | 13 ++-- .../util/validation/DataSizeMaxValidator.java | 24 ++++++ .../util/validation/DataSizeMinValidator.java | 24 ++++++ .../conquery/io/mina/MinaStackTest.java | 2 +- .../validation/DataSizeValidationTest.java | 77 +++++++++++++++++++ 5 files changed, 133 insertions(+), 7 deletions(-) create mode 100644 backend/src/main/java/com/bakdata/conquery/util/validation/DataSizeMaxValidator.java create mode 100644 backend/src/main/java/com/bakdata/conquery/util/validation/DataSizeMinValidator.java create mode 100644 backend/src/test/java/com/bakdata/conquery/util/validation/DataSizeValidationTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java index e94c1ee3ce..db30c61c59 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java @@ -16,6 +16,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.databind.ObjectMapper; import io.dropwizard.core.Configuration; +import io.dropwizard.util.DataSize; import io.dropwizard.util.Duration; import io.dropwizard.validation.PortRange; import lombok.Getter; @@ -51,7 +52,7 @@ public class ClusterConfig extends Configuration { */ @Max(Integer.MAX_VALUE - 4) @Min(64) // not practical - private int maxIoBufferSizeBytes = Integer.MAX_VALUE - 4; + private DataSize maxIoBufferSize = DataSize.bytes(Integer.MAX_VALUE - 4); /** * Defines the starting buffer allocation size. Larger can reduce reallocations, but can cause a greater memory demand. @@ -60,7 +61,7 @@ public class ClusterConfig extends Configuration { */ @Max(Integer.MAX_VALUE - 4) @Min(64) // Mina's default - private int initialIoBufferSizeBytes = 8192; // 8kb + private DataSize initialIoBufferSize = DataSize.bytes(8192); // 8kb /** * @see com.bakdata.conquery.models.messages.namespaces.specific.CollectColumnValuesJob @@ -90,8 +91,8 @@ public NioSocketConnector getClusterConnector(ObjectMapper om, IoHandler ioHandl final NioSocketConnector connector = new NioSocketConnector(); JacksonProtocolEncoder encoder = new JacksonProtocolEncoder(om.writerFor(NetworkMessage.class)); - encoder.setMaxObjectSize(maxIoBufferSizeBytes); - encoder.setInitialBufferCapacityBytes(initialIoBufferSizeBytes); + encoder.setMaxObjectSize(Math.toIntExact(maxIoBufferSize.toBytes())); + encoder.setInitialBufferCapacityBytes(Math.toIntExact(initialIoBufferSize.toBytes())); ProtocolCodecFilter codecFilter = new ProtocolCodecFilter( encoder, @@ -115,8 +116,8 @@ public NioSocketAcceptor getClusterAcceptor(ObjectMapper om, IoHandler ioHandler JacksonProtocolEncoder encoder = new JacksonProtocolEncoder(om.writerFor(NetworkMessage.class)); - encoder.setMaxObjectSize(maxIoBufferSizeBytes); - encoder.setInitialBufferCapacityBytes(initialIoBufferSizeBytes); + encoder.setMaxObjectSize(Math.toIntExact(maxIoBufferSize.toBytes())); + encoder.setInitialBufferCapacityBytes(Math.toIntExact(initialIoBufferSize.toBytes())); ProtocolCodecFilter codecFilter = new ProtocolCodecFilter( encoder, diff --git a/backend/src/main/java/com/bakdata/conquery/util/validation/DataSizeMaxValidator.java b/backend/src/main/java/com/bakdata/conquery/util/validation/DataSizeMaxValidator.java new file mode 100644 index 0000000000..48ad48af15 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/util/validation/DataSizeMaxValidator.java @@ -0,0 +1,24 @@ +package com.bakdata.conquery.util.validation; + +import jakarta.validation.ConstraintValidator; +import jakarta.validation.ConstraintValidatorContext; +import jakarta.validation.constraints.Max; + +import com.google.auto.service.AutoService; +import io.dropwizard.util.DataSize; + +@AutoService(ConstraintValidator.class) +public class DataSizeMaxValidator implements ConstraintValidator { + + private long maxBytes; + + @Override + public void initialize(Max constraintAnnotation) { + maxBytes = constraintAnnotation.value(); + } + + @Override + public boolean isValid(DataSize value, ConstraintValidatorContext context) { + return value.toBytes() <= maxBytes; + } +} diff --git a/backend/src/main/java/com/bakdata/conquery/util/validation/DataSizeMinValidator.java b/backend/src/main/java/com/bakdata/conquery/util/validation/DataSizeMinValidator.java new file mode 100644 index 0000000000..d634cb2a19 --- /dev/null +++ b/backend/src/main/java/com/bakdata/conquery/util/validation/DataSizeMinValidator.java @@ -0,0 +1,24 @@ +package com.bakdata.conquery.util.validation; + +import jakarta.validation.ConstraintValidator; +import jakarta.validation.ConstraintValidatorContext; +import jakarta.validation.constraints.Min; + +import com.google.auto.service.AutoService; +import io.dropwizard.util.DataSize; + +@AutoService(ConstraintValidator.class) +public class DataSizeMinValidator implements ConstraintValidator { + + private long minBytes; + + @Override + public void initialize(Min constraintAnnotation) { + minBytes = constraintAnnotation.value(); + } + + @Override + public boolean isValid(DataSize value, ConstraintValidatorContext context) { + return value.toBytes() >= minBytes; + } +} diff --git a/backend/src/test/java/com/bakdata/conquery/io/mina/MinaStackTest.java b/backend/src/test/java/com/bakdata/conquery/io/mina/MinaStackTest.java index 56b44fad71..e7a79d1cf2 100644 --- a/backend/src/test/java/com/bakdata/conquery/io/mina/MinaStackTest.java +++ b/backend/src/test/java/com/bakdata/conquery/io/mina/MinaStackTest.java @@ -56,7 +56,7 @@ public class MinaStackTest { public static void beforeAll() throws IOException { CLUSTER_CONFIG.setPort(0); - CLUSTER_CONFIG.setMaxIoBufferSizeBytes(toIntExact(DataSize.mebibytes(10).toBytes())); + CLUSTER_CONFIG.setMaxIoBufferSize(DataSize.mebibytes(10)); // This enables the Chunking filter, which triggers for messages > 1 MebiByte CLUSTER_CONFIG.getMina().setSendBufferSize(toIntExact(DataSize.mebibytes(1).toBytes())); diff --git a/backend/src/test/java/com/bakdata/conquery/util/validation/DataSizeValidationTest.java b/backend/src/test/java/com/bakdata/conquery/util/validation/DataSizeValidationTest.java new file mode 100644 index 0000000000..ce4353ab0c --- /dev/null +++ b/backend/src/test/java/com/bakdata/conquery/util/validation/DataSizeValidationTest.java @@ -0,0 +1,77 @@ +package com.bakdata.conquery.util.validation; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Set; +import jakarta.validation.ConstraintViolation; +import jakarta.validation.Validator; +import jakarta.validation.constraints.Max; +import jakarta.validation.constraints.Min; + +import io.dropwizard.jersey.validation.Validators; +import io.dropwizard.util.DataSize; +import lombok.Data; +import org.junit.jupiter.api.Test; + +public class DataSizeValidationTest { + + Validator VALIDATOR = Validators.newValidator(); + + + @Test + void inBounds() { + Container container = new Container(DataSize.bytes(5)); + + Set> validate = VALIDATOR.validate(container); + + assertThat(validate).isEmpty(); + + } + + @Test + void onMaxBound() { + Container container = new Container(DataSize.bytes(6)); + + Set> validate = VALIDATOR.validate(container); + + assertThat(validate).isEmpty(); + + } + + @Test + void onMinBound() { + Container container = new Container(DataSize.bytes(3)); + + Set> validate = VALIDATOR.validate(container); + + assertThat(validate).isEmpty(); + + } + + @Test + void maxedOut() { + Container container = new Container(DataSize.bytes(7)); + + Set> validate = VALIDATOR.validate(container); + + assertThat(validate).hasSize(1); + + } + + @Test + void minedOut() { + Container container = new Container(DataSize.bytes(2)); + + Set> validate = VALIDATOR.validate(container); + + assertThat(validate).hasSize(1); + + } + + @Data + private static class Container { + @Min(3) + @Max(6) + private final DataSize size; + } +} From 86f90f1d95820b04dbfdaaa2812fcac07c357e92 Mon Sep 17 00:00:00 2001 From: Max Thonagel <12283268+thoniTUB@users.noreply.github.com> Date: Tue, 17 Dec 2024 12:38:32 +0100 Subject: [PATCH 2/2] use dw provided validators --- .../conquery/models/config/ClusterConfig.java | 12 +-- .../util/validation/DataSizeMaxValidator.java | 24 ------ .../util/validation/DataSizeMinValidator.java | 24 ------ .../validation/DataSizeValidationTest.java | 77 ------------------- 4 files changed, 7 insertions(+), 130 deletions(-) delete mode 100644 backend/src/main/java/com/bakdata/conquery/util/validation/DataSizeMaxValidator.java delete mode 100644 backend/src/main/java/com/bakdata/conquery/util/validation/DataSizeMinValidator.java delete mode 100644 backend/src/test/java/com/bakdata/conquery/util/validation/DataSizeValidationTest.java diff --git a/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java b/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java index db30c61c59..6f963faaeb 100644 --- a/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java +++ b/backend/src/main/java/com/bakdata/conquery/models/config/ClusterConfig.java @@ -4,7 +4,6 @@ import java.net.InetAddress; import java.net.InetSocketAddress; import jakarta.validation.Valid; -import jakarta.validation.constraints.Max; import jakarta.validation.constraints.Min; import jakarta.validation.constraints.NotNull; @@ -17,7 +16,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import io.dropwizard.core.Configuration; import io.dropwizard.util.DataSize; +import io.dropwizard.util.DataSizeUnit; import io.dropwizard.util.Duration; +import io.dropwizard.validation.MaxDataSize; +import io.dropwizard.validation.MinDataSize; import io.dropwizard.validation.PortRange; import lombok.Getter; import lombok.Setter; @@ -50,8 +52,8 @@ public class ClusterConfig extends Configuration { *

* May only touch this for testing purposes. */ - @Max(Integer.MAX_VALUE - 4) - @Min(64) // not practical + @MaxDataSize(value = Integer.MAX_VALUE - 4, unit = DataSizeUnit.BYTES) + @MinDataSize(value = 64, unit = DataSizeUnit.BYTES) private DataSize maxIoBufferSize = DataSize.bytes(Integer.MAX_VALUE - 4); /** @@ -59,8 +61,8 @@ public class ClusterConfig extends Configuration { *

* May only touch this for testing purposes. */ - @Max(Integer.MAX_VALUE - 4) - @Min(64) // Mina's default + @MaxDataSize(value = Integer.MAX_VALUE - 4, unit = DataSizeUnit.BYTES) + @MinDataSize(value = 64, unit = DataSizeUnit.BYTES) private DataSize initialIoBufferSize = DataSize.bytes(8192); // 8kb /** diff --git a/backend/src/main/java/com/bakdata/conquery/util/validation/DataSizeMaxValidator.java b/backend/src/main/java/com/bakdata/conquery/util/validation/DataSizeMaxValidator.java deleted file mode 100644 index 48ad48af15..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/util/validation/DataSizeMaxValidator.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.bakdata.conquery.util.validation; - -import jakarta.validation.ConstraintValidator; -import jakarta.validation.ConstraintValidatorContext; -import jakarta.validation.constraints.Max; - -import com.google.auto.service.AutoService; -import io.dropwizard.util.DataSize; - -@AutoService(ConstraintValidator.class) -public class DataSizeMaxValidator implements ConstraintValidator { - - private long maxBytes; - - @Override - public void initialize(Max constraintAnnotation) { - maxBytes = constraintAnnotation.value(); - } - - @Override - public boolean isValid(DataSize value, ConstraintValidatorContext context) { - return value.toBytes() <= maxBytes; - } -} diff --git a/backend/src/main/java/com/bakdata/conquery/util/validation/DataSizeMinValidator.java b/backend/src/main/java/com/bakdata/conquery/util/validation/DataSizeMinValidator.java deleted file mode 100644 index d634cb2a19..0000000000 --- a/backend/src/main/java/com/bakdata/conquery/util/validation/DataSizeMinValidator.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.bakdata.conquery.util.validation; - -import jakarta.validation.ConstraintValidator; -import jakarta.validation.ConstraintValidatorContext; -import jakarta.validation.constraints.Min; - -import com.google.auto.service.AutoService; -import io.dropwizard.util.DataSize; - -@AutoService(ConstraintValidator.class) -public class DataSizeMinValidator implements ConstraintValidator { - - private long minBytes; - - @Override - public void initialize(Min constraintAnnotation) { - minBytes = constraintAnnotation.value(); - } - - @Override - public boolean isValid(DataSize value, ConstraintValidatorContext context) { - return value.toBytes() >= minBytes; - } -} diff --git a/backend/src/test/java/com/bakdata/conquery/util/validation/DataSizeValidationTest.java b/backend/src/test/java/com/bakdata/conquery/util/validation/DataSizeValidationTest.java deleted file mode 100644 index ce4353ab0c..0000000000 --- a/backend/src/test/java/com/bakdata/conquery/util/validation/DataSizeValidationTest.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.bakdata.conquery.util.validation; - -import static org.assertj.core.api.Assertions.assertThat; - -import java.util.Set; -import jakarta.validation.ConstraintViolation; -import jakarta.validation.Validator; -import jakarta.validation.constraints.Max; -import jakarta.validation.constraints.Min; - -import io.dropwizard.jersey.validation.Validators; -import io.dropwizard.util.DataSize; -import lombok.Data; -import org.junit.jupiter.api.Test; - -public class DataSizeValidationTest { - - Validator VALIDATOR = Validators.newValidator(); - - - @Test - void inBounds() { - Container container = new Container(DataSize.bytes(5)); - - Set> validate = VALIDATOR.validate(container); - - assertThat(validate).isEmpty(); - - } - - @Test - void onMaxBound() { - Container container = new Container(DataSize.bytes(6)); - - Set> validate = VALIDATOR.validate(container); - - assertThat(validate).isEmpty(); - - } - - @Test - void onMinBound() { - Container container = new Container(DataSize.bytes(3)); - - Set> validate = VALIDATOR.validate(container); - - assertThat(validate).isEmpty(); - - } - - @Test - void maxedOut() { - Container container = new Container(DataSize.bytes(7)); - - Set> validate = VALIDATOR.validate(container); - - assertThat(validate).hasSize(1); - - } - - @Test - void minedOut() { - Container container = new Container(DataSize.bytes(2)); - - Set> validate = VALIDATOR.validate(container); - - assertThat(validate).hasSize(1); - - } - - @Data - private static class Container { - @Min(3) - @Max(6) - private final DataSize size; - } -}