test(#300): determine streaming range issue b/w regular & encryption

pom.xml

@@ -56,7 +56,7 @@
       <dependency>
         <groupId>software.amazon.awssdk</groupId>
         <artifactId>bom</artifactId>
-        <version>2.20.38</version>
+        <version>2.26.7</version>
         <optional>true</optional>
         <type>pom</type>
         <scope>import</scope>
@@ -68,13 +68,13 @@
     <dependency>
       <groupId>software.amazon.awssdk</groupId>
       <artifactId>s3</artifactId>
-      <version>2.20.38</version>
+      <version>2.26.7</version>
     </dependency>
 
     <dependency>
       <groupId>software.amazon.awssdk</groupId>
       <artifactId>kms</artifactId>
-      <version>2.20.38</version>
+      <version>2.26.7</version>
       <optional>true</optional>
     </dependency>
 

src/test/java/software/amazon/encryption/s3/examples/TestEndOfStreamBehavior.java

@@ -0,0 +1,136 @@
+package software.amazon.encryption.s3.examples;
+
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.security.KeyPair;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.http.async.SdkAsyncHttpClient;
+import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.encryption.s3.S3EncryptionClient;
+import software.amazon.encryption.s3.utils.S3EncryptionClientTestResources;
+
+import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.*;
+
+public class TestEndOfStreamBehavior {
+    private static final Region DEFAULT_REGION = KMS_REGION;
+    private static final String KEY = "GHI-300.txt";
+    @SuppressWarnings("SpellCheckingInspection")
+    private static final byte[] CONTENT = new String(new char[4])
+        .replace("\0", "abcdefghijklmnopqrstuvwxyz0123456789")
+        .getBytes();
+    /** The encryption key to use in client-side encryption tests. */
+    protected static final KeyPair KEY_PAIR;
+
+    static {
+        try {
+            KEY_PAIR = S3EncryptionClientTestResources.getRSAKeyPair();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    static Stream<S3Client> clientProvider() {
+        return Stream.of(
+            getClient(DEFAULT_REGION),
+            getEncryptionClient(KEY_PAIR, DEFAULT_REGION));
+    }
+
+    @ParameterizedTest
+    @MethodSource("clientProvider")
+    void testEndOfStreamBehavior(final S3Client client) throws Exception {
+        // Delete the data if it exists
+        final DeleteObjectRequest deleteRequest = DeleteObjectRequest.builder()
+            .bucket(BUCKET)
+            .key(KEY)
+            .build();
+
+        client.deleteObject(deleteRequest);
+
+        // Upload the data
+        final PutObjectRequest uploadRequest =
+            PutObjectRequest.builder().bucket(BUCKET).key(KEY).build();
+        client.putObject(uploadRequest, RequestBody.fromBytes(CONTENT));
+        // wait 5 seconds for the data to be uploaded
+        Thread.sleep(5000);
+
+        // Actual test
+        final GetObjectRequest downloadRequest =
+            GetObjectRequest.builder()
+                .bucket(BUCKET)
+                .key(KEY)
+                .range("bytes=0-15")
+                .build();
+
+        final InputStream stream = client.getObject(downloadRequest);
+
+        // Buffer capacity matters !!!
+        // Behavior difference when the capacity is same as the content length (i.e. 16) of the ranged query
+        final ByteBuffer buffer = ByteBuffer.allocate(16);
+        final byte[] underlyingBuffer = buffer.array();
+        final int capacity = buffer.capacity();
+
+        final int END_OF_STREAM = -1;
+        int byteRead = 0;
+        int startPosition = 0;
+        while (byteRead != END_OF_STREAM) {
+            int lenToRead = capacity - startPosition;
+            System.out.println("Start position: " + startPosition + " Length to read: " + lenToRead);
+            // @NathanEckert , about https://github.com/aws/amazon-s3-encryption-client-java/issues/300
+            // Crypto Tools SOMETIMES got an Assertion Error from
+            // https://github.com/aws/aws-sdk-java-v2/blob/2.20.38/utils/src/main/java/software/amazon/awssdk/utils/async/InputStreamSubscriber.java#L110
+            // when using the Encryption Client.
+            // If we bump our Java SDK dependencies to the latest, which today is 2.26.7,
+            // than we never get the Assertion Error.
+            // Here is the PR that changes InputStreamSubscriber b/w 2.20.38 and 2.26.7:
+            // https://github.com/aws/aws-sdk-java-v2/pull/5201
+            // This makes us suspect that something else is going wrong.
+            // Otherwise, we cannot detect a difference in behavior between
+            // the S3EC V3 Client and the S3 V2 Client with respect to this code.
+            byteRead = stream.read(underlyingBuffer, startPosition, lenToRead);
+            System.out.println("Read " + byteRead + " bytes");
+            startPosition += byteRead;
+            if (byteRead == 0) {
+                // Crypto Tools cannot get this case to ever occur.
+                System.out.println("Looping indefinitely with an encryption client, as startPosition is not increasing");
+                break;
+            }
+        }
+    }
+
+    public static S3Client getEncryptionClient(final KeyPair keyPair, final Region region) {
+        return S3EncryptionClient.builder()
+            .rsaKeyPair(keyPair)
+            .enableLegacyUnauthenticatedModes(true)
+            .wrappedClient(getClient(region))
+            .wrappedAsyncClient(getAsyncClient(region))
+            .build();
+    }
+
+    public static S3Client getClient(final Region region) {
+        return S3Client.builder()
+            .region(region)
+            .credentialsProvider(CREDENTIALS)
+            .httpClient(HTTP_CLIENT)
+            .build();
+    }
+
+    public static S3AsyncClient getAsyncClient(final Region region) {
+        final SdkAsyncHttpClient nettyHttpClient =
+            NettyNioAsyncHttpClient.builder().maxConcurrency(100).build();
+        return S3AsyncClient.builder()
+            .region(region)
+            .credentialsProvider(CREDENTIALS)
+            .httpClient(nettyHttpClient)
+            .build();
+    }
+}