test(#300): @NathanEckert is correct

The modern S3EC and SDK V2 treat reading 0 bytes
from a stream differently in at least one case:
If the Stream has no more content.

.github/workflows/ghi_300.yml

@@ -0,0 +1,49 @@
+name: ghi_300.yml
+on:
+  push:
+    branches:
+      - tony/refactor-tests
+      - '**/ghi-300'
+
+jobs:
+  Build:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: arn:aws:iam::${{ secrets.CI_AWS_ACCOUNT_ID }}:role/service-role/${{ vars.CI_AWS_ROLE }}
+          role-session-name: S3EC-Github-CI-Tests
+          aws-region: ${{ vars.CI_AWS_REGION }}
+
+      - name: Checkout Code
+        uses: actions/checkout@v3
+
+      # TODO: Add OpenJDK
+      # OpenJDK would require a different action than setup-java, so setup is more involved.
+
+      - name: Setup JDK
+        uses: actions/setup-java@v3
+        with:
+          distribution: corretto
+          java-version: 8
+          cache: 'maven'
+
+      - name: Compile
+        run: |
+          mvn --batch-mode -no-transfer-progress clean compile
+          mvn --batch-mode -no-transfer-progress test-compile
+        shell: bash
+
+      - name: Test
+        run: |
+          export AWS_S3EC_TEST_BUCKET=${{ vars.CI_S3_BUCKET }}
+          export AWS_S3EC_TEST_KMS_KEY_ID=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:key/${{ vars.CI_KMS_KEY_ID }}
+          export AWS_S3EC_TEST_KMS_KEY_ALIAS=arn:aws:kms:${{ vars.CI_AWS_REGION }}:${{ secrets.CI_AWS_ACCOUNT_ID }}:alias/${{ vars.CI_KMS_KEY_ALIAS }}
+          export AWS_REGION=${{ vars.CI_AWS_REGION }}
+          mvn -B -ntp -DskipCompile -Dtest=software.amazon.encryption.s3.examples.TestEndOfStreamBehavior test
+        shell: bash

pom.xml

@@ -41,6 +41,7 @@
     <maven.compiler.target>8</maven.compiler.target>
     <maven.compiler.release>8</maven.compiler.release>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+    <aws.java.sdk.version>2.26.7</aws.java.sdk.version>
   </properties>
 
   <dependencyManagement>
@@ -56,7 +57,7 @@
       <dependency>
         <groupId>software.amazon.awssdk</groupId>
         <artifactId>bom</artifactId>
-        <version>2.20.38</version>
+        <version>${aws.java.sdk.version}</version>
         <optional>true</optional>
         <type>pom</type>
         <scope>import</scope>
@@ -68,21 +69,20 @@
     <dependency>
       <groupId>software.amazon.awssdk</groupId>
       <artifactId>s3</artifactId>
-      <version>2.20.38</version>
+      <version>${aws.java.sdk.version}</version>
     </dependency>
 
     <dependency>
       <groupId>software.amazon.awssdk</groupId>
       <artifactId>kms</artifactId>
-      <version>2.20.38</version>
+      <version>${aws.java.sdk.version}</version>
       <optional>true</optional>
     </dependency>
 
-    <!-- Used when enableMultipartPutObject is configured -->
     <dependency>
-      <groupId>software.amazon.awssdk.crt</groupId>
-      <artifactId>aws-crt</artifactId>
-      <version>0.29.24</version>
+      <groupId>software.amazon.awssdk</groupId>
+      <artifactId>aws-crt-client</artifactId>
+      <version>${aws.java.sdk.version}</version>
       <optional>true</optional>
     </dependency>
 

src/test/java/software/amazon/encryption/s3/examples/TestEndOfStreamBehavior.java

@@ -0,0 +1,127 @@
+package software.amazon.encryption.s3.examples;
+
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.security.KeyPair;
+import java.util.stream.Stream;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.http.async.SdkAsyncHttpClient;
+import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3AsyncClient;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest;
+import software.amazon.awssdk.services.s3.model.GetObjectRequest;
+import software.amazon.awssdk.services.s3.model.PutObjectRequest;
+import software.amazon.encryption.s3.S3EncryptionClient;
+import software.amazon.encryption.s3.utils.S3EncryptionClientTestResources;
+
+import static software.amazon.encryption.s3.utils.S3EncryptionClientTestResources.*;
+
+public class TestEndOfStreamBehavior {
+    private static final Region DEFAULT_REGION = KMS_REGION;
+    private static final String KEY = "GHI-300.txt";
+    @SuppressWarnings("SpellCheckingInspection")
+    private static final byte[] CONTENT = new String(new char[4])
+        .replace("\0", "abcdefghijklmnopqrstuvwxyz0123456789")
+        .getBytes();
+    /** The encryption key to use in client-side encryption tests. */
+    protected static final KeyPair KEY_PAIR;
+
+    static {
+        try {
+            KEY_PAIR = S3EncryptionClientTestResources.getRSAKeyPair();
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+    }
+
+    static Stream<S3Client> clientProvider() {
+        return Stream.of(
+            getClient(DEFAULT_REGION),
+            getEncryptionClient(KEY_PAIR, DEFAULT_REGION));
+    }
+
+    @ParameterizedTest
+    @MethodSource("clientProvider")
+    void testEndOfStreamBehavior(final S3Client client) throws Exception {
+        // Delete the data if it exists
+        final DeleteObjectRequest deleteRequest = DeleteObjectRequest.builder()
+            .bucket(BUCKET)
+            .key(KEY)
+            .build();
+
+        client.deleteObject(deleteRequest);
+
+        // Upload the data
+        final PutObjectRequest uploadRequest =
+            PutObjectRequest.builder().bucket(BUCKET).key(KEY).build();
+        client.putObject(uploadRequest, RequestBody.fromBytes(CONTENT));
+        // wait 5 seconds for the data to be uploaded
+        Thread.sleep(5000);
+
+        // Actual test
+        final GetObjectRequest downloadRequest =
+            GetObjectRequest.builder()
+                .bucket(BUCKET)
+                .key(KEY)
+                .range("bytes=0-15")
+                .build();
+
+        final InputStream stream = client.getObject(downloadRequest);
+
+        // Buffer capacity matters !!!
+        // Behavior difference when the capacity is same as the content length (i.e. 16) of the ranged query
+        final ByteBuffer buffer = ByteBuffer.allocate(16);
+        final byte[] underlyingBuffer = buffer.array();
+        final int capacity = buffer.capacity();
+
+        final int END_OF_STREAM = -1;
+        int byteRead = 0;
+        int startPosition = 0;
+        while (byteRead != END_OF_STREAM) {
+            int lenToRead = capacity - startPosition;
+            System.out.println("Start position: " + startPosition + " Length to read: " + lenToRead);
+            byteRead = stream.read(underlyingBuffer, startPosition, lenToRead);
+            System.out.println("Read " + byteRead + " bytes");
+            startPosition += byteRead;
+            if (byteRead == 0) {
+                // Now we always get this error; we probably were always getting this error, but the log was not writing.
+                throw new AssertionError(
+                    String.format("Looping indefinitely with an encryption client, as startPosition is not increasing." +
+                            "\n lenToRead: %s \t byteRead: %s \t startPosition: %s",
+                        lenToRead, byteRead, startPosition));
+            }
+        }
+    }
+
+    public static S3Client getEncryptionClient(final KeyPair keyPair, final Region region) {
+        return S3EncryptionClient.builder()
+            .rsaKeyPair(keyPair)
+            .enableLegacyUnauthenticatedModes(true)
+            .wrappedClient(getClient(region))
+            .wrappedAsyncClient(getAsyncClient(region))
+            .build();
+    }
+
+    public static S3Client getClient(final Region region) {
+        return S3Client.builder()
+            .region(region)
+            .credentialsProvider(CREDENTIALS)
+            .httpClient(HTTP_CLIENT)
+            .build();
+    }
+
+    public static S3AsyncClient getAsyncClient(final Region region) {
+        final SdkAsyncHttpClient nettyHttpClient =
+            NettyNioAsyncHttpClient.builder().maxConcurrency(100).build();
+        return S3AsyncClient.builder()
+            .region(region)
+            .credentialsProvider(CREDENTIALS)
+            .httpClient(nettyHttpClient)
+            .build();
+    }
+}

src/test/java/software/amazon/encryption/s3/utils/S3EncryptionClientTestResources.java

@@ -2,23 +2,44 @@
 // SPDX-License-Identifier: Apache-2.0
 package software.amazon.encryption.s3.utils;
 
+import org.bouncycastle.util.io.pem.PemObject;
+import org.bouncycastle.util.io.pem.PemReader;
+import org.bouncycastle.util.io.pem.PemWriter;
 import org.joda.time.DateTime;
 import org.joda.time.format.DateTimeFormat;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
+import software.amazon.awssdk.http.SdkHttpClient;
+import software.amazon.awssdk.http.apache.ApacheHttpClient;
+import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.s3.S3AsyncClient;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.model.DeleteObjectResponse;
 
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.security.*;
+import java.security.interfaces.RSAPrivateKey;
+import java.security.interfaces.RSAPublicKey;
+import java.security.spec.InvalidKeySpecException;
+import java.security.spec.PKCS8EncodedKeySpec;
+import java.security.spec.X509EncodedKeySpec;
 import java.util.concurrent.CompletableFuture;
 
 /**
  * Determines which AWS resources to use while running tests.
  */
 public class S3EncryptionClientTestResources {
 
+    public static final AwsCredentialsProvider CREDENTIALS = DefaultCredentialsProvider.create();
+    public static final SdkHttpClient HTTP_CLIENT = ApacheHttpClient.create();
     public static final String BUCKET = System.getenv("AWS_S3EC_TEST_BUCKET");
     public static final String KMS_KEY_ID = System.getenv("AWS_S3EC_TEST_KMS_KEY_ID");
     // This alias must point to the same key as KMS_KEY_ID
     public static final String KMS_KEY_ALIAS = System.getenv("AWS_S3EC_TEST_KMS_KEY_ALIAS");
+    public static final Region KMS_REGION = Region.of(System.getenv("AWS_REGION"));
 
     /**
      * For a given string, append a suffix to distinguish it from
@@ -57,4 +78,63 @@ public static void deleteObject(final String bucket, final String objectKey, fin
         // Ensure completion before return
         response.join();
     }
+
+
+    /**
+     * @return If an RSA KeyPair already exists in Test Resources, load and return that.<p>
+     * Otherwise, generate a new key pair, persist that to Resources, and return it.<p>
+     * Assumes working directory is root of the git repo.
+     */
+    public static KeyPair getRSAKeyPair() throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
+        Path resourceDirectory = Paths.get("src","test","resources");
+        if (resourceDirectory.resolve("RSAPrivateKey.pem").toFile().exists() && resourceDirectory.resolve("RSAPublicKey.pem").toFile().exists()) {
+            return readKeyPairFromTestResourcesFile(resourceDirectory);
+        }
+        KeyPair keyPair = generateKeyPair(2048);
+        writeKeyPairToTestResourcesFile(keyPair, resourceDirectory);
+        return keyPair;
+    }
+
+    public static KeyPair generateKeyPair(final int keySize) {
+        if (!(keySize == 2048 || keySize == 4096)) throw new IllegalArgumentException("Only 2048 or 4096 are valid key sizes.");
+        KeyPairGenerator rsaGen;
+        try {
+            rsaGen = KeyPairGenerator.getInstance("RSA");
+        } catch (NoSuchAlgorithmException e) {
+            throw new RuntimeException("No such algorithm", e);
+        }
+        rsaGen.initialize(keySize, new SecureRandom());
+        return rsaGen.generateKeyPair();
+    }
+
+    private static void writePEMFile(Key key, String description, Path filePath) throws IOException {
+        final PemObject pemObject = new PemObject(description, key.getEncoded());
+        try (PemWriter pemWriter = new PemWriter(new OutputStreamWriter(Files.newOutputStream(filePath)))) {
+            pemWriter.writeObject(pemObject);
+        }
+    }
+
+    private static PemObject readPEMFile(Path filePath) throws IOException {
+        try (PemReader pemReader = new PemReader(new InputStreamReader(Files.newInputStream(filePath)))) {
+            return pemReader.readPemObject();
+        }
+    }
+
+    private static void writeKeyPairToTestResourcesFile(final KeyPair keyPair, Path resourceDirectory) throws IOException {
+        RSAPrivateKey privateKey = (RSAPrivateKey) keyPair.getPrivate();
+        writePEMFile(privateKey, "RSA PRIVATE KEY", resourceDirectory.resolve("RSAPrivateKey.pem"));
+        RSAPublicKey publicKey = (RSAPublicKey) keyPair.getPublic();
+        writePEMFile(publicKey, "RSA PUBLIC KEY", resourceDirectory.resolve("RSAPublicKey.pem"));
+    }
+
+    private static KeyPair readKeyPairFromTestResourcesFile(Path resourceDirectory) throws IOException, NoSuchAlgorithmException, InvalidKeySpecException {
+        final KeyFactory factory = KeyFactory.getInstance("RSA");
+        byte[] privateKeyContent = readPEMFile(resourceDirectory.resolve("RSAPrivateKey.pem")).getContent();
+        PKCS8EncodedKeySpec privateKeySpec = new PKCS8EncodedKeySpec(privateKeyContent);
+        final PrivateKey privateKey = factory.generatePrivate(privateKeySpec);
+        byte[] publicKeyContent = readPEMFile(resourceDirectory.resolve("RSAPublicKey.pem")).getContent();
+        X509EncodedKeySpec publicKeySpec = new X509EncodedKeySpec(publicKeyContent);
+        final PublicKey publicKey = factory.generatePublic(publicKeySpec);
+        return new KeyPair(publicKey, privateKey);
+    }
 }

src/test/resources/RSAPrivateKey.pem

@@ -0,0 +1,28 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDDhiKZterVY9+h
+igrZMn8gzfcNEXhqDoCynjWewiQriG3P4DuasePNVjQSIk7tp1/sn6zSeIm/UlAv
+8Mtx+dh2MG9l6jyFiAm8T7heU9CgdSc5Dp5ZJ2QrSwXoQPBnrHTSomxB3sg75Uhn
+hhv1MfVceFxKPUJTXdQ6Mb6hy14RGfx06E3Ugffvy81rNf4e9A7uUbktVocYUqvO
+aiv8JzGYnDXdlhtfZ/DYMnJMH73AFJJ+7+XAfUMU32nvhkcGhiKj6auJXB2makJ3
+FUtwrNXYMc976WrH9Kk2iu33rL7PV6n9+X6xI/WGdf9X0OCORzbk6ih1CEIU0J+M
+KIkRGAirAgMBAAECggEAB9u6REdFevIaqNltejFHXsAob8QF/O08SvGE4i6XWZCQ
+KUyv2JXRvAz85sWuOmsBtfbs8UCa+K+MPYEGDDyocIed0pDJgexnx8PEezYPKoPK
+4cYuoxKsOfk38Y+6mdAameShSTx0+8NJV6/SK9aoL+E+hFVV9xfMUdJyAPq1eyZo
+PTxnUvV4INehN/rQL3X+00XjSzYEUo5IjJldqyvEVOAmcgxUXekgHUIcOa13uTMi
+X9pAcvVH9LV81AFe/s8r7Ob15GWfe8Vyny3hDbj1EtG40vKK512JL2EKJy25DgsR
+sbCWDLLv/2LpOp8mt4X3bxkR9WuKWW4o30ef1OZdkQKBgQDfoy2t1ifBVsmPCEKQ
+3ChAW+hcdRxtD2+yP6L4DZlrIi12UdgojwggUKe81AV/C5NDTr4h2cqzA5EQNT+V
+CdpNND1zcRdbNqqZiOaV1FCaORijfmimmnfBUNKrpKP5E38lV6PfAlF1i45QFryn
+0kuAAA/FlCF+IBhy8C3ksncAZwKBgQDf0XjepXFu8B1zfZP4N4EjASO3+NQY0BGE
+19+rOs5br1bnw3Q82Y89vAC/mPuXRy32ENaSN2RVK8vFds54bsG6NzdRnwWY5deH
+0x1jCZn3/6DicviFA0O0TCMFErTG7DwRHRL58ftyV4lZk9kxq8h/x9deKm0a8cEW
+HZVC7wj7HQKBgF63sQgYVNwpEtMWj4LlC9M+WeqW20RBrnATTcW7lMfwQMsFHQUI
+l0uAfZqXPgCx+VwfhJ23rYcmMpFnzBcmhiP+xSwYsOi7/YNrnSXGN6EqH4pXZqFx
+eNkSjzeNUrmSjV5WgRxZ0gBz7AF1r89wXPPIkuV+uLS/iTtdCEL9ZzNvAoGBALY3
+6Fv7/fn/6zpXhtyS88P37YieQK9i1qB80FCrs83ZVruh2UShK4lrQoC6oDptbPHk
+i4zHJBxjZ6cALuDF61scESGWggwVNAAU1NwIuR27NNSoHcTM/5YOVoSO0jcRpWWZ
+chWj+L8CnYQcZruVy8qcfK7hg6poIHdM5nRz/6/RAoGARtGMAM3CoS8sHB3HYrzZ
+gfzKImHSCADCHz8eo+17SXLf4M4v769M8luicd1a0vaCFrFa5vySe3FiizXtqZa/
+cPpCUxxX4hOnQJ8Mki875JajBgamd60ZJE35ZvlyX8obq4YrSLm2WUQ9aqaHT3dh
+qL/371EPip3eVdvNAyqjwBc=
+-----END RSA PRIVATE KEY-----

src/test/resources/RSAPublicKey.pem

@@ -0,0 +1,9 @@
+-----BEGIN RSA PUBLIC KEY-----
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAw4YimbXq1WPfoYoK2TJ/
+IM33DRF4ag6Asp41nsIkK4htz+A7mrHjzVY0EiJO7adf7J+s0niJv1JQL/DLcfnY
+djBvZeo8hYgJvE+4XlPQoHUnOQ6eWSdkK0sF6EDwZ6x00qJsQd7IO+VIZ4Yb9TH1
+XHhcSj1CU13UOjG+octeERn8dOhN1IH378vNazX+HvQO7lG5LVaHGFKrzmor/Ccx
+mJw13ZYbX2fw2DJyTB+9wBSSfu/lwH1DFN9p74ZHBoYio+mriVwdpmpCdxVLcKzV
+2DHPe+lqx/SpNort96y+z1ep/fl+sSP1hnX/V9Dgjkc25OoodQhCFNCfjCiJERgI
+qwIDAQAB
+-----END RSA PUBLIC KEY-----