s3: Support S3 compatible services like Google Storage

Services like Google Storage expose s3 compatible APIs that can accessed with a non-aws URL. Signed-off-by: Andy Doan <andy@foundries.io>
advancedtelematic · Apr 15, 2019 · 4e9e393 · 4e9e393
1 parent 5b8b259
commit 4e9e393
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 6 deletions.
diff --git a/reposerver/src/main/resources/application.conf b/reposerver/src/main/resources/application.conf
@@ -17,6 +17,7 @@ storage {
     bucketId = ${?TUF_REPOSERVER_AWS_BUCKET_ID}
     region = "eu-central-1"
     region = ${?TUF_REPOSERVER_AWS_REGION}
+    endpointUrl = ${?TUF_REPOSERVER_S3_URL}
   }
 }
 

diff --git a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/Boot.scala b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/Boot.scala
@@ -39,7 +39,8 @@ trait Settings {
     val secretKey = _config.getString("storage.s3.secretKey")
     val bucketId = _config.getString("storage.s3.bucketId")
     val region = Regions.fromName(_config.getString("storage.s3.region"))
-    new S3Credentials(accessKey, secretKey, bucketId, region)
+    val endpointUrl = _config.getString("storage.s3.endpointUrl")
+    new S3Credentials(accessKey, secretKey, bucketId, region, endpointUrl)
   }
 
   lazy val useS3 = _config.getString("storage.type").equals("s3")

diff --git a/...rc/main/scala/com/advancedtelematic/tuf/reposerver/target_store/S3TargetStoreEngine.scala b/...rc/main/scala/com/advancedtelematic/tuf/reposerver/target_store/S3TargetStoreEngine.scala
@@ -12,6 +12,7 @@ import akka.util.ByteString
 import com.advancedtelematic.libtuf.data.TufDataType.{RepoId, TargetFilename}
 import com.advancedtelematic.tuf.reposerver.target_store.TargetStoreEngine.{TargetRedirect, TargetRetrieveResult, TargetStoreResult}
 import com.amazonaws.auth.{AWSCredentials, AWSCredentialsProvider}
+import com.amazonaws.client.builder.AwsClientBuilder
 import com.amazonaws.regions.Regions
 import com.amazonaws.services.s3.AmazonS3ClientBuilder
 import com.amazonaws.services.s3.model.{CannedAccessControlList, PutObjectRequest}
@@ -29,10 +30,20 @@ class S3TargetStoreEngine(credentials: S3Credentials)(implicit val system: Actor
 
   private val log = LoggerFactory.getLogger(this.getClass)
 
-  private lazy val s3client = AmazonS3ClientBuilder.standard()
-    .withCredentials(credentials)
-    .withRegion(credentials.region)
-    .build()
+  protected lazy val s3client = {
+    if(credentials.endpointUrl.length() > 0) {
+      log.info(s"Using custom S3 url: ${credentials.endpointUrl}")
+      AmazonS3ClientBuilder.standard()
+        .withCredentials(credentials)
+        .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(credentials.endpointUrl, credentials.region.getName()))
+        .build()
+    } else {
+      AmazonS3ClientBuilder.standard()
+        .withCredentials(credentials)
+        .withRegion(credentials.region)
+        .build()
+    }
+  }
 
   override def store(repoId: RepoId, filename: TargetFilename, fileData: Source[ByteString, Any]): Future[TargetStoreResult] = {
     val tempFile = File.createTempFile("s3file", ".tmp")
@@ -91,7 +102,7 @@ class S3TargetStoreEngine(credentials: S3Credentials)(implicit val system: Actor
   }
 }
 
-class S3Credentials(accessKey: String, secretKey: String, val bucketId: String, val region: Regions)
+class S3Credentials(accessKey: String, secretKey: String, val bucketId: String, val region: Regions, val endpointUrl: String)
   extends AWSCredentials with AWSCredentialsProvider {
   override def getAWSAccessKeyId: String = accessKey