From 4e9e393382d9db9131600a31ead6b2b052af76bb Mon Sep 17 00:00:00 2001 From: Andy Doan Date: Fri, 12 Apr 2019 14:43:55 -0500 Subject: [PATCH 1/2] s3: Support S3 compatible services like Google Storage Services like Google Storage expose s3 compatible APIs that can accessed with a non-aws URL. Signed-off-by: Andy Doan --- .../src/main/resources/application.conf | 1 + .../tuf/reposerver/Boot.scala | 3 ++- .../target_store/S3TargetStoreEngine.scala | 21 ++++++++++++++----- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/reposerver/src/main/resources/application.conf b/reposerver/src/main/resources/application.conf index 51c3f3f5..7ac81770 100644 --- a/reposerver/src/main/resources/application.conf +++ b/reposerver/src/main/resources/application.conf @@ -17,6 +17,7 @@ storage { bucketId = ${?TUF_REPOSERVER_AWS_BUCKET_ID} region = "eu-central-1" region = ${?TUF_REPOSERVER_AWS_REGION} + endpointUrl = ${?TUF_REPOSERVER_S3_URL} } } diff --git a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/Boot.scala b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/Boot.scala index 6764a6bb..a33eadff 100644 --- a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/Boot.scala +++ b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/Boot.scala @@ -39,7 +39,8 @@ trait Settings { val secretKey = _config.getString("storage.s3.secretKey") val bucketId = _config.getString("storage.s3.bucketId") val region = Regions.fromName(_config.getString("storage.s3.region")) - new S3Credentials(accessKey, secretKey, bucketId, region) + val endpointUrl = _config.getString("storage.s3.endpointUrl") + new S3Credentials(accessKey, secretKey, bucketId, region, endpointUrl) } lazy val useS3 = _config.getString("storage.type").equals("s3") diff --git a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/target_store/S3TargetStoreEngine.scala b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/target_store/S3TargetStoreEngine.scala index 0898f414..0a26c391 100644 --- a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/target_store/S3TargetStoreEngine.scala +++ b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/target_store/S3TargetStoreEngine.scala @@ -12,6 +12,7 @@ import akka.util.ByteString import com.advancedtelematic.libtuf.data.TufDataType.{RepoId, TargetFilename} import com.advancedtelematic.tuf.reposerver.target_store.TargetStoreEngine.{TargetRedirect, TargetRetrieveResult, TargetStoreResult} import com.amazonaws.auth.{AWSCredentials, AWSCredentialsProvider} +import com.amazonaws.client.builder.AwsClientBuilder import com.amazonaws.regions.Regions import com.amazonaws.services.s3.AmazonS3ClientBuilder import com.amazonaws.services.s3.model.{CannedAccessControlList, PutObjectRequest} @@ -29,10 +30,20 @@ class S3TargetStoreEngine(credentials: S3Credentials)(implicit val system: Actor private val log = LoggerFactory.getLogger(this.getClass) - private lazy val s3client = AmazonS3ClientBuilder.standard() - .withCredentials(credentials) - .withRegion(credentials.region) - .build() + protected lazy val s3client = { + if(credentials.endpointUrl.length() > 0) { + log.info(s"Using custom S3 url: ${credentials.endpointUrl}") + AmazonS3ClientBuilder.standard() + .withCredentials(credentials) + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(credentials.endpointUrl, credentials.region.getName())) + .build() + } else { + AmazonS3ClientBuilder.standard() + .withCredentials(credentials) + .withRegion(credentials.region) + .build() + } + } override def store(repoId: RepoId, filename: TargetFilename, fileData: Source[ByteString, Any]): Future[TargetStoreResult] = { val tempFile = File.createTempFile("s3file", ".tmp") @@ -91,7 +102,7 @@ class S3TargetStoreEngine(credentials: S3Credentials)(implicit val system: Actor } } -class S3Credentials(accessKey: String, secretKey: String, val bucketId: String, val region: Regions) +class S3Credentials(accessKey: String, secretKey: String, val bucketId: String, val region: Regions, val endpointUrl: String) extends AWSCredentials with AWSCredentialsProvider { override def getAWSAccessKeyId: String = accessKey From 70e1cdc9ba3216840cb349742791d7bf167db5a0 Mon Sep 17 00:00:00 2001 From: Andy Doan Date: Tue, 23 Apr 2019 12:37:27 -0500 Subject: [PATCH 2/2] Code Review comments Signed-off-by: Andy Doan --- .../target_store/S3TargetStoreEngine.scala | 4 +-- .../S3StorageResourceIntegrationSpec.scala | 32 ++++++++++++++++++- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/target_store/S3TargetStoreEngine.scala b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/target_store/S3TargetStoreEngine.scala index 0a26c391..df989f9f 100644 --- a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/target_store/S3TargetStoreEngine.scala +++ b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/target_store/S3TargetStoreEngine.scala @@ -36,14 +36,12 @@ class S3TargetStoreEngine(credentials: S3Credentials)(implicit val system: Actor AmazonS3ClientBuilder.standard() .withCredentials(credentials) .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(credentials.endpointUrl, credentials.region.getName())) - .build() } else { AmazonS3ClientBuilder.standard() .withCredentials(credentials) .withRegion(credentials.region) - .build() } - } + }.build() override def store(repoId: RepoId, filename: TargetFilename, fileData: Source[ByteString, Any]): Future[TargetStoreResult] = { val tempFile = File.createTempFile("s3file", ".tmp") diff --git a/reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/S3StorageResourceIntegrationSpec.scala b/reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/S3StorageResourceIntegrationSpec.scala index ce04e387..6036c5cc 100644 --- a/reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/S3StorageResourceIntegrationSpec.scala +++ b/reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/S3StorageResourceIntegrationSpec.scala @@ -4,7 +4,7 @@ import akka.http.scaladsl.model.Multipart.FormData.BodyPart import akka.http.scaladsl.model.{HttpEntity, Multipart, StatusCodes} import akka.util.ByteString import com.advancedtelematic.libtuf.data.TufDataType.RepoId -import com.advancedtelematic.tuf.reposerver.target_store.{S3TargetStoreEngine, TargetStore} +import com.advancedtelematic.tuf.reposerver.target_store.{S3Credentials, S3TargetStoreEngine, TargetStore} import com.advancedtelematic.tuf.reposerver.util._ import org.scalatest.{BeforeAndAfterAll, Inspectors} import org.scalatest.concurrent.PatienceConfiguration @@ -12,6 +12,7 @@ import org.scalatest.prop.Whenever import RepoId._ import cats.syntax.show._ import com.advancedtelematic.tuf.reposerver.Settings +import com.amazonaws.regions.Regions class S3StorageResourceIntegrationSpec extends TufReposerverSpec with ResourceSpec with BeforeAndAfterAll with Inspectors with Whenever with PatienceConfiguration { @@ -50,4 +51,33 @@ class S3StorageResourceIntegrationSpec extends TufReposerverSpec header("Location").get.value() should include("amazonaws.com") } } + + test("uploading a target changes targets json with custom endpoint url") { + lazy val credentials = new S3Credentials("", "", "", Regions.fromName("us-central-1"), "https://storage.googleapis.com") + lazy val s3Storage = new S3TargetStoreEngine(credentials) + lazy val targetStore = new TargetStore(fakeKeyserverClient, s3Storage, fakeHttpClient, messageBusPublisher) + + pending // Needs valid s3 credentials to run + + val repoId = RepoId.generate() + fakeKeyserverClient.createRoot(repoId).futureValue + + val entity = HttpEntity(ByteString(""" + |Like all the men of the Library, in my younger days I traveled; + |I have journeyed in quest of a book, perhaps the catalog of catalogs. + """.stripMargin)) + + val fileBodyPart = BodyPart("file", entity, Map("filename" -> "babel.txt")) + + val form = Multipart.FormData(fileBodyPart) + + Put(s"/repo/${repoId.show}/targets/some/target/funky/thing?name=pkgname&version=pkgversion&desc=wat", form) ~> routes ~> check { + status shouldBe StatusCodes.OK + } + + Get(s"/repo/${repoId.show}/targets/some/target/funky/thing") ~> routes ~> check { + status shouldBe StatusCodes.Found + header("Location").get.value() should include("amazonaws.com") + } + } }