diff --git a/reposerver/src/main/resources/application.conf b/reposerver/src/main/resources/application.conf index 51c3f3f5..7ac81770 100644 --- a/reposerver/src/main/resources/application.conf +++ b/reposerver/src/main/resources/application.conf @@ -17,6 +17,7 @@ storage { bucketId = ${?TUF_REPOSERVER_AWS_BUCKET_ID} region = "eu-central-1" region = ${?TUF_REPOSERVER_AWS_REGION} + endpointUrl = ${?TUF_REPOSERVER_S3_URL} } } diff --git a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/Boot.scala b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/Boot.scala index 6764a6bb..a33eadff 100644 --- a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/Boot.scala +++ b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/Boot.scala @@ -39,7 +39,8 @@ trait Settings { val secretKey = _config.getString("storage.s3.secretKey") val bucketId = _config.getString("storage.s3.bucketId") val region = Regions.fromName(_config.getString("storage.s3.region")) - new S3Credentials(accessKey, secretKey, bucketId, region) + val endpointUrl = _config.getString("storage.s3.endpointUrl") + new S3Credentials(accessKey, secretKey, bucketId, region, endpointUrl) } lazy val useS3 = _config.getString("storage.type").equals("s3") diff --git a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/target_store/S3TargetStoreEngine.scala b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/target_store/S3TargetStoreEngine.scala index 0898f414..df989f9f 100644 --- a/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/target_store/S3TargetStoreEngine.scala +++ b/reposerver/src/main/scala/com/advancedtelematic/tuf/reposerver/target_store/S3TargetStoreEngine.scala @@ -12,6 +12,7 @@ import akka.util.ByteString import com.advancedtelematic.libtuf.data.TufDataType.{RepoId, TargetFilename} import com.advancedtelematic.tuf.reposerver.target_store.TargetStoreEngine.{TargetRedirect, TargetRetrieveResult, TargetStoreResult} import com.amazonaws.auth.{AWSCredentials, AWSCredentialsProvider} +import com.amazonaws.client.builder.AwsClientBuilder import com.amazonaws.regions.Regions import com.amazonaws.services.s3.AmazonS3ClientBuilder import com.amazonaws.services.s3.model.{CannedAccessControlList, PutObjectRequest} @@ -29,10 +30,18 @@ class S3TargetStoreEngine(credentials: S3Credentials)(implicit val system: Actor private val log = LoggerFactory.getLogger(this.getClass) - private lazy val s3client = AmazonS3ClientBuilder.standard() - .withCredentials(credentials) - .withRegion(credentials.region) - .build() + protected lazy val s3client = { + if(credentials.endpointUrl.length() > 0) { + log.info(s"Using custom S3 url: ${credentials.endpointUrl}") + AmazonS3ClientBuilder.standard() + .withCredentials(credentials) + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(credentials.endpointUrl, credentials.region.getName())) + } else { + AmazonS3ClientBuilder.standard() + .withCredentials(credentials) + .withRegion(credentials.region) + } + }.build() override def store(repoId: RepoId, filename: TargetFilename, fileData: Source[ByteString, Any]): Future[TargetStoreResult] = { val tempFile = File.createTempFile("s3file", ".tmp") @@ -91,7 +100,7 @@ class S3TargetStoreEngine(credentials: S3Credentials)(implicit val system: Actor } } -class S3Credentials(accessKey: String, secretKey: String, val bucketId: String, val region: Regions) +class S3Credentials(accessKey: String, secretKey: String, val bucketId: String, val region: Regions, val endpointUrl: String) extends AWSCredentials with AWSCredentialsProvider { override def getAWSAccessKeyId: String = accessKey diff --git a/reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/S3StorageResourceIntegrationSpec.scala b/reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/S3StorageResourceIntegrationSpec.scala index ce04e387..6036c5cc 100644 --- a/reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/S3StorageResourceIntegrationSpec.scala +++ b/reposerver/src/test/scala/com/advancedtelematic/tuf/reposerver/http/S3StorageResourceIntegrationSpec.scala @@ -4,7 +4,7 @@ import akka.http.scaladsl.model.Multipart.FormData.BodyPart import akka.http.scaladsl.model.{HttpEntity, Multipart, StatusCodes} import akka.util.ByteString import com.advancedtelematic.libtuf.data.TufDataType.RepoId -import com.advancedtelematic.tuf.reposerver.target_store.{S3TargetStoreEngine, TargetStore} +import com.advancedtelematic.tuf.reposerver.target_store.{S3Credentials, S3TargetStoreEngine, TargetStore} import com.advancedtelematic.tuf.reposerver.util._ import org.scalatest.{BeforeAndAfterAll, Inspectors} import org.scalatest.concurrent.PatienceConfiguration @@ -12,6 +12,7 @@ import org.scalatest.prop.Whenever import RepoId._ import cats.syntax.show._ import com.advancedtelematic.tuf.reposerver.Settings +import com.amazonaws.regions.Regions class S3StorageResourceIntegrationSpec extends TufReposerverSpec with ResourceSpec with BeforeAndAfterAll with Inspectors with Whenever with PatienceConfiguration { @@ -50,4 +51,33 @@ class S3StorageResourceIntegrationSpec extends TufReposerverSpec header("Location").get.value() should include("amazonaws.com") } } + + test("uploading a target changes targets json with custom endpoint url") { + lazy val credentials = new S3Credentials("", "", "", Regions.fromName("us-central-1"), "https://storage.googleapis.com") + lazy val s3Storage = new S3TargetStoreEngine(credentials) + lazy val targetStore = new TargetStore(fakeKeyserverClient, s3Storage, fakeHttpClient, messageBusPublisher) + + pending // Needs valid s3 credentials to run + + val repoId = RepoId.generate() + fakeKeyserverClient.createRoot(repoId).futureValue + + val entity = HttpEntity(ByteString(""" + |Like all the men of the Library, in my younger days I traveled; + |I have journeyed in quest of a book, perhaps the catalog of catalogs. + """.stripMargin)) + + val fileBodyPart = BodyPart("file", entity, Map("filename" -> "babel.txt")) + + val form = Multipart.FormData(fileBodyPart) + + Put(s"/repo/${repoId.show}/targets/some/target/funky/thing?name=pkgname&version=pkgversion&desc=wat", form) ~> routes ~> check { + status shouldBe StatusCodes.OK + } + + Get(s"/repo/${repoId.show}/targets/some/target/funky/thing") ~> routes ~> check { + status shouldBe StatusCodes.Found + header("Location").get.value() should include("amazonaws.com") + } + } }