From b6917a990eee91792e8b2cdd6c62dfd9493268f4 Mon Sep 17 00:00:00 2001 From: Andy Doan Date: Thu, 4 Apr 2019 11:23:04 -0500 Subject: [PATCH] Add Google Storage as a storage backend This works with a Google Storage project configured with S3 interoperability. Signed-off-by: Andy Doan --- src/main/resources/application.conf | 2 +- .../scala/com/advancedtelematic/treehub/Boot.scala | 10 ++++++++-- .../com/advancedtelematic/treehub/Settings.scala | 1 + .../treehub/delta_store/StaticDeltaStorage.scala | 10 +++++++++- .../treehub/object_store/S3BlobStore.scala | 11 ++++++++++- 5 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index 9ad1622..8268e03 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -58,7 +58,7 @@ server = { treehub { storage { - type = "s3" // or local + type = "s3" // or local or gs type = ${?TREEHUB_STORAGE} staleObjectsExpireAfter = 1 hour diff --git a/src/main/scala/com/advancedtelematic/treehub/Boot.scala b/src/main/scala/com/advancedtelematic/treehub/Boot.scala index 764e156..0e9c7e9 100644 --- a/src/main/scala/com/advancedtelematic/treehub/Boot.scala +++ b/src/main/scala/com/advancedtelematic/treehub/Boot.scala @@ -11,7 +11,7 @@ import com.advancedtelematic.libats.slick.db.{BootMigrations, DatabaseConfig} import com.advancedtelematic.libats.slick.monitoring.DatabaseMetrics import com.advancedtelematic.treehub.client._ import com.advancedtelematic.treehub.http.{TreeHubRoutes, Http => TreeHubHttp} -import com.advancedtelematic.treehub.object_store.{LocalFsBlobStore, ObjectStore, S3BlobStore, S3Credentials} +import com.advancedtelematic.treehub.object_store.{GSBlobStore, LocalFsBlobStore, ObjectStore, S3BlobStore, S3Credentials} import com.advancedtelematic.treehub.repo_metrics.UsageMetricsRouter import com.amazonaws.regions.Regions import com.typesafe.config.ConfigFactory @@ -22,7 +22,7 @@ import com.advancedtelematic.libats.http.monitoring.MetricsSupport import com.advancedtelematic.metrics.{AkkaHttpRequestMetrics, InfluxdbMetricsReporterSupport} import com.advancedtelematic.metrics.prometheus.PrometheusMetricsSupport import com.advancedtelematic.treehub.daemon.StaleObjectArchiveActor -import com.advancedtelematic.treehub.delta_store.{LocalDeltaStorage, S3DeltaStorage} +import com.advancedtelematic.treehub.delta_store.{GSDeltaStorage, LocalDeltaStorage, S3DeltaStorage} object Boot extends BootApp with Directives with Settings with VersionInfo @@ -48,6 +48,9 @@ object Boot extends BootApp with Directives with Settings with VersionInfo if(useS3) { log.info("Using s3 storage for object blobs") new S3BlobStore(s3Credentials, allowRedirectsToS3) + } else if (useGoogleStorage) { + log.info("Using GS storage for object blobs") + new GSBlobStore(s3Credentials, allowRedirectsToS3) } else { log.info(s"Using local storage a t$localStorePath for object blobs") LocalFsBlobStore(localStorePath.resolve("object-storage")) @@ -58,6 +61,9 @@ object Boot extends BootApp with Directives with Settings with VersionInfo if(useS3) { log.info("Using s3 storage for object blobs") new S3DeltaStorage(s3Credentials) + } else if (useGoogleStorage) { + log.info("Using GS storage for object blobs") + new GSDeltaStorage(s3Credentials) } else { log.info(s"Using local storage at $localStorePath for object blobs") new LocalDeltaStorage(localStorePath.resolve("delta-storage")) diff --git a/src/main/scala/com/advancedtelematic/treehub/Settings.scala b/src/main/scala/com/advancedtelematic/treehub/Settings.scala index 4e9d376..8a9c6b6 100644 --- a/src/main/scala/com/advancedtelematic/treehub/Settings.scala +++ b/src/main/scala/com/advancedtelematic/treehub/Settings.scala @@ -37,6 +37,7 @@ trait Settings { } lazy val useS3 = _config.getString("treehub.storage.type").equals("s3") + lazy val useGoogleStorage = _config.getString("treehub.storage.type").equals("gs") lazy val staleObjectExpireAfter = _config.getDuration("treehub.storage.staleObjectsExpireAfter") diff --git a/src/main/scala/com/advancedtelematic/treehub/delta_store/StaticDeltaStorage.scala b/src/main/scala/com/advancedtelematic/treehub/delta_store/StaticDeltaStorage.scala index 3d76fcb..031be79 100644 --- a/src/main/scala/com/advancedtelematic/treehub/delta_store/StaticDeltaStorage.scala +++ b/src/main/scala/com/advancedtelematic/treehub/delta_store/StaticDeltaStorage.scala @@ -13,6 +13,7 @@ import com.advancedtelematic.libats.data.DataType.Namespace import com.advancedtelematic.treehub.delta_store.StaticDeltaStorage.{StaticDeltaContent, StaticDeltaRedirectResponse, StaticDeltaResponse} import com.advancedtelematic.treehub.http.Errors import com.advancedtelematic.treehub.object_store.S3Credentials +import com.amazonaws.client.builder.AwsClientBuilder import com.amazonaws.services.s3.AmazonS3ClientBuilder import org.apache.commons.codec.binary.Hex import org.apache.commons.codec.digest.DigestUtils @@ -51,7 +52,7 @@ object StaticDeltaStorage { class S3DeltaStorage(s3Credentials: S3Credentials)(implicit ec: ExecutionContext) extends StaticDeltaStorage { private val _log = LoggerFactory.getLogger(this.getClass) - private lazy val s3client = AmazonS3ClientBuilder.standard() + protected lazy val s3client = AmazonS3ClientBuilder.standard() .withCredentials(s3Credentials) .withRegion(s3Credentials.region) .build() @@ -111,6 +112,13 @@ class S3DeltaStorage(s3Credentials: S3Credentials)(implicit ec: ExecutionContext } } +class GSDeltaStorage(s3Credentials: S3Credentials)(implicit ec: ExecutionContext) extends S3DeltaStorage(s3Credentials) { + override lazy val s3client = AmazonS3ClientBuilder.standard() + .withCredentials(s3Credentials) + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("https://storage.googleapis.com", "Multi-Regional")) + .build() +} + class LocalDeltaStorage(root: Path) extends StaticDeltaStorage { private val _log = LoggerFactory.getLogger(this.getClass) diff --git a/src/main/scala/com/advancedtelematic/treehub/object_store/S3BlobStore.scala b/src/main/scala/com/advancedtelematic/treehub/object_store/S3BlobStore.scala index 42f031c..935ce40 100644 --- a/src/main/scala/com/advancedtelematic/treehub/object_store/S3BlobStore.scala +++ b/src/main/scala/com/advancedtelematic/treehub/object_store/S3BlobStore.scala @@ -17,6 +17,7 @@ import com.advancedtelematic.libats.data.DataType.Namespace import com.advancedtelematic.treehub.object_store.BlobStore.UploadAt import com.amazonaws.HttpMethod import com.amazonaws.auth.{AWSCredentials, AWSCredentialsProvider} +import com.amazonaws.client.builder.AwsClientBuilder import com.amazonaws.regions.Regions import com.amazonaws.services.s3.AmazonS3ClientBuilder import com.amazonaws.services.s3.model._ @@ -32,7 +33,7 @@ class S3BlobStore(s3Credentials: S3Credentials, allowRedirects: Boolean) private val bucketId = s3Credentials.blobBucketId - private lazy val s3client = AmazonS3ClientBuilder.standard() + protected lazy val s3client = AmazonS3ClientBuilder.standard() .withCredentials(s3Credentials) .withRegion(s3Credentials.region) .build() @@ -144,6 +145,14 @@ class S3BlobStore(s3Credentials: S3Credentials, allowRedirects: Boolean) override val supportsOutOfBandStorage: Boolean = true } +class GSBlobStore(s3Credentials: S3Credentials, allowRedirects: Boolean) + (implicit ec: ExecutionContext, mat: Materializer) extends S3BlobStore(s3Credentials, allowRedirects) { + override lazy val s3client = AmazonS3ClientBuilder.standard() + .withCredentials(s3Credentials) + .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("https://storage.googleapis.com", "Multi-Regional")) + .build() +} + class S3Credentials(accessKey: String, secretKey: String, val blobBucketId: String, val deltasBucketId: String,