Add Google Storage as a storage backend

This works with a Google Storage project configured with S3
interoperability.

Signed-off-by: Andy Doan <andy@foundries.io>

src/main/resources/application.conf

@@ -58,7 +58,7 @@ server = {
 
 treehub {
   storage {
-    type = "s3" // or local
+    type = "s3" // or local or gs
     type = ${?TREEHUB_STORAGE}
 
     staleObjectsExpireAfter = 1 hour

src/main/scala/com/advancedtelematic/treehub/Boot.scala

@@ -11,7 +11,7 @@ import com.advancedtelematic.libats.slick.db.{BootMigrations, DatabaseConfig}
 import com.advancedtelematic.libats.slick.monitoring.DatabaseMetrics
 import com.advancedtelematic.treehub.client._
 import com.advancedtelematic.treehub.http.{TreeHubRoutes, Http => TreeHubHttp}
-import com.advancedtelematic.treehub.object_store.{LocalFsBlobStore, ObjectStore, S3BlobStore, S3Credentials}
+import com.advancedtelematic.treehub.object_store.{GSBlobStore, LocalFsBlobStore, ObjectStore, S3BlobStore, S3Credentials}
 import com.advancedtelematic.treehub.repo_metrics.UsageMetricsRouter
 import com.amazonaws.regions.Regions
 import com.typesafe.config.ConfigFactory
@@ -22,7 +22,7 @@ import com.advancedtelematic.libats.http.monitoring.MetricsSupport
 import com.advancedtelematic.metrics.{AkkaHttpRequestMetrics, InfluxdbMetricsReporterSupport}
 import com.advancedtelematic.metrics.prometheus.PrometheusMetricsSupport
 import com.advancedtelematic.treehub.daemon.StaleObjectArchiveActor
-import com.advancedtelematic.treehub.delta_store.{LocalDeltaStorage, S3DeltaStorage}
+import com.advancedtelematic.treehub.delta_store.{GSDeltaStorage, LocalDeltaStorage, S3DeltaStorage}
 
 
 object Boot extends BootApp with Directives with Settings with VersionInfo
@@ -48,6 +48,9 @@ object Boot extends BootApp with Directives with Settings with VersionInfo
     if(useS3) {
       log.info("Using s3 storage for object blobs")
       new S3BlobStore(s3Credentials, allowRedirectsToS3)
+    } else if (useGoogleStorage) {
+      log.info("Using GS storage for object blobs")
+      new GSBlobStore(s3Credentials, allowRedirectsToS3)
     } else {
       log.info(s"Using local storage a t$localStorePath for object blobs")
       LocalFsBlobStore(localStorePath.resolve("object-storage"))
@@ -58,6 +61,9 @@ object Boot extends BootApp with Directives with Settings with VersionInfo
     if(useS3) {
       log.info("Using s3 storage for object blobs")
       new S3DeltaStorage(s3Credentials)
+    } else if (useGoogleStorage) {
+      log.info("Using GS storage for object blobs")
+      new GSDeltaStorage(s3Credentials)
     } else {
       log.info(s"Using local storage at $localStorePath for object blobs")
       new LocalDeltaStorage(localStorePath.resolve("delta-storage"))

src/main/scala/com/advancedtelematic/treehub/Settings.scala

@@ -37,6 +37,7 @@ trait Settings {
   }
 
   lazy val useS3 = _config.getString("treehub.storage.type").equals("s3")
+  lazy val useGoogleStorage = _config.getString("treehub.storage.type").equals("gs")
 
   lazy val staleObjectExpireAfter = _config.getDuration("treehub.storage.staleObjectsExpireAfter")
 

src/main/scala/com/advancedtelematic/treehub/delta_store/StaticDeltaStorage.scala

@@ -13,6 +13,7 @@ import com.advancedtelematic.libats.data.DataType.Namespace
 import com.advancedtelematic.treehub.delta_store.StaticDeltaStorage.{StaticDeltaContent, StaticDeltaRedirectResponse, StaticDeltaResponse}
 import com.advancedtelematic.treehub.http.Errors
 import com.advancedtelematic.treehub.object_store.S3Credentials
+import com.amazonaws.client.builder.AwsClientBuilder
 import com.amazonaws.services.s3.AmazonS3ClientBuilder
 import org.apache.commons.codec.binary.Hex
 import org.apache.commons.codec.digest.DigestUtils
@@ -51,7 +52,7 @@ object StaticDeltaStorage {
 class S3DeltaStorage(s3Credentials: S3Credentials)(implicit ec: ExecutionContext) extends StaticDeltaStorage {
   private val _log = LoggerFactory.getLogger(this.getClass)
 
-  private lazy val s3client = AmazonS3ClientBuilder.standard()
+  protected lazy val s3client = AmazonS3ClientBuilder.standard()
     .withCredentials(s3Credentials)
     .withRegion(s3Credentials.region)
     .build()
@@ -111,6 +112,13 @@ class S3DeltaStorage(s3Credentials: S3Credentials)(implicit ec: ExecutionContext
   }
 }
 
+class GSDeltaStorage(s3Credentials: S3Credentials)(implicit ec: ExecutionContext) extends S3DeltaStorage(s3Credentials) {
+  override lazy val s3client = AmazonS3ClientBuilder.standard()
+      .withCredentials(s3Credentials)
+      .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("https://storage.googleapis.com", "Multi-Regional"))
+      .build()
+}
+
 class LocalDeltaStorage(root: Path) extends StaticDeltaStorage {
   private val _log = LoggerFactory.getLogger(this.getClass)
 

src/main/scala/com/advancedtelematic/treehub/object_store/S3BlobStore.scala

@@ -17,6 +17,7 @@ import com.advancedtelematic.libats.data.DataType.Namespace
 import com.advancedtelematic.treehub.object_store.BlobStore.UploadAt
 import com.amazonaws.HttpMethod
 import com.amazonaws.auth.{AWSCredentials, AWSCredentialsProvider}
+import com.amazonaws.client.builder.AwsClientBuilder
 import com.amazonaws.regions.Regions
 import com.amazonaws.services.s3.AmazonS3ClientBuilder
 import com.amazonaws.services.s3.model._
@@ -32,7 +33,7 @@ class S3BlobStore(s3Credentials: S3Credentials, allowRedirects: Boolean)
 
   private val bucketId = s3Credentials.blobBucketId
 
-  private lazy val s3client = AmazonS3ClientBuilder.standard()
+  protected lazy val s3client = AmazonS3ClientBuilder.standard()
       .withCredentials(s3Credentials)
       .withRegion(s3Credentials.region)
       .build()
@@ -144,6 +145,14 @@ class S3BlobStore(s3Credentials: S3Credentials, allowRedirects: Boolean)
   override val supportsOutOfBandStorage: Boolean = true
 }
 
+class GSBlobStore(s3Credentials: S3Credentials, allowRedirects: Boolean)
+                 (implicit ec: ExecutionContext, mat: Materializer) extends S3BlobStore(s3Credentials, allowRedirects) {
+  override lazy val s3client = AmazonS3ClientBuilder.standard()
+      .withCredentials(s3Credentials)
+      .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("https://storage.googleapis.com", "Multi-Regional"))
+      .build()
+}
+
 class S3Credentials(accessKey: String, secretKey: String,
                     val blobBucketId: String,
                     val deltasBucketId: String,