[Enhancement] Support aws.glue.catalog_id parameters

Signed-off-by: Smith Cruise <chendingchao1@126.com>
Smith-Cruise · Dec 19, 2024 · c0fad0b · c0fad0b
1 parent a3f1168
commit c0fad0b
Show file tree

Hide file tree

Showing 3 changed files with 16 additions and 2 deletions.
diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/hive/glue/util/MetastoreClientUtils.java b/fe/fe-core/src/main/java/com/starrocks/connector/hive/glue/util/MetastoreClientUtils.java
@@ -17,6 +17,7 @@
 
 import com.google.common.collect.Maps;
 import com.starrocks.connector.hive.glue.metastore.GlueMetastoreClientDelegate;
+import com.starrocks.connector.share.credential.CloudConfigurationConstants;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
@@ -122,8 +123,13 @@ public static boolean isExternalTable(Table table) {
     }
 
     public static String getCatalogId(Configuration conf) {
-        if (StringUtils.isNotEmpty(conf.get(GlueMetastoreClientDelegate.CATALOG_ID_CONF))) {
-            return conf.get(GlueMetastoreClientDelegate.CATALOG_ID_CONF);
+        String catalogId = conf.get(GlueMetastoreClientDelegate.CATALOG_ID_CONF);
+        if (StringUtils.isNotEmpty(catalogId)) {
+            return catalogId;
+        }
+        catalogId = conf.get(CloudConfigurationConstants.AWS_GLUE_CATALOG_ID);
+        if (StringUtils.isNotEmpty(catalogId)) {
+            return catalogId;
         }
         // This case defaults to using the caller's account Id as Catalog Id.
         return null;

diff --git a/...t/src/main/java/com/starrocks/connector/share/credential/CloudConfigurationConstants.java b/...t/src/main/java/com/starrocks/connector/share/credential/CloudConfigurationConstants.java
@@ -72,6 +72,8 @@ public class CloudConfigurationConstants {
     public static final String AWS_GLUE_EXTERNAL_ID = "aws.glue.external_id";
     public static final String AWS_GLUE_REGION = "aws.glue.region";
     public static final String AWS_GLUE_ENDPOINT = "aws.glue.endpoint";
+    // https://docs.aws.amazon.com/glue/latest/dg/aws-glue-api-catalog-databases.html
+    public static final String AWS_GLUE_CATALOG_ID = "aws.glue.catalog_id";
 
     // Credential for Azure storage
     // For Azure Blob Storage

diff --git a/...doop-ext/src/main/java/com/starrocks/connector/share/iceberg/IcebergAwsClientFactory.java b/...doop-ext/src/main/java/com/starrocks/connector/share/iceberg/IcebergAwsClientFactory.java
@@ -14,6 +14,7 @@
 
 package com.starrocks.connector.share.iceberg;
 
+import com.starrocks.connector.share.credential.CloudConfigurationConstants;
 import org.apache.iceberg.aws.AwsClientFactory;
 import org.apache.iceberg.aws.AwsProperties;
 import org.slf4j.Logger;
@@ -68,6 +69,7 @@
 import static com.starrocks.connector.share.credential.CloudConfigurationConstants.AWS_S3_USE_AWS_SDK_DEFAULT_BEHAVIOR;
 import static com.starrocks.connector.share.credential.CloudConfigurationConstants.AWS_S3_USE_INSTANCE_PROFILE;
 import static com.starrocks.connector.share.credential.CloudConfigurationConstants.DEFAULT_AWS_REGION;
+import static com.starrocks.connector.share.credential.CloudConfigurationConstants.AWS_GLUE_CATALOG_ID;
 
 public class IcebergAwsClientFactory implements AwsClientFactory {
     private static final Logger LOG = LoggerFactory.getLogger(IcebergAwsClientFactory.class);
@@ -129,6 +131,10 @@ public void initialize(Map<String, String> properties) {
         glueExternalId = properties.getOrDefault(AWS_GLUE_EXTERNAL_ID, "");
         glueRegion = properties.getOrDefault(AWS_GLUE_REGION, "");
         glueEndpoint = properties.getOrDefault(AWS_GLUE_ENDPOINT, "");
+        String glueCatalogId = properties.get(AWS_GLUE_CATALOG_ID);
+        if (glueCatalogId != null) {
+            this.awsProperties.setGlueCatalogId(glueCatalogId);
+        }
     }
 
     private StsAssumeRoleCredentialsProvider getAssumeRoleCredentialsProvider(AwsCredentialsProvider baseCredentials,