Cloning extra-files on launcher vms.

GoogleCloudPlatform · Jan 24, 2025 · dfc3c6f · dfc3c6f
1 parent cf43da3
commit dfc3c6f
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 0 deletions.
diff --git a/...ogle/cloud/teleport/v2/source/reader/io/cassandra/iowrapper/CassandraIOWrapperHelper.java b/...ogle/cloud/teleport/v2/source/reader/io/cassandra/iowrapper/CassandraIOWrapperHelper.java
@@ -32,7 +32,11 @@
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.util.List;
+import java.util.stream.Collectors;
 import org.apache.beam.sdk.transforms.PTransform;
 import org.apache.beam.sdk.values.PBegin;
 import org.apache.beam.sdk.values.PCollection;
@@ -45,6 +49,16 @@ class CassandraIOWrapperHelper {
   private static final Logger LOG = LoggerFactory.getLogger(CassandraIOWrapperHelper.class);
 
   static DataSource buildDataSource(String gcsPath) {
+    // DO NOT SUBMIT
+    try {
+      LOG.info(
+          "Extra Files are {}",
+          Files.list(Paths.get("/extra_files"))
+              .map(p -> p.getFileName())
+              .collect(Collectors.toList()));
+    } catch (IOException e) {
+      throw new RuntimeException(e);
+    }
     DataSource dataSource;
     try {
       dataSource =

diff --git a/...db-to-spanner/src/main/java/com/google/cloud/teleport/v2/templates/SourceDbToSpanner.java b/...db-to-spanner/src/main/java/com/google/cloud/teleport/v2/templates/SourceDbToSpanner.java
@@ -17,6 +17,7 @@
 
 import com.google.cloud.teleport.metadata.Template;
 import com.google.cloud.teleport.metadata.TemplateCategory;
+import com.google.cloud.teleport.v2.common.CommonTemplateJvmInitializer;
 import com.google.cloud.teleport.v2.common.UncaughtExceptionLogger;
 import com.google.cloud.teleport.v2.options.SourceDbToSpannerOptions;
 import com.google.cloud.teleport.v2.spanner.migrations.shard.Shard;
@@ -85,6 +86,9 @@ public static void main(String[] args) {
     // Parse the user options passed from the command-line
     SourceDbToSpannerOptions options =
         PipelineOptionsFactory.fromArgs(args).withValidation().as(SourceDbToSpannerOptions.class);
+    // Stage SSL certificates to extraFiles if required as per the pipeline options.
+    // Ref https://cloud.google.com/dataflow/docs/guides/templates/ssl-certificates
+    new CommonTemplateJvmInitializer().beforeProcessing(options);
     run(options);
   }