Move spinner init in load_data_to_hdfs.py to avoid error if /data is …

…not populated
adisve · Apr 10, 2024 · 7969010 · 7969010
1 parent 48c8b66
commit 7969010
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/scripts/spark/setup/load_data_to_hdfs.py b/scripts/spark/setup/load_data_to_hdfs.py
@@ -70,13 +70,14 @@ def get_schema(self):
         return self.parse_schema_file()
 
     def transfer_data(self):
+        spinner = Halo(text=f"Reading and writing data from /data/output.csv to {self.hdfs_path}")
+        spinner.start()
         try:
             self.start_spark_session()
             schema = self.get_schema()
 
             logging.info(f"Reading and writing data from /data/output.csv to {self.hdfs_path}")
-            spinner = Halo(text=f"Reading and writing data from /data/output.csv to {self.hdfs_path}")
-            spinner.start()
+
             df = (self.spark.read.option("header", "true")
                   .option("mode", "DROPMALFORMED")
                   .option("overwrite", "true")