wip

src/main/com/codely/lesson_02_tests_in_spark/video_02__unit_testing/config/CmdArgs.scala

@@ -1,3 +1,3 @@
 package com.codely.lesson_02_tests_in_spark.video_02__unit_testing.config
 
-case class CmdArgs(configFile: Option[String] = None)
+case class CmdArgs(configFile: Option[String] = None)

src/main/com/codely/lesson_02_tests_in_spark/video_02__unit_testing/job/AvgSpendingJob.scala

@@ -16,12 +16,7 @@ case class AvgSpendingJob(
     val data = streamReader.read(context.source.format, context.source.options)
 
     val avgSpendingPerUserDF =
-      data
-        .parseJson
-        .addDateColum
-        .explodeProducts
-        .transformForAggregation
-        .calculateAvgSpending
+      data.parseJson.addDateColum.explodeProducts.transformForAggregation.calculateAvgSpending
 
     val query = streamWriter.write(avgSpendingPerUserDF, context.sink.mode, context.sink.format, context.sink.options)
 

src/main/com/codely/lesson_02_tests_in_spark/video_02__unit_testing/service/StreamWriter.scala

@@ -5,11 +5,11 @@ import org.apache.spark.sql.DataFrame
 
 case class StreamWriter() {
   def write(
-             df: DataFrame,
-             mode: String,
-             format: String,
-             options: Map[String, String]
-           ): StreamingQuery = {
+      df: DataFrame,
+      mode: String,
+      format: String,
+      options: Map[String, String]
+  ): StreamingQuery = {
     df.writeStream.outputMode(mode).format(format).options(options).start()
   }
 }

src/main/com/codely/lesson_02_tests_in_spark/z_practical_exercise/app/AvgSpendingApp.scala

@@ -5,7 +5,6 @@ import com.codely.lesson_02_tests_in_spark.z_practical_exercise.job.AvgSpendingJ
 import com.codely.lesson_02_tests_in_spark.z_practical_exercise.service.{Reader, StreamWriter}
 import org.apache.spark.sql.SparkSession
 
-
 object AvgSpendingApp extends App {
 
   private val context = AppConfig.load(args)
@@ -16,7 +15,7 @@ object AvgSpendingApp extends App {
     .enableHiveSupport()
     .getOrCreate()
 
-  private val reader = Reader()
+  private val reader      = Reader()
   private val deltaWriter = StreamWriter()
 
   val job = AvgSpendingJob(context, reader, deltaWriter)

src/main/com/codely/lesson_02_tests_in_spark/z_practical_exercise/config/CmdArgs.scala

@@ -1,3 +1,3 @@
 package com.codely.lesson_02_tests_in_spark.z_practical_exercise.config
 
-case class CmdArgs(configFile: Option[String] = None)
+case class CmdArgs(configFile: Option[String] = None)

src/main/com/codely/lesson_02_tests_in_spark/z_practical_exercise/service/StreamWriter.scala

@@ -5,11 +5,11 @@ import org.apache.spark.sql.DataFrame
 
 case class StreamWriter() {
   def write(
-             df: DataFrame,
-             mode: String,
-             format: String,
-             options: Map[String, String]
-           ): StreamingQuery = {
+      df: DataFrame,
+      mode: String,
+      format: String,
+      options: Map[String, String]
+  ): StreamingQuery = {
     df.writeStream.outputMode(mode).format(format).options(options).start()
   }
 }

src/main/com/codely/lesson_03_basics_spark_execution_model/video_01__how_spark_works/HowSparkWorks.scala

@@ -7,7 +7,7 @@ object HowSparkWorks extends App {
     .appName("Spark Example")
     .getOrCreate()
 
-  val sc      = spark.sparkContext
+  val sc = spark.sparkContext
 
   val numbers = sc.parallelize(1 to 1000)
   numbers.count()

src/main/com/codely/lesson_05_monitoring_and_optimizations/z_practical_exercise/SparkApp.scala

@@ -4,7 +4,7 @@ trait SparkApp extends App {
   val spark = org.apache.spark.sql.SparkSession.builder
     .master("local[*]")
     .appName("Spark Broadcast Join")
-    //.config("spark.sql.autoBroadcastJoinThreshold", -1) descomentar primera vez
-    //.config("spark.sql.adaptive.enabled", "false")  descomentar primera vez
+    // .config("spark.sql.autoBroadcastJoinThreshold", -1) descomentar primera vez
+    // .config("spark.sql.adaptive.enabled", "false")  descomentar primera vez
     .getOrCreate()
 }

src/test/com/codely/spark_best_practices_and_deploy/lesson_02_tests_in_spark/video_02__unit_testing/scala/job/SparkTestHelper.scala

@@ -10,10 +10,7 @@ import java.io.File
 import java.nio.file.Files
 import scala.reflect.io.Directory
 
-trait SparkTestHelper
-    extends AnyFlatSpec
-    with BeforeAndAfterEach
-    with BeforeAndAfterAll {
+trait SparkTestHelper extends AnyFlatSpec with BeforeAndAfterEach with BeforeAndAfterAll {
 
   private val sparkSession = SparkSession
     .builder()

src/test/com/codely/spark_best_practices_and_deploy/lesson_02_tests_in_spark/z_practical_exercise/job/SparkTestHelper.scala

@@ -10,10 +10,7 @@ import java.io.File
 import java.nio.file.Files
 import scala.reflect.io.Directory
 
-trait SparkTestHelper
-    extends AnyFlatSpec
-    with BeforeAndAfterEach
-    with BeforeAndAfterAll {
+trait SparkTestHelper extends AnyFlatSpec with BeforeAndAfterEach with BeforeAndAfterAll {
 
   private val sparkSession = SparkSession
     .builder()