Add new content

docker/docker-compose.yml

@@ -38,9 +38,8 @@ services:
       - spark-master
     environment:
       - SPARK_MASTER=spark://spark-master:7077
-      - SPARK_WORKER_CORES=2
-      - SPARK_WORKER_MEMORY=2G
-      - SPARK_DRIVER_MEMORY=2G
+      - SPARK_WORKER_CORES=3
+      - SPARK_WORKER_MEMORY=3G
       - SPARK_EXECUTOR_MEMORY=1G
       - SPARK_WORKLOAD=worker
       - SPARK_LOCAL_IP=172.19.0.2
@@ -63,9 +62,8 @@ services:
       - spark-master
     environment:
       - SPARK_MASTER=spark://spark-master:7077
-      - SPARK_WORKER_CORES=2
-      - SPARK_WORKER_MEMORY=2G
-      - SPARK_DRIVER_MEMORY=2G
+      - SPARK_WORKER_CORES=3
+      - SPARK_WORKER_MEMORY=3G
       - SPARK_EXECUTOR_MEMORY=1G
       - SPARK_WORKLOAD=worker
       - SPARK_LOCAL_IP=172.19.0.3

project/Settings.scala

@@ -6,7 +6,7 @@ object Settings {
   val settings = Seq(
     name                 := "spark-best_practises_and_deploy-course",
     version              := "0.1.0-SNAPSHOT",
-    scalaVersion         := "2.12.12",
+    scalaVersion         := "2.12.19",
     organization         := "com.codely",
     organizationName     := "com.codely, Inc.",
     organizationHomepage := Some(url("https://com.codely")),

src/main/com/codely/lesson_02_tests_in_spark/video_02__unit_testing/config/CmdArgs.scala

@@ -1,3 +1,3 @@
 package com.codely.lesson_02_tests_in_spark.video_02__unit_testing.config
 
-case class CmdArgs(configFile: Option[String] = None)
+case class CmdArgs(configFile: Option[String] = None)

src/main/com/codely/lesson_02_tests_in_spark/video_02__unit_testing/job/AvgSpendingJob.scala

@@ -4,15 +4,14 @@ import com.codely.lesson_02_tests_in_spark.video_02__unit_testing.config.AppCont
 import com.codely.lesson_02_tests_in_spark.video_02__unit_testing.extensions.DataFrameExtensions._
 import com.codely.lesson_02_tests_in_spark.video_02__unit_testing.service.{StreamReader, StreamWriter}
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.streaming.StreamingQuery
 
 case class AvgSpendingJob(
     context: AppContext,
     streamReader: StreamReader,
     streamWriter: StreamWriter
 )(implicit spark: SparkSession) {
 
-  def run(): StreamingQuery = {
+  def run() = {
 
     val data = streamReader.read(context.source.format, context.source.options)
 
@@ -21,7 +20,7 @@ case class AvgSpendingJob(
 
     val query = streamWriter.write(avgSpendingPerUserDF, context.sink.mode, context.sink.format, context.sink.options)
 
-    query
+    query.awaitTermination()
   }
 
 }

src/main/com/codely/lesson_02_tests_in_spark/video_02__unit_testing/service/StreamWriter.scala

@@ -5,11 +5,11 @@ import org.apache.spark.sql.DataFrame
 
 case class StreamWriter() {
   def write(
-             df: DataFrame,
-             mode: String,
-             format: String,
-             options: Map[String, String]
-           ): StreamingQuery = {
+      df: DataFrame,
+      mode: String,
+      format: String,
+      options: Map[String, String]
+  ): StreamingQuery = {
     df.writeStream.outputMode(mode).format(format).options(options).start()
   }
 }

src/main/com/codely/lesson_02_tests_in_spark/z_practical_exercise/app/AvgSpendingApp.scala

@@ -5,7 +5,6 @@ import com.codely.lesson_02_tests_in_spark.z_practical_exercise.job.AvgSpendingJ
 import com.codely.lesson_02_tests_in_spark.z_practical_exercise.service.{Reader, StreamWriter}
 import org.apache.spark.sql.SparkSession
 
-
 object AvgSpendingApp extends App {
 
   private val context = AppConfig.load(args)
@@ -16,7 +15,7 @@ object AvgSpendingApp extends App {
     .enableHiveSupport()
     .getOrCreate()
 
-  private val reader = Reader()
+  private val reader      = Reader()
   private val deltaWriter = StreamWriter()
 
   val job = AvgSpendingJob(context, reader, deltaWriter)

src/main/com/codely/lesson_02_tests_in_spark/z_practical_exercise/config/CmdArgs.scala

@@ -1,3 +1,3 @@
 package com.codely.lesson_02_tests_in_spark.z_practical_exercise.config
 
-case class CmdArgs(configFile: Option[String] = None)
+case class CmdArgs(configFile: Option[String] = None)

src/main/com/codely/lesson_02_tests_in_spark/z_practical_exercise/service/StreamWriter.scala

@@ -5,11 +5,11 @@ import org.apache.spark.sql.DataFrame
 
 case class StreamWriter() {
   def write(
-             df: DataFrame,
-             mode: String,
-             format: String,
-             options: Map[String, String]
-           ): StreamingQuery = {
+      df: DataFrame,
+      mode: String,
+      format: String,
+      options: Map[String, String]
+  ): StreamingQuery = {
     df.writeStream.outputMode(mode).format(format).options(options).start()
   }
 }

src/main/com/codely/lesson_03_basics_spark_execution_model/video_01__how_spark_works/HowSparkWorks.scala

@@ -2,20 +2,16 @@ package com.codely.lesson_03_basics_spark_execution_model.video_01__how_spark_wo
 
 object HowSparkWorks extends App {
 
-  // 1. docker exec -it spark-ecosystem-cluster-spark-master-1 bash
-  // 2. ./bin/spark-shell --master spark://spark-master:7077 --total-executor-cores 2 --executor-memory 1024m
-
   val spark = org.apache.spark.sql.SparkSession.builder
     .master("local")
     .appName("Spark Example")
     .getOrCreate()
 
-  val sc      = spark.sparkContext
+  val sc = spark.sparkContext
+
   val numbers = sc.parallelize(1 to 1000)
   numbers.count()
 
-  // localhost:4040
-
   val doubledNumbers = numbers.map(_ * 2)
   doubledNumbers.count()
 

src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/1-export-datasource.sh → src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/1-import-datasource.sh

@@ -1,5 +1,5 @@
 curl -X POST http://localhost:3000/api/datasources \
 -H "Content-Type: application/json" \
--H "Authorization: Bearer glsa_OvsNen9bJllcjVPQ1VSfCouJcukdcVcL_4a16a12d" \
+-H "Authorization: Bearer glsa_nUPnN9yJcw2JBMI20B06Rdj7HqZahreX_57a07450" \
 --data-binary "@my_datasource.json"
 

src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/2-export-dashboard.sh → src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/2-import-dashboard.sh

@@ -1,5 +1,5 @@
 curl -X POST http://localhost:3000/api/dashboards/db \
 -H "Content-Type: application/json" \
--H "Authorization: Bearer glsa_OvsNen9bJllcjVPQ1VSfCouJcukdcVcL_4a16a12d" \
+-H "Authorization: Bearer glsa_nUPnN9yJcw2JBMI20B06Rdj7HqZahreX_57a07450" \
 --data-binary "@dashboard.json"
 

src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/1.import.sh → src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/3-export.sh

@@ -6,11 +6,11 @@ curl -X GET \
 
  curl -X GET \
   -H "Content-Type: application/json" \
-  -H "Authorization: Bearer glsa_OvsNen9bJllcjVPQ1VSfCouJcukdcVcL_4a16a12d" \
+  -H "Authorization: Bearer glsa_nUPnN9yJcw2JBMI20B06Rdj7HqZahreX_57a07450" \
   http://localhost:3000/api/datasources/1 \
   -o my_datasource.json
 
   curl -X GET "http://localhost:3000/api/dashboards/uid/bdqiuxl7bh98gc" \
--H "Authorization: Bearer glsa_OvsNen9bJllcjVPQ1VSfCouJcukdcVcL_4a16a12d" \
+-H "Authorization: Bearer glsa_nUPnN9yJcw2JBMI20B06Rdj7HqZahreX_57a07450" \
   -H "Content-Type: application/json" \
-  -o dashboard.json
+  -o dashboard2.json