diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index 47ae915..1f58faf 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -38,9 +38,8 @@ services:
       - spark-master
     environment:
       - SPARK_MASTER=spark://spark-master:7077
-      - SPARK_WORKER_CORES=2
-      - SPARK_WORKER_MEMORY=2G
-      - SPARK_DRIVER_MEMORY=2G
+      - SPARK_WORKER_CORES=3
+      - SPARK_WORKER_MEMORY=3G
       - SPARK_EXECUTOR_MEMORY=1G
       - SPARK_WORKLOAD=worker
       - SPARK_LOCAL_IP=172.19.0.2
@@ -63,9 +62,8 @@ services:
       - spark-master
     environment:
       - SPARK_MASTER=spark://spark-master:7077
-      - SPARK_WORKER_CORES=2
-      - SPARK_WORKER_MEMORY=2G
-      - SPARK_DRIVER_MEMORY=2G
+      - SPARK_WORKER_CORES=3
+      - SPARK_WORKER_MEMORY=3G
       - SPARK_EXECUTOR_MEMORY=1G
       - SPARK_WORKLOAD=worker
       - SPARK_LOCAL_IP=172.19.0.3
diff --git a/project/Settings.scala b/project/Settings.scala
index b91edf1..2716e51 100644
--- a/project/Settings.scala
+++ b/project/Settings.scala
@@ -6,7 +6,7 @@ object Settings {
   val settings = Seq(
     name                 := "spark-best_practises_and_deploy-course",
     version              := "0.1.0-SNAPSHOT",
-    scalaVersion         := "2.12.12",
+    scalaVersion         := "2.12.19",
     organization         := "com.codely",
     organizationName     := "com.codely, Inc.",
     organizationHomepage := Some(url("https://com.codely")),
diff --git a/src/main/com/codely/lesson_02_tests_in_spark/video_02__unit_testing/config/CmdArgs.scala b/src/main/com/codely/lesson_02_tests_in_spark/video_02__unit_testing/config/CmdArgs.scala
index cecbf7e..0d3c7c2 100644
--- a/src/main/com/codely/lesson_02_tests_in_spark/video_02__unit_testing/config/CmdArgs.scala
+++ b/src/main/com/codely/lesson_02_tests_in_spark/video_02__unit_testing/config/CmdArgs.scala
@@ -1,3 +1,3 @@
 package com.codely.lesson_02_tests_in_spark.video_02__unit_testing.config
 
-case class CmdArgs(configFile: Option[String] = None)
\ No newline at end of file
+case class CmdArgs(configFile: Option[String] = None)
diff --git a/src/main/com/codely/lesson_02_tests_in_spark/video_02__unit_testing/job/AvgSpendingJob.scala b/src/main/com/codely/lesson_02_tests_in_spark/video_02__unit_testing/job/AvgSpendingJob.scala
index 755b081..474acca 100644
--- a/src/main/com/codely/lesson_02_tests_in_spark/video_02__unit_testing/job/AvgSpendingJob.scala
+++ b/src/main/com/codely/lesson_02_tests_in_spark/video_02__unit_testing/job/AvgSpendingJob.scala
@@ -4,7 +4,6 @@ import com.codely.lesson_02_tests_in_spark.video_02__unit_testing.config.AppCont
 import com.codely.lesson_02_tests_in_spark.video_02__unit_testing.extensions.DataFrameExtensions._
 import com.codely.lesson_02_tests_in_spark.video_02__unit_testing.service.{StreamReader, StreamWriter}
 import org.apache.spark.sql.SparkSession
-import org.apache.spark.sql.streaming.StreamingQuery
 
 case class AvgSpendingJob(
     context: AppContext,
@@ -12,7 +11,7 @@ case class AvgSpendingJob(
     streamWriter: StreamWriter
 )(implicit spark: SparkSession) {
 
-  def run(): StreamingQuery = {
+  def run() = {
 
     val data = streamReader.read(context.source.format, context.source.options)
 
@@ -21,7 +20,7 @@ case class AvgSpendingJob(
 
     val query = streamWriter.write(avgSpendingPerUserDF, context.sink.mode, context.sink.format, context.sink.options)
 
-    query
+    query.awaitTermination()
   }
 
 }
diff --git a/src/main/com/codely/lesson_02_tests_in_spark/video_02__unit_testing/service/StreamWriter.scala b/src/main/com/codely/lesson_02_tests_in_spark/video_02__unit_testing/service/StreamWriter.scala
index 5399563..ef23983 100644
--- a/src/main/com/codely/lesson_02_tests_in_spark/video_02__unit_testing/service/StreamWriter.scala
+++ b/src/main/com/codely/lesson_02_tests_in_spark/video_02__unit_testing/service/StreamWriter.scala
@@ -5,11 +5,11 @@ import org.apache.spark.sql.DataFrame
 
 case class StreamWriter() {
   def write(
-             df: DataFrame,
-             mode: String,
-             format: String,
-             options: Map[String, String]
-           ): StreamingQuery = {
+      df: DataFrame,
+      mode: String,
+      format: String,
+      options: Map[String, String]
+  ): StreamingQuery = {
     df.writeStream.outputMode(mode).format(format).options(options).start()
   }
 }
diff --git a/src/main/com/codely/lesson_02_tests_in_spark/z_practical_exercise/app/AvgSpendingApp.scala b/src/main/com/codely/lesson_02_tests_in_spark/z_practical_exercise/app/AvgSpendingApp.scala
index 2c2e1f6..9201790 100644
--- a/src/main/com/codely/lesson_02_tests_in_spark/z_practical_exercise/app/AvgSpendingApp.scala
+++ b/src/main/com/codely/lesson_02_tests_in_spark/z_practical_exercise/app/AvgSpendingApp.scala
@@ -5,7 +5,6 @@ import com.codely.lesson_02_tests_in_spark.z_practical_exercise.job.AvgSpendingJ
 import com.codely.lesson_02_tests_in_spark.z_practical_exercise.service.{Reader, StreamWriter}
 import org.apache.spark.sql.SparkSession
 
-
 object AvgSpendingApp extends App {
 
   private val context = AppConfig.load(args)
@@ -16,7 +15,7 @@ object AvgSpendingApp extends App {
     .enableHiveSupport()
     .getOrCreate()
 
-  private val reader = Reader()
+  private val reader      = Reader()
   private val deltaWriter = StreamWriter()
 
   val job = AvgSpendingJob(context, reader, deltaWriter)
diff --git a/src/main/com/codely/lesson_02_tests_in_spark/z_practical_exercise/config/CmdArgs.scala b/src/main/com/codely/lesson_02_tests_in_spark/z_practical_exercise/config/CmdArgs.scala
index 6bfbbac..ff3c211 100644
--- a/src/main/com/codely/lesson_02_tests_in_spark/z_practical_exercise/config/CmdArgs.scala
+++ b/src/main/com/codely/lesson_02_tests_in_spark/z_practical_exercise/config/CmdArgs.scala
@@ -1,3 +1,3 @@
 package com.codely.lesson_02_tests_in_spark.z_practical_exercise.config
 
-case class CmdArgs(configFile: Option[String] = None)
\ No newline at end of file
+case class CmdArgs(configFile: Option[String] = None)
diff --git a/src/main/com/codely/lesson_02_tests_in_spark/z_practical_exercise/service/StreamWriter.scala b/src/main/com/codely/lesson_02_tests_in_spark/z_practical_exercise/service/StreamWriter.scala
index f877f16..39a07d9 100644
--- a/src/main/com/codely/lesson_02_tests_in_spark/z_practical_exercise/service/StreamWriter.scala
+++ b/src/main/com/codely/lesson_02_tests_in_spark/z_practical_exercise/service/StreamWriter.scala
@@ -5,11 +5,11 @@ import org.apache.spark.sql.DataFrame
 
 case class StreamWriter() {
   def write(
-             df: DataFrame,
-             mode: String,
-             format: String,
-             options: Map[String, String]
-           ): StreamingQuery = {
+      df: DataFrame,
+      mode: String,
+      format: String,
+      options: Map[String, String]
+  ): StreamingQuery = {
     df.writeStream.outputMode(mode).format(format).options(options).start()
   }
 }
diff --git a/src/main/com/codely/lesson_03_basics_spark_execution_model/video_01__how_spark_works/HowSparkWorks.scala b/src/main/com/codely/lesson_03_basics_spark_execution_model/video_01__how_spark_works/HowSparkWorks.scala
index e7639fd..9d9fb80 100644
--- a/src/main/com/codely/lesson_03_basics_spark_execution_model/video_01__how_spark_works/HowSparkWorks.scala
+++ b/src/main/com/codely/lesson_03_basics_spark_execution_model/video_01__how_spark_works/HowSparkWorks.scala
@@ -2,20 +2,16 @@ package com.codely.lesson_03_basics_spark_execution_model.video_01__how_spark_wo
 
 object HowSparkWorks extends App {
 
-  // 1. docker exec -it spark-ecosystem-cluster-spark-master-1 bash
-  // 2. ./bin/spark-shell --master spark://spark-master:7077 --total-executor-cores 2 --executor-memory 1024m
-
   val spark = org.apache.spark.sql.SparkSession.builder
     .master("local")
     .appName("Spark Example")
     .getOrCreate()
 
-  val sc      = spark.sparkContext
+  val sc = spark.sparkContext
+
   val numbers = sc.parallelize(1 to 1000)
   numbers.count()
 
-  // localhost:4040
-
   val doubledNumbers = numbers.map(_ * 2)
   doubledNumbers.count()
 
diff --git a/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/1-export-datasource.sh b/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/1-import-datasource.sh
similarity index 62%
rename from src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/1-export-datasource.sh
rename to src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/1-import-datasource.sh
index 0d39153..dee619d 100644
--- a/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/1-export-datasource.sh
+++ b/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/1-import-datasource.sh
@@ -1,5 +1,5 @@
 curl -X POST http://localhost:3000/api/datasources \
 -H "Content-Type: application/json" \
--H "Authorization: Bearer glsa_OvsNen9bJllcjVPQ1VSfCouJcukdcVcL_4a16a12d" \
+-H "Authorization: Bearer glsa_nUPnN9yJcw2JBMI20B06Rdj7HqZahreX_57a07450" \
 --data-binary "@my_datasource.json"
 
diff --git a/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/2-export-dashboard.sh b/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/2-import-dashboard.sh
similarity index 62%
rename from src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/2-export-dashboard.sh
rename to src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/2-import-dashboard.sh
index b76b5f1..8e29207 100644
--- a/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/2-export-dashboard.sh
+++ b/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/2-import-dashboard.sh
@@ -1,5 +1,5 @@
 curl -X POST http://localhost:3000/api/dashboards/db \
 -H "Content-Type: application/json" \
--H "Authorization: Bearer glsa_OvsNen9bJllcjVPQ1VSfCouJcukdcVcL_4a16a12d" \
+-H "Authorization: Bearer glsa_nUPnN9yJcw2JBMI20B06Rdj7HqZahreX_57a07450" \
 --data-binary "@dashboard.json"
 
diff --git a/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/1.import.sh b/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/3-export.sh
similarity index 70%
rename from src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/1.import.sh
rename to src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/3-export.sh
index 9416e65..5586665 100644
--- a/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/1.import.sh
+++ b/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/3-export.sh
@@ -6,11 +6,11 @@ curl -X GET \
 
  curl -X GET \
   -H "Content-Type: application/json" \
-  -H "Authorization: Bearer glsa_OvsNen9bJllcjVPQ1VSfCouJcukdcVcL_4a16a12d" \
+  -H "Authorization: Bearer glsa_nUPnN9yJcw2JBMI20B06Rdj7HqZahreX_57a07450" \
   http://localhost:3000/api/datasources/1 \
   -o my_datasource.json
 
   curl -X GET "http://localhost:3000/api/dashboards/uid/bdqiuxl7bh98gc" \
--H "Authorization: Bearer glsa_OvsNen9bJllcjVPQ1VSfCouJcukdcVcL_4a16a12d" \
+-H "Authorization: Bearer glsa_nUPnN9yJcw2JBMI20B06Rdj7HqZahreX_57a07450" \
   -H "Content-Type: application/json" \
-  -o dashboard.json
+  -o dashboard2.json
diff --git a/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/dashboard.json b/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/dashboard.json
index dd50c81..c95d17e 100644
--- a/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/dashboard.json
+++ b/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/dashboard.json
@@ -1 +1 @@
-{"meta":{"type":"db","canSave":true,"canEdit":true,"canAdmin":true,"canStar":true,"canDelete":true,"slug":"spark-cluster-metrics-demo","url":"/d/bdqiuxl7bh98gc/spark-cluster-metrics-demo","expires":"0001-01-01T00:00:00Z","created":"2024-07-28T17:16:17Z","updated":"2024-07-28T17:16:17Z","updatedBy":"admin","createdBy":"admin","version":1,"hasAcl":false,"isFolder":false,"folderId":0,"folderUid":"","folderTitle":"General","folderUrl":"","provisioned":false,"provisionedExternalId":"","annotationsPermissions":{"dashboard":{"canAdd":true,"canEdit":true,"canDelete":true},"organization":{"canAdd":true,"canEdit":true,"canDelete":true}}},"dashboard":{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations \u0026 Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":1,"links":[],"panels":[{"datasource":{"type":"prometheus","uid":"edt4uxwwo5kowa"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":4,"x":0,"y":0},"id":4,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"edt4uxwwo5kowa"},"disableTextWrap":false,"editorMode":"builder","expr":"metrics_master_workers_Value","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"legendFormat":"__auto","range":true,"refId":"A","useBackend":false}],"title":"Total Workers","type":"stat"},{"datasource":{"type":"prometheus","uid":"edt4uxwwo5kowa"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":4,"x":4,"y":0},"id":6,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"edt4uxwwo5kowa"},"disableTextWrap":false,"editorMode":"builder","expr":"metrics_master_aliveWorkers_Value","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"legendFormat":"__auto","range":true,"refId":"A","useBackend":false}],"title":"Workers alive","type":"stat"},{"datasource":{"type":"prometheus","uid":"edt4uxwwo5kowa"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":7,"x":8,"y":0},"id":1,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto"},"pluginVersion":"11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"edt4uxwwo5kowa"},"disableTextWrap":false,"editorMode":"builder","expr":"sum by(instance) (metrics_worker_memUsed_MB_Number)","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"legendFormat":"__auto","range":true,"refId":"A","useBackend":false}],"title":"Workers memory used","type":"gauge"},{"datasource":{"type":"prometheus","uid":"edt4uxwwo5kowa"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":8,"x":15,"y":0},"id":2,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto"},"pluginVersion":"11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"edt4uxwwo5kowa"},"disableTextWrap":false,"editorMode":"builder","expr":"sum by(instance) (metrics_worker_coresUsed_Value)","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"legendFormat":"__auto","range":true,"refId":"A","useBackend":false}],"title":"Workers cores used","type":"gauge"},{"datasource":{"type":"prometheus","uid":"edt4uxwwo5kowa"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":4,"x":0,"y":7},"id":5,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"edt4uxwwo5kowa"},"disableTextWrap":false,"editorMode":"builder","expr":"metrics_master_waitingApps_Value","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"legendFormat":"__auto","range":true,"refId":"A","useBackend":false}],"title":"Waiting apps","type":"stat"},{"datasource":{"type":"prometheus","uid":"edt4uxwwo5kowa"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":4,"x":4,"y":7},"id":8,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"edt4uxwwo5kowa"},"disableTextWrap":false,"editorMode":"builder","expr":"metrics_master_apps_Number","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"legendFormat":"__auto","range":true,"refId":"A","useBackend":false}],"title":"Total apps","type":"stat"},{"datasource":{"type":"prometheus","uid":"edt4uxwwo5kowa"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[{"matcher":{"id":"byName","options":"app-20240702165007-0020"},"properties":[]}]},"gridPos":{"h":7,"w":15,"x":8,"y":7},"id":7,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"edt4uxwwo5kowa"},"disableTextWrap":false,"editorMode":"builder","expr":"sum by(application_id) (metrics_executor_failedTasks_total)","format":"time_series","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"legendFormat":"failed","range":true,"refId":"A","useBackend":false},{"datasource":{"type":"prometheus","uid":"edt4uxwwo5kowa"},"disableTextWrap":false,"editorMode":"builder","expr":"sum by(application_id) (metrics_executor_activeTasks)","fullMetaSearch":false,"hide":false,"includeNullMetadata":true,"instant":false,"legendFormat":"active","range":true,"refId":"B","useBackend":false},{"datasource":{"type":"prometheus","uid":"edt4uxwwo5kowa"},"disableTextWrap":false,"editorMode":"builder","expr":"sum by(application_id) (metrics_executor_completedTasks_total)","fullMetaSearch":false,"hide":false,"includeNullMetadata":true,"instant":false,"legendFormat":"completed","range":true,"refId":"C","useBackend":false}],"title":"Tasks completed vs failed","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"edt4uxwwo5kowa"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":10,"w":23,"x":0,"y":14},"id":9,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"edt4uxwwo5kowa"},"disableTextWrap":false,"editorMode":"builder","expr":"sum(metrics_worker_memFree_MB_Value)","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"legendFormat":"__auto","range":true,"refId":"A","useBackend":false}],"title":"Panel Title","type":"timeseries"}],"schemaVersion":39,"tags":[],"templating":{"list":[]},"time":{"from":"now-15m","to":"now"},"timepicker":{},"timezone":"browser","title":"Spark Cluster metrics Demo","uid":"bdqiuxl7bh98gc","version":1,"weekStart":""}}
\ No newline at end of file
+{"meta":{"type":"db","canSave":true,"canEdit":true,"canAdmin":true,"canStar":true,"canDelete":true,"slug":"spark-cluster-metrics-demo","url":"/d/bdqiuxl7bh98gc/spark-cluster-metrics-demo","expires":"0001-01-01T00:00:00Z","created":"2024-07-31T15:58:10Z","updated":"2024-07-31T15:58:10Z","updatedBy":"admin","createdBy":"admin","version":1,"hasAcl":false,"isFolder":false,"folderId":0,"folderUid":"","folderTitle":"General","folderUrl":"","provisioned":false,"provisionedExternalId":"","annotationsPermissions":{"dashboard":{"canAdd":true,"canEdit":true,"canDelete":true},"organization":{"canAdd":true,"canEdit":true,"canDelete":true}}},"dashboard":{"annotations":{"list":[{"builtIn":1,"datasource":{"type":"grafana","uid":"-- Grafana --"},"enable":true,"hide":true,"iconColor":"rgba(0, 211, 255, 1)","name":"Annotations \u0026 Alerts","type":"dashboard"}]},"editable":true,"fiscalYearStartMonth":0,"graphTooltip":0,"id":2,"links":[],"panels":[{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":0},"id":14,"panels":[{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":5,"w":4,"x":0,"y":9},"id":10,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto"},"pluginVersion":"11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"disableTextWrap":false,"editorMode":"builder","expr":"sum(metrics_worker_coresFree_Value)","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"legendFormat":"__auto","range":true,"refId":"A","useBackend":false}],"title":"Free cores","type":"gauge"},{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":5,"w":4,"x":4,"y":9},"id":11,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto"},"pluginVersion":"11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"disableTextWrap":false,"editorMode":"builder","expr":"sum(metrics_worker_memFree_MB_Value)","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"legendFormat":"__auto","range":true,"refId":"A","useBackend":false}],"title":"Free memory","type":"gauge"},{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":5,"w":16,"x":8,"y":9},"id":9,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"disableTextWrap":false,"editorMode":"builder","expr":"sum(metrics_worker_memFree_MB_Value)","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"legendFormat":"__auto","range":true,"refId":"A","useBackend":false}],"title":"Free memory in cluster","type":"timeseries"}],"title":"Cluster general","type":"row"},{"collapsed":true,"gridPos":{"h":1,"w":24,"x":0,"y":1},"id":12,"panels":[{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":4,"x":0,"y":10},"id":4,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"disableTextWrap":false,"editorMode":"builder","expr":"metrics_master_workers_Value","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"legendFormat":"__auto","range":true,"refId":"A","useBackend":false}],"title":"Total Workers","type":"stat"},{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":4,"x":4,"y":10},"id":6,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"disableTextWrap":false,"editorMode":"builder","expr":"metrics_master_aliveWorkers_Value","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"legendFormat":"__auto","range":true,"refId":"A","useBackend":false}],"title":"Workers alive","type":"stat"},{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":16,"x":8,"y":10},"id":18,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"disableTextWrap":false,"editorMode":"builder","expr":"sum(metrics_executor_activeTasks)","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"legendFormat":"__auto","range":true,"refId":"A","useBackend":false},{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"disableTextWrap":false,"editorMode":"builder","expr":"sum(metrics_executor_failedTasks_total)","fullMetaSearch":false,"hide":false,"includeNullMetadata":true,"instant":false,"legendFormat":"__auto","range":true,"refId":"B","useBackend":false}],"title":"Tasks","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":8,"x":0,"y":17},"id":2,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto"},"pluginVersion":"11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"disableTextWrap":false,"editorMode":"builder","expr":"sum by(instance) (metrics_worker_coresUsed_Value)","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"legendFormat":"__auto","range":true,"refId":"A","useBackend":false}],"title":"Workers cores used","type":"gauge"},{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":7,"x":8,"y":17},"id":1,"options":{"minVizHeight":75,"minVizWidth":75,"orientation":"auto","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showThresholdLabels":false,"showThresholdMarkers":true,"sizing":"auto"},"pluginVersion":"11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"disableTextWrap":false,"editorMode":"builder","expr":"sum by(instance) (metrics_worker_memUsed_MB_Number)","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"legendFormat":"__auto","range":true,"refId":"A","useBackend":false}],"title":"Workers memory used","type":"gauge"}],"title":"Workers","type":"row"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":2},"id":13,"panels":[],"title":"Applications","type":"row"},{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":4,"x":0,"y":3},"id":8,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"disableTextWrap":false,"editorMode":"builder","expr":"metrics_master_apps_Number","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"legendFormat":"__auto","range":true,"refId":"A","useBackend":false}],"title":"Total apps","type":"stat"},{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"fieldConfig":{"defaults":{"color":{"mode":"thresholds"},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":4,"x":4,"y":3},"id":5,"options":{"colorMode":"value","graphMode":"area","justifyMode":"auto","orientation":"auto","percentChangeColorMode":"standard","reduceOptions":{"calcs":["lastNotNull"],"fields":"","values":false},"showPercentChange":false,"textMode":"auto","wideLayout":true},"pluginVersion":"11.1.0","targets":[{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"disableTextWrap":false,"editorMode":"builder","expr":"metrics_master_waitingApps_Value","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"legendFormat":"__auto","range":true,"refId":"A","useBackend":false}],"title":"Waiting apps","type":"stat"},{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":7,"w":16,"x":8,"y":3},"id":19,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"disableTextWrap":false,"editorMode":"builder","expr":"sum by(application_name) (metrics_executor_totalShuffleWrite_bytes_total)","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"legendFormat":"__auto","range":true,"refId":"A","useBackend":false}],"title":"Shuffle write","type":"timeseries"},{"collapsed":false,"gridPos":{"h":1,"w":24,"x":0,"y":10},"id":16,"panels":[],"title":"Hive Metastore","type":"row"},{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":11},"id":15,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"disableTextWrap":false,"editorMode":"builder","expr":"metrics_app_20240728171930_0004_driver_HiveExternalCatalog_partitionsFetched_Count","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"legendFormat":"__auto","range":true,"refId":"A","useBackend":false}],"title":"Partitions fetched Hive Metastore","type":"timeseries"},{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisBorderShow":false,"axisCenteredZero":false,"axisColorMode":"text","axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":0,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"insertNulls":false,"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{"group":"A","mode":"none"},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[{"color":"green","value":null},{"color":"red","value":80}]}},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":11},"id":17,"options":{"legend":{"calcs":[],"displayMode":"list","placement":"bottom","showLegend":true},"tooltip":{"mode":"single","sort":"none"}},"targets":[{"datasource":{"type":"prometheus","uid":"fdtfel5wbkb28b"},"disableTextWrap":false,"editorMode":"builder","expr":"metrics_HiveExternalCatalog_hiveClientCalls_Count","fullMetaSearch":false,"includeNullMetadata":true,"instant":false,"legendFormat":"__auto","range":true,"refId":"A","useBackend":false}],"title":"Hive Metastore calls","type":"timeseries"}],"schemaVersion":39,"tags":[],"templating":{"list":[]},"time":{"from":"now-1h","to":"now"},"timepicker":{},"timezone":"browser","title":"Spark Cluster metrics Demo","uid":"bdqiuxl7bh98gc","version":1,"weekStart":""}}
\ No newline at end of file
diff --git a/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/my_dashboard.json b/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/my_dashboard.json
index 844deb7..7a166b2 100644
--- a/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/my_dashboard.json
+++ b/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/my_dashboard.json
@@ -34,7 +34,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "edt4uxwwo5kowa"
+            "uid": "fdtfel5wbkb28b"
           },
           "fieldConfig": {
             "defaults": {
@@ -85,7 +85,7 @@
             {
               "datasource": {
                 "type": "prometheus",
-                "uid": "edt4uxwwo5kowa"
+                "uid": "fdtfel5wbkb28b"
               },
               "disableTextWrap": false,
               "editorMode": "builder",
@@ -105,7 +105,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "edt4uxwwo5kowa"
+            "uid": "fdtfel5wbkb28b"
           },
           "fieldConfig": {
             "defaults": {
@@ -156,7 +156,7 @@
             {
               "datasource": {
                 "type": "prometheus",
-                "uid": "edt4uxwwo5kowa"
+                "uid": "fdtfel5wbkb28b"
               },
               "disableTextWrap": false,
               "editorMode": "builder",
@@ -176,7 +176,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "edt4uxwwo5kowa"
+            "uid": "fdtfel5wbkb28b"
           },
           "fieldConfig": {
             "defaults": {
@@ -255,7 +255,7 @@
             {
               "datasource": {
                 "type": "prometheus",
-                "uid": "edt4uxwwo5kowa"
+                "uid": "fdtfel5wbkb28b"
               },
               "disableTextWrap": false,
               "editorMode": "builder",
@@ -289,7 +289,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "edt4uxwwo5kowa"
+            "uid": "fdtfel5wbkb28b"
           },
           "fieldConfig": {
             "defaults": {
@@ -342,7 +342,7 @@
             {
               "datasource": {
                 "type": "prometheus",
-                "uid": "edt4uxwwo5kowa"
+                "uid": "fdtfel5wbkb28b"
               },
               "disableTextWrap": false,
               "editorMode": "builder",
@@ -362,7 +362,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "edt4uxwwo5kowa"
+            "uid": "fdtfel5wbkb28b"
           },
           "fieldConfig": {
             "defaults": {
@@ -415,7 +415,7 @@
             {
               "datasource": {
                 "type": "prometheus",
-                "uid": "edt4uxwwo5kowa"
+                "uid": "fdtfel5wbkb28b"
               },
               "disableTextWrap": false,
               "editorMode": "builder",
@@ -435,7 +435,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "edt4uxwwo5kowa"
+            "uid": "fdtfel5wbkb28b"
           },
           "fieldConfig": {
             "defaults": {
@@ -514,7 +514,7 @@
             {
               "datasource": {
                 "type": "prometheus",
-                "uid": "edt4uxwwo5kowa"
+                "uid": "fdtfel5wbkb28b"
               },
               "disableTextWrap": false,
               "editorMode": "builder",
@@ -530,7 +530,7 @@
             {
               "datasource": {
                 "type": "prometheus",
-                "uid": "edt4uxwwo5kowa"
+                "uid": "fdtfel5wbkb28b"
               },
               "disableTextWrap": false,
               "editorMode": "builder",
@@ -551,7 +551,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "edt4uxwwo5kowa"
+            "uid": "fdtfel5wbkb28b"
           },
           "fieldConfig": {
             "defaults": {
@@ -602,7 +602,7 @@
             {
               "datasource": {
                 "type": "prometheus",
-                "uid": "edt4uxwwo5kowa"
+                "uid": "fdtfel5wbkb28b"
               },
               "disableTextWrap": false,
               "editorMode": "builder",
@@ -622,7 +622,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "edt4uxwwo5kowa"
+            "uid": "fdtfel5wbkb28b"
           },
           "fieldConfig": {
             "defaults": {
@@ -673,7 +673,7 @@
             {
               "datasource": {
                 "type": "prometheus",
-                "uid": "edt4uxwwo5kowa"
+                "uid": "fdtfel5wbkb28b"
               },
               "disableTextWrap": false,
               "editorMode": "builder",
@@ -710,7 +710,7 @@
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "edt4uxwwo5kowa"
+        "uid": "fdtfel5wbkb28b"
       },
       "fieldConfig": {
         "defaults": {
@@ -763,7 +763,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "edt4uxwwo5kowa"
+            "uid": "fdtfel5wbkb28b"
           },
           "disableTextWrap": false,
           "editorMode": "builder",
@@ -783,7 +783,7 @@
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "edt4uxwwo5kowa"
+        "uid": "fdtfel5wbkb28b"
       },
       "fieldConfig": {
         "defaults": {
@@ -836,7 +836,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "edt4uxwwo5kowa"
+            "uid": "fdtfel5wbkb28b"
           },
           "disableTextWrap": false,
           "editorMode": "builder",
@@ -856,7 +856,7 @@
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "edt4uxwwo5kowa"
+        "uid": "fdtfel5wbkb28b"
       },
       "fieldConfig": {
         "defaults": {
@@ -935,7 +935,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "edt4uxwwo5kowa"
+            "uid": "fdtfel5wbkb28b"
           },
           "disableTextWrap": false,
           "editorMode": "builder",
@@ -968,7 +968,7 @@
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "edt4uxwwo5kowa"
+        "uid": "fdtfel5wbkb28b"
       },
       "fieldConfig": {
         "defaults": {
@@ -1047,7 +1047,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "edt4uxwwo5kowa"
+            "uid": "fdtfel5wbkb28b"
           },
           "disableTextWrap": false,
           "editorMode": "builder",
@@ -1067,7 +1067,7 @@
     {
       "datasource": {
         "type": "prometheus",
-        "uid": "edt4uxwwo5kowa"
+        "uid": "fdtfel5wbkb28b"
       },
       "fieldConfig": {
         "defaults": {
@@ -1146,7 +1146,7 @@
         {
           "datasource": {
             "type": "prometheus",
-            "uid": "edt4uxwwo5kowa"
+            "uid": "fdtfel5wbkb28b"
           },
           "disableTextWrap": false,
           "editorMode": "builder",
diff --git a/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/my_datasource.json b/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/my_datasource.json
new file mode 100644
index 0000000..aa4873a
--- /dev/null
+++ b/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_01__monitor_your_app_with_grafana/scripts/my_datasource.json
@@ -0,0 +1 @@
+{"id":1,"uid":"fdtfel5wbkb28b","orgId":1,"name":"prometheus","type":"prometheus","typeLogoUrl":"public/app/plugins/datasource/prometheus/img/prometheus_logo.svg","access":"proxy","url":"http://172.19.0.12:9090","user":"","database":"","basicAuth":false,"basicAuthUser":"","withCredentials":false,"isDefault":true,"jsonData":{"httpMethod":"POST"},"secureJsonFields":{},"version":4,"readOnly":false,"apiVersion":""}
\ No newline at end of file
diff --git a/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_02_broadcast_join/JoinOptimizationApp.scala b/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_02_broadcast_join/JoinOptimizationApp.scala
new file mode 100644
index 0000000..35ee8a1
--- /dev/null
+++ b/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_02_broadcast_join/JoinOptimizationApp.scala
@@ -0,0 +1,36 @@
+package com.codely.lesson_05_monitoring_and_optimizations.video_02_broadcast_join
+
+import org.apache.spark.sql.functions.{broadcast, col}
+
+object JoinOptimizationApp extends SparkApp {
+
+  // ./bin/spark-shell --master spark://spark-master:7077 --driver-memory 4g --executor-memory 1024mb --conf spark.sql.adaptive.enabled=false
+
+  spark.sparkContext.setLogLevel("WARN")
+
+  import spark.implicits._
+
+  spark.sparkContext.setJobGroup("join without optimization", "join without optimization")
+
+  val largeDF = spark
+    .range(0, 10000000L, 3) // 3.3 M
+    .map(i => (i, s"Name$i"))
+    .toDF("id", "name")
+
+  val veryLargeDF = spark
+    .range(0, 100000000L, 2) // 50 M
+    .map(i => (i, s"Other$i"))
+    .toDF("id", "other")
+
+  veryLargeDF.join(largeDF, "id").filter(col("id") === 1).show(false)
+
+  spark.sparkContext.clearJobGroup()
+
+  spark.sparkContext.setJobGroup("join with optimization", "join with optimization")
+
+  veryLargeDF.join(broadcast(largeDF), "id").show()
+
+  spark.sparkContext.clearJobGroup()
+
+  Thread.sleep(1000000)
+}
diff --git a/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_02_join_optimization/SparkApp.scala b/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_02_broadcast_join/SparkApp.scala
similarity index 56%
rename from src/main/com/codely/lesson_05_monitoring_and_optimizations/video_02_join_optimization/SparkApp.scala
rename to src/main/com/codely/lesson_05_monitoring_and_optimizations/video_02_broadcast_join/SparkApp.scala
index 51d1af0..47edfc4 100644
--- a/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_02_join_optimization/SparkApp.scala
+++ b/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_02_broadcast_join/SparkApp.scala
@@ -1,10 +1,9 @@
-package com.codely.lesson_05_monitoring_and_optimizations.video_02_join_optimization
+package com.codely.lesson_05_monitoring_and_optimizations.video_02_broadcast_join
 
 trait SparkApp extends App {
   val spark = org.apache.spark.sql.SparkSession.builder
     .master("local[*]")
     .appName("Spark Broadcast Join")
-    //.config("spark.sql.autoBroadcastJoinThreshold", -1) descomentar primera vez
-    //.config("spark.sql.adaptive.enabled", "false")  descomentar primera vez
+    .config("spark.sql.adaptive.enabled", "false")
     .getOrCreate()
 }
diff --git a/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_02_join_optimization/JoinOptimizationApp.scala b/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_02_join_optimization/JoinOptimizationApp.scala
deleted file mode 100644
index db46713..0000000
--- a/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_02_join_optimization/JoinOptimizationApp.scala
+++ /dev/null
@@ -1,30 +0,0 @@
-package com.codely.lesson_05_monitoring_and_optimizations.video_02_join_optimization
-
-import org.apache.spark.sql.functions.broadcast
-
-object JoinOptimizationApp extends SparkApp {
-
-  spark.sparkContext.setLogLevel("WARN")
-
-  import spark.implicits._
-
-  val dataFrame1 =
-    Seq((1, "Alice", 50), (2, "Bob", 80), (3, "Javi", 99))
-      .toDF("id", "name", "score")
-
-  val largeDataFrame = spark
-    .range(1, 100000000L)
-    .map(i => (i, s"Name$i"))
-    .toDF("id", "other")
-
-  // val result = largeDataFrame.join(dataFrame1, "id")
-  // result.explain()
-  // result.show()
-
-  val result = largeDataFrame.join(broadcast(dataFrame1), "id")
-  result.explain()
-  result.show()
-
-  Thread.sleep(1000000)
-
-}
diff --git a/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_03_skew_join/SkewJoinApp.scala b/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_03_skew_join/SkewJoinApp.scala
new file mode 100644
index 0000000..8d30676
--- /dev/null
+++ b/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_03_skew_join/SkewJoinApp.scala
@@ -0,0 +1,58 @@
+package com.codely.lesson_05_monitoring_and_optimizations.video_03_skew_join
+
+import org.apache.spark.sql.functions.when
+
+object SkewJoinApp extends SparkApp {
+
+  // ./bin/spark-shell --master spark://spark-master:7077  --driver-memory 4g --executor-memory 1024mb --conf spark.sql.autoBroadcastJoinThreshold=-1 --conf spark.sql.adaptive.enabled=false
+
+  spark.sparkContext.setLogLevel("WARN")
+
+  import spark.implicits._
+
+  spark.sparkContext.setJobGroup("skewed data", "skewed data")
+
+  val skewedData = spark
+    .range(0, 10000000) // 10M
+    .withColumn("key", when($"id" < 10, $"id").otherwise(999))
+    .withColumn("value", $"id")
+
+  val uniformData = spark
+    .range(0, 1000000) // 1M
+    .withColumn("key", $"id")
+    .withColumn("value", $"id")
+
+  val joined = skewedData.join(uniformData, "key")
+
+  val res = joined.filter($"key" === 999).count()
+  println(s"Count for skew key (999): $res")
+
+  spark.sparkContext.clearJobGroup()
+
+  spark.sparkContext.setJobGroup("adaptative query execution", "adaptative query execution")
+
+  spark.conf.set("spark.sql.adaptive.enabled", "true")
+
+  import org.apache.spark.sql.functions._
+
+  val skewedDataAQE = spark
+    .range(0, 10000000) // 10M
+    .withColumn("key", when($"id" < 10, $"id").otherwise(999))
+    .withColumn("value", $"id")
+
+  val uniformDataAQE = spark
+    .range(0, 1000000) // 1M
+    .withColumn("key", $"id")
+    .withColumn("value", $"id")
+
+  val joinedAQE = skewedDataAQE.join(uniformDataAQE, "key")
+
+  joinedAQE.explain(true)
+
+  val resAQE = joinedAQE.filter($"key" === 999).count()
+  println(s"Count for skew key (999): $resAQE")
+
+  spark.sparkContext.clearJobGroup()
+
+  Thread.sleep(1000000)
+}
diff --git a/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_03_skew_join/SparkApp.scala b/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_03_skew_join/SparkApp.scala
new file mode 100644
index 0000000..291eac6
--- /dev/null
+++ b/src/main/com/codely/lesson_05_monitoring_and_optimizations/video_03_skew_join/SparkApp.scala
@@ -0,0 +1,9 @@
+package com.codely.lesson_05_monitoring_and_optimizations.video_03_skew_join
+
+trait SparkApp extends App {
+  val spark = org.apache.spark.sql.SparkSession.builder
+    .master("local[*]")
+    .appName("Spark Broadcast Join")
+    .config("spark.sql.adaptive.enabled", "false")
+    .getOrCreate()
+}
diff --git a/src/main/com/codely/lesson_05_monitoring_and_optimizations/z_practical_exercise/SparkApp.scala b/src/main/com/codely/lesson_05_monitoring_and_optimizations/z_practical_exercise/SparkApp.scala
index df75ab3..dc56173 100644
--- a/src/main/com/codely/lesson_05_monitoring_and_optimizations/z_practical_exercise/SparkApp.scala
+++ b/src/main/com/codely/lesson_05_monitoring_and_optimizations/z_practical_exercise/SparkApp.scala
@@ -4,7 +4,7 @@ trait SparkApp extends App {
   val spark = org.apache.spark.sql.SparkSession.builder
     .master("local[*]")
     .appName("Spark Broadcast Join")
-    //.config("spark.sql.autoBroadcastJoinThreshold", -1) descomentar primera vez
-    //.config("spark.sql.adaptive.enabled", "false")  descomentar primera vez
+    // .config("spark.sql.autoBroadcastJoinThreshold", -1) descomentar primera vez
+    // .config("spark.sql.adaptive.enabled", "false")  descomentar primera vez
     .getOrCreate()
 }
diff --git a/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/0-delete-all.sh b/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/0-delete-all.sh
deleted file mode 100644
index 2fc4dfe..0000000
--- a/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/0-delete-all.sh
+++ /dev/null
@@ -1,82 +0,0 @@
-#!/bin/bash
-
-# Detach policies from roles if they exist
-detach_role_policy() {
-  local role_name=$1
-  local policy_arn=$2
-  if aws iam get-role --role-name "$role_name" --profile emr-user > /dev/null 2>&1; then
-    echo "Detaching policy $policy_arn from role $role_name"
-    aws iam detach-role-policy --role-name "$role_name" --policy-arn "$policy_arn" --profile emr-user
-  fi
-}
-
-detach_user_policy() {
-  local user_name=$1
-  local policy_arn=$2
-  if aws iam get-user --user-name "$user_name" --profile emr-user > /dev/null 2>&1; then
-    echo "Detaching policy $policy_arn from user $user_name"
-    aws iam detach-user-policy --user-name "$user_name" --policy-arn "$policy_arn" --profile emr-user
-  fi
-}
-
-# Detach policies from roles
-detach_role_policy EMR_DefaultRole arn:aws:iam::010928190667:policy/EMR_S3_FullAccessPolicy
-detach_role_policy EMR_EC2_DefaultRole arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role
-detach_role_policy EMR_EC2_DefaultRole arn:aws:iam::aws:policy/AmazonS3FullAccess
-
-# Detach policies from user
-detach_user_policy emr-user arn:aws:iam::010928190667:policy/EMR_RunJobFlow_Policy
-detach_user_policy emr-user arn:aws:iam::010928190667:policy/EMR_FullAccess_Policy
-
-# Detach policies from Step Functions role
-detach_role_policy EMR_StepFunctions_Role arn:aws:iam::010928190667:policy/EMR_StepFunctions_Policy
-
-# Delete policies if they exist
-delete_policy() {
-  local policy_arn=$1
-  if aws iam get-policy --policy-arn "$policy_arn" --profile emr-user > /dev/null 2>&1; then
-    echo "Deleting policy $policy_arn"
-    aws iam delete-policy --policy-arn "$policy_arn" --profile emr-user
-  fi
-}
-
-delete_policy arn:aws:iam::010928190667:policy/EMR_RunJobFlow_Policy
-delete_policy arn:aws:iam::010928190667:policy/EMR_FullAccess_Policy
-delete_policy arn:aws:iam::010928190667:policy/EMR_StepFunctions_Policy
-
-# Delete roles if they exist
-delete_role() {
-  local role_name=$1
-  if aws iam get-role --role-name "$role_name" --profile emr-user > /dev/null 2>&1; then
-    echo "Deleting role $role_name"
-    aws iam delete-role --role-name "$role_name" --profile emr-user
-  fi
-}
-
-delete_role EMR_DefaultRole
-delete_role EMR_EC2_DefaultRole
-delete_role EMR_StepFunctions_Role
-
-# Delete instance profile if it exists
-delete_instance_profile() {
-  local profile_name=$1
-  if aws iam get-instance-profile --instance-profile-name "$profile_name" --profile emr-user > /dev/null 2>&1; then
-    echo "Deleting instance profile $profile_name"
-    aws iam remove-role-from-instance-profile --instance-profile-name "$profile_name" --role-name EMR_EC2_DefaultRole --profile emr-user
-    aws iam delete-instance-profile --instance-profile-name "$profile_name" --profile emr-user
-  fi
-}
-
-delete_instance_profile EMR_EC2_DefaultRole
-
-# Delete service-linked role for EMR if it exists
-if aws iam get-role --role-name AWSServiceRoleForEMRCleanup --profile emr-user > /dev/null 2>&1; then
-  echo "Deleting service-linked role AWSServiceRoleForEMRCleanup"
-  aws iam delete-service-linked-role --role-name AWSServiceRoleForEMRCleanup --profile emr-user
-fi
-
-# Delete S3 bucket if it exists
-if aws s3 ls s3://my-bucket-codely --profile emr-user > /dev/null 2>&1; then
-  echo "Deleting S3 bucket my-bucket-codely"
-  aws s3 rb s3://my-bucket-codely --force --profile emr-user
-fi
diff --git a/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/1-setup-resources.sh b/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/1-setup-resources.sh
deleted file mode 100644
index c2a920d..0000000
--- a/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/1-setup-resources.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/bin/bash
-
-# Create S3 bucket
-aws s3 mb s3://my-bucket-codely --profile emr-user
-
-# Create EMR-S3-FullAccessPolicy
-aws iam create-policy --policy-name EMR_FullAccessPolicy --policy-document file://EMR_FullAccessPolicy.json --profile emr-user
-
-# Create EMR_DefaultRole
-aws iam create-role --role-name EMR_DefaultRole --assume-role-policy-document file://EMR_DefaultRoleTrustPolicy.json --profile emr-user
-
-# Attach EMR_FullAccessPolicy to EMR_DefaultRole
-aws iam attach-role-policy --role-name EMR_DefaultRole --policy-arn arn:aws:iam::010928190667:policy/EMR_FullAccessPolicy --profile emr-user
-
-# Create EMR_EC2_DefaultRole
-aws iam create-role --role-name EMR_EC2_DefaultRole --assume-role-policy-document file://EMR_EC2_DefaultRoleTrustPolicy.json --profile emr-user
-
-# Create instance profile and add role
-aws iam create-instance-profile --instance-profile-name EMR_EC2_DefaultRole --profile emr-user
-aws iam add-role-to-instance-profile --instance-profile-name EMR_EC2_DefaultRole --role-name EMR_EC2_DefaultRole --profile emr-user
-
-# Attach policies to EMR_EC2_DefaultRole
-aws iam attach-role-policy --role-name EMR_EC2_DefaultRole --policy-arn arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role --profile emr-user
-aws iam attach-role-policy --role-name EMR_EC2_DefaultRole --policy-arn arn:aws:iam::aws:policy/AmazonS3FullAccess --profile emr-user
-
-# Create Step Functions policy
-aws iam create-policy --policy-name EMR_StepFunctions_Policy --policy-document file://EMR_StepFunctions_Policy.json --profile emr-user
-
-# Create Step Functions role and attach policy
-aws iam create-role --role-name EMR_StepFunctions_Role --assume-role-policy-document file://StepFunctionsTrustPolicy.json --profile emr-user
-aws iam attach-role-policy --role-name EMR_StepFunctions_Role --policy-arn arn:aws:iam::010928190667:policy/EMR_StepFunctions_Policy --profile emr-user
-
-# Create service-linked role for EMR
-aws iam create-service-linked-role --aws-service-name elasticmapreduce.amazonaws.com --description "Role for EMR cleanup tasks" --profile emr-user
-
-# Create EMR cluster using Step Functions
-aws stepfunctions create-state-machine --name "EMR_StepFunctions_Machine" --definition file://state_machine_definition.json --role-arn arn:aws:iam::010928190667:role/EMR_StepFunctions_Role --profile emr-user
-
-# Start execution of Step Functions
-# aws stepfunctions start-execution --state-machine-arn arn:aws:states:eu-west-1:010928190667:stateMachine:EMR_StepFunctions_Machine --profile emr-user
diff --git a/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/scripts/1-setup-resources.sh b/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/scripts/1-setup-resources.sh
new file mode 100644
index 0000000..ac53c6f
--- /dev/null
+++ b/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/scripts/1-setup-resources.sh
@@ -0,0 +1,71 @@
+#!/bin/bash
+
+# Define the profile to be used for AWS CLI commands
+AWS_PROFILE="emr-user"
+
+# Define the bucket name
+S3_BUCKET_NAME="my-bucket-codely"
+
+# Define IAM policies and roles names
+EMR_FULL_ACCESS_POLICY="EMR_FullAccessPolicy"
+EMR_DEFAULT_ROLE="EMR_DefaultRole"
+EMR_EC2_DEFAULT_ROLE="EMR_EC2_DefaultRole"
+EMR_STEP_FUNCTIONS_POLICY="EMR_StepFunctions_Policy"
+EMR_STEP_FUNCTIONS_ROLE="EMR_StepFunctions_Role"
+
+# Define AWS Account ID
+AWS_ACCOUNT_ID="010928190667"
+
+# Create S3 bucket
+echo "Creating S3 bucket: $S3_BUCKET_NAME"
+aws s3 mb s3://$S3_BUCKET_NAME --profile $AWS_PROFILE
+
+# Create EMR-FullAccessPolicy
+echo "Creating IAM policy: $EMR_FULL_ACCESS_POLICY"
+aws iam create-policy --policy-name $EMR_FULL_ACCESS_POLICY --policy-document file://policies/EMR_FullAccessPolicy.json --profile $AWS_PROFILE
+
+# Create EMR_DefaultRole
+echo "Creating IAM role: $EMR_DEFAULT_ROLE"
+aws iam create-role --role-name $EMR_DEFAULT_ROLE --assume-role-policy-document file://policies/EMR_DefaultRoleTrustPolicy.json --profile $AWS_PROFILE
+
+# Attach EMR_FullAccessPolicy to EMR_DefaultRole
+echo "Attaching policy $EMR_FULL_ACCESS_POLICY to role $EMR_DEFAULT_ROLE"
+aws iam attach-role-policy --role-name $EMR_DEFAULT_ROLE --policy-arn arn:aws:iam::$AWS_ACCOUNT_ID:policy/$EMR_FULL_ACCESS_POLICY --profile $AWS_PROFILE
+
+# Create EMR_EC2_DefaultRole
+echo "Creating IAM role: $EMR_EC2_DEFAULT_ROLE"
+aws iam create-role --role-name $EMR_EC2_DEFAULT_ROLE --assume-role-policy-document file://policies/EMR_EC2_DefaultRoleTrustPolicy.json --profile $AWS_PROFILE
+
+# Create instance profile and add role
+echo "Creating instance profile and adding role: $EMR_EC2_DEFAULT_ROLE"
+aws iam create-instance-profile --instance-profile-name $EMR_EC2_DEFAULT_ROLE --profile $AWS_PROFILE
+aws iam add-role-to-instance-profile --instance-profile-name $EMR_EC2_DEFAULT_ROLE --role-name $EMR_EC2_DEFAULT_ROLE --profile $AWS_PROFILE
+
+# Attach policies to EMR_EC2_DefaultRole
+echo "Attaching policies to role $EMR_EC2_DEFAULT_ROLE"
+aws iam attach-role-policy --role-name $EMR_EC2_DEFAULT_ROLE --policy-arn arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role --profile $AWS_PROFILE
+aws iam attach-role-policy --role-name $EMR_EC2_DEFAULT_ROLE --policy-arn arn:aws:iam::aws:policy/AmazonS3FullAccess --profile $AWS_PROFILE
+
+# Create Step Functions policy
+echo "Creating IAM policy: $EMR_STEP_FUNCTIONS_POLICY"
+aws iam create-policy --policy-name $EMR_STEP_FUNCTIONS_POLICY --policy-document file://policies/EMR_StepFunctions_Policy.json --profile $AWS_PROFILE
+
+# Create Step Functions role and attach policy
+echo "Creating IAM role: $EMR_STEP_FUNCTIONS_ROLE"
+aws iam create-role --role-name $EMR_STEP_FUNCTIONS_ROLE --assume-role-policy-document file://policies/StepFunctionsTrustPolicy.json --profile $AWS_PROFILE
+aws iam attach-role-policy --role-name $EMR_STEP_FUNCTIONS_ROLE --policy-arn arn:aws:iam::$AWS_ACCOUNT_ID:policy/$EMR_STEP_FUNCTIONS_POLICY --profile $AWS_PROFILE
+
+# Create service-linked role for EMR
+echo "Creating service-linked role for EMR"
+aws iam create-service-linked-role --aws-service-name elasticmapreduce.amazonaws.com --description "Role for EMR cleanup tasks" --profile $AWS_PROFILE
+
+# Create EMR cluster using Step Functions
+echo "Creating Step Functions state machine for EMR cluster"
+aws stepfunctions create-state-machine --name "EMR_StepFunctions_Machine" --definition file://policies/state_machine_definition.json --role-arn arn:aws:iam::$AWS_ACCOUNT_ID:role/$EMR_STEP_FUNCTIONS_ROLE --profile $AWS_PROFILE
+
+# Start execution of Step Functions (Uncomment to enable execution)
+# echo "Starting execution of Step Functions state machine"
+
+aws stepfunctions start-execution --state-machine-arn arn:aws:states:eu-west-1:010928190667:stateMachine:EMR_StepFunctions_Machine --profile emr-user
+
+echo "Setup complete."
diff --git a/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/scripts/2-cleanup-resources.sh b/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/scripts/2-cleanup-resources.sh
new file mode 100644
index 0000000..c6d8b87
--- /dev/null
+++ b/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/scripts/2-cleanup-resources.sh
@@ -0,0 +1,73 @@
+#!/bin/bash
+
+# Define the profile to be used for AWS CLI commands
+AWS_PROFILE="emr-user"
+
+# Define the bucket name
+S3_BUCKET_NAME="my-bucket-codely"
+
+# Define IAM policies and roles names
+EMR_FULL_ACCESS_POLICY="EMR_FullAccessPolicy"
+EMR_DEFAULT_ROLE="EMR_DefaultRole"
+EMR_EC2_DEFAULT_ROLE="EMR_EC2_DefaultRole"
+EMR_STEP_FUNCTIONS_POLICY="EMR_StepFunctions_Policy"
+EMR_STEP_FUNCTIONS_ROLE="EMR_StepFunctions_Role"
+
+# Define AWS Account ID
+AWS_ACCOUNT_ID="010928190667"
+
+# Function to detach and delete IAM policies
+function detach_and_delete_policy() {
+    local role_name=$1
+    local policy_name=$2
+
+    echo "Detaching policy $policy_name from role $role_name"
+    aws iam detach-role-policy --role-name $role_name --policy-arn arn:aws:iam::$AWS_ACCOUNT_ID:policy/$policy_name --profile $AWS_PROFILE
+
+    echo "Deleting IAM policy: $policy_name"
+    aws iam delete-policy --policy-arn arn:aws:iam::$AWS_ACCOUNT_ID:policy/$policy_name --profile $AWS_PROFILE
+}
+
+# Delete EMR Step Functions State Machine
+echo "Deleting Step Functions state machine"
+STATE_MACHINE_ARN=$(aws stepfunctions list-state-machines --profile $AWS_PROFILE --query "stateMachines[?name=='EMR_StepFunctions_Machine'].stateMachineArn" --output text)
+if [ -n "$STATE_MACHINE_ARN" ]; then
+    aws stepfunctions delete-state-machine --state-machine-arn $STATE_MACHINE_ARN --profile $AWS_PROFILE
+else
+    echo "State Machine not found."
+fi
+
+# Detach and delete Step Functions role and policy
+detach_and_delete_policy $EMR_STEP_FUNCTIONS_ROLE $EMR_STEP_FUNCTIONS_POLICY
+echo "Deleting IAM role: $EMR_STEP_FUNCTIONS_ROLE"
+aws iam delete-role --role-name $EMR_STEP_FUNCTIONS_ROLE --profile $AWS_PROFILE
+
+# Delete service-linked role for EMR
+echo "Deleting service-linked role for EMR"
+aws iam delete-service-linked-role --role-name AWSServiceRoleForEMRCleanup --profile $AWS_PROFILE
+
+# Detach policies from EMR_EC2_DefaultRole
+echo "Detaching policies from role $EMR_EC2_DEFAULT_ROLE"
+aws iam detach-role-policy --role-name $EMR_EC2_DEFAULT_ROLE --policy-arn arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforEC2Role --profile $AWS_PROFILE
+aws iam detach-role-policy --role-name $EMR_EC2_DEFAULT_ROLE --policy-arn arn:aws:iam::aws:policy/AmazonS3FullAccess --profile $AWS_PROFILE
+
+# Delete instance profile and role for EMR_EC2_DefaultRole
+echo "Removing role from instance profile: $EMR_EC2_DEFAULT_ROLE"
+aws iam remove-role-from-instance-profile --instance-profile-name $EMR_EC2_DEFAULT_ROLE --role-name $EMR_EC2_DEFAULT_ROLE --profile $AWS_PROFILE
+
+echo "Deleting instance profile: $EMR_EC2_DEFAULT_ROLE"
+aws iam delete-instance-profile --instance-profile-name $EMR_EC2_DEFAULT_ROLE --profile $AWS_PROFILE
+
+echo "Deleting IAM role: $EMR_EC2_DEFAULT_ROLE"
+aws iam delete-role --role-name $EMR_EC2_DEFAULT_ROLE --profile $AWS_PROFILE
+
+# Detach and delete EMR_DefaultRole and policy
+detach_and_delete_policy $EMR_DEFAULT_ROLE $EMR_FULL_ACCESS_POLICY
+echo "Deleting IAM role: $EMR_DEFAULT_ROLE"
+aws iam delete-role --role-name $EMR_DEFAULT_ROLE --profile $AWS_PROFILE
+
+# Delete S3 bucket
+echo "Deleting S3 bucket: $S3_BUCKET_NAME"
+aws s3 rb s3://$S3_BUCKET_NAME --force --profile $AWS_PROFILE
+
+echo "Cleanup complete."
diff --git a/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/EMR_DefaultRoleTrustPolicy.json b/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/scripts/policies/EMR_DefaultRoleTrustPolicy.json
similarity index 100%
rename from src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/EMR_DefaultRoleTrustPolicy.json
rename to src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/scripts/policies/EMR_DefaultRoleTrustPolicy.json
diff --git a/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/EMR_EC2_DefaultRoleTrustPolicy.json b/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/scripts/policies/EMR_EC2_DefaultRoleTrustPolicy.json
similarity index 100%
rename from src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/EMR_EC2_DefaultRoleTrustPolicy.json
rename to src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/scripts/policies/EMR_EC2_DefaultRoleTrustPolicy.json
diff --git a/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/EMR_FullAccessPolicy.json b/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/scripts/policies/EMR_FullAccessPolicy.json
similarity index 100%
rename from src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/EMR_FullAccessPolicy.json
rename to src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/scripts/policies/EMR_FullAccessPolicy.json
diff --git a/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/EMR_StepFunctions_Policy.json b/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/scripts/policies/EMR_StepFunctions_Policy.json
similarity index 100%
rename from src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/EMR_StepFunctions_Policy.json
rename to src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/scripts/policies/EMR_StepFunctions_Policy.json
diff --git a/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/StepFunctionsTrustPolicy.json b/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/scripts/policies/StepFunctionsTrustPolicy.json
similarity index 100%
rename from src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/StepFunctionsTrustPolicy.json
rename to src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/scripts/policies/StepFunctionsTrustPolicy.json
diff --git a/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/state_machine_definition.json b/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/scripts/policies/state_machine_definition.json
similarity index 97%
rename from src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/state_machine_definition.json
rename to src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/scripts/policies/state_machine_definition.json
index de2fcd0..dc35d62 100644
--- a/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/state_machine_definition.json
+++ b/src/main/com/codely/lesson_06__deploy_to_cloud/video_01__infra_aws/scripts/policies/state_machine_definition.json
@@ -6,7 +6,7 @@
       "Type": "Task",
       "Resource": "arn:aws:states:::elasticmapreduce:createCluster.sync",
       "Parameters": {
-        "Name": "WordCountStreamingCluster",
+        "Name": "MyAppCluster",
         "ReleaseLabel": "emr-7.1.0",
         "Applications": [
           {
diff --git a/src/test/com/codely/spark_best_practices_and_deploy/lesson_02_tests_in_spark/video_02__unit_testing/scala/job/SparkTestHelper.scala b/src/test/com/codely/spark_best_practices_and_deploy/lesson_02_tests_in_spark/video_02__unit_testing/scala/job/SparkTestHelper.scala
index ae356f5..b50e2e1 100644
--- a/src/test/com/codely/spark_best_practices_and_deploy/lesson_02_tests_in_spark/video_02__unit_testing/scala/job/SparkTestHelper.scala
+++ b/src/test/com/codely/spark_best_practices_and_deploy/lesson_02_tests_in_spark/video_02__unit_testing/scala/job/SparkTestHelper.scala
@@ -10,10 +10,7 @@ import java.io.File
 import java.nio.file.Files
 import scala.reflect.io.Directory
 
-trait SparkTestHelper
-    extends AnyFlatSpec
-    with BeforeAndAfterEach
-    with BeforeAndAfterAll {
+trait SparkTestHelper extends AnyFlatSpec with BeforeAndAfterEach with BeforeAndAfterAll {
 
   private val sparkSession = SparkSession
     .builder()
diff --git a/src/test/com/codely/spark_best_practices_and_deploy/lesson_02_tests_in_spark/z_practical_exercise/job/SparkTestHelper.scala b/src/test/com/codely/spark_best_practices_and_deploy/lesson_02_tests_in_spark/z_practical_exercise/job/SparkTestHelper.scala
index c0a44ca..3cafd72 100644
--- a/src/test/com/codely/spark_best_practices_and_deploy/lesson_02_tests_in_spark/z_practical_exercise/job/SparkTestHelper.scala
+++ b/src/test/com/codely/spark_best_practices_and_deploy/lesson_02_tests_in_spark/z_practical_exercise/job/SparkTestHelper.scala
@@ -10,10 +10,7 @@ import java.io.File
 import java.nio.file.Files
 import scala.reflect.io.Directory
 
-trait SparkTestHelper
-    extends AnyFlatSpec
-    with BeforeAndAfterEach
-    with BeforeAndAfterAll {
+trait SparkTestHelper extends AnyFlatSpec with BeforeAndAfterEach with BeforeAndAfterAll {
 
   private val sparkSession = SparkSession
     .builder()