[refactor] Refactored into multiple modules to be compatible with dif…

…ferent spark version features (#239)
apache · Dec 3, 2024 · a75cf3b · a75cf3b
1 parent a9d5e25
commit a75cf3b
Show file tree

Hide file tree

Showing 176 changed files with 8,975 additions and 1,053 deletions.
diff --git a/.github/workflows/build-extension.yml b/.github/workflows/build-extension.yml
@@ -42,58 +42,42 @@ jobs:
         java-version: '8'
         cache: 'maven'
 
-    - name: Build spark connector 2.3
-      run: |
-        cd spark-doris-connector && mvn clean package ${MVN_OPT} \
-          -Dspark.version=2.3.4 \
-          -Dscala.version=2.11.8 \
-          -Dspark.major.version=2.3 \
-          -Dscala.major.version=2.11
-
     - name: Build spark connector 2.4 2.11
       run: |
-        cd spark-doris-connector && mvn clean package ${MVN_OPT} \
-          -Dspark.version=2.4.0 \
-          -Dscala.version=2.11.8 \
-          -Dspark.major.version=2.4 \
-          -Dscala.major.version=2.11  
+        cd spark-doris-connector && mvn clean install ${MVN_OPT} \
+          -Pspark-2 -pl spark-doris-connector-dist -am
 
     - name: Build spark connector 2.4 2.12
       run: |
-        cd spark-doris-connector && mvn clean package ${MVN_OPT} \
-          -Dspark.version=2.4.0 \
-          -Dscala.version=2.12.10 \
-          -Dspark.major.version=2.4 \
-          -Dscala.major.version=2.12       
+        cd spark-doris-connector && mvn clean install ${MVN_OPT} \
+          -Pspark-2 -pl spark-doris-connector-dist -am      
+
+    - name: Build spark connector 3.0
+      run: |
+        cd spark-doris-connector && mvn clean install ${MVN_OPT} \
+          -Pspark-3,spark-3.0 -pl spark-doris-connector-dist -am
 
     - name: Build spark connector 3.1
       run: |
-        cd spark-doris-connector && mvn clean package ${MVN_OPT} \
-          -Dspark.version=3.1.2 \
-          -Dscala.version=2.12.10 \
-          -Dspark.major.version=3.1 \
-          -Dscala.major.version=2.12
+        cd spark-doris-connector && mvn clean install ${MVN_OPT} \
+          -Pspark-3,spark-3.1 -pl spark-doris-connector-dist -am
 
     - name: Build spark connector 3.2
       run: |
-        cd spark-doris-connector && mvn clean package ${MVN_OPT} \
-          -Dspark.version=3.2.0 \
-          -Dscala.version=2.12.10 \
-          -Dspark.major.version=3.2 \
-          -Dscala.major.version=2.12
+        cd spark-doris-connector && mvn clean install ${MVN_OPT} \
+          -Pspark-3,spark-3.2 -pl spark-doris-connector-dist -am
 
     - name: Build spark connector 3.3
       run: |
-        cd spark-doris-connector && mvn clean package ${MVN_OPT} \
-          -Dspark.version=3.3.0 \
-          -Dscala.version=2.12.10 \
-          -Dspark.major.version=3.3 \
-          -Dscala.major.version=2.12
+        cd spark-doris-connector && mvn clean install ${MVN_OPT} \
+          -Pspark-3,spark-3.3 -pl spark-doris-connector-dist -am
 
     - name: Build spark connector 3.4
       run: |
-        cd spark-doris-connector && mvn clean package ${MVN_OPT} \
-          -Dspark.version=3.4.0 \
-          -Dscala.version=2.12.10 \
-          -Dspark.major.version=3.4 \
-          -Dscala.major.version=2.12
+        cd spark-doris-connector && mvn clean install ${MVN_OPT} \
+          -Pspark-3,spark-3.4 -pl spark-doris-connector-dist -am
+
+    - name: Build spark connector 3.5
+      run: |
+        cd spark-doris-connector && mvn clean install ${MVN_OPT} \
+          -Pspark-3,spark-3.5 -pl spark-doris-connector-dist -am
diff --git a/.github/workflows/run-itcase-12.yml b/.github/workflows/run-itcase-12.yml
@@ -38,7 +38,10 @@ jobs:
         distribution: adopt
         java-version: '8'
 
-    - name: Run ITCases
+    - name: Run ITCases for spark 2
       run: |
-        cd spark-doris-connector && mvn test -Dtest="*ITCase" -Dimage="adamlee489/doris:1.2.7.1_x86"
+        cd spark-doris-connector && mvn clean test -pl spark-doris-connector-it -Pspark-2 -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:1.2.7.1_x86"
 
+    - name: Run ITCases for spark 3
+      run: |
+        cd spark-doris-connector && mvn clean test -pl spark-doris-connector-it -Pspark-3,spark-3.0 -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:1.2.7.1_x86"
diff --git a/.github/workflows/run-itcase-20.yml b/.github/workflows/run-itcase-20.yml
@@ -37,8 +37,12 @@ jobs:
       with:
         distribution: adopt
         java-version: '8'
-
-    - name: Run ITCases
+ 
+    - name: Run ITCases for spark 2
       run: |
-        cd spark-doris-connector && mvn test -Dtest="*ITCase" -Dimage="adamlee489/doris:2.0.3"
+        cd spark-doris-connector && mvn clean test -pl spark-doris-connector-it -Pspark-2 -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:2.0.3"
 
+    - name: Run ITCases for spark 3
+      run: |
+        cd spark-doris-connector && mvn clean test -pl spark-doris-connector-it -Pspark-3,spark-3.0 -am -DfailIfNoTests=false -Dtest="*ITCase" -Dimage="adamlee489/doris:2.0.3"
+    
diff --git a/.gitignore b/.gitignore
@@ -8,6 +8,9 @@ spark-doris-connector/output/
 spark-doris-connector/target/
 spark-doris-connector/.idea/
 
+dependency-reduced-pom.xml
+target
+.flattened-pom.xml
 
 ### Java template
 # Compiled class file

diff --git a/spark-doris-connector/build.sh b/spark-doris-connector/build.sh
@@ -142,30 +142,33 @@ selectScala() {
 
 selectSpark() {
   echo 'Spark-Doris-Connector supports multiple versions of spark. Which version do you need ?'
-  select spark in "2.3.x" "2.4.x" "3.1.x" "3.2.x" "3.3.x" "3.4.x" "other"
+  select spark in "2.4" "3.0" "3.1" "3.2" "3.3" "3.4" "3.5"  "other"
   do
     case $spark in
-      "2.3.x")
+      "2.4")
         return 1
         ;;
-      "2.4.x")
+      "3.0")
         return 2
         ;;
-      "3.1.x")
+      "3.1")
         return 3
         ;;
-      "3.2.x")
+      "3.2")
         return 4
         ;;
-      "3.3.x")
+      "3.3")
         return 5
         ;;
-      "3.4.x")
+      "3.4")
         return 6
         ;;
-      "other")
+      "3.5")
         return 7
         ;;
+      "other")
+        return 8
+        ;;
     esac
   done
 }
@@ -174,38 +177,37 @@ SCALA_VERSION=0
 selectScala
 ScalaVer=$?
 if [ ${ScalaVer} -eq 1 ]; then
-    SCALA_VERSION="2.11.8"
+    SCALA_VERSION="2.11.12"
 elif [ ${ScalaVer} -eq 2 ]; then
-    SCALA_VERSION="2.12.10"
+    SCALA_VERSION="2.12.18"
 fi
 
 
 SPARK_VERSION=0
 selectSpark
 SparkVer=$?
 if [ ${SparkVer} -eq 1 ]; then
-    SPARK_VERSION="2.3.4"
-elif [ ${SparkVer} -eq 2 ]; then
     SPARK_VERSION="2.4.8"
+elif [ ${SparkVer} -eq 2 ]; then
+    SPARK_VERSION="3.0.3"
 elif [ ${SparkVer} -eq 3 ]; then
     SPARK_VERSION="3.1.3"
 elif [ ${SparkVer} -eq 4 ]; then
     SPARK_VERSION="3.2.4"
 elif [ ${SparkVer} -eq 5 ]; then
-    SPARK_VERSION="3.3.3"
+    SPARK_VERSION="3.3.4"
 elif [ ${SparkVer} -eq 6 ]; then
-    SPARK_VERSION="3.4.1"
+    SPARK_VERSION="3.4.3"
 elif [ ${SparkVer} -eq 7 ]; then
+    SPARK_VERSION="3.5.3"
+elif [ ${SparkVer} -eq 8 ]; then
     # shellcheck disable=SC2162
     read -p 'Which spark version do you need? please input
     :' ver
     SPARK_VERSION=$ver
 fi
 
-if [[ $SPARK_VERSION =~ ^2.3 && $SCALA_VERSION == "2.12.10"  ]]; then
-  echo_r "Spark 2.3 is not compatible with scala 2.12, will exit."
-  exit 1
-elif [[ $SPARK_VERSION =~ ^3.* && $SCALA_VERSION == "2.11.8" ]]; then
+if [[ $SPARK_VERSION =~ ^3.* && $SCALA_VERSION == "2.11.12" ]]; then
   echo_r "Spark 3.x is not compatible with scala 2.11, will exit."
   exit 1
 fi
@@ -221,19 +223,23 @@ echo_g " scala version: ${SCALA_VERSION}, major version: ${SCALA_MAJOR_VERSION}"
 echo_g " spark version: ${SPARK_VERSION}, major version: ${SPARK_MAJOR_VERSION}"
 echo_g " build starting..."
 
-${MVN_BIN} clean package \
+SPARK_PRIMARY_VERSION=0
+[ ${SPARK_MAJOR_VERSION} != 0 ] && SPARK_PRIMARY_VERSION=${SPARK_MAJOR_VERSION%.*}
+
+${MVN_BIN} clean install \
   -Dspark.version=${SPARK_VERSION} \
   -Dscala.version=${SCALA_VERSION} \
   -Dspark.major.version=${SPARK_MAJOR_VERSION} \
-  -Dscala.major.version=${SCALA_MAJOR_VERSION} "$@"
+  -Dscala.major.version=${SCALA_MAJOR_VERSION} \
+  -Pspark-${SPARK_PRIMARY_VERSION} -pl spark-doris-connector-dist -am "$@"
 
 EXIT_CODE=$?
 if [ $EXIT_CODE -eq 0 ]; then
   DIST_DIR=${DORIS_HOME}/dist
   [ ! -d "$DIST_DIR" ] && mkdir "$DIST_DIR"
-  dist_jar=$(ls "${ROOT}"/target | grep "spark-doris-" | grep -v "sources.jar" | grep -v "original-")
-  rm -rf "${DIST_DIR}"/"${dist_jar}"
-  cp "${ROOT}"/target/"${dist_jar}" "$DIST_DIR"
+  dist_jar=$(ls "${ROOT}"/spark-doris-connector-dist/target | grep "spark-doris-" | grep -v "sources.jar" | grep -v "original-")
+  rm -rf "${DIST_DIR}"/spark-doris-connector-dist/"${dist_jar}"
+  cp "${ROOT}"/spark-doris-connector-dist/target/"${dist_jar}" "$DIST_DIR"
 
   echo_g "*****************************************************************"
   echo_g "Successfully build Spark-Doris-Connector"