-
Notifications
You must be signed in to change notification settings - Fork 6
/
Copy pathbuild.sbt
86 lines (75 loc) · 3 KB
/
build.sbt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
import Dependencies.versions._
val sparkStreamingOld = "spark-streaming-old"
val sparkStreamingLatest = "spark-streaming-latest"
val akkaStreamsKafka = "akka-streams-kafka"
val akkaActorsKafka = "akka-actors-kafka"
val structuredStreamingModule = "structured-streaming"
val streamingWriter = "spark-streaming-writer"
val sparkCommon = Seq("org.apache.spark" %% "spark-core" % sparkVersion,
"org.apache.spark" %% "spark-streaming" % sparkVersion)
lazy val sparkKafka08 = (project in file(sparkStreamingOld))
.settings(commonSettings(sparkStreamingOld): _*)
.settings(
libraryDependencies ++= sparkCommon ++ Seq(
"org.apache.spark" % "spark-streaming-kafka-0-8_2.11" % sparkVersion
)
)
lazy val sparkKafka010 = (project in file(sparkStreamingLatest))
.settings(commonSettings(sparkStreamingLatest): _*)
.settings(
libraryDependencies ++= sparkCommon ++ Seq(
"org.apache.spark" % "spark-streaming-kafka-0-10_2.11" % sparkVersion,
"com.twitter" %% "bijection-avro" % "0.8.1",
"joda-time" % "joda-time" % "2.8.1",
"org.scalatest" %% "scalatest" % "3.0.1" % "test",
"org.specs2" % "specs2-core_2.11" % "3.9.5" % "test"
)
)
lazy val structuredStreaming = (project in file(structuredStreamingModule))
.settings(commonSettings(sparkStreamingLatest): _*)
.settings(
libraryDependencies ++= Seq(
"org.apache.spark" % "spark-sql-kafka-0-10_2.11" % sparkVersion,
"org.apache.spark" %% "spark-core" % sparkVersion,
"org.apache.spark" %% "spark-sql" % sparkVersion
)
)
lazy val sparkStreamingWriter = (project in file(streamingWriter))
.settings(commonSettings(sparkStreamingLatest): _*)
.settings(
libraryDependencies ++= Seq(
"org.apache.kafka" % "kafka_2.11" % "0.10.2.1",
"org.apache.spark" %% "spark-streaming-kafka-0-10" % sparkVersion,
"org.apache.spark" %% "spark-streaming" % sparkVersion,
"com.github.benfradet" %% "spark-kafka-0-10-writer" % "0.3.0",
"org.apache.spark" %% "spark-core" % sparkVersion,
"org.apache.spark" %% "spark-sql" % sparkVersion
)
)
lazy val akkaKafkaModule = (project in file(akkaStreamsKafka))
.settings(commonSettings(akkaStreamsKafka): _*)
.settings(
libraryDependencies ++= Seq(
"com.typesafe.akka" %% "akka-stream-kafka" % akkaStreamsKafkaVersion
)
)
lazy val akkaActorsKafkaModule = (project in file(akkaActorsKafka))
.settings(commonSettings(akkaActorsKafka): _*)
.settings(
libraryDependencies ++= Seq(
"net.cakesolutions" %% "scala-kafka-client" % scalaKafkaClient,
"net.cakesolutions" %% "scala-kafka-client-akka" % scalaKafkaClient
)
)
def commonSettings(_name: String) = Seq(
name := _name,
scalaVersion := "2.11.8",
version := "1.0",
parallelExecution in Test := false,
cancelable in Global := true,
resolvers ++= List(
Resolver.jcenterRepo,
Resolver.sonatypeRepo("releases"), Resolver.bintrayRepo("cakesolutions", "maven"),
"apache-snapshots" at "http://repository.apache.org/snapshots/"
)
)