diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..3edd6ed --- /dev/null +++ b/pom.xml @@ -0,0 +1,276 @@ + + + 4.0.0 + + com.luweizheng.flink + flink-tutorials + 0.1 + jar + + Flink Quickstart Job + http://www.myorganization.org + + + + apache.snapshots + Apache Development Snapshot Repository + https://repository.apache.org/content/repositories/snapshots/ + + false + + + true + + + + + + UTF-8 + 1.9.1 + 2.11 + 2.11.12 + + + + + + + org.apache.flink + flink-scala_${scala.binary.version} + ${flink.version} + provided + + + org.apache.flink + flink-streaming-scala_${scala.binary.version} + ${flink.version} + provided + + + + + org.scala-lang + scala-library + ${scala.version} + provided + + + + + + + + + + org.slf4j + slf4j-log4j12 + 1.7.7 + runtime + + + log4j + log4j + 1.2.17 + runtime + + + + + + + + + org.apache.maven.plugins + maven-shade-plugin + 3.0.0 + + + + package + + shade + + + + + org.apache.flink:force-shading + com.google.code.findbugs:jsr305 + org.slf4j:* + log4j:* + + + + + + *:* + + META-INF/*.SF + META-INF/*.DSA + META-INF/*.RSA + + + + + + quickstart.StreamingJob + + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.1 + + 1.8 + 1.8 + + + + + + net.alchim31.maven + scala-maven-plugin + 3.2.2 + + + + compile + testCompile + + + + + + + + org.apache.maven.plugins + maven-eclipse-plugin + 2.8 + + true + + org.scala-ide.sdt.core.scalanature + org.eclipse.jdt.core.javanature + + + org.scala-ide.sdt.core.scalabuilder + + + org.scala-ide.sdt.launching.SCALA_CONTAINER + org.eclipse.jdt.launching.JRE_CONTAINER + + + org.scala-lang:scala-library + org.scala-lang:scala-compiler + + + **/*.scala + **/*.java + + + + + org.codehaus.mojo + build-helper-maven-plugin + 1.7 + + + + add-source + generate-sources + + add-source + + + + src/main/scala + + + + + + add-test-source + generate-test-sources + + add-test-source + + + + src/test/scala + + + + + + + + + + + + + + add-dependencies-for-IDEA + + + + idea.version + + + + + + org.apache.flink + flink-scala_${scala.binary.version} + ${flink.version} + compile + + + org.apache.flink + flink-streaming-scala_${scala.binary.version} + ${flink.version} + compile + + + org.scala-lang + scala-library + ${scala.version} + compile + + + + + + diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties new file mode 100644 index 0000000..da32ea0 --- /dev/null +++ b/src/main/resources/log4j.properties @@ -0,0 +1,23 @@ +################################################################################ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +################################################################################ + +log4j.rootLogger=INFO, console + +log4j.appender.console=org.apache.log4j.ConsoleAppender +log4j.appender.console.layout=org.apache.log4j.PatternLayout +log4j.appender.console.layout.ConversionPattern=%d{HH:mm:ss,SSS} %-5p %-60c %x - %m%n diff --git a/src/main/scala/quickstart/BatchJob.scala b/src/main/scala/quickstart/BatchJob.scala new file mode 100644 index 0000000..36892eb --- /dev/null +++ b/src/main/scala/quickstart/BatchJob.scala @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package quickstart + +import org.apache.flink.api.scala._ + +/** + * Skeleton for a Flink Batch Job. + * + * For a tutorial how to write a Flink batch application, check the + * tutorials and examples on the Flink Website. + * + * To package your application into a JAR file for execution, + * change the main class in the POM.xml file to this class (simply search for 'mainClass') + * and run 'mvn clean package' on the command line. + */ +object BatchJob { + + def main(args: Array[String]) { + // set up the batch execution environment + val env = ExecutionEnvironment.getExecutionEnvironment + + /* + * Here, you can start creating your execution plan for Flink. + * + * Start with getting some data from the environment, like + * env.readTextFile(textPath); + * + * then, transform the resulting DataSet[String] using operations + * like + * .filter() + * .flatMap() + * .join() + * .group() + * + * and many more. + * Have a look at the programming guide: + * + * http://flink.apache.org/docs/latest/apis/batch/index.html + * + * and the examples + * + * http://flink.apache.org/docs/latest/apis/batch/examples.html + * + */ + + // execute program + env.execute("Flink Batch Scala API Skeleton") + } +} diff --git a/src/main/scala/quickstart/StreamingJob.scala b/src/main/scala/quickstart/StreamingJob.scala new file mode 100644 index 0000000..ed87b16 --- /dev/null +++ b/src/main/scala/quickstart/StreamingJob.scala @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package quickstart + +import org.apache.flink.streaming.api.scala._ + +/** + * Skeleton for a Flink Streaming Job. + * + * For a tutorial how to write a Flink streaming application, check the + * tutorials and examples on the Flink Website. + * + * To package your application into a JAR file for execution, run + * 'mvn clean package' on the command line. + * + * If you change the name of the main class (with the public static void main(String[] args)) + * method, change the respective entry in the POM.xml file (simply search for 'mainClass'). + */ +object StreamingJob { + def main(args: Array[String]) { + // set up the streaming execution environment + val env = StreamExecutionEnvironment.getExecutionEnvironment + + /* + * Here, you can start creating your execution plan for Flink. + * + * Start with getting some data from the environment, like + * env.readTextFile(textPath); + * + * then, transform the resulting DataStream[String] using operations + * like + * .filter() + * .flatMap() + * .join() + * .group() + * + * and many more. + * Have a look at the programming guide: + * + * http://flink.apache.org/docs/latest/apis/streaming/index.html + * + */ + + // execute program + env.execute("Flink Streaming Scala API Skeleton") + } +}