Skip to content

Commit 0c1bba5

Browse files
author
Kyle Dong
committed
Added WordCount example
1 parent 11f46bb commit 0c1bba5

File tree

10 files changed

+351
-1
lines changed

10 files changed

+351
-1
lines changed

‎.gitignore‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
# Created by .ignore support plugin (hsz.mobi)
2+
### Scala template
3+
*.class
4+
*.log
5+
6+
### Java template
7+
# Compiled class file
8+
*.class
9+
10+
# Log file
11+
*.log
12+
13+
# BlueJ files
14+
*.ctxt
15+
16+
# Mobile Tools for Java (J2ME)
17+
.mtj.tmp/
18+
19+
# Package Files #
20+
*.jar
21+
*.war
22+
*.nar
23+
*.ear
24+
*.zip
25+
*.tar.gz
26+
*.rar
27+
28+
# virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
29+
hs_err_pid*
30+
31+
### JetBrains template
32+
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
33+
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
34+
35+
# User-specific stuff
36+
.idea/**/workspace.xml
37+
.idea/**/tasks.xml
38+
.idea/**/usage.statistics.xml
39+
.idea/**/dictionaries
40+
.idea/**/shelf
41+
42+
# Generated files
43+
.idea/**/contentModel.xml
44+
45+
# Sensitive or high-churn files
46+
.idea/**/dataSources/
47+
.idea/**/dataSources.ids
48+
.idea/**/dataSources.local.xml
49+
.idea/**/sqlDataSources.xml
50+
.idea/**/dynamic.xml
51+
.idea/**/uiDesigner.xml
52+
.idea/**/dbnavigator.xml
53+
54+
# Gradle
55+
.idea/**/gradle.xml
56+
.idea/**/libraries
57+
58+
# Gradle and Maven with auto-import
59+
# When using Gradle or Maven with auto-import, you should exclude module files,
60+
# since they will be recreated, and may cause churn. Uncomment if using
61+
# auto-import.
62+
# .idea/artifacts
63+
# .idea/compiler.xml
64+
# .idea/jarRepositories.xml
65+
# .idea/modules.xml
66+
# .idea/*.iml
67+
# .idea/modules
68+
*.iml
69+
# *.ipr
70+
71+
# CMake
72+
cmake-build-*/
73+
74+
# Mongo Explorer plugin
75+
.idea/
76+
77+
# File-based project format
78+
*.iws
79+
80+
# IntelliJ
81+
out/
82+
83+
# mpeltonen/sbt-idea plugin
84+
.idea_modules/
85+
86+
# JIRA plugin
87+
atlassian-ide-plugin.xml
88+
89+
# Cursive Clojure plugin
90+
.idea/replstate.xml
91+
92+
# Crashlytics plugin (for Android Studio and IntelliJ)
93+
com_crashlytics_export_strings.xml
94+
crashlytics.properties
95+
crashlytics-build.properties
96+
fabric.properties
97+
98+
# Editor-based Rest Client
99+
.idea/httpRequests
100+
101+
# Android studio 3.1+ serialized cache file
102+
.idea/caches/build_file_checksums.ser
103+
104+
target/

‎README.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,5 @@
11
# flink-hello-world
2-
Flink 示例程序
2+
Flink 示例程序包, 用于快速构建和运行一个 Flink JAR 作业。
3+
4+
目前包含如下用例：
5+
1. WordCount

‎pom.xml‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
<!-- Licensed to the Apache Software Foundation (ASF) under one or more contributor
2+
license agreements. See the NOTICE file distributed with this work for additional
3+
information regarding copyright ownership. The ASF licenses this file to
4+
you under the Apache License, Version 2.0 (the "License"); you may not use
5+
this file except in compliance with the License. You may obtain a copy of
6+
the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required
7+
by applicable law or agreed to in writing, software distributed under the
8+
License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS
9+
OF ANY KIND, either express or implied. See the License for the specific
10+
language governing permissions and limitations under the License. -->
11+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
12+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
13+
<modelVersion>4.0.0</modelVersion>
14+
15+
<artifactId>flink-hello-world</artifactId>
16+
<packaging>jar</packaging>
17+
18+
<name>Flink Hello World</name>
19+
<groupId>com.tencent.cloud.oceanus</groupId>
20+
<version>1.0.0</version>
21+
22+
<properties>
23+
<flink.version>1.11.0</flink.version>
24+
<scala.binary.version>2.11</scala.binary.version>
25+
</properties>
26+
27+
<dependencies>
28+
<dependency>
29+
<groupId>org.apache.flink</groupId>
30+
<artifactId>flink-streaming-java_2.11</artifactId>
31+
<version>${flink.version}</version>
32+
<scope>provided</scope>
33+
</dependency>
34+
35+
<dependency>
36+
<groupId>org.apache.flink</groupId>
37+
<artifactId>flink-streaming-scala_2.11</artifactId>
38+
<version>${flink.version}</version>
39+
<scope>provided</scope>
40+
</dependency>
41+
</dependencies>
42+
43+
<build>
44+
<plugins>
45+
<plugin>
46+
<groupId>org.apache.maven.plugins</groupId>
47+
<artifactId>maven-compiler-plugin</artifactId>
48+
<version>3.1</version>
49+
<configuration>
50+
<source>8</source> <!-- If you want to use Java 8, change this to "1.8" -->
51+
<target>8</target> <!-- If you want to use Java 8, change this to "1.8" -->
52+
</configuration>
53+
</plugin>
54+
55+
<plugin>
56+
<groupId>org.apache.maven.plugins</groupId>
57+
<artifactId>maven-source-plugin</artifactId>
58+
<version>2.1</version>
59+
<executions>
60+
<execution>
61+
<id>attach-sources</id>
62+
<goals>
63+
<goal>jar-no-fork</goal>
64+
</goals>
65+
</execution>
66+
</executions>
67+
</plugin>
68+
</plugins>
69+
70+
<!-- If you want to use Java 8 Lambda Expressions uncomment the following
71+
lines -->
72+
<!-- <pluginManagement> <plugins> <plugin> <artifactId>maven-compiler-plugin</artifactId>
73+
<configuration> <source>1.8</source> <target>1.8</target> <compilerId>jdt</compilerId>
74+
</configuration> <dependencies> <dependency> <groupId>org.eclipse.tycho</groupId>
75+
<artifactId>tycho-compiler-jdt</artifactId> <version>0.21.0</version> </dependency>
76+
</dependencies> </plugin> <plugin> <groupId>org.eclipse.m2e</groupId> <artifactId>lifecycle-mapping</artifactId>
77+
<version>1.0.0</version> <configuration> <lifecycleMappingMetadata> <pluginExecutions>
78+
<pluginExecution> <pluginExecutionFilter> <groupId>org.apache.maven.plugins</groupId>
79+
<artifactId>maven-assembly-plugin</artifactId> <versionRange>[2.4,)</versionRange>
80+
<goals> <goal>single</goal> </goals> </pluginExecutionFilter> <action> <ignore/>
81+
</action> </pluginExecution> <pluginExecution> <pluginExecutionFilter> <groupId>org.apache.maven.plugins</groupId>
82+
<artifactId>maven-compiler-plugin</artifactId> <versionRange>[3.1,)</versionRange>
83+
<goals> <goal>testCompile</goal> <goal>compile</goal> </goals> </pluginExecutionFilter>
84+
<action> <ignore/> </action> </pluginExecution> </pluginExecutions> </lifecycleMappingMetadata>
85+
</configuration> </plugin> </plugins> </pluginManagement> -->
86+
</build>
87+
</project>

‎src/main/java/com/tencent/cloud/oceanus/wordcount/CountDownSource.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
package com.tencent.cloud.oceanus.wordcount;
2+
3+
import org.apache.flink.streaming.api.functions.source.SourceFunction;
4+
5+
public class CountDownSource implements SourceFunction<String> {
6+
private long countDown;
7+
8+
public CountDownSource(long countDownSeconds) {
9+
if (countDownSeconds > 0) {
10+
this.countDown = countDownSeconds;
11+
} else {
12+
this.countDown = Long.MAX_VALUE;
13+
}
14+
}
15+
16+
@SuppressWarnings("BusyWait")
17+
@Override
18+
public void run(SourceContext<String> ctx) throws Exception {
19+
while (countDown > 0) {
20+
ctx.collect("This is a test, a really simple test. Can you write a better one? Try it please.");
21+
Thread.sleep(1000L);
22+
countDown--;
23+
}
24+
}
25+
26+
@Override
27+
public void cancel() {
28+
}
29+
}

‎src/main/java/com/tencent/cloud/oceanus/wordcount/Counter.java‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package com.tencent.cloud.oceanus.wordcount;
2+
3+
public class Counter {
4+
private String word;
5+
private int count;
6+
7+
public Counter() {
8+
}
9+
10+
public String getWord() {
11+
return word;
12+
}
13+
14+
public void setWord(String word) {
15+
this.word = word;
16+
}
17+
18+
public int getCount() {
19+
return count;
20+
}
21+
22+
public void setCount(int count) {
23+
this.count = count;
24+
}
25+
26+
Counter(String word, int count) {
27+
this.word = word;
28+
this.count = count;
29+
}
30+
31+
public String toString() {
32+
return "word: " + this.word + ", count: " + this.count;
33+
}
34+
}

‎src/main/java/com/tencent/cloud/oceanus/wordcount/LoggingSink.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package com.tencent.cloud.oceanus.wordcount;
2+
3+
import org.apache.flink.streaming.api.functions.sink.SinkFunction;
4+
import org.slf4j.Logger;
5+
import org.slf4j.LoggerFactory;
6+
7+
public class LoggingSink implements SinkFunction<Counter> {
8+
private static final Logger LOGGER = LoggerFactory.getLogger(LoggingSink.class);
9+
10+
@Override
11+
public void invoke(Counter value, Context context) throws Exception {
12+
LOGGER.info("{}: {}", value.getWord(), value.getCount());
13+
}
14+
}

‎src/main/java/com/tencent/cloud/oceanus/wordcount/WordCount.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
package com.tencent.cloud.oceanus.wordcount;
2+
3+
import org.apache.flink.streaming.api.datastream.DataStream;
4+
import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;
5+
6+
public class WordCount {
7+
8+
public static void main(String[] args) throws Exception {
9+
10+
StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
11+
12+
// 关闭 Operator Chaining, 令运行图更容易初学者理解
13+
env.disableOperatorChaining();
14+
15+
// 支持传入参数作为倒计时 (秒)
16+
int countDownSeconds = -1;
17+
if (args.length == 1) {
18+
countDownSeconds = Integer.parseInt(args[0]);
19+
}
20+
21+
// 定义数据源
22+
DataStream<String> streamSource = env
23+
.addSource(new CountDownSource(countDownSeconds))
24+
.name("Count down for " + (countDownSeconds <= 0 ? "infinite" : countDownSeconds) + " seconds");
25+
26+
// 数据处理算子
27+
DataStream<Counter> dataStream = streamSource
28+
.flatMap(new WordSplitter())
29+
.keyBy(new WordKeySelector())
30+
.reduce(new WordCountReducer()
31+
);
32+
33+
// 定义数据目的
34+
dataStream
35+
.addSink(new LoggingSink())
36+
.name("Logging Sink");
37+
38+
env.execute(WordCount.class.getSimpleName());
39+
}
40+
}
41+
42+

‎src/main/java/com/tencent/cloud/oceanus/wordcount/WordCountReducer.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
package com.tencent.cloud.oceanus.wordcount;
2+
3+
import org.apache.flink.api.common.functions.ReduceFunction;
4+
5+
public class WordCountReducer implements ReduceFunction<Counter> {
6+
7+
public Counter reduce(Counter value1, Counter value2) throws Exception {
8+
return new Counter(value1.getWord(), value1.getCount() + value2.getCount());
9+
}
10+
}

‎src/main/java/com/tencent/cloud/oceanus/wordcount/WordKeySelector.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package com.tencent.cloud.oceanus.wordcount;
2+
3+
import org.apache.flink.api.java.functions.KeySelector;
4+
5+
public class WordKeySelector implements KeySelector<Counter, String> {
6+
7+
@Override
8+
public String getKey(Counter wordWithCount) throws Exception {
9+
return wordWithCount.getWord();
10+
}
11+
12+
}

‎src/main/java/com/tencent/cloud/oceanus/wordcount/WordSplitter.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package com.tencent.cloud.oceanus.wordcount;
2+
3+
import org.apache.flink.api.common.functions.FlatMapFunction;
4+
import org.apache.flink.util.Collector;
5+
6+
public class WordSplitter implements FlatMapFunction<String, Counter> {
7+
8+
public void flatMap(String value, Collector<Counter> out) throws Exception {
9+
String[] words = value.toLowerCase().split("\\W+");
10+
for (String word : words) {
11+
Counter wc = new Counter(word, 1);
12+
out.collect(wc);
13+
}
14+
}
15+
}

0 commit comments

Comments
 (0)