本文整理汇总了Java中org.apache.spark.streaming.api.java.JavaReceiverInputDStream.mapToPair方法的典型用法代码示例。如果您正苦于以下问题：Java JavaReceiverInputDStream.mapToPair方法的具体用法？Java JavaReceiverInputDStream.mapToPair怎么用？Java JavaReceiverInputDStream.mapToPair使用的例子？那么, 这里精选的方法代码示例或许可以为您提供帮助。您也可以进一步了解该方法所在类org.apache.spark.streaming.api.java.JavaReceiverInputDStream的用法示例。
在下文中一共展示了JavaReceiverInputDStream.mapToPair方法的1个代码示例，这些例子默认根据受欢迎程度排序。您可以为喜欢或者感觉有用的代码点赞，您的评价将有助于系统推荐出更棒的Java代码示例。
示例1: main
import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; //导入方法依赖的package包/类
public static void main(String[] args) throws Exception {
if (args.length < 2) {
System.err.println("Usage: JavaCustomReceiver <hostname> <port>");
System.exit(1);
}
// StreamingExamples.setStreamingLogLevels();
// https://github.com/apache/spark/blob/39e2bad6a866d27c3ca594d15e574a1da3ee84cc/examples/src/main/scala/org/apache/spark/examples/streaming/StreamingExamples.scala
boolean log4jInitialized = Logger.getRootLogger().getAllAppenders().hasMoreElements();
if (!log4jInitialized) {
// We first log something to initialize Spark's default logging, then we override the
// logging level.
/// logInfo("Setting log level to [WARN] for streaming example." +
/// " To override add a custom log4j.properties to the classpath.")
Logger.getRootLogger().setLevel(Level.WARN);
}
// Create the context with a 1 second batch size
SparkConf sparkConf = new SparkConf().setAppName("JavaCustomReceiver").setMaster("local[*]").set("spark.driver.host", "localhost"); // https://issues.apache.org/jira/browse/
JavaStreamingContext ssc = new JavaStreamingContext(sparkConf, new Duration(5000));
// Create an input stream with the custom receiver on target ip:port and count the
// words in input stream of \n delimited text (eg. generated by 'nc')
final JavaReceiverInputDStream<Tuple2<String, Long>> receiverStream = ssc.receiverStream(new JavaCustomReceiver(args[0], Integer.parseInt(args[1])));
PairFunction mapFunction = new PairFunction() {
@Override
public Tuple2 call(Object arg0) throws Exception {
return (Tuple2) arg0;
}
};
final JavaPairDStream<String, Long> keyValues = receiverStream.mapToPair(mapFunction);
JavaPairDStream<String, Long> byKeys = keyValues.reduceByKey((a, b) -> a + b);
byKeys.print();
/* JavaDStream<String> words = lines.flatMap(new FlatMapFunction<String, String>() {
@Override
public Iterable<String> call(String x) {
// return Arrays.asList(SPACE.split(x)).iterator();
return Arrays.asList(SPACE.split(x));
}
});
JavaPairDStream<String, Integer> wordCounts = words.mapToPair(
new PairFunction<String, String, Integer>() {
@Override public Tuple2<String, Integer> call(String s) {
return new Tuple2<>(s, 1);
}
}).reduceByKey(new Function2<Integer, Integer, Integer>() {
@Override
public Integer call(Integer i1, Integer i2) {
return i1 + i2;
}
});
wordCounts.print();*/
ssc.start();
ssc.awaitTermination();
}