Java JavaReceiverInputDStream类代码示例

import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; //导入依赖的package包/类
/**
 * Test that the flow works correctly
 */
@Test
public void testValidTwitchStreamBuilder() {
    Set<String> gamesList = new HashSet<>();
    gamesList.add("League+of+Legends");
    Set<String> channelsList = new HashSet<>();
    channelsList.add("#TSM_Dyrus");

    JavaReceiverInputDStream<Message> stream = new TwitchStreamBuilder()
            .setGames(gamesList)
            .setChannels(channelsList)
            .setLanguage("es")
            .setStorageLevel(StorageLevel.MEMORY_AND_DISK_SER_2())
            .setSchedulingInterval(300)
            .build(jssc);
}
 

import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; //导入依赖的package包/类
/**
 * This will start the spark stream that is reading from the MQTT queue
 *
 * @param broker     - MQTT broker url
 * @param topic      - MQTT topic name
 * @param numSeconds - Number of seconds between batch size
 */
public void processMQTT(final String broker, final String topic, final int numSeconds) {

    LOG.info("************ SparkStreamingMQTTOutside.processMQTT start");

    // Create the spark application and set the name to MQTT
    SparkConf sparkConf = new SparkConf().setAppName("MQTT");

    // Create the spark streaming context with a 'numSeconds' second batch size
    jssc = new JavaStreamingContext(sparkConf, Durations.seconds(numSeconds));
    jssc.checkpoint(checkpointDirectory);

    LOG.info("************ SparkStreamingMQTTOutside.processMQTT about to read the MQTTUtils.createStream");
    //2. MQTTUtils to collect MQTT messages
    JavaReceiverInputDStream<String> messages = MQTTUtils.createStream(jssc, broker, topic);

    LOG.info("************ SparkStreamingMQTTOutside.processMQTT about to do foreachRDD");
    //process the messages on the queue and save them to the database
    messages.foreachRDD(new SaveRDD());

    LOG.info("************ SparkStreamingMQTTOutside.processMQTT prior to context.strt");
    // Start the context
    jssc.start();
    jssc.awaitTermination();
}
 

import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; //导入依赖的package包/类
private static JavaDStream<String> createDStream(JavaStreamingContext javaStreamingContext, String hostName, int port) {
        
        JavaReceiverInputDStream<SparkFlumeEvent> flumeEventStream = FlumeUtils.createStream(javaStreamingContext, hostName, port);
        
        // Set different storage level 
//        flumeEventStream.persist(StorageLevel.MEMORY_AND_DISK_SER());
        
        JavaDStream<String> dStream = flumeEventStream.map(new Function<SparkFlumeEvent, String>() {

            @Override
            public String call(SparkFlumeEvent sparkFlumeEvent) throws Exception {

                byte[] bodyArray = sparkFlumeEvent.event().getBody().array();
                String logTxt = new String(bodyArray, "UTF-8");
                logger.info(logTxt);

                return logTxt;
            }
        });
        // dStream.print();
        
        return dStream;
    }
 

import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; //导入依赖的package包/类
@Test(timeout=6000)
public void testNatsToSparkConnectorWithAdditionalPropertiesAndSubjects() throws InterruptedException {
	
	JavaStreamingContext ssc = new JavaStreamingContext(sc, new Duration(200));

	final Properties properties = new Properties();
	properties.setProperty(PROP_URL, NATS_SERVER_URL);
	final JavaReceiverInputDStream<String> messages =  
			NatsToSparkConnector
				.receiveFromNats(String.class, StorageLevel.MEMORY_ONLY())
				.withProperties(properties)
				.withSubjects(DEFAULT_SUBJECT)
				.asStreamOf(ssc);

	validateTheReceptionOfMessages(ssc, messages);
}
 

import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; //导入依赖的package包/类
@Test(timeout=6000)
public void testNatsToSparkConnectorWithAdditionalPropertiesAndMultipleSubjects() throws InterruptedException {
	
	JavaStreamingContext ssc = new JavaStreamingContext(sc, new Duration(200));

	final Properties properties = new Properties();
	final JavaReceiverInputDStream<String> messages = 
			NatsToSparkConnector
				.receiveFromNats(String.class, StorageLevel.MEMORY_ONLY())
				.withNatsURL(NATS_SERVER_URL)
				.withProperties(properties)
				.withSubjects(DEFAULT_SUBJECT, "EXTRA_SUBJECT")
				.asStreamOf(ssc);

	validateTheReceptionOfMessages(ssc, messages);
}
 

import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; //导入依赖的package包/类
@Test(timeout=6000)
public void testNatsToSparkConnectorWithAdditionalProperties() throws InterruptedException {
	
	JavaStreamingContext ssc = new JavaStreamingContext(sc, new Duration(200));

	final Properties properties = new Properties();
	properties.setProperty(PROP_SUBJECTS, "sub1,"+DEFAULT_SUBJECT+" , sub2");
	properties.setProperty(PROP_URL, NATS_SERVER_URL);
	final JavaReceiverInputDStream<String> messages = 
			NatsToSparkConnector
				.receiveFromNats(String.class, StorageLevel.MEMORY_ONLY())
				.withProperties(properties)
				.asStreamOf(ssc);

	validateTheReceptionOfMessages(ssc, messages);
}
 

import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; //导入依赖的package包/类
@Test(timeout=6000)
public void testNatsToSparkConnectorWithAdditionalPropertiesAndSubjects() throws InterruptedException {
	
	JavaStreamingContext ssc = new JavaStreamingContext(sc, new Duration(200));

	final Properties properties = new Properties();
	properties.setProperty(PROP_URL, NATS_SERVER_URL);
	final JavaReceiverInputDStream<Integer> messages =  
			NatsToSparkConnector
				.receiveFromNats(Integer.class, StorageLevel.MEMORY_ONLY())
				.withProperties(properties)
				.withSubjects(DEFAULT_SUBJECT)
				.asStreamOf(ssc);

	validateTheReceptionOfIntegerMessages(ssc, messages);
}
 

import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; //导入依赖的package包/类
protected void queryTweets(JavaReceiverInputDStream<Status> tweets, int rank) {
	
	// Compute sketches
	JavaPairDStream<ImageInfo, ImageFeature> imFeatures = computeImageFeatures(tweets);
	
	JavaPairDStream<ImageInfo, ImageFeature> sketches = imFeatures.mapValues(new SketchProcedure(indParams.getSketchFunction(), 
			indParams.getNumTables()));
	
	// Query specific and filter by hamming distance
	JavaPairDStream<ImageFeature, ImageFeature> candidates = system.queryFeaturesStreaming(conn,indParams, sketches);
	JavaPairDStream<ImageFeature, ImageFeature> filteredHamming = 
			candidates.filter(new HammingFiltering(indParams.getHammingDistance()));
	
	// Group by image and assign weights
	JavaDStream<ImageMatch> matchedIds = filteredHamming.map(new MatchExtractorStreaming());
	JavaPairDStream<ImageMatch, Long> result = matchedIds.countByValue();

	// Filter by weight if requested
	if (rank > 0) {
		result = result.filter(new WeightFiltering(rank));
	}
	
	// Print results
	result.print();
}
 

import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; //导入依赖的package包/类
protected JavaPairDStream<ImageInfo, ImageFeature> computeImageFeatures(JavaReceiverInputDStream<Status> tweets) {
	
	JavaDStream<ImageInfo> imgs = tweets.mapPartitions(new TweetsToImagesTask());
	
	JavaPairDStream<ImageInfo, ImageFeature> features = imgs.flatMapToPair(new ComputeFeatures(descParams, ProviderType.TWITTER));
	JavaPairDStream<ImageInfo, ImageFeature> filtered = features;
	
	// Filter descriptors if needed
	if (filtParams.getFilteringType().equals(FilteringType.ENTROPY)) {
		filtered = features.filter(new EntropyFiltering(filtParams.getThresh()));
	}
	else if (filtParams.getFilteringType().equals(FilteringType.VARIANCE)) {
		filtered = features.filter(new VarianceFiltering(filtParams.getThresh()));
	}
	
	// Logscale features if needed
	if (filtParams.isLogScaleEnabled()) {
		filtered = filtered.mapValues(new LogScaleFunction());
	}
	
	// Build sketches
	return filtered;
}
 

import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
  
      System.setProperty("hadoop.home.dir", "E:\\hadoop");
	
   SparkConf sparkConf = new SparkConf().setAppName("WordCountSocketEx").setMaster("local[*]");
   JavaStreamingContext streamingContext = new JavaStreamingContext(sparkConf, Durations.seconds(1));
   Logger rootLogger = LogManager.getRootLogger();
 		rootLogger.setLevel(Level.WARN); 
   List<Tuple2<String, Integer>> tuples = Arrays.asList(new Tuple2<>("hello", 10), new Tuple2<>("world", 10));
   JavaPairRDD<String, Integer> initialRDD = streamingContext.sparkContext().parallelizePairs(tuples);
	    

   JavaReceiverInputDStream<String> StreamingLines = streamingContext.socketTextStream( "10.0.75.1", Integer.parseInt("9000"), StorageLevels.MEMORY_AND_DISK_SER);
   
   JavaDStream<String> words = StreamingLines.flatMap( str -> Arrays.asList(str.split(" ")).iterator() );
  
   JavaPairDStream<String, Integer> wordCounts = words.mapToPair(str-> new Tuple2<>(str, 1)).reduceByKey((count1,count2) ->count1+count2 );
  
   wordCounts.print();
   
JavaPairDStream<String, Integer> joinedDstream = wordCounts
		.transformToPair(new Function<JavaPairRDD<String, Integer>, JavaPairRDD<String, Integer>>() {
			@Override
			public JavaPairRDD<String, Integer> call(JavaPairRDD<String, Integer> rdd) throws Exception {
				JavaPairRDD<String, Integer> modRDD = rdd.join(initialRDD).mapToPair(
						new PairFunction<Tuple2<String, Tuple2<Integer, Integer>>, String, Integer>() {
							@Override
							public Tuple2<String, Integer> call(
									Tuple2<String, Tuple2<Integer, Integer>> joinedTuple) throws Exception {
								return new Tuple2<>(joinedTuple._1(),(joinedTuple._2()._1() + joinedTuple._2()._2()));
							}
						});
				return modRDD;
			}
		});

   joinedDstream.print();
   streamingContext.start();
   streamingContext.awaitTermination();
 }
 

import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
 System.setProperty("hadoop.home.dir", "E:\\hadoop");

   SparkConf sparkConf = new SparkConf().setAppName("WordCountSocketEx").setMaster("local[*]");
   JavaStreamingContext streamingContext = new JavaStreamingContext(sparkConf, Durations.seconds(1));
   streamingContext.checkpoint("E:\\hadoop\\checkpoint");
// Initial state RDD input to mapWithState
   @SuppressWarnings("unchecked")
   List<Tuple2<String, Integer>> tuples =Arrays.asList(new Tuple2<>("hello", 1), new Tuple2<>("world", 1));
   JavaPairRDD<String, Integer> initialRDD = streamingContext.sparkContext().parallelizePairs(tuples);
   
   JavaReceiverInputDStream<String> StreamingLines = streamingContext.socketTextStream( "10.0.75.1", Integer.parseInt("9000"), StorageLevels.MEMORY_AND_DISK_SER);
   
   JavaDStream<String> words = StreamingLines.flatMap( str -> Arrays.asList(str.split(" ")).iterator() );
  
   JavaPairDStream<String, Integer> wordCounts = words.mapToPair(str-> new Tuple2<>(str, 1)).reduceByKey((count1,count2) ->count1+count2 );
  


  // Update the cumulative count function
  Function3<String, Optional<Integer>, State<Integer>, Tuple2<String, Integer>> mappingFunc =
      new Function3<String, Optional<Integer>, State<Integer>, Tuple2<String, Integer>>() {
        @Override
        public Tuple2<String, Integer> call(String word, Optional<Integer> one,
            State<Integer> state) {
          int sum = one.orElse(0) + (state.exists() ? state.get() : 0);
          Tuple2<String, Integer> output = new Tuple2<>(word, sum);
          state.update(sum);
          return output;
        }
      };

  // DStream made of get cumulative counts that get updated in every batch
  JavaMapWithStateDStream<String, Integer, Integer, Tuple2<String, Integer>> stateDstream = wordCounts.mapWithState(StateSpec.function(mappingFunc).initialState(initialRDD));

  stateDstream.print();
  streamingContext.start();
  streamingContext.awaitTermination();
}
 

import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; //导入依赖的package包/类
public static void main(String[] args) throws InterruptedException {

		System.setProperty("hadoop.home.dir", "C:\\softwares\\Winutils");

		SparkSession sparkSession = SparkSession.builder().master("local[*]").appName("Stateful Streaming Example")
				.config("spark.sql.warehouse.dir", "file:////C:/Users/sgulati/spark-warehouse").getOrCreate();

		JavaStreamingContext jssc= new JavaStreamingContext(new JavaSparkContext(sparkSession.sparkContext()),
				Durations.milliseconds(1000));
		JavaReceiverInputDStream<String> inStream = jssc.socketTextStream("10.204.136.223", 9999);
		jssc.checkpoint("C:\\Users\\sgulati\\spark-checkpoint");

		JavaDStream<FlightDetails> flightDetailsStream = inStream.map(x -> {
			ObjectMapper mapper = new ObjectMapper();
			return mapper.readValue(x, FlightDetails.class);
		});
		
		

		JavaPairDStream<String, FlightDetails> flightDetailsPairStream = flightDetailsStream
				.mapToPair(f -> new Tuple2<String, FlightDetails>(f.getFlightId(), f));

		Function3<String, Optional<FlightDetails>, State<List<FlightDetails>>, Tuple2<String, Double>> mappingFunc = (
				flightId, curFlightDetail, state) -> {
			List<FlightDetails> details = state.exists() ? state.get() : new ArrayList<>();

			boolean isLanded = false;

			if (curFlightDetail.isPresent()) {
				details.add(curFlightDetail.get());
				if (curFlightDetail.get().isLanded()) {
					isLanded = true;
				}
			}
			Double avgSpeed = details.stream().mapToDouble(f -> f.getTemperature()).average().orElse(0.0);

			if (isLanded) {
				state.remove();
			} else {
				state.update(details);
			}
			return new Tuple2<String, Double>(flightId, avgSpeed);
		};

		JavaMapWithStateDStream<String, FlightDetails, List<FlightDetails>, Tuple2<String, Double>> streamWithState = flightDetailsPairStream
				.mapWithState(StateSpec.function(mappingFunc).timeout(Durations.minutes(5)));
		
		streamWithState.print();
		jssc.start();
		jssc.awaitTermination();
	}
 

import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; //导入依赖的package包/类
public static void main(String[] args) throws Exception {
 
     System.setProperty("hadoop.home.dir", "E:\\hadoop");
	
  SparkConf sparkConf = new SparkConf().setAppName("WordCountSocketEx").setMaster("local[*]");
  JavaStreamingContext streamingContext = new JavaStreamingContext(sparkConf, Durations.seconds(1));
  
  List<Tuple2<String, Integer>> tuples = Arrays.asList(new Tuple2<>("hello", 10), new Tuple2<>("world", 10));
  JavaPairRDD<String, Integer> initialRDD = streamingContext.sparkContext().parallelizePairs(tuples);
    

  JavaReceiverInputDStream<String> StreamingLines = streamingContext.socketTextStream( "10.0.75.1", Integer.parseInt("9000"), StorageLevels.MEMORY_AND_DISK_SER);
  
  JavaDStream<String> words = StreamingLines.flatMap( str -> Arrays.asList(str.split(" ")).iterator() );
 
  JavaPairDStream<String, Integer> wordCounts = words.mapToPair(str-> new Tuple2<>(str, 1)).reduceByKey((count1,count2) ->count1+count2 );
 
  wordCounts.print();
  
JavaPairDStream<String, Integer> joinedDstream = wordCounts.transformToPair(
   new Function<JavaPairRDD<String, Integer>, JavaPairRDD<String, Integer>>() {
	    @Override public JavaPairRDD<String, Integer> call(JavaPairRDD<String, Integer> rdd) throws Exception {
	    	rdd.join(initialRDD).mapToPair(new PairFunction<Tuple2<String,Tuple2<Integer,Integer>>, String, Integer>() {
				@Override
				public Tuple2<String, Integer> call(Tuple2<String, Tuple2<Integer, Integer>> joinedTuple)
						throws Exception {
					// TODO Auto-generated method stub
					return new Tuple2<>( joinedTuple._1(), (joinedTuple._2()._1()+joinedTuple._2()._2()) );
				}
			});
		
		return rdd; 				     
	    }
	  });
 
joinedDstream.print();
  streamingContext.start();
  streamingContext.awaitTermination();
}
 

import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; //导入依赖的package包/类
protected static JavaStreamingContext createContext(String ip, int port, String checkpointDirectory) {
	SparkConf sparkConf = new SparkConf().setAppName("WordCountRecoverableEx").setMaster("local[*]");
	JavaStreamingContext streamingContext = new JavaStreamingContext(sparkConf, Durations.seconds(1));
	streamingContext.checkpoint(checkpointDirectory);
	// Initial state RDD input to mapWithState
	@SuppressWarnings("unchecked")
	List<Tuple2<String, Integer>> tuples = Arrays.asList(new Tuple2<>("hello", 1), new Tuple2<>("world", 1));
	JavaPairRDD<String, Integer> initialRDD = streamingContext.sparkContext().parallelizePairs(tuples);

	JavaReceiverInputDStream<String> StreamingLines = streamingContext.socketTextStream(ip,port, StorageLevels.MEMORY_AND_DISK_SER);

	JavaDStream<String> words = StreamingLines.flatMap(str -> Arrays.asList(str.split(" ")).iterator());

	JavaPairDStream<String, Integer> wordCounts = words.mapToPair(str -> new Tuple2<>(str, 1))
			.reduceByKey((count1, count2) -> count1 + count2);

	// Update the cumulative count function
	Function3<String, Optional<Integer>, State<Integer>, Tuple2<String, Integer>> mappingFunc = new Function3<String, Optional<Integer>, State<Integer>, Tuple2<String, Integer>>() {
		@Override
		public Tuple2<String, Integer> call(String word, Optional<Integer> one, State<Integer> state) {
			int sum = one.orElse(0) + (state.exists() ? state.get() : 0);
			Tuple2<String, Integer> output = new Tuple2<>(word, sum);
			state.update(sum);
			return output;
		}
	};

	// DStream made of get cumulative counts that get updated in every batch
	JavaMapWithStateDStream<String, Integer, Integer, Tuple2<String, Integer>> stateDstream = wordCounts
			.mapWithState(StateSpec.function(mappingFunc).initialState(initialRDD));

	stateDstream.print();
	return streamingContext;
}
 

import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; //导入依赖的package包/类
@Test
public void testNoChannelsOrGames() {
    try {
        JavaReceiverInputDStream<Message> stream = new TwitchStreamBuilder()
                .setLanguage("es")
                .setStorageLevel(StorageLevel.MEMORY_AND_DISK_SER_2())
                .setSchedulingInterval(300)
                .build(jssc);
    } catch (IllegalStateException e) {
        Assert.assertEquals(e.getMessage(), NO_GAMES_OR_CHANNELS);

    }
}