Java DataStream.assignTimestampsAndWatermarks方法代碼示例

import org.apache.flink.streaming.api.datastream.DataStream; //導入方法依賴的package包/類
/***
 * Setup the kayed stream of a raw stream.
 * 
 * @param env
 * @param streamSource
 * @param parsingConfig
 * @return
 */
private static KeyedStream<Tuple3<String, Long, String>, Tuple> setupKayedRawMessagesStream(
    final StreamExecutionEnvironment env, String parsingConfig) {
  DataStream<Tuple3<String, Long, String>> rawStream =
      env.addSource(
          new FileLinesStreamSource(configs.getStringProp("aisDataSetFilePath"), parsingConfig,true))
          .flatMap(new RawStreamMapper(parsingConfig)).setParallelism(1);

  // assign the timestamp of the AIS messages based on their timestamps
  DataStream<Tuple3<String, Long, String>> rawStreamWithTimeStamp =
      rawStream.assignTimestampsAndWatermarks(new RawMessageTimestampAssigner());

  // Construct the keyed stream (i.e., trajectories stream) of the raw messages by grouping them
  // based on the message ID (MMSI for vessels)
  KeyedStream<Tuple3<String, Long, String>, Tuple> kaydAisMessagesStream =
      rawStreamWithTimeStamp.keyBy(0).process(new RawMessagesSorter()).keyBy(0);
  return kaydAisMessagesStream;
}
 

import org.apache.flink.streaming.api.datastream.DataStream; //導入方法依賴的package包/類
public static DataStream<EventPostComment> getEvents(StreamExecutionEnvironment env, AppConfiguration config) {
	String postSource = config.getPosts();
	String commentSource = config.getComments();
	
	DataStream<EventPostComment> events = null;
	
	if (postSource == null || commentSource == null) {
		List<EventPostComment> list = EventPostCommentStreamgen.getDefault();
		events = env.fromCollection(list); 
	} else {
		events = env.addSource(new EventPostCommentSource(postSource, commentSource), "events-pc-source");
	}			
	
	events.assignTimestampsAndWatermarks(new AscendingTimestamper<EventPostComment>());
	
	return events;		
}
 

import org.apache.flink.streaming.api.datastream.DataStream; //導入方法依賴的package包/類
public static DataStream<EventCommentFriendshipLike> getEvents(StreamExecutionEnvironment env, AppConfiguration config) {
	String commentSource = config.getComments();
	String friendshipSource = config.getFriendships();
	String likeSource = config.getLikes();		
	
	DataStream<EventCommentFriendshipLike> events = null;
	
	if (commentSource == null || friendshipSource == null || likeSource == null) {
		List<EventCommentFriendshipLike> list = EventCommentFriendshipLikeStreamgen.getDefault();
		events = env.fromCollection(list); 
	} else {
		events = env.addSource(new EventCommentFriendshipLikeSource(commentSource, friendshipSource, likeSource), "events-cfl-source");
	}			
	
	events.assignTimestampsAndWatermarks(new AscendingTimestamper<EventCommentFriendshipLike>());
	
	return events;		
}
 

import org.apache.flink.streaming.api.datastream.DataStream; //導入方法依賴的package包/類
@Test
public void testUnboundedPojoStreamAndReturnPojo() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	DataStream<Event> input = env.addSource(new RandomEventSource(5));
	input.assignTimestampsAndWatermarks(new AscendingTimestampExtractor<Event>() {
		@Override
		public long extractAscendingTimestamp(Event element) {
			return element.getTimestamp();
		}
	});

	DataStream<Event> output = SiddhiCEP
		.define("inputStream", input, "id", "name", "price", "timestamp")
		.cql("from inputStream select timestamp, id, name, price insert into  outputStream")
		.returns("outputStream", Event.class);

	String resultPath = tempFolder.newFile().toURI().toString();
	output.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);
	env.execute();
	assertEquals(5, getLineCount(resultPath));
}
 

import org.apache.flink.streaming.api.datastream.DataStream; //導入方法依賴的package包/類
@Test
public void testUnboundedPojoStreamAndReturnPojo() throws Exception {
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DataStream<Event> input = env.addSource(new RandomEventSource(5));
    input.assignTimestampsAndWatermarks(new AscendingTimestampExtractor<Event>() {
        @Override
        public long extractAscendingTimestamp(Event element) {
            return element.getTimestamp();
        }
    });

    DataStream<Event> output = SiddhiCEP
        .define("inputStream", input, "id", "name", "price", "timestamp")
        .cql("from inputStream select timestamp, id, name, price insert into  outputStream")
        .returns("outputStream", Event.class);

    String resultPath = tempFolder.newFile().toURI().toString();
    output.writeAsText(resultPath, FileSystem.WriteMode.OVERWRITE);
    env.execute();
    assertEquals(5, getLineCount(resultPath));
}
 

import org.apache.flink.streaming.api.datastream.DataStream; //導入方法依賴的package包/類
/**
 * Get the AIS messages stream from file or kafka stream
 * 
 * @param env
 * @param streamSource
 * @param filePathOrTopicProperty the data file path or the topic name of the input kafka stream
 * @param parsingConfig
 * @param areas
 * @return
 */
public static DataStream<AisMessage> getAisMessagesStream(StreamExecutionEnvironment env,
    StreamSourceType streamSource, String filePathOrTopicProperty, String parsingConfig,
    String outputLineDelimiter) {
  DataStream<AisMessage> aisMessagesStream = null;
  String fileOrTopicName = configs.getStringProp(filePathOrTopicProperty);
  switch (streamSource) {
    case KAFKA:
      Properties kafakaProps = getKafkaConsumerProperties();
      // create a Kafka consumer
      FlinkKafkaConsumer010<AisMessage> kafkaConsumer =
          new FlinkKafkaConsumer010<AisMessage>(fileOrTopicName, new AisMessageCsvSchema(
              parsingConfig, outputLineDelimiter), kafakaProps);

      kafkaConsumer.assignTimestampsAndWatermarks(new AisMessagesTimeAssigner());
      aisMessagesStream = env.addSource(kafkaConsumer);
      break;
    case FILE:

      DataStream<AisMessage> aisMessagesStreamWithoutTime =
          env.addSource(new FileLinesStreamSource(fileOrTopicName, parsingConfig))
              .flatMap(new CsvLineToAisMessageMapper(parsingConfig)).setParallelism(1);

      // Assign the timestamp of the AIS messages based on their timestamps
      aisMessagesStream =
          aisMessagesStreamWithoutTime
              .assignTimestampsAndWatermarks(new AisMessagesTimeAssigner());

      break;

    case HDFS:
      aisMessagesStream =
          env.readTextFile(fileOrTopicName).flatMap(new CsvLineToAisMessageMapper(parsingConfig))
              .assignTimestampsAndWatermarks(new AisMessagesTimeAssigner());
      break;
    default:
      return null;
  }
  return aisMessagesStream;
}
 

import org.apache.flink.streaming.api.datastream.DataStream; //導入方法依賴的package包/類
public static void main(String[] args) throws Exception {
    ParameterTool params = ParameterTool.fromArgs(args);
    FlinkPravegaParams helper = new FlinkPravegaParams(params);
    StreamId stream = helper.createStreamFromParam("input", "examples/turbineHeatTest");

    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);

    // 1. read and decode the sensor events from a Pravega stream
    long startTime = params.getLong("start", 0L);
    FlinkPravegaReader<String> reader = helper.newReader(stream, startTime, String.class);
    DataStream<SensorEvent> events = env.addSource(reader, "input").map(new SensorMapper()).name("events");

    // 2. extract timestamp information to support 'event-time' processing
    SingleOutputStreamOperator<SensorEvent> timestamped = events.assignTimestampsAndWatermarks(
            new BoundedOutOfOrdernessTimestampExtractor<SensorEvent>(Time.seconds(10)) {
        @Override
        public long extractTimestamp(SensorEvent element) {
            return element.getTimestamp();
        }
    });
    timestamped.print();

    // 3. summarize the temperature data for each sensor
    SingleOutputStreamOperator<SensorAggregate> summaries = timestamped
            .keyBy("sensorId")
            .window(TumblingEventTimeWindows.of(Time.days(1), Time.hours(8)))
            .fold(null, new SensorAggregator()).name("summaries");

    // 4. save to HDFS and print to stdout.  Refer to the TaskManager's 'Stdout' view in the Flink UI.
    summaries.print().name("stdout");
    if (params.has("output")) {
        summaries.writeAsCsv(params.getRequired("output"), FileSystem.WriteMode.OVERWRITE);
    }

    env.execute("TurbineHeatProcessor_" + stream);
}
 

import org.apache.flink.streaming.api.datastream.DataStream; //導入方法依賴的package包/類
/**
 * This tests whether timestamps are properly extracted in the timestamp
 * extractor and whether watermarks are also correctly forwarded from this with the auto watermark
 * interval.
 */
@Test
public void testTimestampExtractorWithAutoInterval() throws Exception {
	final int numElements = 10;

	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();

	env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
	env.getConfig().setAutoWatermarkInterval(10);
	env.setParallelism(1);
	env.getConfig().disableSysoutLogging();

	DataStream<Integer> source1 = env.addSource(new SourceFunction<Integer>() {
		@Override
		public void run(SourceContext<Integer> ctx) throws Exception {
			int index = 1;
			while (index <= numElements) {
				ctx.collect(index);
				latch.await();
				index++;
			}
		}

		@Override
		public void cancel() {}
	});

	DataStream<Integer> extractOp = source1.assignTimestampsAndWatermarks(
			new AscendingTimestampExtractor<Integer>() {
				@Override
				public long extractAscendingTimestamp(Integer element) {
					return element;
				}
			});

	extractOp
			.transform("Watermark Check", BasicTypeInfo.INT_TYPE_INFO, new CustomOperator(true))
			.transform("Timestamp Check",
					BasicTypeInfo.INT_TYPE_INFO,
					new TimestampCheckingOperator());

	// verify that extractor picks up source parallelism
	Assert.assertEquals(extractOp.getTransformation().getParallelism(), source1.getTransformation().getParallelism());

	env.execute();

	// verify that we get NUM_ELEMENTS watermarks
	for (int j = 0; j < numElements; j++) {
		if (!CustomOperator.finalWatermarks[0].get(j).equals(new Watermark(j))) {
			long wm = CustomOperator.finalWatermarks[0].get(j).getTimestamp();
			Assert.fail("Wrong watermark. Expected: " + j + " Found: " + wm + " All: " + CustomOperator.finalWatermarks[0]);
		}
	}

	// the input is finite, so it should have a MAX Watermark
	assertEquals(Watermark.MAX_WATERMARK,
			CustomOperator.finalWatermarks[0].get(CustomOperator.finalWatermarks[0].size() - 1));
}