Java Watermark类代码示例

import org.apache.flink.streaming.api.watermark.Watermark; //导入依赖的package包/类
@Test
public void testCount() throws Exception {
	StreamCounter<String> operator = new StreamCounter<String>();

	OneInputStreamOperatorTestHarness<String, Long> testHarness = new OneInputStreamOperatorTestHarness<String, Long>(operator);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<String>("eins", initialTime + 1));
	testHarness.processElement(new StreamRecord<String>("zwei", initialTime + 2));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<String>("drei", initialTime + 3));

	expectedOutput.add(new StreamRecord<Long>(1L, initialTime + 1));
	expectedOutput.add(new StreamRecord<Long>(2L, initialTime + 2));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<Long>(3L, initialTime + 3));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 

import org.apache.flink.streaming.api.watermark.Watermark; //导入依赖的package包/类
private void findAndOutputNewMinWatermarkAcrossAlignedChannels() {
	long newMinWatermark = Long.MAX_VALUE;
	boolean hasAlignedChannels = false;

	// determine new overall watermark by considering only watermark-aligned channels across all channels
	for (InputChannelStatus channelStatus : channelStatuses) {
		if (channelStatus.isWatermarkAligned) {
			hasAlignedChannels = true;
			newMinWatermark = Math.min(channelStatus.watermark, newMinWatermark);
		}
	}

	// we acknowledge and output the new overall watermark if it really is aggregated
	// from some remaining aligned channel, and is also larger than the last output watermark
	if (hasAlignedChannels && newMinWatermark > lastOutputWatermark) {
		lastOutputWatermark = newMinWatermark;
		outputHandler.handleWatermark(new Watermark(lastOutputWatermark));
	}
}
 

import org.apache.flink.streaming.api.watermark.Watermark; //导入依赖的package包/类
private Tuple2<Long, Long> validateElement(Object element, long nextElementValue, long currentWatermark) {
	if (element instanceof StreamRecord) {
		@SuppressWarnings("unchecked")
		StreamRecord<Long> record = (StreamRecord<Long>) element;
		assertEquals(nextElementValue, record.getValue().longValue());
		assertEquals(nextElementValue, record.getTimestamp());
		return new Tuple2<>(nextElementValue + 1, currentWatermark);
	}
	else if (element instanceof Watermark) {
		long wt = ((Watermark) element).getTimestamp();
		assertTrue(wt > currentWatermark);
		return new Tuple2<>(nextElementValue, wt);
	}
	else {
		throw new IllegalArgumentException("unrecognized element: " + element);
	}
}
 

import org.apache.flink.streaming.api.watermark.Watermark; //导入依赖的package包/类
@Override
public int compare(Object o1, Object o2) {
	if (o1 instanceof Watermark || o2 instanceof Watermark) {
		return 0;
	} else {
		StreamRecord<Tuple2<String, Integer>> sr0 = (StreamRecord<Tuple2<String, Integer>>) o1;
		StreamRecord<Tuple2<String, Integer>> sr1 = (StreamRecord<Tuple2<String, Integer>>) o2;
		if (sr0.getTimestamp() != sr1.getTimestamp()) {
			return (int) (sr0.getTimestamp() - sr1.getTimestamp());
		}
		int comparison = sr0.getValue().f0.compareTo(sr1.getValue().f0);
		if (comparison != 0) {
			return comparison;
		} else {
			return sr0.getValue().f1 - sr1.getValue().f1;
		}
	}
}
 

import org.apache.flink.streaming.api.watermark.Watermark; //导入依赖的package包/类
@Override
public void run(SourceContext<Tuple2<Long, Long>> ctx) throws Exception {

	ctx.emitWatermark(new Watermark(0));

	synchronized (ctx.getCheckpointLock()) {
		for (long i = 0; i < numElements; i++) {
			ctx.collect(new Tuple2<>(i, i));
		}
	}

	// don't emit a final watermark so that we don't trigger the registered event-time
	// timers
	while (isRunning) {
		Thread.sleep(20);
	}
}
 

import org.apache.flink.streaming.api.watermark.Watermark; //导入依赖的package包/类
@Override
public void onProcessingTime(long timestamp) throws Exception {

	long minAcrossAll = Long.MAX_VALUE;
	for (KafkaTopicPartitionStateWithPeriodicWatermarks<?, ?> state : allPartitions) {
		
		// we access the current watermark for the periodic assigners under the state
		// lock, to prevent concurrent modification to any internal variables
		final long curr;
		//noinspection SynchronizationOnLocalVariableOrMethodParameter
		synchronized (state) {
			curr = state.getCurrentWatermarkTimestamp();
		}
		
		minAcrossAll = Math.min(minAcrossAll, curr);
	}
	
	// emit next watermark, if there is one
	if (minAcrossAll > lastWatermarkTimestamp) {
		lastWatermarkTimestamp = minAcrossAll;
		emitter.emitWatermark(new Watermark(minAcrossAll));
	}
	
	// schedule the next watermark
	timerService.registerTimer(timerService.getCurrentProcessingTime() + interval, this);
}
 

import org.apache.flink.streaming.api.watermark.Watermark; //导入依赖的package包/类
@Test
public void testKeyedCEPOperatorWatermarkForwarding() throws Exception {

	OneInputStreamOperatorTestHarness<Event, Map<String, List<Event>>> harness = getCepTestHarness(false);

	try {
		harness.open();

		Watermark expectedWatermark = new Watermark(42L);

		harness.processWatermark(expectedWatermark);

		verifyWatermark(harness.getOutput().poll(), 42L);
	} finally {
		harness.close();
	}
}
 

import org.apache.flink.streaming.api.watermark.Watermark; //导入依赖的package包/类
/**
 * Tests that the watermark of an input channel remains intact while in the IDLE status.
 */
@Test
public void testSingleInputWatermarksIntactDuringIdleness() {
	BufferedValveOutputHandler valveOutput = new BufferedValveOutputHandler();
	StatusWatermarkValve valve = new StatusWatermarkValve(1, valveOutput);

	valve.inputWatermark(new Watermark(25), 0);
	assertEquals(new Watermark(25), valveOutput.popLastSeenOutput());
	assertEquals(null, valveOutput.popLastSeenOutput());

	valve.inputStreamStatus(StreamStatus.IDLE, 0);
	assertEquals(StreamStatus.IDLE, valveOutput.popLastSeenOutput());

	valve.inputWatermark(new Watermark(50), 0);
	assertEquals(null, valveOutput.popLastSeenOutput());
	assertEquals(25, valve.getInputChannelStatus(0).watermark);

	valve.inputStreamStatus(StreamStatus.ACTIVE, 0);
	assertEquals(StreamStatus.ACTIVE, valveOutput.popLastSeenOutput());
	assertEquals(null, valveOutput.popLastSeenOutput());

	valve.inputWatermark(new Watermark(50), 0);
	assertEquals(new Watermark(50), valveOutput.popLastSeenOutput());
	assertEquals(null, valveOutput.popLastSeenOutput());
}
 

import org.apache.flink.streaming.api.watermark.Watermark; //导入依赖的package包/类
@Test
public void testMap() throws Exception {
	StreamMap<Integer, String> operator = new StreamMap<Integer, String>(new Map());

	OneInputStreamOperatorTestHarness<Integer, String> testHarness = new OneInputStreamOperatorTestHarness<Integer, String>(operator);

	long initialTime = 0L;
	ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<Object>();

	testHarness.open();

	testHarness.processElement(new StreamRecord<Integer>(1, initialTime + 1));
	testHarness.processElement(new StreamRecord<Integer>(2, initialTime + 2));
	testHarness.processWatermark(new Watermark(initialTime + 2));
	testHarness.processElement(new StreamRecord<Integer>(3, initialTime + 3));

	expectedOutput.add(new StreamRecord<String>("+2", initialTime + 1));
	expectedOutput.add(new StreamRecord<String>("+3", initialTime + 2));
	expectedOutput.add(new Watermark(initialTime + 2));
	expectedOutput.add(new StreamRecord<String>("+4", initialTime + 3));

	TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput());
}
 

import org.apache.flink.streaming.api.watermark.Watermark; //导入依赖的package包/类
@Override
public int compare(Object o1, Object o2) {
	if (o1 instanceof Watermark || o2 instanceof Watermark) {
		return 0;
	} else {
		StreamRecord<Integer> sr0 = (StreamRecord<Integer>) o1;
		StreamRecord<Integer> sr1 = (StreamRecord<Integer>) o2;

		if (sr0.getTimestamp() != sr1.getTimestamp()) {
			return (int) (sr0.getTimestamp() - sr1.getTimestamp());
		}

		int comparison = sr0.getValue().compareTo(sr1.getValue());
		if (comparison != 0) {
			return comparison;
		} else {
			return sr0.getValue() - sr1.getValue();
		}
	}
}
 

import org.apache.flink.streaming.api.watermark.Watermark; //导入依赖的package包/类
/**
 * Tests that for multiple inputs, when some inputs are idle, the min watermark
 * is correctly computed and advanced from the remaining active inputs.
 */
@Test
public void testMultipleInputWatermarkAdvancingWithPartiallyIdleChannels() {
	BufferedValveOutputHandler valveOutput = new BufferedValveOutputHandler();
	StatusWatermarkValve valve = new StatusWatermarkValve(3, valveOutput);

	valve.inputWatermark(new Watermark(15), 0);
	valve.inputWatermark(new Watermark(10), 1);
	assertEquals(null, valveOutput.popLastSeenOutput());

	valve.inputStreamStatus(StreamStatus.IDLE, 2);
	// min watermark should be computed from remaining ACTIVE channels
	assertEquals(new Watermark(10), valveOutput.popLastSeenOutput());
	assertEquals(null, valveOutput.popLastSeenOutput());

	valve.inputWatermark(new Watermark(18), 1);
	// now, min watermark should be 15 from channel #0
	assertEquals(new Watermark(15), valveOutput.popLastSeenOutput());
	assertEquals(null, valveOutput.popLastSeenOutput());

	valve.inputWatermark(new Watermark(20), 0);
	// now, min watermark should be 18 from channel #1
	assertEquals(new Watermark(18), valveOutput.popLastSeenOutput());
	assertEquals(null, valveOutput.popLastSeenOutput());
}
 

import org.apache.flink.streaming.api.watermark.Watermark; //导入依赖的package包/类
@Test
public void testSerialization() throws Exception {
	final MultiplexingStreamRecordSerializer<String> serializer = 
			new MultiplexingStreamRecordSerializer<String>(StringSerializer.INSTANCE);

	StreamRecord<String> withoutTimestamp = new StreamRecord<>("test 1 2 分享基督耶穌的愛給們，開拓雙贏!");
	assertEquals(withoutTimestamp, serializeAndDeserialize(withoutTimestamp, serializer));

	StreamRecord<String> withTimestamp = new StreamRecord<>("one more test 拓 們 分", 77L);
	assertEquals(withTimestamp, serializeAndDeserialize(withTimestamp, serializer));

	StreamRecord<String> negativeTimestamp = new StreamRecord<>("他", Long.MIN_VALUE);
	assertEquals(negativeTimestamp, serializeAndDeserialize(negativeTimestamp, serializer));

	Watermark positiveWatermark = new Watermark(13);
	assertEquals(positiveWatermark, serializeAndDeserialize(positiveWatermark, serializer));

	Watermark negativeWatermark = new Watermark(-4647654567676555876L);
	assertEquals(negativeWatermark, serializeAndDeserialize(negativeWatermark, serializer));
}
 

import org.apache.flink.streaming.api.watermark.Watermark; //导入依赖的package包/类
/**
 * Record emission, if a timestamp will be attached from an assigner that is
 * also a punctuated watermark generator.
 */
protected void emitRecordWithTimestampAndPunctuatedWatermark(
		T record, KafkaTopicPartitionState<KPH> partitionState, long offset, long kafkaEventTimestamp)
{
	@SuppressWarnings("unchecked")
	final KafkaTopicPartitionStateWithPunctuatedWatermarks<T, KPH> withWatermarksState =
			(KafkaTopicPartitionStateWithPunctuatedWatermarks<T, KPH>) partitionState;

	// only one thread ever works on accessing timestamps and watermarks
	// from the punctuated extractor
	final long timestamp = withWatermarksState.getTimestampForRecord(record, kafkaEventTimestamp);
	final Watermark newWatermark = withWatermarksState.checkAndGetNewWatermark(record, timestamp);

	// emit the record with timestamp, using the usual checkpoint lock to guarantee
	// atomicity of record emission and offset state update 
	synchronized (checkpointLock) {
		sourceContext.collectWithTimestamp(record, timestamp);
		partitionState.setOffset(offset);
	}

	// if we also have a new per-partition watermark, check if that is also a
	// new cross-partition watermark
	if (newWatermark != null) {
		updateMinPunctuatedWatermark(newWatermark);
	}
}
 

import org.apache.flink.streaming.api.watermark.Watermark; //导入依赖的package包/类
@Override
public Watermark checkAndGetNextWatermark(String arg0, long arg1) {
	if (null != arg0 && arg0.contains(",")) {
		String parts[] = arg0.split(",");
		return new Watermark(Long.parseLong(parts[0]));
	}
	return null;
}
 

import org.apache.flink.streaming.api.watermark.Watermark; //导入依赖的package包/类
@Override
public void run(SourceContext<UnkeyedDataPoint> ctx) throws Exception {
	while (running) {
		synchronized (ctx.getCheckpointLock()) {
			double stepHeight = (double) currentStep / numberOfSteps;
			currentStep = ++currentStep % numberOfSteps;

			ctx.collectWithTimestamp(new UnkeyedDataPoint(currentTimeMs, stepHeight), currentTimeMs);
			ctx.emitWatermark(new Watermark(currentTimeMs));
			currentTimeMs += intervalMillis;
		}
		timeSync();
	}
}